Squashed 'external/toxcore/c-toxcore/' content from commit 67badf69

git-subtree-dir: external/toxcore/c-toxcore
git-subtree-split: 67badf69416a74e74f6d7eb51dd96f37282b8455

other/docker/coverage/Dockerfile

@@ -0,0 +1,78 @@
+FROM toxchat/c-toxcore:sources AS src
+FROM ubuntu:20.04 AS build
+
+RUN apt-get update && \
+ DEBIAN_FRONTEND="noninteractive" apt-get install -y --no-install-recommends \
+ clang \
+ cmake \
+ gcc \
+ git \
+ golang \
+ libconfig-dev \
+ libgtest-dev \
+ libopus-dev \
+ libsodium-dev \
+ libvpx-dev \
+ llvm-dev \
+ make \
+ ninja-build \
+ pkg-config \
+ python3-pip \
+ python3-pygments \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/* \
+ && pip3 install --no-cache-dir gcovr
+
+ENV CC=clang \
+    CXX=clang++ \
+    PYTHONUNBUFFERED=1
+SHELL ["/bin/bash", "-c"]
+
+WORKDIR /work
+COPY --from=src /src/ /work/
+RUN source .github/scripts/flags-coverage.sh \
+ && go get github.com/things-go/go-socks5 \
+ && go build other/proxy/proxy_server.go \
+ && cmake -B_build -H. -GNinja \
+ -DCMAKE_C_FLAGS="$C_FLAGS" \
+ -DCMAKE_CXX_FLAGS="$CXX_FLAGS" \
+ -DCMAKE_EXE_LINKER_FLAGS="$LD_FLAGS" \
+ -DCMAKE_UNITY_BUILD=ON \
+ -DENABLE_SHARED=OFF \
+ -DMIN_LOGGER_LEVEL=TRACE \
+ -DMUST_BUILD_TOXAV=ON \
+ -DNON_HERMETIC_TESTS=ON \
+ -DSTRICT_ABI=ON \
+ -DAUTOTEST=ON \
+ -DPROXY_TEST=ON \
+ -DUSE_IPV6=OFF \
+ && cmake --build _build --parallel 8 --target install
+
+WORKDIR /work/_build
+RUN /work/proxy_server \
+ & ctest -j50 --output-on-failure --rerun-failed --repeat until-pass:6
+
+WORKDIR /work/mallocfail
+RUN ["git", "clone", "--depth=1", "https://github.com/ralight/mallocfail", "/work/mallocfail"]
+COPY run_mallocfail /usr/local/bin/
+COPY syscall_funcs.c src/
+RUN gcc -fPIC -shared -O2 -g3 -Wall -Ideps/uthash -Ideps/sha3 deps/*/*.c src/*.c -o mallocfail.so -ldl -lbacktrace \
+ && install mallocfail.so /usr/local/lib/mallocfail.so
+
+WORKDIR /work/_build
+RUN ["run_mallocfail", "--ctest=2", "--jobs=8"]
+RUN ["gcovr", \
+ "--sort-percentage", \
+ "--gcov-executable=llvm-cov gcov", \
+ "--html-details=html/", \
+ "--root=..", \
+ "--exclude=CMakeFiles/", \
+ "--exclude=_deps/", \
+ "--exclude=(.+/)?auto_tests/", \
+ "--exclude=.+_test.cc?$", \
+ "--exclude=(.+/)?other/", \
+ "--exclude=(.+/)?testing/"]
+
+FROM nginx:alpine
+COPY --from=build /work/_build/html/coverage_details.html /usr/share/nginx/html/index.html
+COPY --from=build /work/_build/html/ /usr/share/nginx/html/

other/docker/coverage/mallocfail.h

@@ -0,0 +1,3 @@
+// Dummy, actual one is here: https://github.com/ralight/mallocfail/blob/master/src/mallocfail.h
+
+int should_malloc_fail(void);

other/docker/coverage/run

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -eux
+
+docker build -t toxchat/c-toxcore:sources -f other/docker/sources/Dockerfile .
+docker build -t toxchat/c-toxcore:coverage other/docker/coverage
+docker run --rm -it -p "28192:80" toxchat/c-toxcore:coverage

other/docker/coverage/run_mallocfail

@@ -0,0 +1,166 @@
+#!/usr/bin/env python3
+"""Run a test repeatedly with mallocfail.
+
+Usage: run_mallocfail [--ctest=<cost>] [<exe>...]
+
+This runs the programs with mallocfail until there are no more additional
+stack hashes for mallocfail to try out.
+
+Passing "--ctest" additionally runs all the tests with a cost lower than the
+given flag value. Cost is measured in seconds of runtime.
+
+You need to build mallocfail (https://github.com/ralight/mallocfail) and install
+it to /usr/local/lib/mallocfail. Change _MALLOCFAIL_SO below if you want it
+elsewhere.
+"""
+import glob
+import multiprocessing
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import time
+from typing import Dict
+from typing import List
+from typing import NoReturn
+from typing import Optional
+from typing import Tuple
+
+_PRIMER = "./unit_util_test"
+_MALLOCFAIL_SO = "/usr/local/lib/mallocfail.so"
+_HASHES = "mallocfail_hashes"
+_HASHES_PREV = "mallocfail_hashes.prev"
+_TIMEOUT = 3.0
+
+_ENV = {
+    "LD_PRELOAD": _MALLOCFAIL_SO,
+    "UBSAN_OPTIONS": "color=always,print_stacktrace=1,exitcode=11",
+}
+
+
+def run_mallocfail(tmpdir: str, timeout: float, exe: str,
+                   iteration: int) -> bool:
+    """Run a program with mallocfail."""
+    print(f"\x1b[1;33mmallocfail '{exe}' run #{iteration}\x1b[0m")
+    hashes = os.path.join(tmpdir, _HASHES)
+    hashes_prev = os.path.join(tmpdir, _HASHES_PREV)
+    if os.path.exists(hashes):
+        shutil.copy(hashes, hashes_prev)
+    try:
+        proc = subprocess.run([exe], timeout=timeout, env=_ENV, cwd=tmpdir)
+    except subprocess.TimeoutExpired:
+        print(f"\x1b[1;34mProgram {exe} timed out\x1b[0m")
+        return True
+    finally:
+        assert os.path.exists(hashes)
+        if os.path.exists(hashes_prev):
+            with open(hashes_prev, "r") as prev:
+                with open(hashes, "r") as cur:
+                    if prev.read() == cur.read():
+                        # Done: no new stack hashes.
+                        return False
+
+    if proc.returncode in (0, 1):
+        # Process exited cleanly (success or failure).
+        pass
+    elif proc.returncode == -6:
+        # Assertion failed.
+        pass
+    elif proc.returncode == -14:
+        print(f"\x1b[0;34mProgram '{exe}' timed out\x1b[0m")
+    else:
+        print(
+            f"\x1b[1;32mProgram '{exe}' failed to handle OOM situation cleanly\x1b[0m"
+        )
+        raise Exception("Aborting test")
+
+    return True
+
+
+def ctest_costs() -> Dict[str, float]:
+    with open("Testing/Temporary/CTestCostData.txt", "r") as fh:
+        costs = {}
+        for line in fh.readlines():
+            if line.startswith("---"):
+                break
+            prog, _, cost = line.rstrip().split(" ")
+            costs[prog] = float(cost)
+        return costs
+
+
+def find_prog(name: str) -> Tuple[Optional[str], ...]:
+    def attempt(path: str) -> Optional[str]:
+        if os.path.exists(path):
+            return path
+        return None
+
+    return (attempt(f"./unit_{name}_test"), attempt(f"./auto_{name}_test"))
+
+
+def parse_flags(args: List[str]) -> Tuple[Dict[str, str], List[str]]:
+    flags: Dict[str, str] = {}
+    exes: List[str] = []
+    for arg in args:
+        if arg.startswith("--"):
+            flag, value = arg.split("=", 1)
+            flags[flag] = value
+        else:
+            exes.append(arg)
+    return flags, exes
+
+
+def loop_mallocfail(tmpdir: str, timeout: float, exe: str) -> None:
+    i = 1
+    while run_mallocfail(tmpdir, timeout, exe, i):
+        i += 1
+
+
+def isolated_mallocfail(timeout: int, exe: str) -> None:
+    with tempfile.TemporaryDirectory(prefix="mallocfail") as tmpdir:
+        print(f"\x1b[1;33mRunning for {exe} in isolated path {tmpdir}\x1b[0m")
+        shutil.copy(exe, os.path.join(tmpdir, exe))
+        shutil.copy(_HASHES, os.path.join(tmpdir, _HASHES))
+        loop_mallocfail(tmpdir, timeout, exe)
+
+
+def main(args: List[str]) -> None:
+    """Run a program repeatedly under mallocfail."""
+    if len(args) == 1:
+        print(f"Usage: {args[0]} <exe>")
+        sys.exit(1)
+
+    flags, exes = parse_flags(args[1:])
+
+    timeout = _TIMEOUT
+    if "--ctest" in flags:
+        costs = ctest_costs()
+        max_cost = float(flags["--ctest"])
+        timeout = max(max_cost + 1, timeout)
+        exes.extend(prog for test in costs.keys() for prog in find_prog(test)
+                    if costs[test] <= max_cost and prog)
+    if "--jobs" in flags:
+        jobs = int(flags["--jobs"])
+    else:
+        jobs = 1
+
+    # Start by running util_test, which allocates no memory of its own, just
+    # to prime the mallocfail hashes so it doesn't make global initialisers
+    # such as llvm_gcov_init fail.
+    if os.path.exists(_PRIMER):
+        print(f"\x1b[1;33mPriming hashes with unit_util_test\x1b[0m")
+        loop_mallocfail(".", timeout, _PRIMER)
+
+    print(f"\x1b[1;33m--------------------------------\x1b[0m")
+    print(f"\x1b[1;33mStarting mallocfail for {len(exes)} programs:\x1b[0m")
+    print(f"\x1b[1;33m{exes}\x1b[0m")
+    print(f"\x1b[1;33m--------------------------------\x1b[0m")
+    time.sleep(1)
+    with multiprocessing.Pool(jobs) as p:
+        done = tuple(
+            p.starmap(isolated_mallocfail, ((timeout, exe) for exe in exes)))
+        print(f"\x1b[1;32mCompleted {len(done)} programs\x1b[0m")
+
+
+if __name__ == "__main__":
+    main(sys.argv)

other/docker/coverage/syscall_funcs.c

@@ -0,0 +1,149 @@
+#define _GNU_SOURCE
+
+#include "mallocfail.h"
+
+#include <dlfcn.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <stdarg.h>
+#include <sys/ioctl.h>
+#include <sys/socket.h>
+#include <sys/types.h>
+
+static int (*libc_ioctl)(int fd, unsigned long request, ...);
+static int (*libc_bind)(int sockfd, const struct sockaddr *addr,
+        socklen_t addrlen);
+static int (*libc_getsockopt)(int sockfd, int level, int optname,
+        void *optval, socklen_t *optlen);
+static int (*libc_setsockopt)(int sockfd, int level, int optname,
+        const void *optval, socklen_t optlen);
+static ssize_t (*libc_recv)(int sockfd, void *buf, size_t len, int flags);
+static ssize_t (*libc_recvfrom)(int sockfd, void *buf, size_t len, int flags,
+        struct sockaddr *src_addr, socklen_t *addrlen);
+static ssize_t (*libc_send)(int sockfd, const void *buf, size_t len, int flags);
+static ssize_t(*libc_sendto)(int sockfd, const void *buf, size_t len, int flags,
+        const struct sockaddr *dest_addr, socklen_t addrlen);
+static int (*libc_socket)(int domain, int type, int protocol);
+static int (*libc_listen)(int sockfd, int backlog);
+
+__attribute__((__constructor__))
+static void init(void)
+{
+    libc_ioctl = dlsym(RTLD_NEXT, "ioctl");
+    libc_bind = dlsym(RTLD_NEXT, "bind");
+    libc_getsockopt = dlsym(RTLD_NEXT, "getsockopt");
+    libc_setsockopt = dlsym(RTLD_NEXT, "setsockopt");
+    libc_recv = dlsym(RTLD_NEXT, "recv");
+    libc_recvfrom = dlsym(RTLD_NEXT, "recvfrom");
+    libc_send = dlsym(RTLD_NEXT, "send");
+    libc_sendto = dlsym(RTLD_NEXT, "sendto");
+    libc_socket = dlsym(RTLD_NEXT, "socket");
+    libc_listen = dlsym(RTLD_NEXT, "listen");
+}
+
+int ioctl(int fd, unsigned long request, ...)
+{
+    if (should_malloc_fail()) {
+        errno = ENOMEM;
+        return -1;
+    }
+
+    va_list ap;
+    va_start(ap, request);
+    const int ret = libc_ioctl(fd, SIOCGIFCONF, va_arg(ap, void *));
+    va_end(ap);
+    return ret;
+}
+
+int bind(int sockfd, const struct sockaddr *addr,
+        socklen_t addrlen)
+{
+    // Unlike all others, if bind should fail once, it should fail always, because in toxcore we try
+    // many ports before giving up. If this only fails once, we'll never reach the code path where
+    // we give up.
+    static int should_fail = -1;
+    if (should_fail == -1) {
+        should_fail = should_malloc_fail();
+    }
+    if (should_fail) {
+        errno = ENOMEM;
+        return -1;
+    }
+    return libc_bind(sockfd, addr, addrlen);
+}
+
+int getsockopt(int sockfd, int level, int optname,
+        void *optval, socklen_t *optlen)
+{
+    if (should_malloc_fail()) {
+        errno = ENOMEM;
+        return -1;
+    }
+    return libc_getsockopt(sockfd, level, optname, optval, optlen);
+}
+
+int setsockopt(int sockfd, int level, int optname,
+        const void *optval, socklen_t optlen)
+{
+    if (should_malloc_fail()) {
+        errno = ENOMEM;
+        return -1;
+    }
+    return libc_setsockopt(sockfd, level, optname, optval, optlen);
+}
+
+ssize_t recv(int sockfd, void *buf, size_t len, int flags)
+{
+    if (should_malloc_fail()) {
+        errno = ENOMEM;
+        return -1;
+    }
+    return libc_recv(sockfd, buf, len, flags);
+}
+
+ssize_t recvfrom(int sockfd, void *buf, size_t len, int flags,
+        struct sockaddr *src_addr, socklen_t *addrlen)
+{
+    if (should_malloc_fail()) {
+        errno = ENOMEM;
+        return -1;
+    }
+    return libc_recvfrom(sockfd, buf, len, flags, src_addr, addrlen);
+}
+
+ssize_t send(int sockfd, const void *buf, size_t len, int flags)
+{
+    if (should_malloc_fail()) {
+        errno = ENOMEM;
+        return -1;
+    }
+    return libc_send(sockfd, buf, len, flags);
+}
+
+ssize_t sendto(int sockfd, const void *buf, size_t len, int flags,
+        const struct sockaddr *dest_addr, socklen_t addrlen)
+{
+    if (should_malloc_fail()) {
+        errno = ENOMEM;
+        return -1;
+    }
+    return libc_sendto(sockfd, buf, len, flags, dest_addr, addrlen);
+}
+
+int socket(int domain, int type, int protocol)
+{
+    if (should_malloc_fail()) {
+        errno = ENOMEM;
+        return -1;
+    }
+    return libc_socket(domain, type, protocol);
+}
+
+int listen(int sockfd, int backlog)
+{
+    if (should_malloc_fail()) {
+        errno = ENOMEM;
+        return -1;
+    }
+    return libc_listen(sockfd, backlog);
+}