Tha_14/tomato-testing
1
0
Fork 0
forked from Green-Sky/tomato
Code Pull Requests Actions Packages Releases Activity

Squashed 'external/toxcore/c-toxcore/' changes from 1828c5356..c9cdae001

Browse Source 
c9cdae001 fix(toxav): remove extra copy of video frame on encode
4f6d4546b test: Improve the fake network library.
a2581e700 refactor(toxcore): generate `Friend_Request` and `Dht_Nodes_Response`
2aaa11770 refactor(toxcore): use Tox_Memory in generated events
5c367452b test(toxcore): fix incorrect mutex in tox_scenario_get_time
8f92e710f perf: Add a timed limit of number of cookie requests.
695b6417a test: Add some more simulated network support.
815ae9ce9 test(toxcore): fix thread-safety in scenario framework
6d85c754e test(toxcore): add unit tests for net_crypto
9c22e79cc test(support): add SimulatedEnvironment for deterministic testing
f34fcb195 chore: Update windows Dockerfile to debian stable (trixie).
ece0e8980 fix(group_moderation): allow validating unsorted sanction list signatures
a4fa754d7 refactor: rename struct Packet to struct Net_Packet
d6f330f85 cleanup: Fix some warnings from coverity.
e206bffa2 fix(group_chats): fix sync packets reverting topics
0e4715598 test: Add new scenario testing framework.
668291f44 refactor(toxcore): decouple Network_Funcs from sockaddr via IP_Port
fc4396cef fix: potential division by zero in toxav and unsafe hex parsing
8e8b352ab refactor: Add nullable annotations to struct members.
7740bb421 refactor: decouple net_crypto from DHT
1936d4296 test: add benchmark for toxav audio and video
46bfdc2df fix: correct printf format specifiers for unsigned integers
REVERT: 1828c5356 fix(toxav): remove extra copy of video frame on encode

git-subtree-dir: external/toxcore/c-toxcore
git-subtree-split: c9cdae001341e701fca980c9bb9febfeb95d2902
This commit is contained in:
Green Sky
2026-01-11 14:42:31 +01:00
parent e95f2cbb1c
commit 565efa4f39
328 changed files with 19057 additions and 13982 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
toxav/BUILD.bazel
View File

@@ -1,6 +1,5 @@
load("@rules_cc//cc:defs.bzl", "cc_test")
load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test")
load("@rules_fuzzing//fuzzing:cc_defs.bzl", "cc_fuzz_test")
load("//tools:no_undefined.bzl", "cc_library")
exports_files(
srcs = ["toxav.h"],
@@ -80,7 +79,7 @@ cc_fuzz_test(
copts = ["-UNDEBUG"],
deps = [
":rtp",
"//c-toxcore/testing/fuzzing:fuzz_support",
"//c-toxcore/testing/support",
"//c-toxcore/toxcore:logger",
"//c-toxcore/toxcore:mono_time",
"//c-toxcore/toxcore:os_memory",
@@ -131,12 +130,29 @@ cc_library(
],
)
cc_library(
name = "av_test_support",
testonly = True,
srcs = ["av_test_support.cc"],
hdrs = ["av_test_support.hh"],
deps = [
":audio",
":rtp",
":video",
"//c-toxcore/toxcore:logger",
"//c-toxcore/toxcore:mono_time",
"//c-toxcore/toxcore:os_memory",
"@com_google_googletest//:gtest",
],
)
cc_test(
name = "audio_test",
timeout = "moderate",
srcs = ["audio_test.cc"],
deps = [
":audio",
":av_test_support",
":rtp",
"//c-toxcore/toxcore:logger",
"//c-toxcore/toxcore:mono_time",
@@ -166,6 +182,7 @@ cc_test(
timeout = "moderate",
srcs = ["video_test.cc"],
deps = [
":av_test_support",
":rtp",
":video",
"//c-toxcore/toxcore:logger",
@@ -176,6 +193,50 @@ cc_test(
],
)
cc_binary(
name = "video_bench",
testonly = True,
srcs = ["video_bench.cc"],
deps = [
":av_test_support",
":rtp",
":video",
"//c-toxcore/toxcore:logger",
"//c-toxcore/toxcore:mono_time",
"//c-toxcore/toxcore:os_memory",
"@benchmark",
],
)
cc_binary(
name = "audio_bench",
testonly = True,
srcs = ["audio_bench.cc"],
deps = [
":audio",
":av_test_support",
":rtp",
"//c-toxcore/toxcore:logger",
"//c-toxcore/toxcore:mono_time",
"//c-toxcore/toxcore:os_memory",
"@benchmark",
],
)
cc_binary(
name = "rtp_bench",
testonly = True,
srcs = ["rtp_bench.cc"],
deps = [
":av_test_support",
":rtp",
"//c-toxcore/toxcore:logger",
"//c-toxcore/toxcore:mono_time",
"//c-toxcore/toxcore:os_memory",
"@benchmark",
],
)
cc_library(
name = "msi",
srcs = ["msi.c"],

20
toxav/audio.c
View File

@@ -48,16 +48,16 @@ struct ACSession {
};
static struct JitterBuffer *jbuf_new(uint32_t capacity);
static void jbuf_clear(struct JitterBuffer *q);
static void jbuf_free(struct JitterBuffer *q);
static int jbuf_write(const Logger *log, struct JitterBuffer *q, struct RTPMessage *m);
static struct RTPMessage *jbuf_read(struct JitterBuffer *q, int32_t *success);
static OpusEncoder *create_audio_encoder(const Logger *log, uint32_t bit_rate, uint32_t sampling_rate,
static struct JitterBuffer *_Nullable jbuf_new(uint32_t capacity);
static void jbuf_clear(struct JitterBuffer *_Nonnull q);
static void jbuf_free(struct JitterBuffer *_Nullable q);
static int jbuf_write(const Logger *_Nonnull log, struct JitterBuffer *_Nonnull q, struct RTPMessage *_Nonnull m);
static struct RTPMessage *_Nullable jbuf_read(struct JitterBuffer *_Nonnull q, int32_t *_Nonnull success);
static OpusEncoder *_Nullable create_audio_encoder(const Logger *_Nonnull log, uint32_t bit_rate, uint32_t sampling_rate,
uint8_t channel_count);
static bool reconfigure_audio_encoder(const Logger *log, OpusEncoder **e, uint32_t new_br, uint32_t new_sr,
uint8_t new_ch, uint32_t *old_br, uint32_t *old_sr, uint8_t *old_ch);
static bool reconfigure_audio_decoder(ACSession *ac, uint32_t sampling_rate, uint8_t channels);
static bool reconfigure_audio_encoder(const Logger *_Nonnull log, OpusEncoder *_Nonnull *_Nonnull e, uint32_t new_br, uint32_t new_sr,
uint8_t new_ch, uint32_t *_Nonnull old_br, uint32_t *_Nonnull old_sr, uint8_t *_Nonnull old_ch);
static bool reconfigure_audio_decoder(ACSession *_Nonnull ac, uint32_t sampling_rate, uint8_t channels);
@@ -246,7 +246,7 @@ void ac_iterate(ACSession *ac)
if (rc < 0) {
LOGGER_WARNING(ac->log, "Decoding error: %s", opus_strerror(rc));
} else if (ac->acb != nullptr) {
} else if (ac->acb != nullptr && ac->lp_sampling_rate != 0) {
ac->lp_frame_duration = (rc * 1000) / ac->lp_sampling_rate;
ac->acb(ac->friend_number, temp_audio_buffer, (size_t)rc, ac->lp_channel_count,

22
toxav/audio.h
View File

@@ -5,8 +5,8 @@
#ifndef C_TOXCORE_TOXAV_AUDIO_H
#define C_TOXCORE_TOXAV_AUDIO_H
#include <stdint.h>
#include <stddef.h>
#include <stdint.h>
#include "../toxcore/logger.h"
#include "../toxcore/mono_time.h"
@@ -35,23 +35,23 @@ extern "C" {
#define AUDIO_MAX_BUFFER_SIZE_PCM16 ((AUDIO_MAX_SAMPLE_RATE * AUDIO_MAX_FRAME_DURATION_MS) / 1000)
#define AUDIO_MAX_BUFFER_SIZE_BYTES (AUDIO_MAX_BUFFER_SIZE_PCM16 * 2)
typedef void ac_audio_receive_frame_cb(uint32_t friend_number, const int16_t *pcm, size_t sample_count,
uint8_t channels, uint32_t sampling_rate, void *user_data);
typedef void ac_audio_receive_frame_cb(uint32_t friend_number, const int16_t *_Nonnull pcm, size_t sample_count,
uint8_t channels, uint32_t sampling_rate, void *_Nullable user_data);
typedef struct ACSession ACSession;
struct RTPMessage;
ACSession *ac_new(Mono_Time *mono_time, const Logger *log, uint32_t friend_number,
ac_audio_receive_frame_cb *cb, void *user_data);
void ac_kill(ACSession *ac);
void ac_iterate(ACSession *ac);
int ac_queue_message(const Mono_Time *mono_time, void *cs, struct RTPMessage *msg);
int ac_reconfigure_encoder(ACSession *ac, uint32_t bit_rate, uint32_t sampling_rate, uint8_t channels);
ACSession *_Nullable ac_new(Mono_Time *_Nonnull mono_time, const Logger *_Nonnull log, uint32_t friend_number,
ac_audio_receive_frame_cb *_Nullable cb, void *_Nullable user_data);
void ac_kill(ACSession *_Nullable ac);
void ac_iterate(ACSession *_Nullable ac);
int ac_queue_message(const Mono_Time *_Nonnull mono_time, void *_Nullable cs, struct RTPMessage *_Nullable msg);
int ac_reconfigure_encoder(ACSession *_Nullable ac, uint32_t bit_rate, uint32_t sampling_rate, uint8_t channels);
uint32_t ac_get_lp_frame_duration(const ACSession *ac);
uint32_t ac_get_lp_frame_duration(const ACSession *_Nonnull ac);
int ac_encode(ACSession *ac, const int16_t *pcm, size_t sample_count, uint8_t *dest, size_t dest_max);
int ac_encode(ACSession *_Nonnull ac, const int16_t *_Nonnull pcm, size_t sample_count, uint8_t *_Nonnull dest, size_t dest_max);
#ifdef __cplusplus
} /* extern "C" */

199
toxav/audio_bench.cc Normal file
View File

@@ -0,0 +1,199 @@
/* SPDX-License-Identifier: GPL-3.0-or-later
* Copyright © 2025 The TokTok team.
*/
#include <benchmark/benchmark.h>
#include <cmath>
#include <cstring>
#include <vector>
#include "../toxcore/logger.h"
#include "../toxcore/mono_time.h"
#include "../toxcore/network.h"
#include "../toxcore/os_memory.h"
#include "audio.h"
#include "av_test_support.hh"
#include "rtp.h"
namespace {
class AudioBench : public benchmark::Fixture {
public:
void SetUp(const ::benchmark::State &state) override
{
const Memory *mem = os_memory();
log = logger_new(mem);
tm.t = 1000;
mono_time = mono_time_new(mem, mock_time_cb, &tm);
ac = ac_new(mono_time, log, 123, nullptr, nullptr);
sampling_rate = static_cast<uint32_t>(state.range(0));
channels = static_cast<uint8_t>(state.range(1));
uint32_t bitrate = (channels == 1) ? 32000 : 64000;
ac_reconfigure_encoder(ac, bitrate, sampling_rate, channels);
sample_count = sampling_rate / 50; // 20ms frames
pcm.resize(sample_count * channels);
rtp_mock.capture_packets = false; // Disable capturing for benchmarks
rtp_mock.auto_forward = true;
rtp_mock.recv_session = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet,
&rtp_mock, nullptr, nullptr, nullptr, ac, RtpMock::audio_cb);
}
void TearDown(const ::benchmark::State &state) override
{
const Memory *mem = os_memory();
if (rtp_mock.recv_session) {
rtp_kill(log, rtp_mock.recv_session);
}
if (ac) {
ac_kill(ac);
}
if (mono_time) {
mono_time_free(mem, mono_time);
}
if (log) {
logger_kill(log);
}
}
Logger *log = nullptr;
Mono_Time *mono_time = nullptr;
MockTime tm;
ACSession *ac = nullptr;
RtpMock rtp_mock;
uint32_t sampling_rate = 0;
uint8_t channels = 0;
size_t sample_count = 0;
std::vector<int16_t> pcm;
};
// Benchmark encoding a sequence of silent audio frames.
BENCHMARK_DEFINE_F(AudioBench, EncodeSilentSequence)(benchmark::State &state)
{
std::vector<int16_t> silent_pcm(sample_count * channels);
fill_silent_frame(channels, sample_count, silent_pcm);
std::vector<uint8_t> encoded(2000);
for (auto _ : state) {
int encoded_size
= ac_encode(ac, silent_pcm.data(), sample_count, encoded.data(), encoded.size());
benchmark::DoNotOptimize(encoded_size);
}
}
BENCHMARK_REGISTER_F(AudioBench, EncodeSilentSequence)
->Args({8000, 1})
->Args({48000, 1})
->Args({48000, 2});
// Benchmark encoding a sequence of audio frames.
BENCHMARK_DEFINE_F(AudioBench, EncodeSequence)(benchmark::State &state)
{
int frame_index = 0;
const int num_prefilled = 50;
std::vector<std::vector<int16_t>> pcms(
num_prefilled, std::vector<int16_t>(sample_count * channels));
for (int i = 0; i < num_prefilled; ++i) {
fill_audio_frame(sampling_rate, channels, i, sample_count, pcms[i]);
}
std::vector<uint8_t> encoded(2000);
for (auto _ : state) {
int idx = frame_index % num_prefilled;
int encoded_size
= ac_encode(ac, pcms[idx].data(), sample_count, encoded.data(), encoded.size());
benchmark::DoNotOptimize(encoded_size);
frame_index++;
}
}
BENCHMARK_REGISTER_F(AudioBench, EncodeSequence)
->Args({8000, 1})
->Args({16000, 1})
->Args({24000, 1})
->Args({48000, 1})
->Args({48000, 2});
// Benchmark decoding a sequence of audio frames.
BENCHMARK_DEFINE_F(AudioBench, DecodeSequence)(benchmark::State &state)
{
const int num_frames = 50;
std::vector<std::vector<uint8_t>> encoded_frames(num_frames);
// Pre-encode
std::vector<uint8_t> encoded_tmp(2000);
for (int i = 0; i < num_frames; ++i) {
fill_audio_frame(sampling_rate, channels, i, sample_count, pcm);
int size = ac_encode(ac, pcm.data(), sample_count, encoded_tmp.data(), encoded_tmp.size());
encoded_frames[i].resize(4 + size);
uint32_t net_sr = net_htonl(sampling_rate);
std::memcpy(encoded_frames[i].data(), &net_sr, 4);
std::memcpy(encoded_frames[i].data() + 4, encoded_tmp.data(), size);
}
int frame_index = 0;
for (auto _ : state) {
int idx = frame_index % num_frames;
rtp_send_data(log, rtp_mock.recv_session, encoded_frames[idx].data(),
static_cast<uint32_t>(encoded_frames[idx].size()), false);
ac_iterate(ac);
frame_index++;
}
}
BENCHMARK_REGISTER_F(AudioBench, DecodeSequence)
->Args({8000, 1})
->Args({16000, 1})
->Args({24000, 1})
->Args({48000, 1})
->Args({48000, 2});
// Full end-to-end sequence benchmark (Encode -> RTP -> Decode)
BENCHMARK_DEFINE_F(AudioBench, FullSequence)(benchmark::State &state)
{
int frame_index = 0;
const int num_prefilled = 50;
std::vector<std::vector<int16_t>> pcms(
num_prefilled, std::vector<int16_t>(sample_count * channels));
for (int i = 0; i < num_prefilled; ++i) {
fill_audio_frame(sampling_rate, channels, i, sample_count, pcms[i]);
}
std::vector<uint8_t> encoded_tmp(2000);
for (auto _ : state) {
int idx = frame_index % num_prefilled;
int size
= ac_encode(ac, pcms[idx].data(), sample_count, encoded_tmp.data(), encoded_tmp.size());
std::vector<uint8_t> payload(4 + size);
uint32_t net_sr = net_htonl(sampling_rate);
std::memcpy(payload.data(), &net_sr, 4);
std::memcpy(payload.data() + 4, encoded_tmp.data(), size);
rtp_send_data(log, rtp_mock.recv_session, payload.data(),
static_cast<uint32_t>(payload.size()), false);
ac_iterate(ac);
frame_index++;
}
}
BENCHMARK_REGISTER_F(AudioBench, FullSequence)
->Args({8000, 1})
->Args({16000, 1})
->Args({24000, 1})
->Args({48000, 1})
->Args({48000, 2});
}
BENCHMARK_MAIN();

388
toxav/audio_test.cc
View File

@@ -3,84 +3,19 @@
#include <gtest/gtest.h>
#include <algorithm>
#include <cmath>
#include <vector>
#include "../toxcore/logger.h"
#include "../toxcore/mono_time.h"
#include "../toxcore/network.h"
#include "../toxcore/os_memory.h"
#include "av_test_support.hh"
#include "rtp.h"
namespace {
struct AudioTimeMock {
uint64_t t;
};
uint64_t audio_mock_time_cb(void *ud) { return static_cast<AudioTimeMock *>(ud)->t; }
struct AudioTestData {
uint32_t friend_number = 0;
std::vector<int16_t> last_pcm;
size_t sample_count = 0;
uint8_t channels = 0;
uint32_t sampling_rate = 0;
static void receive_frame(uint32_t friend_number, const int16_t *pcm, size_t sample_count,
uint8_t channels, uint32_t sampling_rate, void *user_data)
{
auto *self = static_cast<AudioTestData *>(user_data);
self->friend_number = friend_number;
self->last_pcm.assign(pcm, pcm + sample_count * channels);
self->sample_count = sample_count;
self->channels = channels;
self->sampling_rate = sampling_rate;
}
};
struct AudioRtpMock {
RTPSession *recv_session = nullptr;
std::vector<std::vector<uint8_t>> captured_packets;
bool auto_forward = true;
static int send_packet(void *user_data, const uint8_t *data, uint16_t length)
{
auto *self = static_cast<AudioRtpMock *>(user_data);
self->captured_packets.push_back(std::vector<uint8_t>(data, data + length));
if (self->auto_forward && self->recv_session) {
rtp_receive_packet(self->recv_session, data, length);
}
return 0;
}
static int audio_cb(const Mono_Time *mono_time, void *cs, RTPMessage *msg)
{
return ac_queue_message(mono_time, cs, msg);
}
};
class AudioTest : public ::testing::Test {
protected:
void SetUp() override
{
const Memory *mem = os_memory();
log = logger_new(mem);
tm.t = 1000;
mono_time = mono_time_new(mem, audio_mock_time_cb, &tm);
mono_time_update(mono_time);
}
void TearDown() override
{
const Memory *mem = os_memory();
mono_time_free(mem, mono_time);
logger_kill(log);
}
Logger *log;
Mono_Time *mono_time;
AudioTimeMock tm;
};
using AudioTest = AvTest;
TEST_F(AudioTest, BasicNewKill)
{
@@ -96,11 +31,11 @@ TEST_F(AudioTest, EncodeDecodeLoop)
ACSession *ac = ac_new(mono_time, log, 123, AudioTestData::receive_frame, &data);
ASSERT_NE(ac, nullptr);
AudioRtpMock rtp_mock;
RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet,
&rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb);
RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet,
&rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb);
RtpMock rtp_mock;
RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock,
nullptr, nullptr, nullptr, ac, RtpMock::audio_cb);
RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock,
nullptr, nullptr, nullptr, ac, RtpMock::audio_cb);
rtp_mock.recv_session = recv_rtp;
uint32_t sampling_rate = 48000;
@@ -144,6 +79,203 @@ TEST_F(AudioTest, EncodeDecodeLoop)
ac_kill(ac);
}
TEST_F(AudioTest, EncodeDecodeRealistic)
{
AudioTestData data;
ACSession *ac = ac_new(mono_time, log, 123, AudioTestData::receive_frame, &data);
ASSERT_NE(ac, nullptr);
RtpMock rtp_mock;
RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock,
nullptr, nullptr, nullptr, ac, RtpMock::audio_cb);
RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock,
nullptr, nullptr, nullptr, ac, RtpMock::audio_cb);
rtp_mock.recv_session = recv_rtp;
uint32_t sampling_rate = 48000;
uint8_t channels = 1;
size_t sample_count = 960; // 20ms at 48kHz
uint32_t bitrate = 48000;
ASSERT_EQ(ac_reconfigure_encoder(ac, bitrate, sampling_rate, channels), 0);
double frequency = 440.0;
double amplitude = 10000.0;
const double pi = std::acos(-1.0);
std::vector<int16_t> all_sent;
std::vector<int16_t> all_recv;
for (int frame = 0; frame < 50; ++frame) {
std::vector<int16_t> pcm(sample_count * channels);
for (size_t i = 0; i < sample_count; ++i) {
double t = static_cast<double>(frame * sample_count + i) / sampling_rate;
pcm[i] = static_cast<int16_t>(std::sin(2.0 * pi * frequency * t) * amplitude);
}
all_sent.insert(all_sent.end(), pcm.begin(), pcm.end());
std::vector<uint8_t> encoded(2000);
int encoded_size = ac_encode(ac, pcm.data(), sample_count, encoded.data(), encoded.size());
ASSERT_GT(encoded_size, 0);
std::vector<uint8_t> payload(4 + static_cast<size_t>(encoded_size));
uint32_t net_sr = net_htonl(sampling_rate);
memcpy(payload.data(), &net_sr, 4);
memcpy(payload.data() + 4, encoded.data(), static_cast<size_t>(encoded_size));
rtp_send_data(log, send_rtp, payload.data(), static_cast<uint32_t>(payload.size()), false);
ac_iterate(ac);
if (data.sample_count > 0) {
all_recv.insert(all_recv.end(), data.last_pcm.begin(), data.last_pcm.end());
}
}
ASSERT_FALSE(all_recv.empty());
// Find the best match by trying different delays.
// Jitter buffer delay (3 frames = 2880 samples) + Opus lookahead (~312 samples) = ~3192.
double min_mse = 1e18;
int best_delay = 0;
// Search around the expected delay
for (int delay = 3000; delay < 3500; ++delay) {
double mse = 0;
int count = 0;
for (size_t i = 0; i < 2000; ++i) { // Compare a decent chunk
if (i + delay < all_sent.size() && i < all_recv.size()) {
int diff = all_sent[i + delay] - all_recv[i];
mse += static_cast<double>(diff) * diff;
count++;
}
}
if (count > 1000) {
mse /= count;
if (mse < min_mse) {
min_mse = mse;
best_delay = delay;
}
}
}
printf("Best audio delay found: %d samples, Min MSE: %f\n", best_delay, min_mse);
// For 48kbps Opus, the MSE for a sine wave should be quite low once aligned.
// 10M is about 20% of the signal power (50M), which is a safe threshold for verification.
EXPECT_LT(min_mse, 10000000.0);
rtp_kill(log, send_rtp);
rtp_kill(log, recv_rtp);
ac_kill(ac);
}
TEST_F(AudioTest, EncodeDecodeSiren)
{
AudioTestData data;
ACSession *ac = ac_new(mono_time, log, 123, AudioTestData::receive_frame, &data);
ASSERT_NE(ac, nullptr);
RtpMock rtp_mock;
RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock,
nullptr, nullptr, nullptr, ac, RtpMock::audio_cb);
RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock,
nullptr, nullptr, nullptr, ac, RtpMock::audio_cb);
rtp_mock.recv_session = recv_rtp;
uint32_t sampling_rate = 48000;
uint8_t channels = 1;
size_t sample_count = 960; // 20ms at 48kHz
uint32_t bitrate = 64000;
ASSERT_EQ(ac_reconfigure_encoder(ac, bitrate, sampling_rate, channels), 0);
double amplitude = 10000.0;
const double pi = std::acos(-1.0);
std::vector<int16_t> all_sent;
std::vector<int16_t> all_recv;
// 1 second of audio (50 frames) is enough for a siren test
for (int frame = 0; frame < 50; ++frame) {
std::vector<int16_t> pcm(sample_count * channels);
for (size_t i = 0; i < sample_count; ++i) {
double t = static_cast<double>(frame * sample_count + i) / sampling_rate;
// Linear frequency sweep from 50Hz to 440Hz over 1 second
// f(t) = 50 + (440-50)/1 * t = 50 + 390t
// phi(t) = 2*pi * integral(f(t)) = 2*pi * (50t + 195t^2)
double phi = 2.0 * pi * (50.0 * t + 195.0 * t * t);
pcm[i] = static_cast<int16_t>(std::sin(phi) * amplitude);
}
all_sent.insert(all_sent.end(), pcm.begin(), pcm.end());
std::vector<uint8_t> encoded(2000);
int encoded_size = ac_encode(ac, pcm.data(), sample_count, encoded.data(), encoded.size());
ASSERT_GT(encoded_size, 0);
std::vector<uint8_t> payload(4 + static_cast<size_t>(encoded_size));
uint32_t net_sr = net_htonl(sampling_rate);
memcpy(payload.data(), &net_sr, 4);
memcpy(payload.data() + 4, encoded.data(), static_cast<size_t>(encoded_size));
rtp_send_data(log, send_rtp, payload.data(), static_cast<uint32_t>(payload.size()), false);
ac_iterate(ac);
if (data.sample_count > 0) {
all_recv.insert(all_recv.end(), data.last_pcm.begin(), data.last_pcm.end());
}
}
ASSERT_FALSE(all_recv.empty());
auto calculate_mse_at = [&](int delay, size_t window) {
double mse = 0;
int count = 0;
for (size_t i = 0; i < window; ++i) {
int sent_idx = static_cast<int>(i) + delay;
if (sent_idx >= 0 && static_cast<size_t>(sent_idx) < all_sent.size()
&& i < all_recv.size()) {
int diff = all_sent[static_cast<size_t>(sent_idx)] - all_recv[i];
mse += static_cast<double>(diff) * diff;
count++;
}
}
return count > 0 ? mse / count : 1e18;
};
// Two-stage search for speed
double min_mse = 1e18;
int coarse_best = 0;
// 1. Coarse search
for (int delay = -5000; delay < 5000; delay += 100) {
double mse = calculate_mse_at(delay, 5000);
if (mse < min_mse) {
min_mse = mse;
coarse_best = delay;
}
}
// 2. Fine search around coarse best
int best_delay = coarse_best;
for (int delay = coarse_best - 100; delay <= coarse_best + 100; ++delay) {
double mse = calculate_mse_at(delay, 10000);
if (mse < min_mse) {
min_mse = mse;
best_delay = delay;
}
}
printf("Best siren audio delay found: %d samples, Min MSE: %f\n", best_delay, min_mse);
// For 64kbps Opus, the MSE for a siren wave should be reasonably low once aligned.
EXPECT_LT(min_mse, 20000000.0);
rtp_kill(log, send_rtp);
rtp_kill(log, recv_rtp);
ac_kill(ac);
}
TEST_F(AudioTest, ReconfigureEncoder)
{
AudioTestData data;
@@ -184,12 +316,12 @@ TEST_F(AudioTest, QueueInvalidMessage)
ACSession *ac = ac_new(mono_time, log, 123, AudioTestData::receive_frame, &data);
ASSERT_NE(ac, nullptr);
AudioRtpMock rtp_mock;
RtpMock rtp_mock;
// Create a video RTP session but try to queue to audio session
RTPSession *video_rtp = rtp_new(log, RTP_TYPE_VIDEO, mono_time, AudioRtpMock::send_packet,
&rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb);
RTPSession *audio_recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet,
&rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb);
RTPSession *video_rtp = rtp_new(log, RTP_TYPE_VIDEO, mono_time, RtpMock::send_packet, &rtp_mock,
nullptr, nullptr, nullptr, ac, RtpMock::audio_cb);
RTPSession *audio_recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet,
&rtp_mock, nullptr, nullptr, nullptr, ac, RtpMock::audio_cb);
rtp_mock.recv_session = audio_recv_rtp;
std::vector<uint8_t> dummy_video(100, 0);
@@ -212,12 +344,12 @@ TEST_F(AudioTest, JitterBufferDuplicate)
ACSession *ac = ac_new(mono_time, log, 123, AudioTestData::receive_frame, &data);
ASSERT_NE(ac, nullptr);
AudioRtpMock rtp_mock;
RtpMock rtp_mock;
rtp_mock.auto_forward = false;
RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet,
&rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb);
RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet,
&rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb);
RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock,
nullptr, nullptr, nullptr, ac, RtpMock::audio_cb);
RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock,
nullptr, nullptr, nullptr, ac, RtpMock::audio_cb);
rtp_mock.recv_session = recv_rtp;
uint8_t dummy_data[100] = {0};
@@ -253,12 +385,12 @@ TEST_F(AudioTest, JitterBufferOutOfOrder)
ACSession *ac = ac_new(mono_time, log, 123, AudioTestData::receive_frame, &data);
ASSERT_NE(ac, nullptr);
AudioRtpMock rtp_mock;
RtpMock rtp_mock;
rtp_mock.auto_forward = false;
RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet,
&rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb);
RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet,
&rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb);
RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock,
nullptr, nullptr, nullptr, ac, RtpMock::audio_cb);
RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock,
nullptr, nullptr, nullptr, ac, RtpMock::audio_cb);
rtp_mock.recv_session = recv_rtp;
uint8_t dummy_data[100] = {0};
@@ -300,12 +432,12 @@ TEST_F(AudioTest, PacketLossConcealment)
ACSession *ac = ac_new(mono_time, log, 123, AudioTestData::receive_frame, &data);
ASSERT_NE(ac, nullptr);
AudioRtpMock rtp_mock;
RtpMock rtp_mock;
rtp_mock.auto_forward = false;
RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet,
&rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb);
RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet,
&rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb);
RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock,
nullptr, nullptr, nullptr, ac, RtpMock::audio_cb);
RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock,
nullptr, nullptr, nullptr, ac, RtpMock::audio_cb);
rtp_mock.recv_session = recv_rtp;
uint8_t dummy_data[100] = {0};
@@ -346,12 +478,12 @@ TEST_F(AudioTest, JitterBufferReset)
ACSession *ac = ac_new(mono_time, log, 123, AudioTestData::receive_frame, &data);
ASSERT_NE(ac, nullptr);
AudioRtpMock rtp_mock;
RtpMock rtp_mock;
rtp_mock.auto_forward = false;
RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet,
&rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb);
RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet,
&rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb);
RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock,
nullptr, nullptr, nullptr, ac, RtpMock::audio_cb);
RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock,
nullptr, nullptr, nullptr, ac, RtpMock::audio_cb);
rtp_mock.recv_session = recv_rtp;
uint8_t dummy_data[100] = {0};
@@ -391,12 +523,12 @@ TEST_F(AudioTest, DecoderReconfigureCooldown)
ACSession *ac = ac_new(mono_time, log, 123, AudioTestData::receive_frame, &data);
ASSERT_NE(ac, nullptr);
AudioRtpMock rtp_mock;
RtpMock rtp_mock;
rtp_mock.auto_forward = false;
RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet,
&rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb);
RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet,
&rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb);
RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock,
nullptr, nullptr, nullptr, ac, RtpMock::audio_cb);
RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock,
nullptr, nullptr, nullptr, ac, RtpMock::audio_cb);
rtp_mock.recv_session = recv_rtp;
uint8_t dummy_data[100] = {0};
@@ -452,12 +584,12 @@ TEST_F(AudioTest, QueueDummyMessage)
ACSession *ac = ac_new(mono_time, log, 123, AudioTestData::receive_frame, &data);
ASSERT_NE(ac, nullptr);
AudioRtpMock rtp_mock;
RtpMock rtp_mock;
// RTP_TYPE_AUDIO + 2 is the dummy type
RTPSession *dummy_rtp = rtp_new(log, RTP_TYPE_AUDIO + 2, mono_time, AudioRtpMock::send_packet,
&rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb);
RTPSession *audio_recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet,
&rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb);
RTPSession *dummy_rtp = rtp_new(log, RTP_TYPE_AUDIO + 2, mono_time, RtpMock::send_packet,
&rtp_mock, nullptr, nullptr, nullptr, ac, RtpMock::audio_cb);
RTPSession *audio_recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet,
&rtp_mock, nullptr, nullptr, nullptr, ac, RtpMock::audio_cb);
rtp_mock.recv_session = audio_recv_rtp;
std::vector<uint8_t> dummy_payload(100, 0);
@@ -480,12 +612,12 @@ TEST_F(AudioTest, LatePacketReset)
ACSession *ac = ac_new(mono_time, log, 123, AudioTestData::receive_frame, &data);
ASSERT_NE(ac, nullptr);
AudioRtpMock rtp_mock;
RtpMock rtp_mock;
rtp_mock.auto_forward = false;
RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet,
&rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb);
RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet,
&rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb);
RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock,
nullptr, nullptr, nullptr, ac, RtpMock::audio_cb);
RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock,
nullptr, nullptr, nullptr, ac, RtpMock::audio_cb);
rtp_mock.recv_session = recv_rtp;
uint8_t dummy_data[100] = {0};
@@ -531,12 +663,12 @@ TEST_F(AudioTest, InvalidSamplingRate)
ACSession *ac = ac_new(mono_time, log, 123, AudioTestData::receive_frame, &data);
ASSERT_NE(ac, nullptr);
AudioRtpMock rtp_mock;
RtpMock rtp_mock;
rtp_mock.auto_forward = false;
RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet,
&rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb);
RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet,
&rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb);
RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock,
nullptr, nullptr, nullptr, ac, RtpMock::audio_cb);
RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock,
nullptr, nullptr, nullptr, ac, RtpMock::audio_cb);
rtp_mock.recv_session = recv_rtp;
// 1. Send a packet with an absurdly large sampling rate.
@@ -578,12 +710,12 @@ TEST_F(AudioTest, ShortPacket)
ACSession *ac = ac_new(mono_time, log, 123, AudioTestData::receive_frame, &data);
ASSERT_NE(ac, nullptr);
AudioRtpMock rtp_mock;
RtpMock rtp_mock;
rtp_mock.auto_forward = false;
RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet,
&rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb);
RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet,
&rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb);
RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock,
nullptr, nullptr, nullptr, ac, RtpMock::audio_cb);
RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock,
nullptr, nullptr, nullptr, ac, RtpMock::audio_cb);
rtp_mock.recv_session = recv_rtp;
// 1. Send a packet that is too short (only sampling rate, no Opus data).
@@ -610,12 +742,12 @@ TEST_F(AudioTest, JitterBufferWrapAround)
ACSession *ac = ac_new(mono_time, log, 123, AudioTestData::receive_frame, &data);
ASSERT_NE(ac, nullptr);
AudioRtpMock rtp_mock;
RtpMock rtp_mock;
rtp_mock.auto_forward = false;
RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet,
&rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb);
RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet,
&rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb);
RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock,
nullptr, nullptr, nullptr, ac, RtpMock::audio_cb);
RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock,
nullptr, nullptr, nullptr, ac, RtpMock::audio_cb);
rtp_mock.recv_session = recv_rtp;
uint8_t dummy_data[100] = {0};

171
toxav/av_test_support.cc Normal file
View File

@@ -0,0 +1,171 @@
#include "av_test_support.hh"
#include <algorithm>
#include <cmath>
#include <cstring>
#include "../toxcore/os_memory.h"
// Mock Time
uint64_t mock_time_cb(void *ud) { return static_cast<MockTime *>(ud)->t; }
// RTP Mock
int RtpMock::send_packet(void *user_data, const uint8_t *data, uint16_t length)
{
auto *self = static_cast<RtpMock *>(user_data);
if (self->capture_packets) {
if (self->store_last_packet_only) {
if (self->captured_packets.empty()) {
self->captured_packets.emplace_back(data, data + length);
} else {
self->captured_packets[0].assign(data, data + length);
}
} else {
self->captured_packets.push_back(std::vector<uint8_t>(data, data + length));
}
}
if (self->auto_forward && self->recv_session) {
rtp_receive_packet(self->recv_session, data, length);
}
return 0;
}
int RtpMock::audio_cb(const Mono_Time *mono_time, void *cs, RTPMessage *msg)
{
return ac_queue_message(mono_time, cs, msg);
}
int RtpMock::video_cb(const Mono_Time *mono_time, void *cs, RTPMessage *msg)
{
return vc_queue_message(mono_time, cs, msg);
}
int RtpMock::noop_cb(const Mono_Time * /*mono_time*/, void * /*cs*/, RTPMessage *msg)
{
std::free(msg);
return 0;
}
// Audio Helpers
void fill_audio_frame(uint32_t sampling_rate, uint8_t channels, int frame_index,
size_t sample_count, std::vector<int16_t> &pcm)
{
const double pi = std::acos(-1.0);
double amplitude = 10000.0;
for (size_t i = 0; i < sample_count; ++i) {
double t = static_cast<double>(frame_index * sample_count + i) / sampling_rate;
// Linear frequency sweep from 50Hz to 440Hz over 1 second (50 frames)
// f(t) = 50 + 390t
// phi(t) = 2*pi * (50t + 195t^2)
double phi = 2.0 * pi * (50.0 * t + 195.0 * t * t);
int16_t val = static_cast<int16_t>(std::sin(phi) * amplitude);
for (uint8_t c = 0; c < channels; ++c) {
pcm[i * channels + c] = val;
}
}
}
void fill_silent_frame(uint8_t channels, size_t sample_count, std::vector<int16_t> &pcm)
{
for (size_t i = 0; i < sample_count * channels; ++i) {
// Very low amplitude white noise (simulating silence with background hiss)
pcm[i] = (std::rand() % 21) - 10;
}
}
AudioTestData::AudioTestData() = default;
AudioTestData::~AudioTestData() = default;
void AudioTestData::receive_frame(uint32_t friend_number, const int16_t *pcm, size_t sample_count,
uint8_t channels, uint32_t sampling_rate, void *user_data)
{
auto *self = static_cast<AudioTestData *>(user_data);
self->friend_number = friend_number;
self->last_pcm.assign(pcm, pcm + sample_count * channels);
self->sample_count = sample_count;
self->channels = channels;
self->sampling_rate = sampling_rate;
}
// Video Helpers
void fill_video_frame(uint16_t width, uint16_t height, int frame_index, std::vector<uint8_t> &y,
std::vector<uint8_t> &u, std::vector<uint8_t> &v)
{
// Background (dark gray)
std::fill(y.begin(), y.end(), 32);
std::fill(u.begin(), u.end(), 128);
std::fill(v.begin(), v.end(), 128);
// Moving square (light gray)
int sq_size = height / 4;
if (sq_size < 16)
sq_size = 16;
int x0 = (frame_index * 8) % (width - sq_size);
int y0 = (frame_index * 4) % (height - sq_size);
for (int r = 0; r < sq_size; ++r) {
for (int c = 0; c < sq_size; ++c) {
y[(y0 + r) * width + (x0 + c)] = 200;
}
}
}
double calculate_video_mse(uint16_t width, uint16_t height, int32_t ystride,
const std::vector<uint8_t> &y_recv, const std::vector<uint8_t> &y_orig)
{
if (y_recv.empty() || y_orig.size() != static_cast<size_t>(width) * height) {
return 1e10;
}
double mse = 0;
for (int r = 0; r < height; ++r) {
for (int c = 0; c < width; ++c) {
int diff = static_cast<int>(y_orig[r * width + c]) - y_recv[r * std::abs(ystride) + c];
mse += diff * diff;
}
}
return mse / (static_cast<size_t>(width) * height);
}
// Video Test Data Helper
VideoTestData::VideoTestData() = default;
VideoTestData::~VideoTestData() = default;
void VideoTestData::receive_frame(uint32_t friend_number, uint16_t width, uint16_t height,
const uint8_t *y, const uint8_t *u, const uint8_t *v, int32_t ystride, int32_t ustride,
int32_t vstride, void *user_data)
{
auto *self = static_cast<VideoTestData *>(user_data);
self->friend_number = friend_number;
self->width = width;
self->height = height;
self->ystride = ystride;
self->ustride = ustride;
self->vstride = vstride;
self->y.assign(y, y + static_cast<size_t>(std::abs(ystride)) * height);
self->u.assign(u, u + static_cast<size_t>(std::abs(ustride)) * (height / 2));
self->v.assign(v, v + static_cast<size_t>(std::abs(vstride)) * (height / 2));
}
double VideoTestData::calculate_mse(const std::vector<uint8_t> &y_orig) const
{
return calculate_video_mse(width, height, ystride, y, y_orig);
}
// Common Test Fixture
void AvTest::SetUp()
{
const Memory *mem = os_memory();
log = logger_new(mem);
tm.t = 1000;
mono_time = mono_time_new(mem, mock_time_cb, &tm);
mono_time_update(mono_time);
}
void AvTest::TearDown()
{
const Memory *mem = os_memory();
mono_time_free(mem, mono_time);
logger_kill(log);
}

89
toxav/av_test_support.hh Normal file
View File

@@ -0,0 +1,89 @@
#ifndef TOXAV_AV_TEST_SUPPORT_H
#define TOXAV_AV_TEST_SUPPORT_H
#include <gtest/gtest.h>
#include <cstdint>
#include <vector>
#include "../toxcore/logger.h"
#include "../toxcore/mono_time.h"
#include "audio.h"
#include "rtp.h"
#include "video.h"
// Mock Time
struct MockTime {
uint64_t t = 1000;
};
uint64_t mock_time_cb(void *ud);
// RTP Mock
struct RtpMock {
RTPSession *recv_session = nullptr;
std::vector<std::vector<uint8_t>> captured_packets;
bool auto_forward = true;
bool capture_packets = true;
bool store_last_packet_only = false;
static int send_packet(void *user_data, const uint8_t *data, uint16_t length);
static int audio_cb(const Mono_Time *mono_time, void *cs, RTPMessage *msg);
static int video_cb(const Mono_Time *mono_time, void *cs, RTPMessage *msg);
static int noop_cb(const Mono_Time *mono_time, void *cs, RTPMessage *msg);
};
// Audio Helpers
void fill_audio_frame(uint32_t sampling_rate, uint8_t channels, int frame_index,
size_t sample_count, std::vector<int16_t> &pcm);
void fill_silent_frame(uint8_t channels, size_t sample_count, std::vector<int16_t> &pcm);
struct AudioTestData {
uint32_t friend_number = 0;
std::vector<int16_t> last_pcm;
size_t sample_count = 0;
uint8_t channels = 0;
uint32_t sampling_rate = 0;
AudioTestData();
~AudioTestData();
static void receive_frame(uint32_t friend_number, const int16_t *pcm, size_t sample_count,
uint8_t channels, uint32_t sampling_rate, void *user_data);
};
// Video Helpers
void fill_video_frame(uint16_t width, uint16_t height, int frame_index, std::vector<uint8_t> &y,
std::vector<uint8_t> &u, std::vector<uint8_t> &v);
double calculate_video_mse(uint16_t width, uint16_t height, int32_t ystride,
const std::vector<uint8_t> &y_recv, const std::vector<uint8_t> &y_orig);
// Video Test Data Helper
struct VideoTestData {
uint32_t friend_number = 0;
uint16_t width = 0;
uint16_t height = 0;
std::vector<uint8_t> y, u, v;
int32_t ystride = 0, ustride = 0, vstride = 0;
VideoTestData();
~VideoTestData();
static void receive_frame(uint32_t friend_number, uint16_t width, uint16_t height,
const uint8_t *y, const uint8_t *u, const uint8_t *v, int32_t ystride, int32_t ustride,
int32_t vstride, void *user_data);
double calculate_mse(const std::vector<uint8_t> &y_orig) const;
};
// Common Test Fixture
class AvTest : public ::testing::Test {
protected:
void SetUp() override;
void TearDown() override;
Logger *log = nullptr;
Mono_Time *mono_time = nullptr;
MockTime tm;
};
#endif // TOXAV_AV_TEST_SUPPORT_H

4
toxav/bwcontroller.c
View File

@@ -58,7 +58,7 @@ struct BWCMessage {
uint32_t recv;
};
static void send_update(BWController *bwc);
static void send_update(BWController *_Nonnull bwc);
BWController *bwc_new(const Logger *log, uint32_t friendnumber,
@@ -163,7 +163,7 @@ static void send_update(BWController *bwc)
}
}
static int on_update(BWController *bwc, const struct BWCMessage *msg)
static int on_update(BWController *_Nonnull bwc, const struct BWCMessage *_Nonnull msg)
{
LOGGER_DEBUG(bwc->log, "%p Got update from peer", (void *)bwc);

22
toxav/bwcontroller.h
View File

@@ -5,8 +5,8 @@
#ifndef C_TOXCORE_TOXAV_BWCONTROLLER_H
#define C_TOXCORE_TOXAV_BWCONTROLLER_H
#include <stdint.h>
#include <stddef.h>
#include <stdint.h>
#include "../toxcore/logger.h"
#include "../toxcore/mono_time.h"
@@ -19,21 +19,21 @@ extern "C" {
typedef struct BWController BWController;
typedef void bwc_loss_report_cb(BWController *bwc, uint32_t friend_number, float loss, void *user_data);
typedef void bwc_loss_report_cb(BWController *_Nonnull bwc, uint32_t friend_number, float loss, void *_Nullable user_data);
typedef int bwc_send_packet_cb(void *user_data, const uint8_t *data, uint16_t length);
typedef int bwc_send_packet_cb(void *_Nullable user_data, const uint8_t *_Nonnull data, uint16_t length);
BWController *bwc_new(const Logger *log, uint32_t friendnumber,
bwc_loss_report_cb *mcb, void *mcb_user_data,
bwc_send_packet_cb *send_packet, void *send_packet_user_data,
Mono_Time *bwc_mono_time);
BWController *_Nullable bwc_new(const Logger *_Nonnull log, uint32_t friendnumber,
bwc_loss_report_cb *_Nullable mcb, void *_Nullable mcb_user_data,
bwc_send_packet_cb *_Nullable send_packet, void *_Nullable send_packet_user_data,
Mono_Time *_Nonnull bwc_mono_time);
void bwc_kill(BWController *bwc);
void bwc_kill(BWController *_Nullable bwc);
void bwc_add_lost(BWController *bwc, uint32_t bytes_lost);
void bwc_add_recv(BWController *bwc, uint32_t recv_bytes);
void bwc_add_lost(BWController *_Nullable bwc, uint32_t bytes_lost);
void bwc_add_recv(BWController *_Nullable bwc, uint32_t recv_bytes);
void bwc_handle_packet(BWController *bwc, const uint8_t *data, size_t length);
void bwc_handle_packet(BWController *_Nullable bwc, const uint8_t *_Nonnull data, size_t length);
#ifdef __cplusplus
} /* extern "C" */

16
toxav/bwcontroller_test.cc
View File

@@ -150,22 +150,6 @@ TEST_F(BwcTest, HandlePacket)
bwc_kill(bwc);
}
TEST_F(BwcTest, NullArgs)
{
// These should just return without crashing
bwc_kill(nullptr);
bwc_add_lost(nullptr, 100);
bwc_add_recv(nullptr, 100);
bwc_handle_packet(nullptr, nullptr, 0);
MockBwcData sd;
BWController *bwc = bwc_new(
log, 123, MockBwcData::loss_report, &sd, MockBwcData::send_packet, &sd, mono_time);
bwc_add_recv(bwc, 0); // Should return early
bwc_add_lost(bwc, 0); // Should return early
bwc_kill(bwc);
}
TEST_F(BwcTest, InvalidPacketSize)
{
MockBwcData sd;

34
toxav/groupav.c
View File

@@ -31,7 +31,7 @@ typedef struct Group_JitterBuffer {
uint64_t last_queued_time;
} Group_JitterBuffer;
static void free_audio_packet(Group_Audio_Packet *pk)
static void free_audio_packet(Group_Audio_Packet *_Nullable pk)
{
if (pk == nullptr) {
return;
@@ -41,7 +41,7 @@ static void free_audio_packet(Group_Audio_Packet *pk)
free(pk);
}
static Group_JitterBuffer *create_queue(unsigned int capacity)
static Group_JitterBuffer *_Nullable create_queue(unsigned int capacity)
{
unsigned int size = 1;
@@ -67,7 +67,7 @@ static Group_JitterBuffer *create_queue(unsigned int capacity)
return q;
}
static void clear_queue(Group_JitterBuffer *q)
static void clear_queue(Group_JitterBuffer *_Nonnull q)
{
while (q->bottom != q->top) {
const size_t idx = q->bottom % q->size;
@@ -77,7 +77,7 @@ static void clear_queue(Group_JitterBuffer *q)
}
}
static void terminate_queue(Group_JitterBuffer *q)
static void terminate_queue(Group_JitterBuffer *_Nullable q)
{
if (q == nullptr) {
return;
@@ -91,7 +91,7 @@ static void terminate_queue(Group_JitterBuffer *q)
/** @retval 0 if packet was queued
* @retval -1 if it wasn't.
*/
static int queue(Group_JitterBuffer *q, const Mono_Time *mono_time, Group_Audio_Packet *pk)
static int queue(Group_JitterBuffer *_Nonnull q, const Mono_Time *_Nonnull mono_time, Group_Audio_Packet *_Nonnull pk)
{
const uint16_t sequnum = pk->sequnum;
@@ -133,7 +133,7 @@ static int queue(Group_JitterBuffer *q, const Mono_Time *mono_time, Group_Audio_
* - 1 when there's a good packet
* - 2 when there's a lost packet
*/
static Group_Audio_Packet *dequeue(Group_JitterBuffer *q, int *success)
static Group_Audio_Packet *_Nullable dequeue(Group_JitterBuffer *_Nonnull q, int *_Nonnull success)
{
if (q->top == q->bottom) {
*success = 0;
@@ -185,7 +185,7 @@ typedef struct Group_Peer_AV {
unsigned int last_packet_samples;
} Group_Peer_AV;
static void kill_group_av(Group_AV *group_av)
static void kill_group_av(Group_AV *_Nonnull group_av)
{
if (group_av->audio_encoder != nullptr) {
opus_encoder_destroy(group_av->audio_encoder);
@@ -194,7 +194,7 @@ static void kill_group_av(Group_AV *group_av)
free(group_av);
}
static int recreate_encoder(Group_AV *group_av)
static int recreate_encoder(Group_AV *_Nonnull group_av)
{
if (group_av->audio_encoder != nullptr) {
opus_encoder_destroy(group_av->audio_encoder);
@@ -232,8 +232,8 @@ static int recreate_encoder(Group_AV *group_av)
return 0;
}
static Group_AV *new_group_av(const Logger *log, Tox *tox, Group_Chats *g_c, audio_data_cb *audio_callback,
void *userdata)
static Group_AV *_Nullable new_group_av(const Logger *_Nonnull log, Tox *_Nonnull tox, Group_Chats *_Nonnull g_c, audio_data_cb *_Nullable audio_callback,
void *_Nullable userdata)
{
if (g_c == nullptr) {
return nullptr;
@@ -255,7 +255,7 @@ static Group_AV *new_group_av(const Logger *log, Tox *tox, Group_Chats *g_c, aud
return group_av;
}
static void group_av_peer_new(void *object, Tox_Conference_Number conference_number, Tox_Conference_Peer_Number peer_number)
static void group_av_peer_new(void *_Nonnull object, Tox_Conference_Number conference_number, Tox_Conference_Peer_Number peer_number)
{
const Group_AV *group_av = (const Group_AV *)object;
Group_Peer_AV *peer_av = (Group_Peer_AV *)calloc(1, sizeof(Group_Peer_AV));
@@ -272,7 +272,7 @@ static void group_av_peer_new(void *object, Tox_Conference_Number conference_num
}
}
static void group_av_peer_delete(void *object, Tox_Conference_Number conference_number, void *peer_object)
static void group_av_peer_delete(void *_Nullable object, Tox_Conference_Number conference_number, void *_Nullable peer_object)
{
Group_Peer_AV *peer_av = (Group_Peer_AV *)peer_object;
@@ -288,7 +288,7 @@ static void group_av_peer_delete(void *object, Tox_Conference_Number conference_
free(peer_object);
}
static void group_av_groupchat_delete(void *object, Tox_Conference_Number conference_number)
static void group_av_groupchat_delete(void *_Nullable object, Tox_Conference_Number conference_number)
{
Group_AV *group_av = (Group_AV *)object;
if (group_av != nullptr) {
@@ -296,7 +296,7 @@ static void group_av_groupchat_delete(void *object, Tox_Conference_Number confer
}
}
static int decode_audio_packet(Group_AV *group_av, Group_Peer_AV *peer_av, Tox_Conference_Number conference_number,
static int decode_audio_packet(Group_AV *_Nonnull group_av, Group_Peer_AV *_Nonnull peer_av, Tox_Conference_Number conference_number,
Tox_Conference_Peer_Number peer_number)
{
if (group_av == nullptr || peer_av == nullptr) {
@@ -402,8 +402,8 @@ static int decode_audio_packet(Group_AV *group_av, Group_Peer_AV *peer_av, Tox_C
return -1;
}
static int handle_group_audio_packet(void *object, Tox_Conference_Number conference_number, Tox_Conference_Peer_Number peer_number, void *peer_object,
const uint8_t *packet, uint16_t length)
static int handle_group_audio_packet(void *_Nonnull object, Tox_Conference_Number conference_number, Tox_Conference_Peer_Number peer_number, void *_Nonnull peer_object,
const uint8_t *_Nonnull packet, uint16_t length)
{
Group_AV *group_av = (Group_AV *)object;
Group_Peer_AV *peer_av = (Group_Peer_AV *)peer_object;
@@ -579,7 +579,7 @@ int join_av_groupchat(const Logger *log, Tox *tox, Group_Chats *g_c, Tox_Friend_
* @retval 0 on success.
* @retval -1 on failure.
*/
static int send_audio_packet(const Group_Chats *g_c, Tox_Conference_Number conference_number, const uint8_t *packet, uint16_t length)
static int send_audio_packet(const Group_Chats *_Nonnull g_c, Tox_Conference_Number conference_number, const uint8_t *_Nonnull packet, uint16_t length)
{
if (length == 0 || length > MAX_CRYPTO_DATA_SIZE - 1 - sizeof(uint16_t)) {
return -1;

20
toxav/groupav.h
View File

@@ -16,30 +16,30 @@
// TODO(iphydf): Use this better typed one instead of the void-pointer one below.
// typedef void audio_data_cb(Tox *tox, uint32_t conference_number, uint32_t peer_number, const int16_t *pcm,
// uint32_t samples, uint8_t channels, uint32_t sample_rate, void *userdata);
typedef void audio_data_cb(void *tox, Tox_Conference_Number conference_number, Tox_Conference_Peer_Number peer_number, const int16_t pcm[],
uint32_t samples, uint8_t channels, uint32_t sample_rate, void *userdata);
typedef void audio_data_cb(void *_Nullable tox, Tox_Conference_Number conference_number, Tox_Conference_Peer_Number peer_number, const int16_t pcm[_Nullable],
uint32_t samples, uint8_t channels, uint32_t sample_rate, void *_Nullable userdata);
/** @brief Create and connect to a new toxav group.
*
* @return conference number on success.
* @retval -1 on failure.
*/
int add_av_groupchat(const Logger *log, Tox *tox, Group_Chats *g_c, audio_data_cb *audio_callback, void *userdata);
int add_av_groupchat(const Logger *_Nonnull log, Tox *_Nonnull tox, Group_Chats *_Nonnull g_c, audio_data_cb *_Nullable audio_callback, void *_Nullable userdata);
/** @brief Join a AV group (you need to have been invited first).
*
* @return conference number on success
* @retval -1 on failure.
*/
int join_av_groupchat(const Logger *log, Tox *tox, Group_Chats *g_c, Tox_Friend_Number friend_number, const uint8_t *data,
uint16_t length, audio_data_cb *audio_callback, void *userdata);
int join_av_groupchat(const Logger *_Nonnull log, Tox *_Nonnull tox, Group_Chats *_Nonnull g_c, Tox_Friend_Number friend_number, const uint8_t *_Nonnull data,
uint16_t length, audio_data_cb *_Nullable audio_callback, void *_Nullable userdata);
/** @brief Send audio to the conference.
*
* @retval 0 on success.
* @retval -1 on failure.
*/
int group_send_audio(const Group_Chats *g_c, Tox_Conference_Number conference_number, const int16_t pcm[], uint32_t samples,
int group_send_audio(const Group_Chats *_Nonnull g_c, Tox_Conference_Number conference_number, const int16_t pcm[_Nonnull], uint32_t samples,
uint8_t channels,
uint32_t sample_rate);
@@ -48,17 +48,17 @@ int group_send_audio(const Group_Chats *g_c, Tox_Conference_Number conference_nu
* @retval 0 on success.
* @retval -1 on failure.
*/
int groupchat_enable_av(const Logger *log, Tox *tox, Group_Chats *g_c, Tox_Conference_Number conference_number,
audio_data_cb *audio_callback, void *userdata);
int groupchat_enable_av(const Logger *_Nonnull log, Tox *_Nonnull tox, Group_Chats *_Nonnull g_c, Tox_Conference_Number conference_number,
audio_data_cb *_Nullable audio_callback, void *_Nullable userdata);
/** @brief Disable A/V in a conference.
*
* @retval 0 on success.
* @retval -1 on failure.
*/
int groupchat_disable_av(const Group_Chats *g_c, Tox_Conference_Number conference_number);
int groupchat_disable_av(const Group_Chats *_Nonnull g_c, Tox_Conference_Number conference_number);
/** Return whether A/V is enabled in the conference. */
bool groupchat_av_enabled(const Group_Chats *g_c, Tox_Conference_Number conference_number);
bool groupchat_av_enabled(const Group_Chats *_Nonnull g_c, Tox_Conference_Number conference_number);
#endif /* C_TOXCORE_TOXAV_GROUPAV_H */

40
toxav/msi.c
View File

@@ -54,19 +54,19 @@ typedef struct MSIMessage {
MSIHeaderCapabilities capabilities;
} MSIMessage;
static void msg_init(MSIMessage *dest, MSIRequest request);
static void kill_call(const Logger *log, MSICall *call);
static int msg_parse_in(const Logger *log, MSIMessage *dest, const uint8_t *data, uint16_t length);
static uint8_t *msg_parse_header_out(MSIHeaderID id, uint8_t *dest, const uint8_t *value, uint8_t value_len,
uint16_t *length);
static int send_message(const Logger *log, MSISession *session, uint32_t friend_number, const MSIMessage *msg);
static int send_error(const Logger *log, MSISession *session, uint32_t friend_number, MSIError error);
static MSICall *get_call(MSISession *session, uint32_t friend_number);
static MSICall *new_call(MSISession *session, uint32_t friend_number);
static bool invoke_callback(const Logger *log, MSICall *call, MSICallbackID cb);
static void handle_init(const Logger *log, MSICall *call, const MSIMessage *msg);
static void handle_push(const Logger *log, MSICall *call, const MSIMessage *msg);
static void handle_pop(const Logger *log, MSICall *call, const MSIMessage *msg);
static void msg_init(MSIMessage *_Nonnull dest, MSIRequest request);
static void kill_call(const Logger *_Nonnull log, MSICall *_Nonnull call);
static int msg_parse_in(const Logger *_Nonnull log, MSIMessage *_Nonnull dest, const uint8_t *_Nonnull data, uint16_t length);
static uint8_t *_Nonnull msg_parse_header_out(MSIHeaderID id, uint8_t *_Nonnull dest, const uint8_t *_Nonnull value, uint8_t value_len,
uint16_t *_Nonnull length);
static int send_message(const Logger *_Nonnull log, MSISession *_Nonnull session, uint32_t friend_number, const MSIMessage *_Nonnull msg);
static int send_error(const Logger *_Nonnull log, MSISession *_Nonnull session, uint32_t friend_number, MSIError error);
static MSICall *_Nullable get_call(MSISession *_Nonnull session, uint32_t friend_number);
static MSICall *_Nullable new_call(MSISession *_Nonnull session, uint32_t friend_number);
static bool invoke_callback(const Logger *_Nonnull log, MSICall *_Nonnull call, MSICallbackID cb);
static void handle_init(const Logger *_Nonnull log, MSICall *_Nonnull call, const MSIMessage *_Nonnull msg);
static void handle_push(const Logger *_Nonnull log, MSICall *_Nonnull call, const MSIMessage *_Nonnull msg);
static void handle_pop(const Logger *_Nonnull log, MSICall *_Nonnull call, const MSIMessage *_Nonnull msg);
/*
* Public functions
@@ -321,7 +321,7 @@ static void msg_init(MSIMessage *dest, MSIRequest request)
dest->request.value = request;
}
static bool check_size(const Logger *log, const uint8_t *bytes, int *constraint, uint8_t size)
static bool check_size(const Logger *_Nonnull log, const uint8_t *_Nonnull bytes, int *_Nonnull constraint, uint8_t size)
{
*constraint -= 2 + size;
@@ -339,7 +339,7 @@ static bool check_size(const Logger *log, const uint8_t *bytes, int *constraint,
}
/** Assumes size == 1 */
static bool check_enum_high(const Logger *log, const uint8_t *bytes, uint8_t enum_high)
static bool check_enum_high(const Logger *_Nonnull log, const uint8_t *_Nonnull bytes, uint8_t enum_high)
{
if (bytes[2] > enum_high) {
LOGGER_ERROR(log, "Failed enum high limit!");
@@ -349,7 +349,7 @@ static bool check_enum_high(const Logger *log, const uint8_t *bytes, uint8_t enu
return true;
}
static const uint8_t *msg_parse_one(const Logger *log, MSIMessage *dest, const uint8_t *it, int *size_constraint)
static const uint8_t *_Nullable msg_parse_one(const Logger *_Nonnull log, MSIMessage *_Nonnull dest, const uint8_t *_Nonnull it, int *_Nonnull size_constraint)
{
switch (*it) {
case ID_REQUEST: {
@@ -451,7 +451,7 @@ static int send_message(const Logger *log, MSISession *session, uint32_t friend_
uint16_t size = 0;
if (msg->request.exists) {
uint8_t cast = msg->request.value;
const uint8_t cast = msg->request.value;
it = msg_parse_header_out(ID_REQUEST, it, &cast,
sizeof(cast), &size);
} else {
@@ -460,7 +460,7 @@ static int send_message(const Logger *log, MSISession *session, uint32_t friend_
}
if (msg->error.exists) {
uint8_t cast = msg->error.value;
const uint8_t cast = msg->error.value;
it = msg_parse_header_out(ID_ERROR, it, &cast,
sizeof(cast), &size);
}
@@ -501,7 +501,7 @@ static int send_error(const Logger *log, MSISession *session, uint32_t friend_nu
return 0;
}
static int invoke_callback_inner(const Logger *log, MSICall *call, MSICallbackID id)
static int invoke_callback_inner(const Logger *_Nonnull log, MSICall *_Nonnull call, MSICallbackID id)
{
MSISession *session = call->session;
LOGGER_DEBUG(log, "invoking callback function: %u", id);
@@ -671,7 +671,7 @@ CLEAR_CONTAINER:
}
static bool try_handle_init(const Logger *log, MSICall *call, const MSIMessage *msg)
static bool try_handle_init(const Logger *_Nonnull log, MSICall *_Nonnull call, const MSIMessage *_Nonnull msg)
{
if (!msg->capabilities.exists) {
LOGGER_WARNING(log, "Session: %p Invalid capabilities on 'init'", (void *)call->session);

37
toxav/msi.h
View File

@@ -6,6 +6,7 @@
#define C_TOXCORE_TOXAV_MSI_H
#include <pthread.h>
#include <stddef.h>
#include <stdint.h>
#include "../toxcore/logger.h"
@@ -64,19 +65,19 @@ typedef enum MSICallbackID {
* The call struct. Please do not modify outside msi.c
*/
typedef struct MSICall {
struct MSISession *session; /* Session pointer */
struct MSISession *_Nonnull session; /* Session pointer */
MSICallState state;
uint8_t peer_capabilities; /* Peer capabilities */
uint8_t self_capabilities; /* Self capabilities */
uint16_t peer_vfpsz; /* Video frame piece size */
uint32_t friend_number; /* Index of this call in MSISession */
MSIError error; /* Last error */
MSICallState state;
uint8_t peer_capabilities; /* Peer capabilities */
uint8_t self_capabilities; /* Self capabilities */
uint16_t peer_vfpsz; /* Video frame piece size */
uint32_t friend_number; /* Index of this call in MSISession */
MSIError error; /* Last error */
void *user_data; /* Pointer to av call handler */
void *_Nullable user_data; /* Pointer to av call handler */
struct MSICall *next;
struct MSICall *prev;
struct MSICall *_Nullable next;
struct MSICall *_Nullable prev;
} MSICall;
/**
@@ -84,14 +85,14 @@ typedef struct MSICall {
* returned the call is considered errored and will be handled
* as such which means it will be terminated without any notice.
*/
typedef int msi_action_cb(void *object, MSICall *call);
typedef int msi_action_cb(void *_Nullable object, MSICall *_Nonnull call);
/**
* Send packet callback.
*
* @return 0 on success, -1 on failure.
*/
typedef int msi_send_packet_cb(void *user_data, uint32_t friend_number, const uint8_t *data, size_t length);
typedef int msi_send_packet_cb(void *_Nullable user_data, uint32_t friend_number, const uint8_t *_Nonnull data, size_t length);
/**
* MSI callbacks.
@@ -110,14 +111,14 @@ typedef struct MSICallbacks {
*/
typedef struct MSISession {
/* Call handlers */
MSICall **calls;
uint32_t calls_tail;
uint32_t calls_head;
MSICall *_Nullable *_Nullable calls;
uint32_t calls_tail;
uint32_t calls_head;
void *user_data;
void *_Nullable user_data;
msi_send_packet_cb *send_packet;
void *send_packet_user_data;
msi_send_packet_cb *_Nonnull send_packet;
void *_Nullable send_packet_user_data;
pthread_mutex_t mutex[1];

18
toxav/ring_buffer.h
View File

@@ -9,20 +9,22 @@
#include <stdbool.h>
#include <stdint.h>
#include "../toxcore/attributes.h"
#ifdef __cplusplus
extern "C" {
#endif
/** Ring buffer */
typedef struct RingBuffer RingBuffer;
bool rb_full(const RingBuffer *b);
bool rb_empty(const RingBuffer *b);
void *rb_write(RingBuffer *b, void *p);
bool rb_read(RingBuffer *b, void **p);
RingBuffer *rb_new(int size);
void rb_kill(RingBuffer *b);
uint16_t rb_size(const RingBuffer *b);
uint16_t rb_data(const RingBuffer *b, void **dest);
bool rb_full(const RingBuffer *_Nonnull b);
bool rb_empty(const RingBuffer *_Nonnull b);
void *_Nullable rb_write(RingBuffer *_Nullable b, void *_Nullable p);
bool rb_read(RingBuffer *_Nonnull b, void *_Nonnull *_Nullable p);
RingBuffer *_Nullable rb_new(int size);
void rb_kill(RingBuffer *_Nullable b);
uint16_t rb_size(const RingBuffer *_Nonnull b);
uint16_t rb_data(const RingBuffer *_Nonnull b, void *_Nonnull *_Nonnull dest);
#ifdef __cplusplus
} /* extern "C" */

61
toxav/rtp.c
View File

@@ -171,6 +171,9 @@ uint32_t rtp_message_data_length_full(const RTPMessage *msg)
bool rtp_session_is_receiving_active(const RTPSession *session)
{
if (session == nullptr) {
return false;
}
return session->rtp_receive_active;
}
@@ -191,8 +194,8 @@ void rtp_session_set_ssrc(RTPSession *session, uint32_t ssrc)
#define VIDEO_KEEP_KEYFRAME_IN_BUFFER_FOR_MS 15
// allocate_len is NOT including header!
static struct RTPMessage *new_message(const Logger *log, const struct RTPHeader *header, size_t allocate_len,
const uint8_t *data, uint16_t data_length)
static struct RTPMessage *_Nullable new_message(const Logger *_Nonnull log, const struct RTPHeader *_Nonnull header, size_t allocate_len,
const uint8_t *_Nonnull data, uint16_t data_length)
{
if (allocate_len < data_length) {
LOGGER_WARNING(log, "new_message: allocate_len (%zu) < data_length (%u)", allocate_len, data_length);
@@ -234,8 +237,8 @@ static struct RTPMessage *new_message(const Logger *log, const struct RTPHeader
* do not kick it out right away if all slots are full instead kick out the new
* incoming interframe.
*/
static int8_t get_slot(const Logger *log, struct RTPWorkBufferList *wkbl, bool is_keyframe,
const struct RTPHeader *header, bool is_multipart)
static int8_t get_slot(const Logger *_Nonnull log, struct RTPWorkBufferList *_Nonnull wkbl, bool is_keyframe,
const struct RTPHeader *_Nonnull header, bool is_multipart)
{
if (is_multipart) {
// This RTP message is part of a multipart frame, so we try to find an
@@ -350,7 +353,7 @@ static int8_t get_slot(const Logger *log, struct RTPWorkBufferList *wkbl, bool i
* non-NULL, it transfers ownership of the message to the caller, i.e. the
* caller is responsible for storing it elsewhere or calling `free()`.
*/
static struct RTPMessage *process_frame(const Logger *log, struct RTPWorkBufferList *wkbl, uint8_t slot_id)
static struct RTPMessage *_Nullable process_frame(const Logger *_Nonnull log, struct RTPWorkBufferList *_Nonnull wkbl, uint8_t slot_id)
{
assert(wkbl->next_free_entry >= 0);
@@ -390,7 +393,7 @@ static struct RTPMessage *process_frame(const Logger *log, struct RTPWorkBufferL
--wkbl->next_free_entry;
// Clear the newly freed entry.
const struct RTPWorkBuffer empty = {0};
const struct RTPWorkBuffer empty = {false};
wkbl->work_buffer[wkbl->next_free_entry] = empty;
// Move ownership of the frame to the caller.
@@ -406,9 +409,9 @@ static struct RTPMessage *process_frame(const Logger *log, struct RTPWorkBufferL
* @param incoming_data The pure payload without header.
* @param incoming_data_length The length in bytes of the incoming data payload.
*/
static bool fill_data_into_slot(const Logger *log, struct RTPWorkBufferList *wkbl, const uint8_t slot_id,
bool is_keyframe, const struct RTPHeader *header,
const uint8_t *incoming_data, uint16_t incoming_data_length)
static bool fill_data_into_slot(const Logger *_Nonnull log, struct RTPWorkBufferList *_Nonnull wkbl, const uint8_t slot_id,
bool is_keyframe, const struct RTPHeader *_Nonnull header,
const uint8_t *_Nonnull incoming_data, uint16_t incoming_data_length)
{
// We're either filling the data into an existing slot, or in a new one that
// is the next free entry.
@@ -483,20 +486,20 @@ static bool fill_data_into_slot(const Logger *log, struct RTPWorkBufferList *wkb
return slot->received_len == header->data_length_full;
}
static void update_bwc_values(RTPSession *session, const struct RTPMessage *msg)
static void update_bwc_values(RTPSession *_Nonnull session, const struct RTPMessage *_Nonnull msg)
{
if (session->first_packets_counter < DISMISS_FIRST_LOST_VIDEO_PACKET_COUNT) {
++session->first_packets_counter;
} else {
const uint32_t data_length_full = msg->header.data_length_full; // without header
const uint32_t received_length_full = msg->header.received_length_full; // without header
if (session->add_recv) {
if (session->add_recv != nullptr) {
session->add_recv(session->bwc_user_data, data_length_full);
}
if (received_length_full < data_length_full) {
LOGGER_DEBUG(session->log, "BWC: full length=%u received length=%u", data_length_full, received_length_full);
if (session->add_lost) {
if (session->add_lost != nullptr) {
session->add_lost(session->bwc_user_data, data_length_full - received_length_full);
}
}
@@ -520,8 +523,8 @@ static void update_bwc_values(RTPSession *session, const struct RTPMessage *msg)
* @retval -1 on error.
* @retval 0 on success.
*/
static int handle_video_packet(const Logger *log, RTPSession *session, const struct RTPHeader *header,
const uint8_t *incoming_data, uint16_t incoming_data_length)
static int handle_video_packet(const Logger *_Nonnull log, RTPSession *_Nonnull session, const struct RTPHeader *_Nonnull header,
const uint8_t *_Nonnull incoming_data, uint16_t incoming_data_length)
{
// Full frame length in bytes. The frame may be split into multiple packets,
// but this value is the complete assembled frame size.
@@ -679,7 +682,7 @@ void rtp_receive_packet(RTPSession *session, const uint8_t *data, size_t length)
/* Message is not late; pick up the latest parameters */
session->rsequnum = header.sequnum;
session->rtimestamp = header.timestamp;
if (session->add_recv) {
if (session->add_recv != nullptr) {
session->add_recv(session->bwc_user_data, payload_size);
}
@@ -723,7 +726,7 @@ void rtp_receive_packet(RTPSession *session, const uint8_t *data, size_t length)
memcpy(session->mp->data + header.offset_lower, &payload[RTP_HEADER_SIZE],
payload_size - RTP_HEADER_SIZE);
session->mp->len += payload_size - RTP_HEADER_SIZE;
if (session->add_recv) {
if (session->add_recv != nullptr) {
session->add_recv(session->bwc_user_data, payload_size);
}
@@ -765,7 +768,7 @@ NEW_MULTIPARTED:
/* Message is not late; pick up the latest parameters */
session->rsequnum = header.sequnum;
session->rtimestamp = header.timestamp;
if (session->add_recv) {
if (session->add_recv != nullptr) {
session->add_recv(session->bwc_user_data, payload_size);
}
@@ -849,10 +852,10 @@ static uint32_t rtp_random_u32(void)
return randombytes_random();
}
RTPSession *rtp_new(const Logger *log, int payload_type, Mono_Time *mono_time,
rtp_send_packet_cb *send_packet, void *send_packet_user_data,
rtp_add_recv_cb *add_recv, rtp_add_lost_cb *add_lost, void *bwc_user_data,
void *cs, rtp_m_cb *mcb)
RTPSession *_Nullable rtp_new(const Logger *_Nonnull log, int payload_type, Mono_Time *_Nonnull mono_time,
rtp_send_packet_cb *_Nullable send_packet, void *_Nullable send_packet_user_data,
rtp_add_recv_cb *_Nullable add_recv, rtp_add_lost_cb *_Nullable add_lost, void *_Nullable bwc_user_data,
void *_Nonnull cs, rtp_m_cb *_Nonnull mcb)
{
assert(mcb != nullptr);
assert(cs != nullptr);
@@ -898,7 +901,7 @@ RTPSession *rtp_new(const Logger *log, int payload_type, Mono_Time *mono_time,
return session;
}
void rtp_kill(const Logger *log, RTPSession *session)
void rtp_kill(const Logger *_Nonnull log, RTPSession *_Nullable session)
{
if (session == nullptr) {
LOGGER_WARNING(log, "No session");
@@ -909,7 +912,7 @@ void rtp_kill(const Logger *log, RTPSession *session)
LOGGER_DEBUG(log, "Terminated RTP session V3 work_buffer_list->next_free_entry: %d",
(int)session->work_buffer_list->next_free_entry);
if (session->work_buffer_list) {
if (session->work_buffer_list != nullptr) {
for (int8_t i = 0; i < session->work_buffer_list->next_free_entry; ++i) {
free(session->work_buffer_list->work_buffer[i].buf);
}
@@ -919,34 +922,34 @@ void rtp_kill(const Logger *log, RTPSession *session)
free(session);
}
void rtp_allow_receiving_mark(RTPSession *session)
void rtp_allow_receiving_mark(RTPSession *_Nullable session)
{
if (session != nullptr) {
session->rtp_receive_active = true;
}
}
void rtp_stop_receiving_mark(RTPSession *session)
void rtp_stop_receiving_mark(RTPSession *_Nullable session)
{
if (session != nullptr) {
session->rtp_receive_active = false;
}
}
static void rtp_send_piece(RTPSession *session, const struct RTPHeader *header,
const uint8_t *data, uint8_t *rdata, uint16_t length)
static void rtp_send_piece(RTPSession *_Nonnull session, const struct RTPHeader *_Nonnull header,
const uint8_t *_Nonnull data, uint8_t *_Nonnull rdata, uint16_t length)
{
rtp_header_pack(rdata + 1, header);
memcpy(rdata + 1 + RTP_HEADER_SIZE, data, length);
const uint16_t rdata_size = length + RTP_HEADER_SIZE + 1;
if (session->send_packet) {
if (session->send_packet != nullptr) {
session->send_packet(session->send_packet_user_data, rdata, rdata_size);
}
}
static struct RTPHeader rtp_default_header(const RTPSession *session, uint32_t length, bool is_keyframe)
static struct RTPHeader rtp_default_header(const RTPSession *_Nonnull session, uint32_t length, bool is_keyframe)
{
uint16_t length_safe = (uint16_t)length;

48
toxav/rtp.h
View File

@@ -56,28 +56,28 @@ typedef struct RTPMessage RTPMessage;
typedef struct RTPSession RTPSession;
/* RTPMessage accessors */
const uint8_t *rtp_message_data(const RTPMessage *msg);
uint32_t rtp_message_len(const RTPMessage *msg);
uint8_t rtp_message_pt(const RTPMessage *msg);
uint16_t rtp_message_sequnum(const RTPMessage *msg);
uint64_t rtp_message_flags(const RTPMessage *msg);
uint32_t rtp_message_data_length_full(const RTPMessage *msg);
const uint8_t *_Nonnull rtp_message_data(const RTPMessage *_Nonnull msg);
uint32_t rtp_message_len(const RTPMessage *_Nonnull msg);
uint8_t rtp_message_pt(const RTPMessage *_Nonnull msg);
uint16_t rtp_message_sequnum(const RTPMessage *_Nonnull msg);
uint64_t rtp_message_flags(const RTPMessage *_Nonnull msg);
uint32_t rtp_message_data_length_full(const RTPMessage *_Nonnull msg);
/* RTPSession accessors */
bool rtp_session_is_receiving_active(const RTPSession *session);
uint32_t rtp_session_get_ssrc(const RTPSession *session);
void rtp_session_set_ssrc(RTPSession *session, uint32_t ssrc);
bool rtp_session_is_receiving_active(const RTPSession *_Nullable session);
uint32_t rtp_session_get_ssrc(const RTPSession *_Nonnull session);
void rtp_session_set_ssrc(RTPSession *_Nonnull session, uint32_t ssrc);
#define USED_RTP_WORKBUFFER_COUNT 3
#define DISMISS_FIRST_LOST_VIDEO_PACKET_COUNT 10
typedef int rtp_m_cb(const Mono_Time *mono_time, void *cs, RTPMessage *msg);
typedef int rtp_m_cb(const Mono_Time *_Nonnull mono_time, void *_Nonnull cs, RTPMessage *_Nonnull msg);
typedef int rtp_send_packet_cb(void *user_data, const uint8_t *data, uint16_t length);
typedef void rtp_add_recv_cb(void *user_data, uint32_t bytes);
typedef void rtp_add_lost_cb(void *user_data, uint32_t bytes);
typedef int rtp_send_packet_cb(void *_Nullable user_data, const uint8_t *_Nonnull data, uint16_t length);
typedef void rtp_add_recv_cb(void *_Nullable user_data, uint32_t bytes);
typedef void rtp_add_lost_cb(void *_Nullable user_data, uint32_t bytes);
void rtp_receive_packet(RTPSession *session, const uint8_t *data, size_t length);
void rtp_receive_packet(RTPSession *_Nonnull session, const uint8_t *_Nonnull data, size_t length);
/**
* Serialise an RTPHeader to bytes to be sent over the network.
@@ -87,7 +87,7 @@ void rtp_receive_packet(RTPSession *session, const uint8_t *data, size_t length)
* to this function.
* @param header The RTPHeader to serialise.
*/
size_t rtp_header_pack(uint8_t *rdata, const struct RTPHeader *header);
size_t rtp_header_pack(uint8_t *_Nonnull rdata, const struct RTPHeader *_Nonnull header);
/**
* Deserialise an RTPHeader from bytes received over the network.
@@ -95,15 +95,15 @@ size_t rtp_header_pack(uint8_t *rdata, const struct RTPHeader *header);
* @param data A byte array of length RTP_HEADER_SIZE.
* @param header The RTPHeader to write the unpacked values to.
*/
size_t rtp_header_unpack(const uint8_t *data, struct RTPHeader *header);
size_t rtp_header_unpack(const uint8_t *_Nonnull data, struct RTPHeader *_Nonnull header);
RTPSession *rtp_new(const Logger *log, int payload_type, Mono_Time *mono_time,
rtp_send_packet_cb *send_packet, void *send_packet_user_data,
rtp_add_recv_cb *add_recv, rtp_add_lost_cb *add_lost, void *bwc_user_data,
void *cs, rtp_m_cb *mcb);
void rtp_kill(const Logger *log, RTPSession *session);
void rtp_allow_receiving_mark(RTPSession *session);
void rtp_stop_receiving_mark(RTPSession *session);
RTPSession *_Nullable rtp_new(const Logger *_Nonnull log, int payload_type, Mono_Time *_Nonnull mono_time,
rtp_send_packet_cb *_Nullable send_packet, void *_Nullable send_packet_user_data,
rtp_add_recv_cb *_Nullable add_recv, rtp_add_lost_cb *_Nullable add_lost, void *_Nullable bwc_user_data,
void *_Nonnull cs, rtp_m_cb *_Nonnull mcb);
void rtp_kill(const Logger *_Nonnull log, RTPSession *_Nullable session);
void rtp_allow_receiving_mark(RTPSession *_Nullable session);
void rtp_stop_receiving_mark(RTPSession *_Nullable session);
/**
* @brief Send a frame of audio or video data, chunked in @ref RTPMessage instances.
@@ -114,7 +114,7 @@ void rtp_stop_receiving_mark(RTPSession *session);
* @param is_keyframe Whether this video frame is a key frame. If it is an
* audio frame, this parameter is ignored.
*/
int rtp_send_data(const Logger *log, RTPSession *session, const uint8_t *data, uint32_t length,
int rtp_send_data(const Logger *_Nonnull log, RTPSession *_Nonnull session, const uint8_t *_Nonnull data, uint32_t length,
bool is_keyframe);
#ifdef __cplusplus

73
toxav/rtp_bench.cc Normal file
View File

@@ -0,0 +1,73 @@
/* SPDX-License-Identifier: GPL-3.0-or-later
* Copyright © 2025 The TokTok team.
*/
#include <benchmark/benchmark.h>
#include <cstring>
#include <vector>
#include "../toxcore/logger.h"
#include "../toxcore/mono_time.h"
#include "../toxcore/os_memory.h"
#include "av_test_support.hh"
#include "rtp.h"
namespace {
class RtpBench : public benchmark::Fixture {
public:
void SetUp(const ::benchmark::State &) override
{
const Memory *mem = os_memory();
log = logger_new(mem);
mono_time = mono_time_new(mem, nullptr, nullptr);
mock.store_last_packet_only = true;
session = rtp_new(log, RTP_TYPE_VIDEO, mono_time, RtpMock::send_packet, &mock, nullptr,
nullptr, nullptr, &mock, RtpMock::noop_cb);
}
void TearDown(const ::benchmark::State &) override
{
const Memory *mem = os_memory();
rtp_kill(log, session);
mono_time_free(mem, mono_time);
logger_kill(log);
}
Logger *log = nullptr;
Mono_Time *mono_time = nullptr;
RTPSession *session = nullptr;
RtpMock mock;
};
BENCHMARK_DEFINE_F(RtpBench, SendData)(benchmark::State &state)
{
size_t data_size = static_cast<size_t>(state.range(0));
std::vector<uint8_t> data(data_size, 0xAA);
for (auto _ : state) {
rtp_send_data(log, session, data.data(), static_cast<uint32_t>(data.size()), false);
benchmark::DoNotOptimize(mock.captured_packets.back());
}
}
BENCHMARK_REGISTER_F(RtpBench, SendData)->Arg(100)->Arg(1000)->Arg(5000);
BENCHMARK_DEFINE_F(RtpBench, ReceivePacket)(benchmark::State &state)
{
size_t data_size = static_cast<size_t>(state.range(0));
std::vector<uint8_t> data(data_size, 0xAA);
rtp_send_data(log, session, data.data(), static_cast<uint32_t>(data.size()), false);
std::vector<uint8_t> packet = mock.captured_packets.back();
for (auto _ : state) {
rtp_receive_packet(session, packet.data(), packet.size());
}
}
BENCHMARK_REGISTER_F(RtpBench, ReceivePacket)->Arg(100)->Arg(1000);
} // namespace
BENCHMARK_MAIN();

5
toxav/rtp_fuzz_test.cc
View File

@@ -1,15 +1,18 @@
#include "rtp.h"
#include <cstdlib>
#include <memory>
#include <vector>
#include "../testing/fuzzing/fuzz_support.hh"
#include "../testing/support/public/fuzz_data.hh"
#include "../toxcore/logger.h"
#include "../toxcore/mono_time.h"
#include "../toxcore/os_memory.h"
namespace {
using tox::test::Fuzz_Data;
struct MockSessionData { };
static int mock_send_packet(void * /*user_data*/, const uint8_t * /*data*/, uint16_t /*length*/)

5
toxav/rtp_test.cc
View File

@@ -183,7 +183,8 @@ TEST_F(RtpPublicTest, HandlingInvalidPackets)
mock_add_recv, mock_add_lost, &sd, &sd, mock_m_cb);
// Packet too short to even contain the Tox packet ID
rtp_receive_packet(session, nullptr, 0);
uint8_t empty[1];
rtp_receive_packet(session, empty, 0);
// Packet too short (less than RTP_HEADER_SIZE + 1)
uint8_t short_pkt[10] = {RTP_TYPE_AUDIO};
@@ -214,6 +215,8 @@ TEST_F(RtpPublicTest, ReceiveActiveToggle)
rtp_allow_receiving_mark(session);
EXPECT_TRUE(rtp_session_is_receiving_active(session));
EXPECT_FALSE(rtp_session_is_receiving_active(nullptr));
rtp_kill(log, session);
}

127
toxav/toxav.c
View File

@@ -9,11 +9,11 @@
#include <stdlib.h>
#include <string.h>
#include "audio.h"
#include "bwcontroller.h"
#include "msi.h"
#include "rtp.h"
#include "audio.h"
#include "video.h"
#include "bwcontroller.h"
#include "../toxcore/Messenger.h"
#include "../toxcore/ccompat.h"
@@ -33,9 +33,9 @@
typedef struct ToxAVCall ToxAVCall;
static ToxAVCall *call_get(ToxAV *av, uint32_t friend_number);
static RTPSession *rtp_session_get(ToxAVCall *call, int payload_type);
static BWController *bwc_controller_get(const ToxAVCall *call);
static ToxAVCall *_Nullable call_get(ToxAV *_Nonnull av, uint32_t friend_number);
static RTPSession *_Nullable rtp_session_get(ToxAVCall *_Nullable call, int payload_type);
static BWController *_Nullable bwc_controller_get(const ToxAVCall *_Nullable call);
struct ToxAVCall {
ToxAV *av;
@@ -63,6 +63,9 @@ struct ToxAVCall {
toxav_audio_receive_frame_cb *acb;
void *acb_user_data;
toxav_video_receive_frame_cb *vcb;
void *vcb_user_data;
pthread_mutex_t toxav_call_mutex[1];
struct ToxAVCall *prev;
@@ -93,6 +96,7 @@ struct ToxAV {
uint32_t calls_tail;
uint32_t calls_head;
pthread_mutex_t mutex[1];
pthread_mutex_t *mutable_mutex;
/* Call callback */
toxav_call_cb *ccb;
@@ -120,9 +124,9 @@ struct ToxAV {
Mono_Time *toxav_mono_time; // ToxAV's own mono_time instance
};
static void callback_bwc(BWController *bwc, Tox_Friend_Number friend_number, float loss, void *user_data);
static void callback_bwc(BWController *_Nonnull bwc, Tox_Friend_Number friend_number, float loss, void *_Nonnull user_data);
static int msi_send_packet(void *user_data, uint32_t friend_number, const uint8_t *data, size_t length)
static int msi_send_packet(void *_Nonnull user_data, uint32_t friend_number, const uint8_t *_Nonnull data, size_t length)
{
Tox *tox = (Tox *)user_data;
const size_t length_new = length + 1;
@@ -135,8 +139,8 @@ static int msi_send_packet(void *user_data, uint32_t friend_number, const uint8_
return error == TOX_ERR_FRIEND_CUSTOM_PACKET_OK ? 0 : -1;
}
static void handle_msi_packet(Tox *tox, uint32_t friend_number, const uint8_t *data, size_t length,
void *user_data)
static void handle_msi_packet(Tox *_Nonnull tox, uint32_t friend_number, const uint8_t *_Nonnull data, size_t length,
void *_Nullable user_data)
{
ToxAV *toxav = (ToxAV *)tox_get_av_object(tox);
@@ -152,7 +156,7 @@ static void handle_msi_packet(Tox *tox, uint32_t friend_number, const uint8_t *d
msi_handle_packet(toxav->msi, toxav->log, friend_number, data + 1, length - 1);
}
static int rtp_send_packet(void *user_data, const uint8_t *data, uint16_t length)
static int rtp_send_packet(void *_Nonnull user_data, const uint8_t *_Nonnull data, uint16_t length)
{
ToxAVCall *call = (ToxAVCall *)user_data;
Tox_Err_Friend_Custom_Packet error;
@@ -160,19 +164,19 @@ static int rtp_send_packet(void *user_data, const uint8_t *data, uint16_t length
return error == TOX_ERR_FRIEND_CUSTOM_PACKET_OK ? 0 : -1;
}
static void rtp_add_recv(void *user_data, uint32_t bytes)
static void rtp_add_recv(void *_Nullable user_data, uint32_t bytes)
{
BWController *bwc = (BWController *)user_data;
bwc_add_recv(bwc, bytes);
}
static void rtp_add_lost(void *user_data, uint32_t bytes)
static void rtp_add_lost(void *_Nullable user_data, uint32_t bytes)
{
BWController *bwc = (BWController *)user_data;
bwc_add_lost(bwc, bytes);
}
static void handle_rtp_packet(Tox *tox, Tox_Friend_Number friend_number, const uint8_t *data, size_t length, void *user_data)
static void handle_rtp_packet(Tox *_Nonnull tox, Tox_Friend_Number friend_number, const uint8_t *_Nonnull data, size_t length, void *_Nullable user_data)
{
ToxAV *toxav = (ToxAV *)tox_get_av_object(tox);
@@ -199,7 +203,7 @@ static void handle_rtp_packet(Tox *tox, Tox_Friend_Number friend_number, const u
rtp_receive_packet(session, data, length);
}
static void handle_bwc_packet(Tox *tox, uint32_t friend_number, const uint8_t *data, size_t length, void *user_data)
static void handle_bwc_packet(Tox *_Nonnull tox, uint32_t friend_number, const uint8_t *_Nonnull data, size_t length, void *_Nullable user_data)
{
ToxAV *toxav = (ToxAV *)tox_get_av_object(tox);
@@ -222,12 +226,14 @@ static void handle_bwc_packet(Tox *tox, uint32_t friend_number, const uint8_t *d
bwc_handle_packet(bwc, data, length);
}
static void handle_audio_frame(uint32_t friend_number, const int16_t *pcm, size_t sample_count, uint8_t channels,
uint32_t sampling_rate, void *user_data)
static void handle_audio_frame(uint32_t friend_number, const int16_t *_Nonnull pcm, size_t sample_count, uint8_t channels,
uint32_t sampling_rate, void *_Nullable user_data)
{
ToxAVCall *call = (ToxAVCall *)user_data;
pthread_mutex_lock(call->toxav_call_mutex);
toxav_audio_receive_frame_cb *acb = call->acb;
void *acb_user_data = call->acb_user_data;
pthread_mutex_unlock(call->toxav_call_mutex);
if (acb != nullptr) {
acb(call->av, friend_number, pcm, sample_count, channels, sampling_rate, acb_user_data);
@@ -235,13 +241,15 @@ static void handle_audio_frame(uint32_t friend_number, const int16_t *pcm, size_
}
static void handle_video_frame(uint32_t friend_number, uint16_t width, uint16_t height,
const uint8_t *y, const uint8_t *u, const uint8_t *v,
const uint8_t *_Nonnull y, const uint8_t *_Nonnull u, const uint8_t *_Nonnull v,
int32_t ystride, int32_t ustride, int32_t vstride,
void *user_data)
void *_Nullable user_data)
{
ToxAVCall *call = (ToxAVCall *)user_data;
toxav_video_receive_frame_cb *vcb = call->av->vcb;
void *vcb_user_data = call->av->vcb_user_data;
pthread_mutex_lock(call->toxav_call_mutex);
toxav_video_receive_frame_cb *vcb = call->vcb;
void *vcb_user_data = call->vcb_user_data;
pthread_mutex_unlock(call->toxav_call_mutex);
if (vcb != nullptr) {
vcb(call->av, friend_number, width, height, y, u, v, ystride, ustride, vstride, vcb_user_data);
@@ -256,11 +264,11 @@ static int callback_capabilities(void *object, MSICall *call);
static bool audio_bit_rate_invalid(uint32_t bit_rate);
static bool video_bit_rate_invalid(uint32_t bit_rate);
static bool invoke_call_state_callback(ToxAV *av, Tox_Friend_Number friend_number, uint32_t state);
static ToxAVCall *call_new(ToxAV *av, Tox_Friend_Number friend_number, Toxav_Err_Call *error);
static ToxAVCall *call_remove(ToxAVCall *call);
static bool call_prepare_transmission(ToxAVCall *call);
static void call_kill_transmission(ToxAVCall *call);
static bool invoke_call_state_callback(ToxAV *_Nonnull av, Tox_Friend_Number friend_number, uint32_t state);
static ToxAVCall *_Nullable call_new(ToxAV *_Nonnull av, Tox_Friend_Number friend_number, Toxav_Err_Call *_Nullable error);
static ToxAVCall *_Nullable call_remove(ToxAVCall *_Nullable call);
static bool call_prepare_transmission(ToxAVCall *_Nullable call);
static void call_kill_transmission(ToxAVCall *_Nullable call);
static ToxAVCall *call_get(ToxAV *av, uint32_t friend_number)
{
@@ -302,7 +310,7 @@ static BWController *bwc_controller_get(const ToxAVCall *call)
* @brief initialize d with default values
* @param d struct to be initialized, must not be nullptr
*/
static void init_decode_time_stats(DecodeTimeStats *d)
static void init_decode_time_stats(DecodeTimeStats *_Nonnull d)
{
assert(d != nullptr);
d->count = 0;
@@ -341,6 +349,7 @@ ToxAV *toxav_new(Tox *tox, Toxav_Err_New *error)
rc = TOXAV_ERR_NEW_MALLOC;
goto RETURN;
}
av->mutable_mutex = av->mutex;
av->mem = tox->sys.mem;
av->log = tox->m->log;
@@ -431,7 +440,11 @@ void toxav_kill(ToxAV *av)
Tox *toxav_get_tox(const ToxAV *av)
{
return av->tox;
Tox *tox;
pthread_mutex_lock(av->mutable_mutex);
tox = av->tox;
pthread_mutex_unlock(av->mutable_mutex);
return tox;
}
uint32_t toxav_audio_iteration_interval(const ToxAV *av)
@@ -457,7 +470,7 @@ uint32_t toxav_iteration_interval(const ToxAV *av)
* @param frame_time the duration of the current frame in ms
* @param start_time the timestamp when decoding of this frame started
*/
static void calc_interval(const ToxAV *av, DecodeTimeStats *stats, int32_t frame_time, uint64_t start_time)
static void calc_interval(const ToxAV *_Nonnull av, DecodeTimeStats *_Nonnull stats, int32_t frame_time, uint64_t start_time)
{
stats->interval = frame_time < stats->average ? 0 : (frame_time - stats->average);
stats->total += current_time_monotonic(av->toxav_mono_time) - start_time;
@@ -475,7 +488,7 @@ static void calc_interval(const ToxAV *av, DecodeTimeStats *stats, int32_t frame
* @param av pointer to ToxAV structure of current instance
* @param audio if true, iterate audio, video else
*/
static void iterate_common(ToxAV *av, bool audio)
static void iterate_common(ToxAV *_Nonnull av, bool audio)
{
pthread_mutex_lock(av->mutex);
@@ -671,7 +684,7 @@ void toxav_callback_call_state(ToxAV *av, toxav_call_state_cb *callback, void *u
pthread_mutex_unlock(av->mutex);
}
static Toxav_Err_Call_Control call_control_handle_resume(const ToxAVCall *call)
static Toxav_Err_Call_Control call_control_handle_resume(const ToxAVCall *_Nonnull call)
{
/* Only act if paused and had media transfer active before */
if (call->msi_call->self_capabilities != 0 || call->previous_self_capabilities == 0) {
@@ -688,7 +701,7 @@ static Toxav_Err_Call_Control call_control_handle_resume(const ToxAVCall *call)
return TOXAV_ERR_CALL_CONTROL_OK;
}
static Toxav_Err_Call_Control call_control_handle_pause(ToxAVCall *call)
static Toxav_Err_Call_Control call_control_handle_pause(ToxAVCall *_Nonnull call)
{
/* Only act if not already paused */
if (call->msi_call->self_capabilities == 0) {
@@ -706,7 +719,7 @@ static Toxav_Err_Call_Control call_control_handle_pause(ToxAVCall *call)
return TOXAV_ERR_CALL_CONTROL_OK;
}
static Toxav_Err_Call_Control call_control_handle_cancel(ToxAVCall *call)
static Toxav_Err_Call_Control call_control_handle_cancel(ToxAVCall *_Nonnull call)
{
/* Hang up */
pthread_mutex_lock(call->toxav_call_mutex);
@@ -725,7 +738,7 @@ static Toxav_Err_Call_Control call_control_handle_cancel(ToxAVCall *call)
return TOXAV_ERR_CALL_CONTROL_OK;
}
static Toxav_Err_Call_Control call_control_handle_mute_audio(const ToxAVCall *call)
static Toxav_Err_Call_Control call_control_handle_mute_audio(const ToxAVCall *_Nonnull call)
{
if ((call->msi_call->self_capabilities & MSI_CAP_R_AUDIO) == 0) {
return TOXAV_ERR_CALL_CONTROL_INVALID_TRANSITION;
@@ -740,7 +753,7 @@ static Toxav_Err_Call_Control call_control_handle_mute_audio(const ToxAVCall *ca
rtp_stop_receiving_mark(call->audio_rtp);
return TOXAV_ERR_CALL_CONTROL_OK;
}
static Toxav_Err_Call_Control call_control_handle_unmute_audio(const ToxAVCall *call)
static Toxav_Err_Call_Control call_control_handle_unmute_audio(const ToxAVCall *_Nonnull call)
{
if ((call->msi_call->self_capabilities ^ MSI_CAP_R_AUDIO) == 0) {
return TOXAV_ERR_CALL_CONTROL_INVALID_TRANSITION;
@@ -754,7 +767,7 @@ static Toxav_Err_Call_Control call_control_handle_unmute_audio(const ToxAVCall *
rtp_allow_receiving_mark(call->audio_rtp);
return TOXAV_ERR_CALL_CONTROL_OK;
}
static Toxav_Err_Call_Control call_control_handle_hide_video(const ToxAVCall *call)
static Toxav_Err_Call_Control call_control_handle_hide_video(const ToxAVCall *_Nonnull call)
{
if ((call->msi_call->self_capabilities & MSI_CAP_R_VIDEO) == 0) {
return TOXAV_ERR_CALL_CONTROL_INVALID_TRANSITION;
@@ -768,7 +781,7 @@ static Toxav_Err_Call_Control call_control_handle_hide_video(const ToxAVCall *ca
rtp_stop_receiving_mark(call->video_rtp);
return TOXAV_ERR_CALL_CONTROL_OK;
}
static Toxav_Err_Call_Control call_control_handle_show_video(const ToxAVCall *call)
static Toxav_Err_Call_Control call_control_handle_show_video(const ToxAVCall *_Nonnull call)
{
if ((call->msi_call->self_capabilities ^ MSI_CAP_R_VIDEO) == 0) {
return TOXAV_ERR_CALL_CONTROL_INVALID_TRANSITION;
@@ -782,7 +795,7 @@ static Toxav_Err_Call_Control call_control_handle_show_video(const ToxAVCall *ca
rtp_allow_receiving_mark(call->video_rtp);
return TOXAV_ERR_CALL_CONTROL_OK;
}
static Toxav_Err_Call_Control call_control_handle(ToxAVCall *call, Toxav_Call_Control control)
static Toxav_Err_Call_Control call_control_handle(ToxAVCall *_Nonnull call, Toxav_Call_Control control)
{
switch (control) {
case TOXAV_CALL_CONTROL_RESUME:
@@ -809,7 +822,7 @@ static Toxav_Err_Call_Control call_control_handle(ToxAVCall *call, Toxav_Call_Co
return TOXAV_ERR_CALL_CONTROL_INVALID_TRANSITION;
}
static Toxav_Err_Call_Control call_control(ToxAV *av, Tox_Friend_Number friend_number, Toxav_Call_Control control)
static Toxav_Err_Call_Control call_control(ToxAV *_Nonnull av, Tox_Friend_Number friend_number, Toxav_Call_Control control)
{
if (!tox_friend_exists(av->tox, friend_number)) {
return TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_FOUND;
@@ -1092,7 +1105,7 @@ static Toxav_Err_Send_Frame send_frames(const ToxAV *av, ToxAVCall *call)
uint32_t size;
bool is_keyframe;
while (vc_get_cx_data(call->video, &data, &size, &is_keyframe)) {
while (vc_get_cx_data(call->video, &data, &size, &is_keyframe) != 0) {
const int res = rtp_send_data(
av->log,
call->video_rtp,
@@ -1218,6 +1231,16 @@ void toxav_callback_video_receive_frame(ToxAV *av, toxav_video_receive_frame_cb
pthread_mutex_lock(av->mutex);
av->vcb = callback;
av->vcb_user_data = user_data;
if (av->calls != nullptr) {
for (ToxAVCall *i = av->calls[av->calls_head]; i != nullptr; i = i->next) {
pthread_mutex_lock(i->toxav_call_mutex);
i->vcb = callback;
i->vcb_user_data = user_data;
pthread_mutex_unlock(i->toxav_call_mutex);
}
}
pthread_mutex_unlock(av->mutex);
}
@@ -1272,7 +1295,7 @@ static void callback_bwc(BWController *bwc, Tox_Friend_Number friend_number, flo
pthread_mutex_unlock(call->av->mutex);
}
static int callback_invite(void *object, MSICall *call)
static int callback_invite(void *_Nonnull object, MSICall *_Nonnull call)
{
ToxAV *toxav = (ToxAV *)object;
pthread_mutex_lock(toxav->mutex);
@@ -1289,8 +1312,8 @@ static int callback_invite(void *object, MSICall *call)
av_call->msi_call = call;
if (toxav->ccb != nullptr) {
toxav->ccb(toxav, call->friend_number, call->peer_capabilities & MSI_CAP_S_AUDIO,
call->peer_capabilities & MSI_CAP_S_VIDEO, toxav->ccb_user_data);
toxav->ccb(toxav, call->friend_number, (call->peer_capabilities & MSI_CAP_S_AUDIO) != 0,
(call->peer_capabilities & MSI_CAP_S_VIDEO) != 0, toxav->ccb_user_data);
} else {
/* No handler to capture the call request, send failure */
pthread_mutex_unlock(toxav->mutex);
@@ -1301,7 +1324,7 @@ static int callback_invite(void *object, MSICall *call)
return 0;
}
static int callback_start(void *object, MSICall *call)
static int callback_start(void *_Nonnull object, MSICall *_Nonnull call)
{
ToxAV *toxav = (ToxAV *)object;
pthread_mutex_lock(toxav->mutex);
@@ -1330,7 +1353,7 @@ static int callback_start(void *object, MSICall *call)
return 0;
}
static int callback_end(void *object, MSICall *call)
static int callback_end(void *_Nonnull object, MSICall *_Nonnull call)
{
ToxAV *toxav = (ToxAV *)object;
pthread_mutex_lock(toxav->mutex);
@@ -1346,7 +1369,7 @@ static int callback_end(void *object, MSICall *call)
return 0;
}
static int callback_error(void *object, MSICall *call)
static int callback_error(void *_Nonnull object, MSICall *_Nonnull call)
{
ToxAV *toxav = (ToxAV *)object;
pthread_mutex_lock(toxav->mutex);
@@ -1362,7 +1385,7 @@ static int callback_error(void *object, MSICall *call)
return 0;
}
static int callback_capabilities(void *object, MSICall *call)
static int callback_capabilities(void *_Nonnull object, MSICall *_Nonnull call)
{
ToxAV *toxav = (ToxAV *)object;
pthread_mutex_lock(toxav->mutex);
@@ -1404,7 +1427,7 @@ static bool video_bit_rate_invalid(uint32_t bit_rate)
return bit_rate > 1000000;
}
static bool invoke_call_state_callback(ToxAV *av, Tox_Friend_Number friend_number, uint32_t state)
static bool invoke_call_state_callback(ToxAV *_Nonnull av, Tox_Friend_Number friend_number, uint32_t state)
{
if (av->scb != nullptr) {
av->scb(av, friend_number, state, av->scb_user_data);
@@ -1415,7 +1438,7 @@ static bool invoke_call_state_callback(ToxAV *av, Tox_Friend_Number friend_numbe
return true;
}
static ToxAVCall *call_new(ToxAV *av, Tox_Friend_Number friend_number, Toxav_Err_Call *error)
static ToxAVCall *_Nullable call_new(ToxAV *_Nonnull av, Tox_Friend_Number friend_number, Toxav_Err_Call *_Nullable error)
{
/* Assumes mutex locked */
Toxav_Err_Call rc = TOXAV_ERR_CALL_OK;
@@ -1510,7 +1533,7 @@ RETURN:
return call;
}
static ToxAVCall *call_remove(ToxAVCall *call)
static ToxAVCall *_Nullable call_remove(ToxAVCall *_Nullable call)
{
if (call == nullptr) {
return nullptr;
@@ -1560,7 +1583,7 @@ CLEAR:
return nullptr;
}
static bool call_prepare_transmission(ToxAVCall *call)
static bool call_prepare_transmission(ToxAVCall *_Nullable call)
{
/* Assumes mutex locked */
@@ -1612,6 +1635,8 @@ static bool call_prepare_transmission(ToxAVCall *call)
}
}
{ /* Prepare video */
call->vcb = av->vcb;
call->vcb_user_data = av->vcb_user_data;
call->video = vc_new(av->log, av->toxav_mono_time, call->friend_number, handle_video_frame, call);
if (call->video == nullptr) {
@@ -1649,7 +1674,7 @@ FAILURE_2:
return false;
}
static void call_kill_transmission(ToxAVCall *call)
static void call_kill_transmission(ToxAVCall *_Nullable call)
{
if (call == nullptr || !call->active) {
return;

22
toxav/video.c
View File

@@ -42,6 +42,7 @@ struct VCSession {
void *user_data;
pthread_mutex_t queue_mutex[1];
pthread_mutex_t *mutable_queue_mutex;
const Logger *log;
vpx_codec_iter_t iter;
@@ -71,11 +72,11 @@ struct VCSession {
#define VIDEO_MAX_FRAME_SIZE (10 * 1024 * 1024)
#define VIDEO_MAX_RESOLUTION_LIMIT 4096
static vpx_codec_iface_t *video_codec_decoder_interface(void)
static vpx_codec_iface_t *_Nonnull video_codec_decoder_interface(void)
{
return vpx_codec_vp8_dx();
}
static vpx_codec_iface_t *video_codec_encoder_interface(void)
static vpx_codec_iface_t *_Nonnull video_codec_encoder_interface(void)
{
return vpx_codec_vp8_cx();
}
@@ -89,7 +90,7 @@ static vpx_codec_iface_t *video_codec_encoder_interface(void)
#define VPX_MAX_DECODER_THREADS 4
#define VIDEO_VP8_DECODER_POST_PROCESSING_ENABLED 0
static void vc_init_encoder_cfg(const Logger *log, vpx_codec_enc_cfg_t *cfg, int16_t kf_max_dist)
static void vc_init_encoder_cfg(const Logger *_Nonnull log, vpx_codec_enc_cfg_t *_Nonnull cfg, int16_t kf_max_dist)
{
const vpx_codec_err_t rc = vpx_codec_enc_config_default(video_codec_encoder_interface(), cfg, 0);
@@ -178,6 +179,7 @@ VCSession *vc_new(const Logger *log, const Mono_Time *mono_time, uint32_t friend
free(vc);
return nullptr;
}
vc->mutable_queue_mutex = vc->queue_mutex;
const int cpu_used_value = VP8E_SET_CPUUSED_VALUE;
@@ -224,7 +226,7 @@ VCSession *vc_new(const Logger *log, const Mono_Time *mono_time, uint32_t friend
}
} else {
vp8_postproc_cfg_t pp = {0, 0, 0};
vpx_codec_err_t cc_res = vpx_codec_control(vc->decoder, VP8_SET_POSTPROC, &pp);
const vpx_codec_err_t cc_res = vpx_codec_control(vc->decoder, VP8_SET_POSTPROC, &pp);
if (cc_res != VPX_CODEC_OK) {
LOGGER_WARNING(log, "Failed to turn OFF postproc");
@@ -509,9 +511,9 @@ int vc_encode(VCSession *vc, uint16_t width, uint16_t height, const uint8_t *y,
/* I420 "It comprises an NxM Y plane followed by (N/2)x(M/2) V and U planes."
* http://fourcc.org/yuv.php#IYUV
*/
memcpy(img.planes[VPX_PLANE_Y], y, width * height);
memcpy(img.planes[VPX_PLANE_U], u, (width / 2) * (height / 2));
memcpy(img.planes[VPX_PLANE_V], v, (width / 2) * (height / 2));
memcpy(img.planes[VPX_PLANE_Y], y, (size_t)width * height);
memcpy(img.planes[VPX_PLANE_U], u, ((size_t)width / 2) * (height / 2));
memcpy(img.planes[VPX_PLANE_V], v, ((size_t)width / 2) * (height / 2));
}
int vpx_flags = 0;
@@ -555,7 +557,11 @@ int vc_get_cx_data(VCSession *vc, uint8_t **data, uint32_t *size, bool *is_keyfr
uint32_t vc_get_lcfd(const VCSession *vc)
{
return vc->lcfd;
uint32_t lcfd;
pthread_mutex_lock(vc->mutable_queue_mutex);
lcfd = vc->lcfd;
pthread_mutex_unlock(vc->mutable_queue_mutex);
return lcfd;
}
pthread_mutex_t *vc_get_queue_mutex(VCSession *vc)

28
toxav/video.h
View File

@@ -16,9 +16,9 @@ extern "C" {
#endif
typedef void vc_video_receive_frame_cb(uint32_t friend_number, uint16_t width, uint16_t height,
const uint8_t *y, const uint8_t *u, const uint8_t *v,
const uint8_t *_Nonnull y, const uint8_t *_Nonnull u, const uint8_t *_Nonnull v,
int32_t ystride, int32_t ustride, int32_t vstride,
void *user_data);
void *_Nullable user_data);
typedef struct VCSession VCSession;
@@ -27,21 +27,21 @@ typedef struct VCSession VCSession;
struct RTPMessage;
VCSession *vc_new(const Logger *log, const Mono_Time *mono_time, uint32_t friend_number,
vc_video_receive_frame_cb *cb, void *user_data);
void vc_kill(VCSession *vc);
void vc_iterate(VCSession *vc);
VCSession *_Nullable vc_new(const Logger *_Nonnull log, const Mono_Time *_Nonnull mono_time, uint32_t friend_number,
vc_video_receive_frame_cb *_Nullable cb, void *_Nullable user_data);
void vc_kill(VCSession *_Nullable vc);
void vc_iterate(VCSession *_Nullable vc);
int vc_queue_message(const Mono_Time *mono_time, void *cs, struct RTPMessage *msg);
int vc_reconfigure_encoder(VCSession *vc, uint32_t bit_rate, uint16_t width, uint16_t height, int16_t kf_max_dist);
int vc_queue_message(const Mono_Time *_Nonnull mono_time, void *_Nullable cs, struct RTPMessage *_Nullable msg);
int vc_reconfigure_encoder(VCSession *_Nullable vc, uint32_t bit_rate, uint16_t width, uint16_t height, int16_t kf_max_dist);
int vc_encode(VCSession *vc, uint16_t width, uint16_t height, const uint8_t *y,
const uint8_t *u, const uint8_t *v, int encode_flags);
int vc_encode(VCSession *_Nonnull vc, uint16_t width, uint16_t height, const uint8_t *_Nonnull y,
const uint8_t *_Nonnull u, const uint8_t *_Nonnull v, int encode_flags);
int vc_get_cx_data(VCSession *vc, uint8_t **data, uint32_t *size, bool *is_keyframe);
uint32_t vc_get_lcfd(const VCSession *vc);
pthread_mutex_t *vc_get_queue_mutex(VCSession *vc);
void vc_increment_frame_counter(VCSession *vc);
int vc_get_cx_data(VCSession *_Nonnull vc, uint8_t *_Nonnull *_Nonnull data, uint32_t *_Nonnull size, bool *_Nonnull is_keyframe);
uint32_t vc_get_lcfd(const VCSession *_Nonnull vc);
pthread_mutex_t *_Nonnull vc_get_queue_mutex(VCSession *_Nonnull vc);
void vc_increment_frame_counter(VCSession *_Nonnull vc);
#ifdef __cplusplus
} /* extern "C" */

196
toxav/video_bench.cc Normal file
View File

@@ -0,0 +1,196 @@
/* SPDX-License-Identifier: GPL-3.0-or-later
* Copyright © 2025 The TokTok team.
*/
#include <benchmark/benchmark.h>
#include <vector>
#include "../toxcore/logger.h"
#include "../toxcore/mono_time.h"
#include "../toxcore/os_memory.h"
#include "av_test_support.hh"
#include "rtp.h"
#include "video.h"
namespace {
class VideoBench : public benchmark::Fixture {
public:
void SetUp(const ::benchmark::State &state) override
{
const Memory *mem = os_memory();
log = logger_new(mem);
tm.t = 1000;
mono_time = mono_time_new(mem, mock_time_cb, &tm);
vc = vc_new(log, mono_time, 123, nullptr, nullptr);
width = static_cast<uint16_t>(state.range(0));
height = static_cast<uint16_t>(state.range(1));
// Use a standard bitrate for benchmarks
vc_reconfigure_encoder(vc, 2000, width, height, -1);
y.resize(static_cast<size_t>(width) * height);
u.resize((static_cast<size_t>(width) / 2) * (static_cast<size_t>(height) / 2));
v.resize((static_cast<size_t>(width) / 2) * (static_cast<size_t>(height) / 2));
rtp_mock.capture_packets = false; // Disable capturing for benchmarks
rtp_mock.auto_forward = true;
rtp_mock.recv_session = rtp_new(log, RTP_TYPE_VIDEO, mono_time, RtpMock::send_packet,
&rtp_mock, nullptr, nullptr, nullptr, vc, RtpMock::video_cb);
}
void TearDown(const ::benchmark::State &state) override
{
const Memory *mem = os_memory();
if (rtp_mock.recv_session) {
rtp_kill(log, rtp_mock.recv_session);
}
if (vc) {
vc_kill(vc);
}
if (mono_time) {
mono_time_free(mem, mono_time);
}
if (log) {
logger_kill(log);
}
}
Logger *log = nullptr;
Mono_Time *mono_time = nullptr;
MockTime tm;
VCSession *vc = nullptr;
RtpMock rtp_mock;
uint16_t width = 0, height = 0;
std::vector<uint8_t> y, u, v;
};
// Benchmark encoding a sequence of frames.
// Measures how the encoder performs as it builds up temporal state.
BENCHMARK_DEFINE_F(VideoBench, EncodeSequence)(benchmark::State &state)
{
int frame_index = 0;
// Pre-fill frames to avoid measuring fill_frame time
const int num_prefilled = 100;
std::vector<std::vector<uint8_t>> ys(num_prefilled, std::vector<uint8_t>(width * height));
std::vector<std::vector<uint8_t>> us(
num_prefilled, std::vector<uint8_t>((width / 2) * (height / 2)));
std::vector<std::vector<uint8_t>> vs(
num_prefilled, std::vector<uint8_t>((width / 2) * (height / 2)));
for (int i = 0; i < num_prefilled; ++i) {
fill_video_frame(width, height, i, ys[i], us[i], vs[i]);
}
for (auto _ : state) {
int idx = frame_index % num_prefilled;
// Force a keyframe every 100 frames to simulate real-world periodic keyframes
int flags = (frame_index % 100 == 0) ? VC_EFLAG_FORCE_KF : VC_EFLAG_NONE;
vc_encode(vc, width, height, ys[idx].data(), us[idx].data(), vs[idx].data(), flags);
vc_increment_frame_counter(vc);
uint8_t *pkt_data;
uint32_t pkt_size;
bool is_keyframe;
while (vc_get_cx_data(vc, &pkt_data, &pkt_size, &is_keyframe)) {
benchmark::DoNotOptimize(pkt_data);
benchmark::DoNotOptimize(pkt_size);
}
frame_index++;
}
}
BENCHMARK_REGISTER_F(VideoBench, EncodeSequence)
->Args({320, 240})
->Args({640, 480})
->Args({1280, 720})
->Args({1920, 1080});
// Benchmark decoding a sequence of frames.
// First pre-encodes a sequence, then measures decoding performance.
BENCHMARK_DEFINE_F(VideoBench, DecodeSequence)(benchmark::State &state)
{
const int num_frames = 100;
std::vector<std::vector<uint8_t>> encoded_frames(num_frames);
std::vector<bool> is_keyframe_list(num_frames);
// Pre-encode
for (int i = 0; i < num_frames; ++i) {
fill_video_frame(width, height, i, y, u, v);
int flags = (i == 0) ? VC_EFLAG_FORCE_KF : VC_EFLAG_NONE;
vc_encode(vc, width, height, y.data(), u.data(), v.data(), flags);
vc_increment_frame_counter(vc);
uint8_t *pkt_data;
uint32_t pkt_size;
bool is_kf;
while (vc_get_cx_data(vc, &pkt_data, &pkt_size, &is_kf)) {
encoded_frames[i].insert(encoded_frames[i].end(), pkt_data, pkt_data + pkt_size);
is_keyframe_list[i] = is_kf;
}
}
int frame_index = 0;
for (auto _ : state) {
int idx = frame_index % num_frames;
const auto &encoded_data = encoded_frames[idx];
rtp_send_data(log, rtp_mock.recv_session, encoded_data.data(),
static_cast<uint32_t>(encoded_data.size()), is_keyframe_list[idx]);
vc_iterate(vc);
frame_index++;
}
}
BENCHMARK_REGISTER_F(VideoBench, DecodeSequence)
->Args({320, 240})
->Args({640, 480})
->Args({1280, 720})
->Args({1920, 1080});
// Full end-to-end sequence benchmark (Encode -> RTP -> Decode)
BENCHMARK_DEFINE_F(VideoBench, FullSequence)(benchmark::State &state)
{
int frame_index = 0;
const int num_prefilled = 100;
std::vector<std::vector<uint8_t>> ys(num_prefilled, std::vector<uint8_t>(width * height));
std::vector<std::vector<uint8_t>> us(
num_prefilled, std::vector<uint8_t>((width / 2) * (height / 2)));
std::vector<std::vector<uint8_t>> vs(
num_prefilled, std::vector<uint8_t>((width / 2) * (height / 2)));
for (int i = 0; i < num_prefilled; ++i) {
fill_video_frame(width, height, i, ys[i], us[i], vs[i]);
}
for (auto _ : state) {
int idx = frame_index % num_prefilled;
int flags = (frame_index % 100 == 0) ? VC_EFLAG_FORCE_KF : VC_EFLAG_NONE;
vc_encode(vc, width, height, ys[idx].data(), us[idx].data(), vs[idx].data(), flags);
vc_increment_frame_counter(vc);
uint8_t *pkt_data;
uint32_t pkt_size;
bool is_keyframe = false;
// We need to collect all packets for the frame before sending to decoder
std::vector<uint8_t> frame_data;
while (vc_get_cx_data(vc, &pkt_data, &pkt_size, &is_keyframe)) {
frame_data.insert(frame_data.end(), pkt_data, pkt_data + pkt_size);
}
rtp_send_data(log, rtp_mock.recv_session, frame_data.data(),
static_cast<uint32_t>(frame_data.size()), is_keyframe);
vc_iterate(vc);
frame_index++;
}
}
BENCHMARK_REGISTER_F(VideoBench, FullSequence)
->Args({320, 240})
->Args({640, 480})
->Args({1280, 720})
->Args({1920, 1080});
}
BENCHMARK_MAIN();

344
toxav/video_test.cc
View File

@@ -3,123 +3,19 @@
#include <gtest/gtest.h>
#include <algorithm>
#include <cmath>
#include <vector>
#include "../toxcore/logger.h"
#include "../toxcore/mono_time.h"
#include "../toxcore/network.h"
#include "../toxcore/os_memory.h"
#include "av_test_support.hh"
#include "rtp.h"
namespace {
struct VideoTimeMock {
uint64_t t;
};
uint64_t video_mock_time_cb(void *ud) { return static_cast<VideoTimeMock *>(ud)->t; }
void test_logger_cb(void *context, Logger_Level level, const char *file, uint32_t line,
const char *func, const char *message, void *userdata)
{
(void)context;
(void)userdata;
const char *level_str = "UNKNOWN";
switch (level) {
case LOGGER_LEVEL_TRACE:
level_str = "TRACE";
break;
case LOGGER_LEVEL_DEBUG:
level_str = "DEBUG";
break;
case LOGGER_LEVEL_INFO:
level_str = "INFO";
break;
case LOGGER_LEVEL_WARNING:
level_str = "WARN";
break;
case LOGGER_LEVEL_ERROR:
level_str = "ERROR";
break;
}
printf("[%s] %s:%u %s: %s\n", level_str, file, line, func, message);
}
struct VideoTestData {
uint32_t friend_number = 0;
uint16_t width = 0;
uint16_t height = 0;
std::vector<uint8_t> y, u, v;
int32_t ystride = 0, ustride = 0, vstride = 0;
VideoTestData();
~VideoTestData();
static void receive_frame(uint32_t friend_number, uint16_t width, uint16_t height,
const uint8_t *y, const uint8_t *u, const uint8_t *v, int32_t ystride, int32_t ustride,
int32_t vstride, void *user_data)
{
auto *self = static_cast<VideoTestData *>(user_data);
self->friend_number = friend_number;
self->width = width;
self->height = height;
self->ystride = ystride;
self->ustride = ustride;
self->vstride = vstride;
self->y.assign(y, y + static_cast<size_t>(std::abs(ystride)) * height);
self->u.assign(u, u + static_cast<size_t>(std::abs(ustride)) * (height / 2));
self->v.assign(v, v + static_cast<size_t>(std::abs(vstride)) * (height / 2));
}
};
VideoTestData::VideoTestData() = default;
VideoTestData::~VideoTestData() = default;
struct VideoRtpMock {
RTPSession *recv_session = nullptr;
std::vector<std::vector<uint8_t>> captured_packets;
bool auto_forward = true;
static int send_packet(void *user_data, const uint8_t *data, uint16_t length)
{
auto *self = static_cast<VideoRtpMock *>(user_data);
self->captured_packets.push_back(std::vector<uint8_t>(data, data + length));
if (self->auto_forward && self->recv_session) {
rtp_receive_packet(self->recv_session, data, length);
}
return 0;
}
static int video_cb(const Mono_Time *mono_time, void *cs, RTPMessage *msg)
{
return vc_queue_message(mono_time, cs, msg);
}
};
class VideoTest : public ::testing::Test {
protected:
void SetUp() override
{
const Memory *mem = os_memory();
log = logger_new(mem);
logger_callback_log(log, test_logger_cb, nullptr, nullptr);
tm.t = 1000;
mono_time = mono_time_new(mem, video_mock_time_cb, &tm);
mono_time_update(mono_time);
}
void TearDown() override
{
const Memory *mem = os_memory();
mono_time_free(mem, mono_time);
logger_kill(log);
}
Logger *log;
Mono_Time *mono_time;
VideoTimeMock tm;
};
using VideoTest = AvTest;
TEST_F(VideoTest, BasicNewKill)
{
@@ -135,11 +31,11 @@ TEST_F(VideoTest, EncodeDecodeLoop)
VCSession *vc = vc_new(log, mono_time, 123, VideoTestData::receive_frame, &data);
ASSERT_NE(vc, nullptr);
VideoRtpMock rtp_mock;
RTPSession *send_rtp = rtp_new(log, RTP_TYPE_VIDEO, mono_time, VideoRtpMock::send_packet,
&rtp_mock, nullptr, nullptr, nullptr, vc, VideoRtpMock::video_cb);
RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_VIDEO, mono_time, VideoRtpMock::send_packet,
&rtp_mock, nullptr, nullptr, nullptr, vc, VideoRtpMock::video_cb);
RtpMock rtp_mock;
RTPSession *send_rtp = rtp_new(log, RTP_TYPE_VIDEO, mono_time, RtpMock::send_packet, &rtp_mock,
nullptr, nullptr, nullptr, vc, RtpMock::video_cb);
RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_VIDEO, mono_time, RtpMock::send_packet, &rtp_mock,
nullptr, nullptr, nullptr, vc, RtpMock::video_cb);
rtp_mock.recv_session = recv_rtp;
uint16_t width = 320;
@@ -176,6 +72,197 @@ TEST_F(VideoTest, EncodeDecodeLoop)
vc_kill(vc);
}
TEST_F(VideoTest, EncodeDecodeSequence)
{
VideoTestData data;
VCSession *vc = vc_new(log, mono_time, 123, VideoTestData::receive_frame, &data);
ASSERT_NE(vc, nullptr);
RtpMock rtp_mock;
RTPSession *send_rtp = rtp_new(log, RTP_TYPE_VIDEO, mono_time, RtpMock::send_packet, &rtp_mock,
nullptr, nullptr, nullptr, vc, RtpMock::video_cb);
RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_VIDEO, mono_time, RtpMock::send_packet, &rtp_mock,
nullptr, nullptr, nullptr, vc, RtpMock::video_cb);
rtp_mock.recv_session = recv_rtp;
uint16_t width = 320;
uint16_t height = 240;
uint32_t bitrate = 2000;
ASSERT_EQ(vc_reconfigure_encoder(vc, bitrate, width, height, -1), 0);
for (int i = 0; i < 20; ++i) {
std::vector<uint8_t> y(width * height);
std::vector<uint8_t> u((width / 2) * (height / 2));
std::vector<uint8_t> v((width / 2) * (height / 2));
// Background
std::fill(y.begin(), y.end(), 16);
std::fill(u.begin(), u.end(), 128);
std::fill(v.begin(), v.end(), 128);
// Moving square
int sq_size = 64;
int x0 = (i * 8) % (width - sq_size);
int y0 = (i * 4) % (height - sq_size);
for (int r = 0; r < sq_size; ++r) {
for (int c = 0; c < sq_size; ++c) {
y[(y0 + r) * width + (x0 + c)] = 200;
}
}
ASSERT_EQ(vc_encode(vc, width, height, y.data(), u.data(), v.data(),
i == 0 ? VC_EFLAG_FORCE_KF : VC_EFLAG_NONE),
0);
vc_increment_frame_counter(vc);
uint8_t *pkt_data;
uint32_t pkt_size;
bool is_keyframe;
while (vc_get_cx_data(vc, &pkt_data, &pkt_size, &is_keyframe)) {
rtp_send_data(log, send_rtp, pkt_data, pkt_size, is_keyframe);
}
vc_iterate(vc);
ASSERT_EQ(data.width, width);
ASSERT_EQ(data.height, height);
double mse = data.calculate_mse(y);
// Expect MSE to be reasonably low for high bitrate
EXPECT_LT(mse, 100.0) << "Frame " << i << " MSE too high: " << mse;
}
rtp_kill(log, send_rtp);
rtp_kill(log, recv_rtp);
vc_kill(vc);
}
TEST_F(VideoTest, EncodeDecodeResolutionChange)
{
VideoTestData data;
VCSession *vc = vc_new(log, mono_time, 123, VideoTestData::receive_frame, &data);
ASSERT_NE(vc, nullptr);
RtpMock rtp_mock;
RTPSession *send_rtp = rtp_new(log, RTP_TYPE_VIDEO, mono_time, RtpMock::send_packet, &rtp_mock,
nullptr, nullptr, nullptr, vc, RtpMock::video_cb);
RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_VIDEO, mono_time, RtpMock::send_packet, &rtp_mock,
nullptr, nullptr, nullptr, vc, RtpMock::video_cb);
rtp_mock.recv_session = recv_rtp;
uint16_t widths[] = {320, 160, 480};
uint16_t heights[] = {240, 120, 360};
for (int res = 0; res < 3; ++res) {
uint16_t width = widths[res];
uint16_t height = heights[res];
ASSERT_EQ(vc_reconfigure_encoder(vc, 2000, width, height, -1), 0);
for (int i = 0; i < 5; ++i) {
std::vector<uint8_t> y(width * height);
std::vector<uint8_t> u((width / 2) * (height / 2));
std::vector<uint8_t> v((width / 2) * (height / 2));
std::fill(y.begin(), y.end(), static_cast<uint8_t>((res * 50 + i * 10) % 256));
std::fill(u.begin(), u.end(), 128);
std::fill(v.begin(), v.end(), 128);
ASSERT_EQ(vc_encode(vc, width, height, y.data(), u.data(), v.data(),
i == 0 ? VC_EFLAG_FORCE_KF : VC_EFLAG_NONE),
0);
vc_increment_frame_counter(vc);
uint8_t *pkt_data;
uint32_t pkt_size;
bool is_keyframe;
while (vc_get_cx_data(vc, &pkt_data, &pkt_size, &is_keyframe)) {
rtp_send_data(log, send_rtp, pkt_data, pkt_size, is_keyframe);
}
vc_iterate(vc);
ASSERT_EQ(data.width, width);
ASSERT_EQ(data.height, height);
double mse = data.calculate_mse(y);
EXPECT_LT(mse, 100.0) << "Res " << res << " Frame " << i << " MSE too high: " << mse;
}
}
rtp_kill(log, send_rtp);
rtp_kill(log, recv_rtp);
vc_kill(vc);
}
TEST_F(VideoTest, EncodeDecodeBitrateImpact)
{
uint32_t bitrates[] = {100, 500, 2000};
double mses[3];
uint16_t width = 320;
uint16_t height = 240;
for (int b = 0; b < 3; ++b) {
VideoTestData data;
VCSession *vc = vc_new(log, mono_time, 123, VideoTestData::receive_frame, &data);
ASSERT_NE(vc, nullptr);
RtpMock rtp_mock;
RTPSession *send_rtp = rtp_new(log, RTP_TYPE_VIDEO, mono_time, RtpMock::send_packet,
&rtp_mock, nullptr, nullptr, nullptr, vc, RtpMock::video_cb);
RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_VIDEO, mono_time, RtpMock::send_packet,
&rtp_mock, nullptr, nullptr, nullptr, vc, RtpMock::video_cb);
rtp_mock.recv_session = recv_rtp;
ASSERT_EQ(vc_reconfigure_encoder(vc, bitrates[b], width, height, -1), 0);
double total_mse = 0;
int frames = 10;
for (int i = 0; i < frames; ++i) {
std::vector<uint8_t> y(width * height);
std::vector<uint8_t> u((width / 2) * (height / 2));
std::vector<uint8_t> v((width / 2) * (height / 2));
for (size_t j = 0; j < y.size(); ++j)
y[j] = static_cast<uint8_t>((j + i * 10) % 256);
std::fill(u.begin(), u.end(), 128);
std::fill(v.begin(), v.end(), 128);
ASSERT_EQ(vc_encode(vc, width, height, y.data(), u.data(), v.data(),
i == 0 ? VC_EFLAG_FORCE_KF : VC_EFLAG_NONE),
0);
vc_increment_frame_counter(vc);
uint8_t *pkt_data;
uint32_t pkt_size;
bool is_keyframe;
while (vc_get_cx_data(vc, &pkt_data, &pkt_size, &is_keyframe)) {
rtp_send_data(log, send_rtp, pkt_data, pkt_size, is_keyframe);
}
vc_iterate(vc);
total_mse += data.calculate_mse(y);
}
mses[b] = total_mse / frames;
rtp_kill(log, send_rtp);
rtp_kill(log, recv_rtp);
vc_kill(vc);
}
// Quality should generally improve (MSE decrease) as bitrate increases.
// 100kbps should have significantly higher MSE than 2000kbps for this complex pattern.
EXPECT_GT(mses[0], mses[1]);
EXPECT_GT(mses[1], mses[2]);
printf("MSE results: 100kbps: %f, 500kbps: %f, 2000kbps: %f\n", mses[0], mses[1], mses[2]);
}
TEST_F(VideoTest, ReconfigureEncoder)
{
VideoTestData data;
@@ -215,12 +302,12 @@ TEST_F(VideoTest, QueueInvalidMessage)
VCSession *vc = vc_new(log, mono_time, 123, VideoTestData::receive_frame, &data);
ASSERT_NE(vc, nullptr);
VideoRtpMock rtp_mock;
RtpMock rtp_mock;
// Create an audio RTP session but try to queue to video session
RTPSession *audio_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, VideoRtpMock::send_packet,
&rtp_mock, nullptr, nullptr, nullptr, vc, VideoRtpMock::video_cb);
RTPSession *video_recv_rtp = rtp_new(log, RTP_TYPE_VIDEO, mono_time, VideoRtpMock::send_packet,
&rtp_mock, nullptr, nullptr, nullptr, vc, VideoRtpMock::video_cb);
RTPSession *audio_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock,
nullptr, nullptr, nullptr, vc, RtpMock::video_cb);
RTPSession *video_recv_rtp = rtp_new(log, RTP_TYPE_VIDEO, mono_time, RtpMock::send_packet,
&rtp_mock, nullptr, nullptr, nullptr, vc, RtpMock::video_cb);
rtp_mock.recv_session = video_recv_rtp;
std::vector<uint8_t> dummy_audio(100, 0);
@@ -262,9 +349,9 @@ TEST_F(VideoTest, LcfdAndSpecialPackets)
VCSession *vc = vc_new(log, mono_time, 123, VideoTestData::receive_frame, &data);
ASSERT_NE(vc, nullptr);
VideoRtpMock rtp_mock;
RTPSession *video_recv_rtp = rtp_new(log, RTP_TYPE_VIDEO, mono_time, VideoRtpMock::send_packet,
&rtp_mock, nullptr, nullptr, nullptr, vc, VideoRtpMock::video_cb);
RtpMock rtp_mock;
RTPSession *video_recv_rtp = rtp_new(log, RTP_TYPE_VIDEO, mono_time, RtpMock::send_packet,
&rtp_mock, nullptr, nullptr, nullptr, vc, RtpMock::video_cb);
rtp_mock.recv_session = video_recv_rtp;
// 1. Test lcfd update
@@ -286,8 +373,8 @@ TEST_F(VideoTest, LcfdAndSpecialPackets)
EXPECT_EQ(vc_get_lcfd(vc), 50u); // Should still be 50
// 3. Test dummy packet PT = (RTP_TYPE_VIDEO + 2) % 128
RTPSession *dummy_rtp = rtp_new(log, (RTP_TYPE_VIDEO + 2), mono_time, VideoRtpMock::send_packet,
&rtp_mock, nullptr, nullptr, nullptr, vc, VideoRtpMock::video_cb);
RTPSession *dummy_rtp = rtp_new(log, (RTP_TYPE_VIDEO + 2), mono_time, RtpMock::send_packet,
&rtp_mock, nullptr, nullptr, nullptr, vc, RtpMock::video_cb);
rtp_mock.recv_session = dummy_rtp;
rtp_send_data(
log, dummy_rtp, dummy_frame.data(), static_cast<uint32_t>(dummy_frame.size()), false);
@@ -321,13 +408,6 @@ TEST_F(VideoTest, MultiReconfigureEncode)
vc_kill(vc);
}
TEST_F(VideoTest, NewWithNullMonoTime)
{
VideoTestData data;
VCSession *vc = vc_new(log, nullptr, 123, VideoTestData::receive_frame, &data);
EXPECT_EQ(vc, nullptr);
}
TEST_F(VideoTest, ReconfigureFailDoS)
{
VideoTestData data;
@@ -354,9 +434,9 @@ TEST_F(VideoTest, LyingLengthOOB)
VCSession *vc = vc_new(log, mono_time, 123, VideoTestData::receive_frame, &data);
ASSERT_NE(vc, nullptr);
VideoRtpMock rtp_mock;
RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_VIDEO, mono_time, VideoRtpMock::send_packet,
&rtp_mock, nullptr, nullptr, nullptr, vc, VideoRtpMock::video_cb);
RtpMock rtp_mock;
RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_VIDEO, mono_time, RtpMock::send_packet, &rtp_mock,
nullptr, nullptr, nullptr, vc, RtpMock::video_cb);
rtp_mock.recv_session = recv_rtp;
// Craft a malicious RTP packet