Green-Sky/tomato
1
0
Fork 1
Code Issues Pull Requests 1 Actions Packages Projects Releases 1 Wiki Activity

Squashed 'external/toxcore/c-toxcore/' content from commit 67badf69

Browse Source 
git-subtree-dir: external/toxcore/c-toxcore
git-subtree-split: 67badf69416a74e74f6d7eb51dd96f37282b8455
This commit is contained in:
Green Sky
2023-07-25 11:53:09 +02:00
commit 227425b90e
467 changed files with 116591 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

170
toxav/BUILD.bazel Normal file
View File

@ -0,0 +1,170 @@
load("@rules_cc//cc:defs.bzl", "cc_test")
load("//tools:no_undefined.bzl", "cc_library")
package(features = ["layering_check"])
exports_files(
srcs = ["toxav.h"],
visibility = ["//c-toxcore:__pkg__"],
)
# Private library with the public API header in it because in toxav, lots of
# things depend on the public API header.
cc_library(
name = "public_api",
hdrs = ["toxav.h"],
)
cc_library(
name = "ring_buffer",
srcs = ["ring_buffer.c"],
hdrs = ["ring_buffer.h"],
deps = ["//c-toxcore/toxcore:ccompat"],
)
cc_test(
name = "ring_buffer_test",
size = "small",
srcs = ["ring_buffer_test.cc"],
deps = [
":ring_buffer",
"@com_google_googletest//:gtest",
"@com_google_googletest//:gtest_main",
],
)
cc_library(
name = "ring_buffer_srcs",
hdrs = [
"ring_buffer.c",
"ring_buffer.h",
],
visibility = ["//c-toxcore/testing:__pkg__"],
deps = ["//c-toxcore/toxcore:ccompat"],
)
cc_library(
name = "bwcontroller",
srcs = ["bwcontroller.c"],
hdrs = ["bwcontroller.h"],
deps = [
":ring_buffer",
"//c-toxcore/toxcore",
"//c-toxcore/toxcore:Messenger",
"//c-toxcore/toxcore:ccompat",
"//c-toxcore/toxcore:logger",
"//c-toxcore/toxcore:mono_time",
"//c-toxcore/toxcore:util",
],
)
cc_library(
name = "rtp",
srcs = ["rtp.c"],
hdrs = ["rtp.h"],
deps = [
":bwcontroller",
"//c-toxcore/toxcore:Messenger",
"//c-toxcore/toxcore:ccompat",
"//c-toxcore/toxcore:logger",
"//c-toxcore/toxcore:mono_time",
"//c-toxcore/toxcore:util",
],
)
cc_test(
name = "rtp_test",
size = "small",
srcs = ["rtp_test.cc"],
deps = [
":rtp",
"//c-toxcore/toxcore:crypto_core",
"@com_google_googletest//:gtest",
"@com_google_googletest//:gtest_main",
],
)
cc_library(
name = "audio",
srcs = ["audio.c"],
hdrs = ["audio.h"],
deps = [
":public_api",
":rtp",
"//c-toxcore/toxcore:ccompat",
"//c-toxcore/toxcore:logger",
"//c-toxcore/toxcore:mono_time",
"@opus",
],
)
cc_library(
name = "video",
srcs = [
"msi.c",
"video.c",
],
hdrs = [
"msi.h",
"video.h",
],
deps = [
":audio",
":public_api",
":ring_buffer",
":rtp",
"//c-toxcore/toxcore:ccompat",
"//c-toxcore/toxcore:logger",
"//c-toxcore/toxcore:mono_time",
"//c-toxcore/toxcore:network",
"//c-toxcore/toxcore:util",
"@libvpx",
],
)
cc_library(
name = "groupav",
srcs = ["groupav.c"],
hdrs = ["groupav.h"],
deps = [
"//c-toxcore/toxcore",
"//c-toxcore/toxcore:ccompat",
"//c-toxcore/toxcore:logger",
"//c-toxcore/toxcore:mono_time",
"//c-toxcore/toxcore:tox",
"//c-toxcore/toxcore:util",
"@opus",
],
)
cc_library(
name = "toxav",
srcs = [
"toxav.c",
"toxav_old.c",
],
hdrs = [
"toxav.h",
],
visibility = ["//c-toxcore:__subpackages__"],
deps = [
":groupav",
":rtp",
":video",
"//c-toxcore/toxcore:Messenger",
"//c-toxcore/toxcore:ccompat",
"//c-toxcore/toxcore:logger",
"//c-toxcore/toxcore:mono_time",
"//c-toxcore/toxcore:tox",
"//c-toxcore/toxcore:util",
],
)
sh_library(
name = "cimple_files",
srcs = glob([
"*.c",
"*.h",
]),
visibility = ["//c-toxcore/testing:__pkg__"],
)

50
toxav/Makefile.inc Normal file
View File

@ -0,0 +1,50 @@
if BUILD_AV
lib_LTLIBRARIES += libtoxav.la
libtoxav_la_include_HEADERS = ../toxav/toxav.h
libtoxav_la_includedir = $(includedir)/tox
libtoxav_la_SOURCES = ../toxav/rtp.h \
../toxav/rtp.c \
../toxav/msi.h \
../toxav/msi.c \
../toxav/groupav.h \
../toxav/groupav.c \
../toxav/audio.h \
../toxav/audio.c \
../toxav/video.h \
../toxav/video.c \
../toxav/bwcontroller.h \
../toxav/bwcontroller.c \
../toxav/ring_buffer.h \
../toxav/ring_buffer.c \
../toxav/toxav.h \
../toxav/toxav.c \
../toxav/toxav_old.c
libtoxav_la_CFLAGS = -I../toxcore \
-I../toxav \
$(LIBSODIUM_CFLAGS) \
$(NACL_CFLAGS) \
$(AV_CFLAGS) \
$(PTHREAD_CFLAGS)
libtoxav_la_LDFLAGS = $(LT_LDFLAGS) \
$(LIBSODIUM_LDFLAGS) \
$(NACL_LDFLAGS) \
$(EXTRA_LT_LDFLAGS) \
$(WINSOCK2_LIBS)
libtoxav_la_LIBADD = libtoxcore.la \
$(LIBSODIUM_LIBS) \
$(NACL_LIBS) \
$(PTHREAD_LIBS) \
$(AV_LIBS)
if SET_SO_VERSION
EXTRA_libtoxav_la_DEPENDENCIES = ../so.version
endif
endif

504
toxav/audio.c Normal file
View File

@ -0,0 +1,504 @@
/* SPDX-License-Identifier: GPL-3.0-or-later
* Copyright © 2016-2018 The TokTok team.
* Copyright © 2013-2015 Tox project.
*/
#include "audio.h"
#include <assert.h>
#include <stdlib.h>
#include <string.h>
#include "rtp.h"
#include "../toxcore/ccompat.h"
#include "../toxcore/logger.h"
#include "../toxcore/mono_time.h"
static struct JitterBuffer *jbuf_new(uint32_t capacity);
static void jbuf_clear(struct JitterBuffer *q);
static void jbuf_free(struct JitterBuffer *q);
static int jbuf_write(const Logger *log, struct JitterBuffer *q, struct RTPMessage *m);
static struct RTPMessage *jbuf_read(struct JitterBuffer *q, int32_t *success);
static OpusEncoder *create_audio_encoder(const Logger *log, uint32_t bit_rate, uint32_t sampling_rate,
uint8_t channel_count);
static bool reconfigure_audio_encoder(const Logger *log, OpusEncoder **e, uint32_t new_br, uint32_t new_sr,
uint8_t new_ch, uint32_t *old_br, uint32_t *old_sr, uint8_t *old_ch);
static bool reconfigure_audio_decoder(ACSession *ac, uint32_t sampling_rate, uint8_t channels);
ACSession *ac_new(Mono_Time *mono_time, const Logger *log, ToxAV *av, uint32_t friend_number,
toxav_audio_receive_frame_cb *cb, void *cb_data)
{
ACSession *ac = (ACSession *)calloc(1, sizeof(ACSession));
if (ac == nullptr) {
LOGGER_WARNING(log, "Allocation failed! Application might misbehave!");
return nullptr;
}
if (create_recursive_mutex(ac->queue_mutex) != 0) {
LOGGER_WARNING(log, "Failed to create recursive mutex!");
free(ac);
return nullptr;
}
int status;
ac->decoder = opus_decoder_create(AUDIO_DECODER_START_SAMPLE_RATE, AUDIO_DECODER_START_CHANNEL_COUNT, &status);
if (status != OPUS_OK) {
LOGGER_ERROR(log, "Error while starting audio decoder: %s", opus_strerror(status));
goto BASE_CLEANUP;
}
ac->j_buf = jbuf_new(AUDIO_JITTERBUFFER_COUNT);
if (ac->j_buf == nullptr) {
LOGGER_WARNING(log, "Jitter buffer creaton failed!");
opus_decoder_destroy(ac->decoder);
goto BASE_CLEANUP;
}
ac->mono_time = mono_time;
ac->log = log;
/* Initialize encoders with default values */
ac->encoder = create_audio_encoder(log, AUDIO_START_BITRATE, AUDIO_START_SAMPLE_RATE, AUDIO_START_CHANNEL_COUNT);
if (ac->encoder == nullptr) {
goto DECODER_CLEANUP;
}
ac->le_bit_rate = AUDIO_START_BITRATE;
ac->le_sample_rate = AUDIO_START_SAMPLE_RATE;
ac->le_channel_count = AUDIO_START_CHANNEL_COUNT;
ac->ld_channel_count = AUDIO_DECODER_START_CHANNEL_COUNT;
ac->ld_sample_rate = AUDIO_DECODER_START_SAMPLE_RATE;
ac->ldrts = 0; /* Make it possible to reconfigure straight away */
/* These need to be set in order to properly
* do error correction with opus */
ac->lp_frame_duration = AUDIO_MAX_FRAME_DURATION_MS;
ac->lp_sampling_rate = AUDIO_DECODER_START_SAMPLE_RATE;
ac->lp_channel_count = AUDIO_DECODER_START_CHANNEL_COUNT;
ac->av = av;
ac->friend_number = friend_number;
ac->acb = cb;
ac->acb_user_data = cb_data;
return ac;
DECODER_CLEANUP:
opus_decoder_destroy(ac->decoder);
jbuf_free((struct JitterBuffer *)ac->j_buf);
BASE_CLEANUP:
pthread_mutex_destroy(ac->queue_mutex);
free(ac);
return nullptr;
}
void ac_kill(ACSession *ac)
{
if (ac == nullptr) {
return;
}
opus_encoder_destroy(ac->encoder);
opus_decoder_destroy(ac->decoder);
jbuf_free((struct JitterBuffer *)ac->j_buf);
pthread_mutex_destroy(ac->queue_mutex);
LOGGER_DEBUG(ac->log, "Terminated audio handler: %p", (void *)ac);
free(ac);
}
void ac_iterate(ACSession *ac)
{
if (ac == nullptr) {
return;
}
/* TODO: fix this and jitter buffering */
/* Enough space for the maximum frame size (120 ms 48 KHz stereo audio) */
int16_t *temp_audio_buffer = (int16_t *)malloc(AUDIO_MAX_BUFFER_SIZE_PCM16 * AUDIO_MAX_CHANNEL_COUNT * sizeof(int16_t));
if (temp_audio_buffer == nullptr) {
LOGGER_ERROR(ac->log, "Failed to allocate memory for audio buffer");
return;
}
pthread_mutex_lock(ac->queue_mutex);
struct JitterBuffer *const j_buf = (struct JitterBuffer *)ac->j_buf;
int rc = 0;
for (struct RTPMessage *msg = jbuf_read(j_buf, &rc); msg != nullptr || rc == 2; msg = jbuf_read(j_buf, &rc)) {
pthread_mutex_unlock(ac->queue_mutex);
if (rc == 2) {
LOGGER_DEBUG(ac->log, "OPUS correction");
const int fs = (ac->lp_sampling_rate * ac->lp_frame_duration) / 1000;
rc = opus_decode(ac->decoder, nullptr, 0, temp_audio_buffer, fs, 1);
} else {
assert(msg->len > 4);
/* Pick up sampling rate from packet */
memcpy(&ac->lp_sampling_rate, msg->data, 4);
ac->lp_sampling_rate = net_ntohl(ac->lp_sampling_rate);
ac->lp_channel_count = opus_packet_get_nb_channels(msg->data + 4);
/* NOTE: even though OPUS supports decoding mono frames with stereo decoder and vice versa,
* it didn't work quite well.
*/
if (!reconfigure_audio_decoder(ac, ac->lp_sampling_rate, ac->lp_channel_count)) {
LOGGER_WARNING(ac->log, "Failed to reconfigure decoder!");
free(msg);
pthread_mutex_lock(ac->queue_mutex);
continue;
}
/*
* frame_size = opus_decode(dec, packet, len, decoded, max_size, 0);
* where
* packet is the byte array containing the compressed data
* len is the exact number of bytes contained in the packet
* decoded is the decoded audio data in opus_int16 (or float for opus_decode_float())
* max_size is the max duration of the frame in samples (per channel) that can fit
* into the decoded_frame array
*/
rc = opus_decode(ac->decoder, msg->data + 4, msg->len - 4, temp_audio_buffer, 5760, 0);
free(msg);
}
if (rc < 0) {
LOGGER_WARNING(ac->log, "Decoding error: %s", opus_strerror(rc));
} else if (ac->acb != nullptr) {
ac->lp_frame_duration = (rc * 1000) / ac->lp_sampling_rate;
ac->acb(ac->av, ac->friend_number, temp_audio_buffer, rc, ac->lp_channel_count,
ac->lp_sampling_rate, ac->acb_user_data);
}
free(temp_audio_buffer);
return;
}
pthread_mutex_unlock(ac->queue_mutex);
free(temp_audio_buffer);
}
int ac_queue_message(Mono_Time *mono_time, void *acp, struct RTPMessage *msg)
{
if (acp == nullptr || msg == nullptr) {
free(msg);
return -1;
}
ACSession *ac = (ACSession *)acp;
if ((msg->header.pt & 0x7f) == (RTP_TYPE_AUDIO + 2) % 128) {
LOGGER_WARNING(ac->log, "Got dummy!");
free(msg);
return 0;
}
if ((msg->header.pt & 0x7f) != RTP_TYPE_AUDIO % 128) {
LOGGER_WARNING(ac->log, "Invalid payload type!");
free(msg);
return -1;
}
pthread_mutex_lock(ac->queue_mutex);
const int rc = jbuf_write(ac->log, (struct JitterBuffer *)ac->j_buf, msg);
pthread_mutex_unlock(ac->queue_mutex);
if (rc == -1) {
LOGGER_WARNING(ac->log, "Could not queue the message!");
free(msg);
return -1;
}
return 0;
}
int ac_reconfigure_encoder(ACSession *ac, uint32_t bit_rate, uint32_t sampling_rate, uint8_t channels)
{
if (ac == nullptr || !reconfigure_audio_encoder(
ac->log, &ac->encoder, bit_rate,
sampling_rate, channels,
&ac->le_bit_rate,
&ac->le_sample_rate,
&ac->le_channel_count)) {
return -1;
}
return 0;
}
struct JitterBuffer {
struct RTPMessage **queue;
uint32_t size;
uint32_t capacity;
uint16_t bottom;
uint16_t top;
};
static struct JitterBuffer *jbuf_new(uint32_t capacity)
{
unsigned int size = 1;
while (size <= (capacity * 4)) {
size *= 2;
}
struct JitterBuffer *q = (struct JitterBuffer *)calloc(1, sizeof(struct JitterBuffer));
if (q == nullptr) {
return nullptr;
}
q->queue = (struct RTPMessage **)calloc(size, sizeof(struct RTPMessage *));
if (q->queue == nullptr) {
free(q);
return nullptr;
}
q->size = size;
q->capacity = capacity;
return q;
}
static void jbuf_clear(struct JitterBuffer *q)
{
while (q->bottom != q->top) {
free(q->queue[q->bottom % q->size]);
q->queue[q->bottom % q->size] = nullptr;
++q->bottom;
}
}
static void jbuf_free(struct JitterBuffer *q)
{
if (q == nullptr) {
return;
}
jbuf_clear(q);
free(q->queue);
free(q);
}
static int jbuf_write(const Logger *log, struct JitterBuffer *q, struct RTPMessage *m)
{
const uint16_t sequnum = m->header.sequnum;
const unsigned int num = sequnum % q->size;
if ((uint32_t)(sequnum - q->bottom) > q->size) {
LOGGER_DEBUG(log, "Clearing filled jitter buffer: %p", (void *)q);
jbuf_clear(q);
q->bottom = sequnum - q->capacity;
q->queue[num] = m;
q->top = sequnum + 1;
return 0;
}
if (q->queue[num] != nullptr) {
return -1;
}
q->queue[num] = m;
if ((sequnum - q->bottom) >= (q->top - q->bottom)) {
q->top = sequnum + 1;
}
return 0;
}
static struct RTPMessage *jbuf_read(struct JitterBuffer *q, int32_t *success)
{
if (q->top == q->bottom) {
*success = 0;
return nullptr;
}
const unsigned int num = q->bottom % q->size;
if (q->queue[num] != nullptr) {
struct RTPMessage *ret = q->queue[num];
q->queue[num] = nullptr;
++q->bottom;
*success = 1;
return ret;
}
if ((uint32_t)(q->top - q->bottom) > q->capacity) {
++q->bottom;
*success = 2;
return nullptr;
}
*success = 0;
return nullptr;
}
static OpusEncoder *create_audio_encoder(const Logger *log, uint32_t bit_rate, uint32_t sampling_rate,
uint8_t channel_count)
{
int status = OPUS_OK;
/*
* OPUS_APPLICATION_VOIP Process signal for improved speech intelligibility
* OPUS_APPLICATION_AUDIO Favor faithfulness to the original input
* OPUS_APPLICATION_RESTRICTED_LOWDELAY Configure the minimum possible coding delay
*/
OpusEncoder *rc = opus_encoder_create(sampling_rate, channel_count, OPUS_APPLICATION_VOIP, &status);
if (status != OPUS_OK) {
LOGGER_ERROR(log, "Error while starting audio encoder: %s", opus_strerror(status));
return nullptr;
}
/*
* Rates from 500 to 512000 bits per second are meaningful as well as the special
* values OPUS_BITRATE_AUTO and OPUS_BITRATE_MAX. The value OPUS_BITRATE_MAX can
* be used to cause the codec to use as much rate as it can, which is useful for
* controlling the rate by adjusting the output buffer size.
*
* Parameters:
* `[in]` `x` `opus_int32`: bitrate in bits per second.
*/
status = opus_encoder_ctl(rc, OPUS_SET_BITRATE(bit_rate));
if (status != OPUS_OK) {
LOGGER_ERROR(log, "Error while setting encoder ctl: %s", opus_strerror(status));
goto FAILURE;
}
/*
* Configures the encoder's use of inband forward error correction.
* Note:
* This is only applicable to the LPC layer
* Parameters:
* `[in]` `x` `int`: FEC flag, 0 (disabled) is default
*/
/* Enable in-band forward error correction in codec */
status = opus_encoder_ctl(rc, OPUS_SET_INBAND_FEC(1));
if (status != OPUS_OK) {
LOGGER_ERROR(log, "Error while setting encoder ctl: %s", opus_strerror(status));
goto FAILURE;
}
/*
* Configures the encoder's expected packet loss percentage.
* Higher values with trigger progressively more loss resistant behavior in
* the encoder at the expense of quality at a given bitrate in the lossless case,
* but greater quality under loss.
* Parameters:
* `[in]` `x` `int`: Loss percentage in the range 0-100, inclusive.
*/
/* Make codec resistant to up to 10% packet loss
* NOTE This could also be adjusted on the fly, rather than hard-coded,
* with feedback from the receiving client.
*/
status = opus_encoder_ctl(rc, OPUS_SET_PACKET_LOSS_PERC(AUDIO_OPUS_PACKET_LOSS_PERC));
if (status != OPUS_OK) {
LOGGER_ERROR(log, "Error while setting encoder ctl: %s", opus_strerror(status));
goto FAILURE;
}
/*
* Configures the encoder's computational complexity.
*
* The supported range is 0-10 inclusive with 10 representing the highest complexity.
* The default value is 10.
*
* Parameters:
* `[in]` `x` `int`: 0-10, inclusive
*/
/* Set algorithm to the highest complexity, maximizing compression */
status = opus_encoder_ctl(rc, OPUS_SET_COMPLEXITY(AUDIO_OPUS_COMPLEXITY));
if (status != OPUS_OK) {
LOGGER_ERROR(log, "Error while setting encoder ctl: %s", opus_strerror(status));
goto FAILURE;
}
return rc;
FAILURE:
opus_encoder_destroy(rc);
return nullptr;
}
static bool reconfigure_audio_encoder(const Logger *log, OpusEncoder **e, uint32_t new_br, uint32_t new_sr,
uint8_t new_ch, uint32_t *old_br, uint32_t *old_sr, uint8_t *old_ch)
{
/* Values are checked in toxav.c */
if (*old_sr != new_sr || *old_ch != new_ch) {
OpusEncoder *new_encoder = create_audio_encoder(log, new_br, new_sr, new_ch);
if (new_encoder == nullptr) {
return false;
}
opus_encoder_destroy(*e);
*e = new_encoder;
} else if (*old_br == new_br) {
return true; /* Nothing changed */
}
const int status = opus_encoder_ctl(*e, OPUS_SET_BITRATE(new_br));
if (status != OPUS_OK) {
LOGGER_ERROR(log, "Error while setting encoder ctl: %s", opus_strerror(status));
return false;
}
*old_br = new_br;
*old_sr = new_sr;
*old_ch = new_ch;
LOGGER_DEBUG(log, "Reconfigured audio encoder br: %d sr: %d cc:%d", new_br, new_sr, new_ch);
return true;
}
static bool reconfigure_audio_decoder(ACSession *ac, uint32_t sampling_rate, uint8_t channels)
{
if (sampling_rate != ac->ld_sample_rate || channels != ac->ld_channel_count) {
if (current_time_monotonic(ac->mono_time) - ac->ldrts < 500) {
return false;
}
int status;
OpusDecoder *new_dec = opus_decoder_create(sampling_rate, channels, &status);
if (status != OPUS_OK) {
LOGGER_ERROR(ac->log, "Error while starting audio decoder(%d %d): %s", sampling_rate, channels, opus_strerror(status));
return false;
}
ac->ld_sample_rate = sampling_rate;
ac->ld_channel_count = channels;
ac->ldrts = current_time_monotonic(ac->mono_time);
opus_decoder_destroy(ac->decoder);
ac->decoder = new_dec;
LOGGER_DEBUG(ac->log, "Reconfigured audio decoder sr: %d cc: %d", sampling_rate, channels);
}
return true;
}

73
toxav/audio.h Normal file
View File

@ -0,0 +1,73 @@
/* SPDX-License-Identifier: GPL-3.0-or-later
* Copyright © 2016-2018 The TokTok team.
* Copyright © 2013-2015 Tox project.
*/
#ifndef C_TOXCORE_TOXAV_AUDIO_H
#define C_TOXCORE_TOXAV_AUDIO_H
#include <opus.h>
#include <pthread.h>
#include "toxav.h"
#include "../toxcore/logger.h"
#include "../toxcore/util.h"
#include "rtp.h"
#define AUDIO_JITTERBUFFER_COUNT 3
#define AUDIO_MAX_SAMPLE_RATE 48000
#define AUDIO_MAX_CHANNEL_COUNT 2
#define AUDIO_START_SAMPLE_RATE 48000
#define AUDIO_START_BITRATE 48000
#define AUDIO_START_CHANNEL_COUNT 2
#define AUDIO_OPUS_PACKET_LOSS_PERC 10
#define AUDIO_OPUS_COMPLEXITY 10
#define AUDIO_DECODER_START_SAMPLE_RATE 48000
#define AUDIO_DECODER_START_CHANNEL_COUNT 1
#define AUDIO_MAX_FRAME_DURATION_MS 120
// ((sampling_rate_in_hz * frame_duration_in_ms) / 1000) * 2 // because PCM16 needs 2 bytes for 1 sample
// These are per frame and per channel.
#define AUDIO_MAX_BUFFER_SIZE_PCM16 ((AUDIO_MAX_SAMPLE_RATE * AUDIO_MAX_FRAME_DURATION_MS) / 1000)
#define AUDIO_MAX_BUFFER_SIZE_BYTES (AUDIO_MAX_BUFFER_SIZE_PCM16 * 2)
typedef struct ACSession {
Mono_Time *mono_time;
const Logger *log;
/* encoding */
OpusEncoder *encoder;
uint32_t le_sample_rate; /* Last encoder sample rate */
uint8_t le_channel_count; /* Last encoder channel count */
uint32_t le_bit_rate; /* Last encoder bit rate */
/* decoding */
OpusDecoder *decoder;
uint8_t lp_channel_count; /* Last packet channel count */
uint32_t lp_sampling_rate; /* Last packet sample rate */
uint32_t lp_frame_duration; /* Last packet frame duration */
uint32_t ld_sample_rate; /* Last decoder sample rate */
uint8_t ld_channel_count; /* Last decoder channel count */
uint64_t ldrts; /* Last decoder reconfiguration time stamp */
void *j_buf;
pthread_mutex_t queue_mutex[1];
ToxAV *av;
uint32_t friend_number;
/* Audio frame receive callback */
toxav_audio_receive_frame_cb *acb;
void *acb_user_data;
} ACSession;
ACSession *ac_new(Mono_Time *mono_time, const Logger *log, ToxAV *av, uint32_t friend_number,
toxav_audio_receive_frame_cb *cb, void *cb_data);
void ac_kill(ACSession *ac);
void ac_iterate(ACSession *ac);
int ac_queue_message(Mono_Time *mono_time, void *acp, struct RTPMessage *msg);
int ac_reconfigure_encoder(ACSession *ac, uint32_t bit_rate, uint32_t sampling_rate, uint8_t channels);
#endif // C_TOXCORE_TOXAV_AUDIO_H

222
toxav/bwcontroller.c Normal file
View File

@ -0,0 +1,222 @@
/* SPDX-License-Identifier: GPL-3.0-or-later
* Copyright © 2016-2018 The TokTok team.
* Copyright © 2013-2015 Tox project.
*/
#include "bwcontroller.h"
#include <assert.h>
#include <errno.h>
#include <stdlib.h>
#include <string.h>
#include "ring_buffer.h"
#include "../toxcore/ccompat.h"
#include "../toxcore/logger.h"
#include "../toxcore/mono_time.h"
#include "../toxcore/util.h"
#define BWC_PACKET_ID 196
#define BWC_SEND_INTERVAL_MS 950 // 0.95s
#define BWC_AVG_PKT_COUNT 20
#define BWC_AVG_LOSS_OVER_CYCLES_COUNT 30
typedef struct BWCCycle {
uint32_t last_recv_timestamp; /* Last recv update time stamp */
uint32_t last_sent_timestamp; /* Last sent update time stamp */
uint32_t last_refresh_timestamp; /* Last refresh time stamp */
uint32_t lost;
uint32_t recv;
} BWCCycle;
typedef struct BWCRcvPkt {
uint32_t packet_length_array[BWC_AVG_PKT_COUNT];
RingBuffer *rb;
} BWCRcvPkt;
struct BWController {
m_cb *mcb;
void *mcb_user_data;
Messenger *m;
Tox *tox;
uint32_t friend_number;
BWCCycle cycle;
BWCRcvPkt rcvpkt; /* To calculate average received packet (this means split parts, not the full message!) */
uint32_t packet_loss_counted_cycles;
Mono_Time *bwc_mono_time;
};
struct BWCMessage {
uint32_t lost;
uint32_t recv;
};
static int bwc_handle_data(Messenger *m, uint32_t friendnumber, const uint8_t *data, uint16_t length, void *object);
static int bwc_send_custom_lossy_packet(Tox *tox, int32_t friendnumber, const uint8_t *data, uint32_t length);
static void send_update(BWController *bwc);
BWController *bwc_new(Messenger *m, Tox *tox, uint32_t friendnumber, m_cb *mcb, void *mcb_user_data,
Mono_Time *bwc_mono_time)
{
BWController *retu = (BWController *)calloc(1, sizeof(BWController));
if (retu == nullptr) {
return nullptr;
}
LOGGER_DEBUG(m->log, "Creating bandwidth controller");
retu->mcb = mcb;
retu->mcb_user_data = mcb_user_data;
retu->m = m;
retu->friend_number = friendnumber;
retu->bwc_mono_time = bwc_mono_time;
const uint64_t now = current_time_monotonic(bwc_mono_time);
retu->cycle.last_sent_timestamp = now;
retu->cycle.last_refresh_timestamp = now;
retu->tox = tox;
retu->rcvpkt.rb = rb_new(BWC_AVG_PKT_COUNT);
retu->cycle.lost = 0;
retu->cycle.recv = 0;
retu->packet_loss_counted_cycles = 0;
/* Fill with zeros */
for (int i = 0; i < BWC_AVG_PKT_COUNT; ++i) {
rb_write(retu->rcvpkt.rb, &retu->rcvpkt.packet_length_array[i]);
}
m_callback_rtp_packet(m, friendnumber, BWC_PACKET_ID, bwc_handle_data, retu);
return retu;
}
void bwc_kill(BWController *bwc)
{
if (bwc == nullptr) {
return;
}
m_callback_rtp_packet(bwc->m, bwc->friend_number, BWC_PACKET_ID, nullptr, nullptr);
rb_kill(bwc->rcvpkt.rb);
free(bwc);
}
void bwc_add_lost(BWController *bwc, uint32_t bytes_lost)
{
if (bwc == nullptr) {
return;
}
if (bytes_lost > 0) {
LOGGER_DEBUG(bwc->m->log, "BWC lost(1): %d", (int)bytes_lost);
bwc->cycle.lost += bytes_lost;
send_update(bwc);
}
}
void bwc_add_recv(BWController *bwc, uint32_t recv_bytes)
{
if (bwc == nullptr || recv_bytes == 0) {
return;
}
++bwc->packet_loss_counted_cycles;
bwc->cycle.recv += recv_bytes;
send_update(bwc);
}
static void send_update(BWController *bwc)
{
if (bwc->packet_loss_counted_cycles > BWC_AVG_LOSS_OVER_CYCLES_COUNT &&
current_time_monotonic(bwc->bwc_mono_time) - bwc->cycle.last_sent_timestamp > BWC_SEND_INTERVAL_MS) {
bwc->packet_loss_counted_cycles = 0;
if (bwc->cycle.lost != 0) {
LOGGER_DEBUG(bwc->m->log, "%p Sent update rcv: %u lost: %u percent: %f %%",
(void *)bwc, bwc->cycle.recv, bwc->cycle.lost,
((double)bwc->cycle.lost / (bwc->cycle.recv + bwc->cycle.lost)) * 100.0);
uint8_t bwc_packet[sizeof(struct BWCMessage) + 1];
size_t offset = 0;
bwc_packet[offset] = BWC_PACKET_ID; // set packet ID
++offset;
offset += net_pack_u32(bwc_packet + offset, bwc->cycle.lost);
offset += net_pack_u32(bwc_packet + offset, bwc->cycle.recv);
assert(offset == sizeof(bwc_packet));
if (bwc_send_custom_lossy_packet(bwc->tox, bwc->friend_number, bwc_packet, sizeof(bwc_packet)) == -1) {
char *netstrerror = net_new_strerror(net_error());
char *stdstrerror = net_new_strerror(errno);
LOGGER_WARNING(bwc->m->log, "BWC send failed (len: %u)! std error: %s, net error %s",
(unsigned)sizeof(bwc_packet), stdstrerror, netstrerror);
net_kill_strerror(stdstrerror);
net_kill_strerror(netstrerror);
}
}
bwc->cycle.last_sent_timestamp = current_time_monotonic(bwc->bwc_mono_time);
bwc->cycle.lost = 0;
bwc->cycle.recv = 0;
}
}
static int on_update(BWController *bwc, const struct BWCMessage *msg)
{
LOGGER_DEBUG(bwc->m->log, "%p Got update from peer", (void *)bwc);
/* Peers sent update too soon */
if (bwc->cycle.last_recv_timestamp + BWC_SEND_INTERVAL_MS > current_time_monotonic(bwc->bwc_mono_time)) {
LOGGER_INFO(bwc->m->log, "%p Rejecting extra update", (void *)bwc);
return -1;
}
bwc->cycle.last_recv_timestamp = current_time_monotonic(bwc->bwc_mono_time);
const uint32_t lost = msg->lost;
if (lost != 0 && bwc->mcb != nullptr) {
const uint32_t recv = msg->recv;
LOGGER_DEBUG(bwc->m->log, "recved: %u lost: %u percentage: %f %%", recv, lost,
((double)lost / (recv + lost)) * 100.0);
bwc->mcb(bwc, bwc->friend_number,
(float)lost / (recv + lost),
bwc->mcb_user_data);
}
return 0;
}
/*
* return -1 on failure, 0 on success
*
*/
static int bwc_send_custom_lossy_packet(Tox *tox, int32_t friendnumber, const uint8_t *data, uint32_t length)
{
Tox_Err_Friend_Custom_Packet error;
tox_friend_send_lossy_packet(tox, friendnumber, data, (size_t)length, &error);
if (error == TOX_ERR_FRIEND_CUSTOM_PACKET_OK) {
return 0;
}
return -1;
}
static int bwc_handle_data(Messenger *m, uint32_t friendnumber, const uint8_t *data, uint16_t length, void *object)
{
if (length - 1 != sizeof(struct BWCMessage)) {
return -1;
}
size_t offset = 1; // Ignore packet id.
struct BWCMessage msg;
offset += net_unpack_u32(data + offset, &msg.lost);
offset += net_unpack_u32(data + offset, &msg.recv);
assert(offset == length);
return on_update((BWController *)object, &msg);
}

23
toxav/bwcontroller.h Normal file
View File

@ -0,0 +1,23 @@
/* SPDX-License-Identifier: GPL-3.0-or-later
* Copyright © 2016-2018 The TokTok team.
* Copyright © 2013-2015 Tox project.
*/
#ifndef C_TOXCORE_TOXAV_BWCONTROLLER_H
#define C_TOXCORE_TOXAV_BWCONTROLLER_H
#include "../toxcore/Messenger.h"
#include "../toxcore/tox.h"
typedef struct BWController BWController;
typedef void m_cb(BWController *bwc, uint32_t friend_number, float todo, void *user_data);
BWController *bwc_new(Messenger *m, Tox *tox, uint32_t friendnumber, m_cb *mcb, void *mcb_user_data,
Mono_Time *bwc_mono_time);
void bwc_kill(BWController *bwc);
void bwc_add_lost(BWController *bwc, uint32_t bytes_lost);
void bwc_add_recv(BWController *bwc, uint32_t recv_bytes);
#endif // C_TOXCORE_TOXAV_BWCONTROLLER_H

657
toxav/groupav.c Normal file
View File

@ -0,0 +1,657 @@
/* SPDX-License-Identifier: GPL-3.0-or-later
* Copyright © 2016-2018 The TokTok team.
* Copyright © 2014 Tox project.
*/
#include "groupav.h"
#include <stdlib.h>
#include <string.h>
#include "../toxcore/ccompat.h"
#include "../toxcore/logger.h"
#include "../toxcore/mono_time.h"
#include "../toxcore/tox_struct.h"
#include "../toxcore/util.h"
#define GROUP_JBUF_SIZE 6
#define GROUP_JBUF_DEAD_SECONDS 4
typedef struct Group_Audio_Packet {
uint16_t sequnum;
uint16_t length;
uint8_t *data;
} Group_Audio_Packet;
typedef struct Group_JitterBuffer {
Group_Audio_Packet **queue;
uint32_t size;
uint32_t capacity;
uint16_t bottom;
uint16_t top;
uint64_t last_queued_time;
} Group_JitterBuffer;
static void free_audio_packet(Group_Audio_Packet *pk)
{
if (pk == nullptr) {
return;
}
free(pk->data);
free(pk);
}
static Group_JitterBuffer *create_queue(unsigned int capacity)
{
unsigned int size = 1;
while (size <= capacity) {
size *= 2;
}
Group_JitterBuffer *q = (Group_JitterBuffer *)calloc(1, sizeof(Group_JitterBuffer));
if (q == nullptr) {
return nullptr;
}
q->queue = (Group_Audio_Packet **)calloc(size, sizeof(Group_Audio_Packet *));
if (q->queue == nullptr) {
free(q);
return nullptr;
}
q->size = size;
q->capacity = capacity;
return q;
}
static void clear_queue(Group_JitterBuffer *q)
{
while (q->bottom != q->top) {
const size_t idx = q->bottom % q->size;
free_audio_packet(q->queue[idx]);
q->queue[idx] = nullptr;
++q->bottom;
}
}
static void terminate_queue(Group_JitterBuffer *q)
{
if (q == nullptr) {
return;
}
clear_queue(q);
free(q->queue);
free(q);
}
/** @retval 0 if packet was queued
* @retval -1 if it wasn't.
*/
static int queue(Group_JitterBuffer *q, const Mono_Time *mono_time, Group_Audio_Packet *pk)
{
const uint16_t sequnum = pk->sequnum;
const unsigned int num = sequnum % q->size;
if (!mono_time_is_timeout(mono_time, q->last_queued_time, GROUP_JBUF_DEAD_SECONDS)) {
if ((uint32_t)(sequnum - q->bottom) > (1 << 15)) {
/* Drop old packet. */
return -1;
}
}
if ((uint32_t)(sequnum - q->bottom) > q->size) {
clear_queue(q);
q->bottom = sequnum - q->capacity;
q->queue[num] = pk;
q->top = sequnum + 1;
q->last_queued_time = mono_time_get(mono_time);
return 0;
}
if (q->queue[num] != nullptr) {
return -1;
}
q->queue[num] = pk;
if ((sequnum - q->bottom) >= (q->top - q->bottom)) {
q->top = sequnum + 1;
}
q->last_queued_time = mono_time_get(mono_time);
return 0;
}
/**
* success is:
* - 0 when there is nothing to dequeue
* - 1 when there's a good packet
* - 2 when there's a lost packet
*/
static Group_Audio_Packet *dequeue(Group_JitterBuffer *q, int *success)
{
if (q->top == q->bottom) {
*success = 0;
return nullptr;
}
const unsigned int num = q->bottom % q->size;
if (q->queue[num] != nullptr) {
Group_Audio_Packet *ret = q->queue[num];
q->queue[num] = nullptr;
++q->bottom;
*success = 1;
return ret;
}
if ((uint32_t)(q->top - q->bottom) > q->capacity) {
++q->bottom;
*success = 2;
return nullptr;
}
*success = 0;
return nullptr;
}
typedef struct Group_AV {
const Logger *log;
Tox *tox;
Group_Chats *g_c;
OpusEncoder *audio_encoder;
unsigned int audio_channels;
unsigned int audio_sample_rate;
unsigned int audio_bitrate;
uint16_t audio_sequnum;
audio_data_cb *audio_data;
void *userdata;
} Group_AV;
typedef struct Group_Peer_AV {
const Mono_Time *mono_time;
Group_JitterBuffer *buffer;
OpusDecoder *audio_decoder;
int decoder_channels;
unsigned int last_packet_samples;
} Group_Peer_AV;
static void kill_group_av(Group_AV *group_av)
{
if (group_av->audio_encoder != nullptr) {
opus_encoder_destroy(group_av->audio_encoder);
}
free(group_av);
}
static int recreate_encoder(Group_AV *group_av)
{
if (group_av->audio_encoder != nullptr) {
opus_encoder_destroy(group_av->audio_encoder);
group_av->audio_encoder = nullptr;
}
int rc = OPUS_OK;
group_av->audio_encoder = opus_encoder_create(group_av->audio_sample_rate, group_av->audio_channels,
OPUS_APPLICATION_AUDIO, &rc);
if (rc != OPUS_OK) {
LOGGER_ERROR(group_av->log, "Error while starting audio encoder: %s", opus_strerror(rc));
group_av->audio_encoder = nullptr;
return -1;
}
rc = opus_encoder_ctl(group_av->audio_encoder, OPUS_SET_BITRATE(group_av->audio_bitrate));
if (rc != OPUS_OK) {
LOGGER_ERROR(group_av->log, "Error while setting encoder ctl: %s", opus_strerror(rc));
opus_encoder_destroy(group_av->audio_encoder);
group_av->audio_encoder = nullptr;
return -1;
}
rc = opus_encoder_ctl(group_av->audio_encoder, OPUS_SET_COMPLEXITY(10));
if (rc != OPUS_OK) {
LOGGER_ERROR(group_av->log, "Error while setting encoder ctl: %s", opus_strerror(rc));
opus_encoder_destroy(group_av->audio_encoder);
group_av->audio_encoder = nullptr;
return -1;
}
return 0;
}
static Group_AV *new_group_av(const Logger *log, Tox *tox, Group_Chats *g_c, audio_data_cb *audio_callback,
void *userdata)
{
if (g_c == nullptr) {
return nullptr;
}
Group_AV *group_av = (Group_AV *)calloc(1, sizeof(Group_AV));
if (group_av == nullptr) {
return nullptr;
}
group_av->log = log;
group_av->tox = tox;
group_av->g_c = g_c;
group_av->audio_data = audio_callback;
group_av->userdata = userdata;
return group_av;
}
static void group_av_peer_new(void *object, uint32_t groupnumber, uint32_t friendgroupnumber)
{
const Group_AV *group_av = (const Group_AV *)object;
Group_Peer_AV *peer_av = (Group_Peer_AV *)calloc(1, sizeof(Group_Peer_AV));
if (peer_av == nullptr) {
return;
}
peer_av->mono_time = g_mono_time(group_av->g_c);
peer_av->buffer = create_queue(GROUP_JBUF_SIZE);
if (group_peer_set_object(group_av->g_c, groupnumber, friendgroupnumber, peer_av) == -1) {
free(peer_av);
}
}
static void group_av_peer_delete(void *object, uint32_t groupnumber, void *peer_object)
{
Group_Peer_AV *peer_av = (Group_Peer_AV *)peer_object;
if (peer_av == nullptr) {
return;
}
if (peer_av->audio_decoder != nullptr) {
opus_decoder_destroy(peer_av->audio_decoder);
}
terminate_queue(peer_av->buffer);
free(peer_object);
}
static void group_av_groupchat_delete(void *object, uint32_t groupnumber)
{
if (object != nullptr) {
kill_group_av((Group_AV *)object);
}
}
static int decode_audio_packet(Group_AV *group_av, Group_Peer_AV *peer_av, uint32_t groupnumber,
uint32_t friendgroupnumber)
{
if (group_av == nullptr || peer_av == nullptr) {
return -1;
}
int success;
Group_Audio_Packet *pk = dequeue(peer_av->buffer, &success);
if (success == 0) {
return -1;
}
int16_t *out_audio = nullptr;
int out_audio_samples = 0;
const unsigned int sample_rate = 48000;
if (success == 1) {
const int channels = opus_packet_get_nb_channels(pk->data);
if (channels == OPUS_INVALID_PACKET) {
free_audio_packet(pk);
return -1;
}
if (channels != 1 && channels != 2) {
free_audio_packet(pk);
return -1;
}
if (channels != peer_av->decoder_channels) {
if (peer_av->audio_decoder != nullptr) {
opus_decoder_destroy(peer_av->audio_decoder);
peer_av->audio_decoder = nullptr;
}
int rc;
peer_av->audio_decoder = opus_decoder_create(sample_rate, channels, &rc);
if (rc != OPUS_OK) {
LOGGER_ERROR(group_av->log, "Error while starting audio decoder: %s", opus_strerror(rc));
free_audio_packet(pk);
return -1;
}
peer_av->decoder_channels = channels;
}
const int num_samples = opus_decoder_get_nb_samples(peer_av->audio_decoder, pk->data, pk->length);
out_audio = (int16_t *)malloc(num_samples * peer_av->decoder_channels * sizeof(int16_t));
if (out_audio == nullptr) {
free_audio_packet(pk);
return -1;
}
out_audio_samples = opus_decode(peer_av->audio_decoder, pk->data, pk->length, out_audio, num_samples, 0);
free_audio_packet(pk);
if (out_audio_samples <= 0) {
free(out_audio);
return -1;
}
peer_av->last_packet_samples = out_audio_samples;
} else {
if (peer_av->audio_decoder == nullptr) {
return -1;
}
if (peer_av->last_packet_samples == 0) {
return -1;
}
out_audio = (int16_t *)malloc(peer_av->last_packet_samples * peer_av->decoder_channels * sizeof(int16_t));
if (out_audio == nullptr) {
free_audio_packet(pk);
return -1;
}
out_audio_samples = opus_decode(peer_av->audio_decoder, nullptr, 0, out_audio, peer_av->last_packet_samples, 1);
if (out_audio_samples <= 0) {
free(out_audio);
return -1;
}
}
if (out_audio != nullptr) {
if (group_av->audio_data != nullptr) {
group_av->audio_data(group_av->tox, groupnumber, friendgroupnumber, out_audio, out_audio_samples,
peer_av->decoder_channels, sample_rate, group_av->userdata);
}
free(out_audio);
return 0;
}
return -1;
}
static int handle_group_audio_packet(void *object, uint32_t groupnumber, uint32_t friendgroupnumber, void *peer_object,
const uint8_t *packet, uint16_t length)
{
if (peer_object == nullptr || object == nullptr || length <= sizeof(uint16_t)) {
return -1;
}
Group_Peer_AV *peer_av = (Group_Peer_AV *)peer_object;
Group_Audio_Packet *pk = (Group_Audio_Packet *)calloc(1, sizeof(Group_Audio_Packet));
if (pk == nullptr) {
return -1;
}
net_unpack_u16(packet, &pk->sequnum);
pk->length = length - sizeof(uint16_t);
pk->data = (uint8_t *)malloc(pk->length);
if (pk->data == nullptr) {
free_audio_packet(pk);
return -1;
}
memcpy(pk->data, packet + sizeof(uint16_t), pk->length);
if (queue(peer_av->buffer, peer_av->mono_time, pk) == -1) {
free_audio_packet(pk);
return -1;
}
while (decode_audio_packet((Group_AV *)object, peer_av, groupnumber, friendgroupnumber) == 0) {
continue;
}
return 0;
}
/** @brief Enable A/V in a groupchat.
*
* @retval 0 on success.
* @retval -1 on failure.
*/
int groupchat_enable_av(const Logger *log, Tox *tox, Group_Chats *g_c, uint32_t groupnumber,
audio_data_cb *audio_callback, void *userdata)
{
if (group_get_type(g_c, groupnumber) != GROUPCHAT_TYPE_AV
|| group_get_object(g_c, groupnumber) != nullptr) {
return -1;
}
Group_AV *group_av = new_group_av(log, tox, g_c, audio_callback, userdata);
if (group_av == nullptr) {
return -1;
}
if (group_set_object(g_c, groupnumber, group_av) == -1
|| callback_groupchat_peer_new(g_c, groupnumber, group_av_peer_new) == -1
|| callback_groupchat_peer_delete(g_c, groupnumber, group_av_peer_delete) == -1
|| callback_groupchat_delete(g_c, groupnumber, group_av_groupchat_delete) == -1) {
kill_group_av(group_av);
return -1;
}
const int numpeers = group_number_peers(g_c, groupnumber, false);
if (numpeers < 0) {
kill_group_av(group_av);
return -1;
}
for (uint32_t i = 0; i < numpeers; ++i) {
group_av_peer_new(group_av, groupnumber, i);
}
group_lossy_packet_registerhandler(g_c, GROUP_AUDIO_PACKET_ID, &handle_group_audio_packet);
return 0;
}
/** @brief Disable A/V in a groupchat.
*
* @retval 0 on success.
* @retval -1 on failure.
*/
int groupchat_disable_av(const Group_Chats *g_c, uint32_t groupnumber)
{
if (group_get_type(g_c, groupnumber) != GROUPCHAT_TYPE_AV) {
return -1;
}
Group_AV *group_av = (Group_AV *)group_get_object(g_c, groupnumber);
if (group_av == nullptr) {
return -1;
}
const int numpeers = group_number_peers(g_c, groupnumber, false);
if (numpeers < 0) {
kill_group_av(group_av);
return -1;
}
for (uint32_t i = 0; i < numpeers; ++i) {
group_av_peer_delete(group_av, groupnumber, group_peer_get_object(g_c, groupnumber, i));
group_peer_set_object(g_c, groupnumber, i, nullptr);
}
kill_group_av(group_av);
if (group_set_object(g_c, groupnumber, nullptr) == -1
|| callback_groupchat_peer_new(g_c, groupnumber, nullptr) == -1
|| callback_groupchat_peer_delete(g_c, groupnumber, nullptr) == -1
|| callback_groupchat_delete(g_c, groupnumber, nullptr) == -1) {
return -1;
}
return 0;
}
/** Return whether A/V is enabled in the groupchat. */
bool groupchat_av_enabled(const Group_Chats *g_c, uint32_t groupnumber)
{
return group_get_object(g_c, groupnumber) != nullptr;
}
/** @brief Create and connect to a new toxav group.
*
* @return group number on success.
* @retval -1 on failure.
*/
int add_av_groupchat(const Logger *log, Tox *tox, Group_Chats *g_c, audio_data_cb *audio_callback, void *userdata)
{
const int groupnumber = add_groupchat(g_c, &tox->rng, GROUPCHAT_TYPE_AV);
if (groupnumber == -1) {
return -1;
}
if (groupchat_enable_av(log, tox, g_c, groupnumber, audio_callback, userdata) == -1) {
del_groupchat(g_c, groupnumber, true);
return -1;
}
return groupnumber;
}
/** @brief Join a AV group (you need to have been invited first).
*
* @return group number on success
* @retval -1 on failure.
*/
int join_av_groupchat(const Logger *log, Tox *tox, Group_Chats *g_c, uint32_t friendnumber, const uint8_t *data,
uint16_t length, audio_data_cb *audio_callback, void *userdata)
{
const int groupnumber = join_groupchat(g_c, friendnumber, GROUPCHAT_TYPE_AV, data, length);
if (groupnumber == -1) {
return -1;
}
if (groupchat_enable_av(log, tox, g_c, groupnumber, audio_callback, userdata) == -1) {
del_groupchat(g_c, groupnumber, true);
return -1;
}
return groupnumber;
}
/** @brief Send an encoded audio packet to the group chat.
*
* @retval 0 on success.
* @retval -1 on failure.
*/
static int send_audio_packet(const Group_Chats *g_c, uint32_t groupnumber, const uint8_t *packet, uint16_t length)
{
if (length == 0 || length > MAX_CRYPTO_DATA_SIZE - 1 - sizeof(uint16_t)) {
return -1;
}
const uint16_t plen = 1 + sizeof(uint16_t) + length;
Group_AV *const group_av = (Group_AV *)group_get_object(g_c, groupnumber);
if (group_av == nullptr) {
return -1;
}
uint8_t data[MAX_CRYPTO_DATA_SIZE];
uint8_t *ptr = data;
*ptr = GROUP_AUDIO_PACKET_ID;
++ptr;
ptr += net_pack_u16(ptr, group_av->audio_sequnum);
memcpy(ptr, packet, length);
if (send_group_lossy_packet(g_c, groupnumber, data, plen) == -1) {
return -1;
}
++group_av->audio_sequnum;
return 0;
}
/** @brief Send audio to the group chat.
*
* @retval 0 on success.
* @retval -1 on failure.
*/
int group_send_audio(Group_Chats *g_c, uint32_t groupnumber, const int16_t *pcm, unsigned int samples, uint8_t channels,
uint32_t sample_rate)
{
Group_AV *group_av = (Group_AV *)group_get_object(g_c, groupnumber);
if (group_av == nullptr) {
return -1;
}
if (channels != 1 && channels != 2) {
return -1;
}
if (sample_rate != 8000 && sample_rate != 12000 && sample_rate != 16000 && sample_rate != 24000
&& sample_rate != 48000) {
return -1;
}
if (group_av->audio_encoder == nullptr || group_av->audio_channels != channels
|| group_av->audio_sample_rate != sample_rate) {
group_av->audio_channels = channels;
group_av->audio_sample_rate = sample_rate;
if (channels == 1) {
group_av->audio_bitrate = 32000; // TODO(mannol): add way of adjusting bitrate
} else {
group_av->audio_bitrate = 64000; // TODO(mannol): add way of adjusting bitrate
}
if (recreate_encoder(group_av) == -1) {
return -1;
}
}
uint8_t encoded[1024];
const int32_t size = opus_encode(group_av->audio_encoder, pcm, samples, encoded, sizeof(encoded));
if (size <= 0) {
return -1;
}
return send_audio_packet(g_c, groupnumber, encoded, size);
}

64
toxav/groupav.h Normal file
View File

@ -0,0 +1,64 @@
/* SPDX-License-Identifier: GPL-3.0-or-later
* Copyright © 2016-2018 The TokTok team.
* Copyright © 2014 Tox project.
*/
#ifndef C_TOXCORE_TOXAV_GROUPAV_H
#define C_TOXCORE_TOXAV_GROUPAV_H
// Audio encoding/decoding
#include <opus.h>
#include "../toxcore/group.h"
#include "../toxcore/tox.h"
#define GROUP_AUDIO_PACKET_ID 192
// TODO(iphydf): Use this better typed one instead of the void-pointer one below.
// typedef void audio_data_cb(Tox *tox, uint32_t groupnumber, uint32_t peernumber, const int16_t *pcm,
// uint32_t samples, uint8_t channels, uint32_t sample_rate, void *userdata);
typedef void audio_data_cb(void *tox, uint32_t groupnumber, uint32_t peernumber, const int16_t *pcm,
uint32_t samples, uint8_t channels, uint32_t sample_rate, void *userdata);
/** @brief Create and connect to a new toxav group.
*
* @return group number on success.
* @retval -1 on failure.
*/
int add_av_groupchat(const Logger *log, Tox *tox, Group_Chats *g_c, audio_data_cb *audio_callback, void *userdata);
/** @brief Join a AV group (you need to have been invited first).
*
* @return group number on success
* @retval -1 on failure.
*/
int join_av_groupchat(const Logger *log, Tox *tox, Group_Chats *g_c, uint32_t friendnumber, const uint8_t *data,
uint16_t length, audio_data_cb *audio_callback, void *userdata);
/** @brief Send audio to the group chat.
*
* @retval 0 on success.
* @retval -1 on failure.
*/
int group_send_audio(Group_Chats *g_c, uint32_t groupnumber, const int16_t *pcm, unsigned int samples, uint8_t channels,
uint32_t sample_rate);
/** @brief Enable A/V in a groupchat.
*
* @retval 0 on success.
* @retval -1 on failure.
*/
int groupchat_enable_av(const Logger *log, Tox *tox, Group_Chats *g_c, uint32_t groupnumber,
audio_data_cb *audio_callback, void *userdata);
/** @brief Disable A/V in a groupchat.
*
* @retval 0 on success.
* @retval -1 on failure.
*/
int groupchat_disable_av(const Group_Chats *g_c, uint32_t groupnumber);
/** Return whether A/V is enabled in the groupchat. */
bool groupchat_av_enabled(const Group_Chats *g_c, uint32_t groupnumber);
#endif // C_TOXCORE_TOXAV_GROUPAV_H

903
toxav/msi.c Normal file
View File

@ -0,0 +1,903 @@
/* SPDX-License-Identifier: GPL-3.0-or-later
* Copyright © 2016-2018 The TokTok team.
* Copyright © 2013-2015 Tox project.
*/
#include "msi.h"
#include <assert.h>
#include <stdbool.h>
#include <stdlib.h>
#include <string.h>
#include "../toxcore/ccompat.h"
#include "../toxcore/logger.h"
#include "../toxcore/util.h"
#define MSI_MAXMSG_SIZE 256
/**
* Protocol:
*
* `|id [1 byte]| |size [1 byte]| |data [$size bytes]| |...{repeat}| |0 {end byte}|`
*/
typedef enum MSIHeaderID {
ID_REQUEST = 1,
ID_ERROR,
ID_CAPABILITIES,
} MSIHeaderID;
typedef enum MSIRequest {
REQU_INIT,
REQU_PUSH,
REQU_POP,
} MSIRequest;
typedef struct MSIHeaderRequest {
MSIRequest value;
bool exists;
} MSIHeaderRequest;
typedef struct MSIHeaderError {
MSIError value;
bool exists;
} MSIHeaderError;
typedef struct MSIHeaderCapabilities {
uint8_t value;
bool exists;
} MSIHeaderCapabilities;
typedef struct MSIMessage {
MSIHeaderRequest request;
MSIHeaderError error;
MSIHeaderCapabilities capabilities;
} MSIMessage;
static void msg_init(MSIMessage *dest, MSIRequest request);
static int msg_parse_in(const Logger *log, MSIMessage *dest, const uint8_t *data, uint16_t length);
static uint8_t *msg_parse_header_out(MSIHeaderID id, uint8_t *dest, const void *value, uint8_t value_len,
uint16_t *length);
static int send_message(Messenger *m, uint32_t friend_number, const MSIMessage *msg);
static int send_error(Messenger *m, uint32_t friend_number, MSIError error);
static bool invoke_callback(MSICall *call, MSICallbackID cb);
static MSICall *get_call(MSISession *session, uint32_t friend_number);
static MSICall *new_call(MSISession *session, uint32_t friend_number);
static void kill_call(MSICall *call);
static void on_peer_status(Messenger *m, uint32_t friend_number, uint8_t status, void *data);
static void handle_init(MSICall *call, const MSIMessage *msg);
static void handle_push(MSICall *call, const MSIMessage *msg);
static void handle_pop(MSICall *call, const MSIMessage *msg);
static void handle_msi_packet(Messenger *m, uint32_t friend_number, const uint8_t *data, uint16_t length, void *object);
/*
* Public functions
*/
void msi_callback_invite(MSISession *session, msi_action_cb *callback)
{
session->invite_callback = callback;
}
void msi_callback_start(MSISession *session, msi_action_cb *callback)
{
session->start_callback = callback;
}
void msi_callback_end(MSISession *session, msi_action_cb *callback)
{
session->end_callback = callback;
}
void msi_callback_error(MSISession *session, msi_action_cb *callback)
{
session->error_callback = callback;
}
void msi_callback_peertimeout(MSISession *session, msi_action_cb *callback)
{
session->peertimeout_callback = callback;
}
void msi_callback_capabilities(MSISession *session, msi_action_cb *callback)
{
session->capabilities_callback = callback;
}
MSISession *msi_new(Messenger *m)
{
if (m == nullptr) {
return nullptr;
}
MSISession *retu = (MSISession *)calloc(1, sizeof(MSISession));
if (retu == nullptr) {
LOGGER_ERROR(m->log, "Allocation failed! Program might misbehave!");
return nullptr;
}
if (create_recursive_mutex(retu->mutex) != 0) {
LOGGER_ERROR(m->log, "Failed to init mutex! Program might misbehave");
free(retu);
return nullptr;
}
retu->messenger = m;
m_callback_msi_packet(m, handle_msi_packet, retu);
/* This is called when remote terminates session */
m_callback_connectionstatus_internal_av(m, on_peer_status, retu);
LOGGER_DEBUG(m->log, "New msi session: %p ", (void *)retu);
return retu;
}
int msi_kill(MSISession *session, const Logger *log)
{
if (session == nullptr) {
LOGGER_ERROR(log, "Tried to terminate non-existing session");
return -1;
}
m_callback_msi_packet(session->messenger, nullptr, nullptr);
if (pthread_mutex_trylock(session->mutex) != 0) {
LOGGER_ERROR(log, "Failed to acquire lock on msi mutex");
return -1;
}
if (session->calls != nullptr) {
MSIMessage msg;
msg_init(&msg, REQU_POP);
MSICall *it = get_call(session, session->calls_head);
while (it != nullptr) {
send_message(session->messenger, it->friend_number, &msg);
MSICall *temp_it = it;
it = it->next;
kill_call(temp_it); /* This will eventually free session->calls */
}
}
pthread_mutex_unlock(session->mutex);
pthread_mutex_destroy(session->mutex);
LOGGER_DEBUG(log, "Terminated session: %p", (void *)session);
free(session);
return 0;
}
int msi_invite(MSISession *session, MSICall **call, uint32_t friend_number, uint8_t capabilities)
{
if (session == nullptr) {
return -1;
}
LOGGER_DEBUG(session->messenger->log, "Session: %p Inviting friend: %u", (void *)session, friend_number);
if (pthread_mutex_trylock(session->mutex) != 0) {
LOGGER_ERROR(session->messenger->log, "Failed to acquire lock on msi mutex");
return -1;
}
if (get_call(session, friend_number) != nullptr) {
LOGGER_ERROR(session->messenger->log, "Already in a call");
pthread_mutex_unlock(session->mutex);
return -1;
}
MSICall *temp = new_call(session, friend_number);
if (temp == nullptr) {
pthread_mutex_unlock(session->mutex);
return -1;
}
temp->self_capabilities = capabilities;
MSIMessage msg;
msg_init(&msg, REQU_INIT);
msg.capabilities.exists = true;
msg.capabilities.value = capabilities;
send_message(temp->session->messenger, temp->friend_number, &msg);
temp->state = MSI_CALL_REQUESTING;
*call = temp;
LOGGER_DEBUG(session->messenger->log, "Invite sent");
pthread_mutex_unlock(session->mutex);
return 0;
}
int msi_hangup(MSICall *call)
{
if (call == nullptr || call->session == nullptr) {
return -1;
}
MSISession *session = call->session;
LOGGER_DEBUG(session->messenger->log, "Session: %p Hanging up call with friend: %u", (void *)call->session,
call->friend_number);
if (pthread_mutex_trylock(session->mutex) != 0) {
LOGGER_ERROR(session->messenger->log, "Failed to acquire lock on msi mutex");
return -1;
}
if (call->state == MSI_CALL_INACTIVE) {
LOGGER_ERROR(session->messenger->log, "Call is in invalid state!");
pthread_mutex_unlock(session->mutex);
return -1;
}
MSIMessage msg;
msg_init(&msg, REQU_POP);
send_message(session->messenger, call->friend_number, &msg);
kill_call(call);
pthread_mutex_unlock(session->mutex);
return 0;
}
int msi_answer(MSICall *call, uint8_t capabilities)
{
if (call == nullptr || call->session == nullptr) {
return -1;
}
MSISession *session = call->session;
LOGGER_DEBUG(session->messenger->log, "Session: %p Answering call from: %u", (void *)call->session,
call->friend_number);
if (pthread_mutex_trylock(session->mutex) != 0) {
LOGGER_ERROR(session->messenger->log, "Failed to acquire lock on msi mutex");
return -1;
}
if (call->state != MSI_CALL_REQUESTED) {
/* Though sending in invalid state will not cause anything weird
* Its better to not do it like a maniac */
LOGGER_ERROR(session->messenger->log, "Call is in invalid state!");
pthread_mutex_unlock(session->mutex);
return -1;
}
call->self_capabilities = capabilities;
MSIMessage msg;
msg_init(&msg, REQU_PUSH);
msg.capabilities.exists = true;
msg.capabilities.value = capabilities;
send_message(session->messenger, call->friend_number, &msg);
call->state = MSI_CALL_ACTIVE;
pthread_mutex_unlock(session->mutex);
return 0;
}
int msi_change_capabilities(MSICall *call, uint8_t capabilities)
{
if (call == nullptr || call->session == nullptr) {
return -1;
}
MSISession *session = call->session;
LOGGER_DEBUG(session->messenger->log, "Session: %p Trying to change capabilities to friend %u", (void *)call->session,
call->friend_number);
if (pthread_mutex_trylock(session->mutex) != 0) {
LOGGER_ERROR(session->messenger->log, "Failed to acquire lock on msi mutex");
return -1;
}
if (call->state != MSI_CALL_ACTIVE) {
LOGGER_ERROR(session->messenger->log, "Call is in invalid state!");
pthread_mutex_unlock(session->mutex);
return -1;
}
call->self_capabilities = capabilities;
MSIMessage msg;
msg_init(&msg, REQU_PUSH);
msg.capabilities.exists = true;
msg.capabilities.value = capabilities;
send_message(call->session->messenger, call->friend_number, &msg);
pthread_mutex_unlock(session->mutex);
return 0;
}
/**
* Private functions
*/
static void msg_init(MSIMessage *dest, MSIRequest request)
{
memset(dest, 0, sizeof(*dest));
dest->request.exists = true;
dest->request.value = request;
}
static bool check_size(const Logger *log, const uint8_t *bytes, int *constraint, uint8_t size)
{
*constraint -= 2 + size;
if (*constraint < 1) {
LOGGER_ERROR(log, "Read over length!");
return false;
}
if (bytes[1] != size) {
LOGGER_ERROR(log, "Invalid data size!");
return false;
}
return true;
}
/** Assumes size == 1 */
static bool check_enum_high(const Logger *log, const uint8_t *bytes, uint8_t enum_high)
{
if (bytes[2] > enum_high) {
LOGGER_ERROR(log, "Failed enum high limit!");
return false;
}
return true;
}
static int msg_parse_in(const Logger *log, MSIMessage *dest, const uint8_t *data, uint16_t length)
{
/* Parse raw data received from socket into MSIMessage struct */
assert(dest != nullptr);
if (length == 0 || data[length - 1] != 0) { /* End byte must have value 0 */
LOGGER_ERROR(log, "Invalid end byte");
return -1;
}
memset(dest, 0, sizeof(*dest));
const uint8_t *it = data;
int size_constraint = length;
while (*it != 0) {/* until end byte is hit */
switch (*it) {
case ID_REQUEST: {
if (!check_size(log, it, &size_constraint, 1) ||
!check_enum_high(log, it, REQU_POP)) {
return -1;
}
dest->request.value = (MSIRequest)it[2];
dest->request.exists = true;
it += 3;
break;
}
case ID_ERROR: {
if (!check_size(log, it, &size_constraint, 1) ||
!check_enum_high(log, it, MSI_E_UNDISCLOSED)) {
return -1;
}
dest->error.value = (MSIError)it[2];
dest->error.exists = true;
it += 3;
break;
}
case ID_CAPABILITIES: {
if (!check_size(log, it, &size_constraint, 1)) {
return -1;
}
dest->capabilities.value = it[2];
dest->capabilities.exists = true;
it += 3;
break;
}
default: {
LOGGER_ERROR(log, "Invalid id byte");
return -1;
}
}
}
if (!dest->request.exists) {
LOGGER_ERROR(log, "Invalid request field!");
return -1;
}
return 0;
}
static uint8_t *msg_parse_header_out(MSIHeaderID id, uint8_t *dest, const void *value, uint8_t value_len,
uint16_t *length)
{
/* Parse a single header for sending */
assert(dest != nullptr);
assert(value != nullptr);
assert(value_len != 0);
*dest = id;
++dest;
*dest = value_len;
++dest;
memcpy(dest, value, value_len);
*length += 2 + value_len;
return dest + value_len; /* Set to next position ready to be written */
}
static int send_message(Messenger *m, uint32_t friend_number, const MSIMessage *msg)
{
/* Parse and send message */
assert(m != nullptr);
uint8_t parsed [MSI_MAXMSG_SIZE];
uint8_t *it = parsed;
uint16_t size = 0;
if (msg->request.exists) {
uint8_t cast = msg->request.value;
it = msg_parse_header_out(ID_REQUEST, it, &cast,
sizeof(cast), &size);
} else {
LOGGER_DEBUG(m->log, "Must have request field");
return -1;
}
if (msg->error.exists) {
uint8_t cast = msg->error.value;
it = msg_parse_header_out(ID_ERROR, it, &cast,
sizeof(cast), &size);
}
if (msg->capabilities.exists) {
it = msg_parse_header_out(ID_CAPABILITIES, it, &msg->capabilities.value,
sizeof(msg->capabilities.value), &size);
}
if (it == parsed) {
LOGGER_WARNING(m->log, "Parsing message failed; empty message");
return -1;
}
*it = 0;
++size;
if (m_msi_packet(m, friend_number, parsed, size)) {
LOGGER_DEBUG(m->log, "Sent message");
return 0;
}
return -1;
}
static int send_error(Messenger *m, uint32_t friend_number, MSIError error)
{
/* Send error message */
assert(m != nullptr);
LOGGER_DEBUG(m->log, "Sending error: %d to friend: %d", error, friend_number);
MSIMessage msg;
msg_init(&msg, REQU_POP);
msg.error.exists = true;
msg.error.value = error;
send_message(m, friend_number, &msg);
return 0;
}
static int invoke_callback_inner(MSICall *call, MSICallbackID id)
{
MSISession *session = call->session;
LOGGER_DEBUG(session->messenger->log, "invoking callback function: %d", id);
switch (id) {
case MSI_ON_INVITE:
return session->invite_callback(session->av, call);
case MSI_ON_START:
return session->start_callback(session->av, call);
case MSI_ON_END:
return session->end_callback(session->av, call);
case MSI_ON_ERROR:
return session->error_callback(session->av, call);
case MSI_ON_PEERTIMEOUT:
return session->peertimeout_callback(session->av, call);
case MSI_ON_CAPABILITIES:
return session->capabilities_callback(session->av, call);
}
LOGGER_FATAL(session->messenger->log, "invalid callback id: %d", id);
return -1;
}
static bool invoke_callback(MSICall *call, MSICallbackID cb)
{
assert(call != nullptr);
if (invoke_callback_inner(call, cb) != 0) {
LOGGER_WARNING(call->session->messenger->log,
"Callback state handling failed, sending error");
/* If no callback present or error happened while handling,
* an error message will be sent to friend
*/
if (call->error == MSI_E_NONE) {
call->error = MSI_E_HANDLE;
}
return false;
}
return true;
}
static MSICall *get_call(MSISession *session, uint32_t friend_number)
{
assert(session != nullptr);
if (session->calls == nullptr || session->calls_tail < friend_number) {
return nullptr;
}
return session->calls[friend_number];
}
static MSICall *new_call(MSISession *session, uint32_t friend_number)
{
assert(session != nullptr);
MSICall *rc = (MSICall *)calloc(1, sizeof(MSICall));
if (rc == nullptr) {
return nullptr;
}
rc->session = session;
rc->friend_number = friend_number;
if (session->calls == nullptr) { /* Creating */
session->calls = (MSICall **)calloc(friend_number + 1, sizeof(MSICall *));
if (session->calls == nullptr) {
free(rc);
return nullptr;
}
session->calls_tail = friend_number;
session->calls_head = friend_number;
} else if (session->calls_tail < friend_number) { /* Appending */
MSICall **tmp = (MSICall **)realloc(session->calls, sizeof(MSICall *) * (friend_number + 1));
if (tmp == nullptr) {
free(rc);
return nullptr;
}
session->calls = tmp;
/* Set fields in between to null */
for (uint32_t i = session->calls_tail + 1; i < friend_number; ++i) {
session->calls[i] = nullptr;
}
rc->prev = session->calls[session->calls_tail];
session->calls[session->calls_tail]->next = rc;
session->calls_tail = friend_number;
} else if (session->calls_head > friend_number) { /* Inserting at front */
rc->next = session->calls[session->calls_head];
session->calls[session->calls_head]->prev = rc;
session->calls_head = friend_number;
}
session->calls[friend_number] = rc;
return rc;
}
static void kill_call(MSICall *call)
{
/* Assume that session mutex is locked */
if (call == nullptr) {
return;
}
MSISession *session = call->session;
LOGGER_DEBUG(session->messenger->log, "Killing call: %p", (void *)call);
MSICall *prev = call->prev;
MSICall *next = call->next;
if (prev != nullptr) {
prev->next = next;
} else if (next != nullptr) {
session->calls_head = next->friend_number;
} else {
goto CLEAR_CONTAINER;
}
if (next != nullptr) {
next->prev = prev;
} else if (prev != nullptr) {
session->calls_tail = prev->friend_number;
} else {
goto CLEAR_CONTAINER;
}
session->calls[call->friend_number] = nullptr;
free(call);
return;
CLEAR_CONTAINER:
session->calls_head = 0;
session->calls_tail = 0;
free(session->calls);
free(call);
session->calls = nullptr;
}
static void on_peer_status(Messenger *m, uint32_t friend_number, uint8_t status, void *data)
{
if (status != 0) {
// Friend is online.
return;
}
MSISession *session = (MSISession *)data;
LOGGER_DEBUG(m->log, "Friend %d is now offline", friend_number);
pthread_mutex_lock(session->mutex);
MSICall *call = get_call(session, friend_number);
if (call == nullptr) {
pthread_mutex_unlock(session->mutex);
return;
}
invoke_callback(call, MSI_ON_PEERTIMEOUT); /* Failure is ignored */
kill_call(call);
pthread_mutex_unlock(session->mutex);
}
static bool try_handle_init(MSICall *call, const MSIMessage *msg)
{
assert(call != nullptr);
LOGGER_DEBUG(call->session->messenger->log,
"Session: %p Handling 'init' friend: %d", (void *)call->session, call->friend_number);
if (!msg->capabilities.exists) {
LOGGER_WARNING(call->session->messenger->log, "Session: %p Invalid capabilities on 'init'", (void *)call->session);
call->error = MSI_E_INVALID_MESSAGE;
return false;
}
switch (call->state) {
case MSI_CALL_INACTIVE: {
/* Call requested */
call->peer_capabilities = msg->capabilities.value;
call->state = MSI_CALL_REQUESTED;
if (!invoke_callback(call, MSI_ON_INVITE)) {
return false;
}
break;
}
case MSI_CALL_ACTIVE: {
/* If peer sent init while the call is already
* active it's probable that he is trying to
* re-call us while the call is not terminated
* on our side. We can assume that in this case
* we can automatically answer the re-call.
*/
LOGGER_INFO(call->session->messenger->log, "Friend is recalling us");
MSIMessage out_msg;
msg_init(&out_msg, REQU_PUSH);
out_msg.capabilities.exists = true;
out_msg.capabilities.value = call->self_capabilities;
send_message(call->session->messenger, call->friend_number, &out_msg);
/* If peer changed capabilities during re-call they will
* be handled accordingly during the next step
*/
break;
}
case MSI_CALL_REQUESTED: // fall-through
case MSI_CALL_REQUESTING: {
LOGGER_WARNING(call->session->messenger->log, "Session: %p Invalid state on 'init'", (void *)call->session);
call->error = MSI_E_INVALID_STATE;
return false;
}
}
return true;
}
static void handle_init(MSICall *call, const MSIMessage *msg)
{
assert(call != nullptr);
LOGGER_DEBUG(call->session->messenger->log,
"Session: %p Handling 'init' friend: %d", (void *)call->session, call->friend_number);
if (!try_handle_init(call, msg)) {
send_error(call->session->messenger, call->friend_number, call->error);
kill_call(call);
}
}
static void handle_push(MSICall *call, const MSIMessage *msg)
{
assert(call != nullptr);
LOGGER_DEBUG(call->session->messenger->log, "Session: %p Handling 'push' friend: %d", (void *)call->session,
call->friend_number);
if (!msg->capabilities.exists) {
LOGGER_WARNING(call->session->messenger->log, "Session: %p Invalid capabilities on 'push'", (void *)call->session);
call->error = MSI_E_INVALID_MESSAGE;
goto FAILURE;
}
switch (call->state) {
case MSI_CALL_ACTIVE: {
if (call->peer_capabilities != msg->capabilities.value) {
/* Only act if capabilities changed */
LOGGER_INFO(call->session->messenger->log, "Friend is changing capabilities to: %u", msg->capabilities.value);
call->peer_capabilities = msg->capabilities.value;
if (!invoke_callback(call, MSI_ON_CAPABILITIES)) {
goto FAILURE;
}
}
break;
}
case MSI_CALL_REQUESTING: {
LOGGER_INFO(call->session->messenger->log, "Friend answered our call");
/* Call started */
call->peer_capabilities = msg->capabilities.value;
call->state = MSI_CALL_ACTIVE;
if (!invoke_callback(call, MSI_ON_START)) {
goto FAILURE;
}
break;
}
case MSI_CALL_INACTIVE: // fall-through
case MSI_CALL_REQUESTED: {
/* Pushes during initialization state are ignored */
LOGGER_WARNING(call->session->messenger->log, "Ignoring invalid push");
break;
}
}
return;
FAILURE:
send_error(call->session->messenger, call->friend_number, call->error);
kill_call(call);
}
static void handle_pop(MSICall *call, const MSIMessage *msg)
{
assert(call != nullptr);
LOGGER_DEBUG(call->session->messenger->log, "Session: %p Handling 'pop', friend id: %d", (void *)call->session,
call->friend_number);
/* callback errors are ignored */
if (msg->error.exists) {
LOGGER_WARNING(call->session->messenger->log, "Friend detected an error: %d", msg->error.value);
call->error = msg->error.value;
invoke_callback(call, MSI_ON_ERROR);
} else {
switch (call->state) {
case MSI_CALL_INACTIVE: {
LOGGER_FATAL(call->session->messenger->log, "Handling what should be impossible case");
}
case MSI_CALL_ACTIVE: {
/* Hangup */
LOGGER_INFO(call->session->messenger->log, "Friend hung up on us");
invoke_callback(call, MSI_ON_END);
break;
}
case MSI_CALL_REQUESTING: {
/* Reject */
LOGGER_INFO(call->session->messenger->log, "Friend rejected our call");
invoke_callback(call, MSI_ON_END);
break;
}
case MSI_CALL_REQUESTED: {
/* Cancel */
LOGGER_INFO(call->session->messenger->log, "Friend canceled call invite");
invoke_callback(call, MSI_ON_END);
break;
}
}
}
kill_call(call);
}
static void handle_msi_packet(Messenger *m, uint32_t friend_number, const uint8_t *data, uint16_t length, void *object)
{
LOGGER_DEBUG(m->log, "Got msi message");
MSISession *session = (MSISession *)object;
MSIMessage msg;
if (msg_parse_in(m->log, &msg, data, length) == -1) {
LOGGER_WARNING(m->log, "Error parsing message");
send_error(m, friend_number, MSI_E_INVALID_MESSAGE);
return;
}
LOGGER_DEBUG(m->log, "Successfully parsed message");
pthread_mutex_lock(session->mutex);
MSICall *call = get_call(session, friend_number);
if (call == nullptr) {
if (msg.request.value != REQU_INIT) {
send_error(m, friend_number, MSI_E_STRAY_MESSAGE);
pthread_mutex_unlock(session->mutex);
return;
}
call = new_call(session, friend_number);
if (call == nullptr) {
send_error(m, friend_number, MSI_E_SYSTEM);
pthread_mutex_unlock(session->mutex);
return;
}
}
switch (msg.request.value) {
case REQU_INIT: {
handle_init(call, &msg);
break;
}
case REQU_PUSH: {
handle_push(call, &msg);
break;
}
case REQU_POP: {
handle_pop(call, &msg); /* always kills the call */
break;
}
}
pthread_mutex_unlock(session->mutex);
}

147
toxav/msi.h Normal file
View File

@ -0,0 +1,147 @@
/* SPDX-License-Identifier: GPL-3.0-or-later
* Copyright © 2016-2018 The TokTok team.
* Copyright © 2013-2015 Tox project.
*/
#ifndef C_TOXCORE_TOXAV_MSI_H
#define C_TOXCORE_TOXAV_MSI_H
#include <pthread.h>
#include <stdint.h>
#include "audio.h"
#include "video.h"
#include "../toxcore/Messenger.h"
#include "../toxcore/logger.h"
/**
* Error codes.
*/
typedef enum MSIError {
MSI_E_NONE,
MSI_E_INVALID_MESSAGE,
MSI_E_INVALID_PARAM,
MSI_E_INVALID_STATE,
MSI_E_STRAY_MESSAGE,
MSI_E_SYSTEM,
MSI_E_HANDLE,
MSI_E_UNDISCLOSED, /* NOTE: must be last enum otherwise parsing will not work */
} MSIError;
/**
* Supported capabilities
*/
typedef enum MSICapabilities {
MSI_CAP_S_AUDIO = 4, /* sending audio */
MSI_CAP_S_VIDEO = 8, /* sending video */
MSI_CAP_R_AUDIO = 16, /* receiving audio */
MSI_CAP_R_VIDEO = 32, /* receiving video */
} MSICapabilities;
/**
* Call state identifiers.
*/
typedef enum MSICallState {
MSI_CALL_INACTIVE, /* Default */
MSI_CALL_ACTIVE,
MSI_CALL_REQUESTING, /* when sending call invite */
MSI_CALL_REQUESTED, /* when getting call invite */
} MSICallState;
/**
* Callbacks ids that handle the states
*/
typedef enum MSICallbackID {
MSI_ON_INVITE, /* Incoming call */
MSI_ON_START, /* Call (RTP transmission) started */
MSI_ON_END, /* Call that was active ended */
MSI_ON_ERROR, /* On protocol error */
MSI_ON_PEERTIMEOUT, /* Peer timed out; stop the call */
MSI_ON_CAPABILITIES, /* Peer requested capabilities change */
} MSICallbackID;
/**
* The call struct. Please do not modify outside msi.c
*/
typedef struct MSICall {
struct MSISession *session; /* Session pointer */
MSICallState state;
uint8_t peer_capabilities; /* Peer capabilities */
uint8_t self_capabilities; /* Self capabilities */
uint16_t peer_vfpsz; /* Video frame piece size */
uint32_t friend_number; /* Index of this call in MSISession */
MSIError error; /* Last error */
struct ToxAVCall *av_call; /* Pointer to av call handler */
struct MSICall *next;
struct MSICall *prev;
} MSICall;
/**
* Expected return on success is 0, if any other number is
* returned the call is considered errored and will be handled
* as such which means it will be terminated without any notice.
*/
typedef int msi_action_cb(void *av, MSICall *call);
/**
* Control session struct. Please do not modify outside msi.c
*/
typedef struct MSISession {
/* Call handlers */
MSICall **calls;
uint32_t calls_tail;
uint32_t calls_head;
void *av;
Messenger *messenger;
pthread_mutex_t mutex[1];
msi_action_cb *invite_callback;
msi_action_cb *start_callback;
msi_action_cb *end_callback;
msi_action_cb *error_callback;
msi_action_cb *peertimeout_callback;
msi_action_cb *capabilities_callback;
} MSISession;
/**
* Start the control session.
*/
MSISession *msi_new(Messenger *m);
/**
* Terminate control session. NOTE: all calls will be freed
*/
int msi_kill(MSISession *session, const Logger *log);
/**
* Callback setters.
*/
void msi_callback_invite(MSISession *session, msi_action_cb *callback);
void msi_callback_start(MSISession *session, msi_action_cb *callback);
void msi_callback_end(MSISession *session, msi_action_cb *callback);
void msi_callback_error(MSISession *session, msi_action_cb *callback);
void msi_callback_peertimeout(MSISession *session, msi_action_cb *callback);
void msi_callback_capabilities(MSISession *session, msi_action_cb *callback);
/**
* Send invite request to friend_number.
*/
int msi_invite(MSISession *session, MSICall **call, uint32_t friend_number, uint8_t capabilities);
/**
* Hangup call. NOTE: `call` will be freed
*/
int msi_hangup(MSICall *call);
/**
* Answer call request.
*/
int msi_answer(MSICall *call, uint8_t capabilities);
/**
* Change capabilities of the call.
*/
int msi_change_capabilities(MSICall *call, uint8_t capabilities);
#endif // C_TOXCORE_TOXAV_MSI_H

115
toxav/ring_buffer.c Normal file
View File

@ -0,0 +1,115 @@
/* SPDX-License-Identifier: GPL-3.0-or-later
* Copyright © 2016-2018 The TokTok team.
* Copyright © 2013 Tox project.
* Copyright © 2013 plutooo
*/
#include "ring_buffer.h"
#include <stdlib.h>
#include "../toxcore/ccompat.h"
struct RingBuffer {
uint16_t size; /* Max size */
uint16_t start;
uint16_t end;
void **data;
};
bool rb_full(const RingBuffer *b)
{
return (b->end + 1) % b->size == b->start;
}
bool rb_empty(const RingBuffer *b)
{
return b->end == b->start;
}
/**
* @retval NULL on success
* @return input value "p" on failure, so caller can free on failed rb_write
*/
void *rb_write(RingBuffer *b, void *p)
{
if (b == nullptr) {
return p;
}
void *rc = nullptr;
if ((b->end + 1) % b->size == b->start) { /* full */
rc = b->data[b->start];
}
b->data[b->end] = p;
b->end = (b->end + 1) % b->size;
if (b->end == b->start) {
b->start = (b->start + 1) % b->size;
}
return rc;
}
bool rb_read(RingBuffer *b, void **p)
{
if (b->end == b->start) { /* Empty */
*p = nullptr;
return false;
}
*p = b->data[b->start];
b->start = (b->start + 1) % b->size;
return true;
}
RingBuffer *rb_new(int size)
{
RingBuffer *buf = (RingBuffer *)calloc(1, sizeof(RingBuffer));
if (buf == nullptr) {
return nullptr;
}
buf->size = size + 1; /* include empty elem */
buf->data = (void **)calloc(buf->size, sizeof(void *));
if (buf->data == nullptr) {
free(buf);
return nullptr;
}
return buf;
}
void rb_kill(RingBuffer *b)
{
if (b != nullptr) {
free(b->data);
free(b);
}
}
uint16_t rb_size(const RingBuffer *b)
{
if (rb_empty(b)) {
return 0;
}
return
b->end > b->start ?
b->end - b->start :
(b->size - b->start) + b->end;
}
uint16_t rb_data(const RingBuffer *b, void **dest)
{
uint16_t i;
for (i = 0; i < rb_size(b); ++i) {
dest[i] = b->data[(b->start + i) % b->size];
}
return i;
}

31
toxav/ring_buffer.h Normal file
View File

@ -0,0 +1,31 @@
/* SPDX-License-Identifier: GPL-3.0-or-later
* Copyright © 2016-2018 The TokTok team.
* Copyright © 2013 Tox project.
* Copyright © 2013 plutooo
*/
#ifndef C_TOXCORE_TOXAV_RING_BUFFER_H
#define C_TOXCORE_TOXAV_RING_BUFFER_H
#include <stdbool.h>
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
#endif
/** Ring buffer */
typedef struct RingBuffer RingBuffer;
bool rb_full(const RingBuffer *b);
bool rb_empty(const RingBuffer *b);
void *rb_write(RingBuffer *b, void *p);
bool rb_read(RingBuffer *b, void **p);
RingBuffer *rb_new(int size);
void rb_kill(RingBuffer *b);
uint16_t rb_size(const RingBuffer *b);
uint16_t rb_data(const RingBuffer *b, void **dest);
#ifdef __cplusplus
}
#endif
#endif // C_TOXCORE_TOXAV_RING_BUFFER_H

214
toxav/ring_buffer_test.cc Normal file
View File

@ -0,0 +1,214 @@
#include "ring_buffer.h"
#include <gtest/gtest.h>
#include <algorithm>
#include <cassert>
#include <vector>
namespace {
template <typename T>
class TypedRingBuffer;
template <typename T>
class TypedRingBuffer<T *> {
public:
explicit TypedRingBuffer(int size)
: rb_(rb_new(size))
{
}
~TypedRingBuffer() { rb_kill(rb_); }
TypedRingBuffer(TypedRingBuffer const &) = delete;
bool full() const { return rb_full(rb_); }
bool empty() const { return rb_empty(rb_); }
T *write(T *p) { return static_cast<T *>(rb_write(rb_, p)); }
bool read(T **p)
{
void *vp;
bool res = rb_read(rb_, &vp);
*p = static_cast<T *>(vp);
return res;
}
uint16_t size() const { return rb_size(rb_); }
uint16_t data(T **dest) const
{
std::vector<void *> vdest(size());
uint16_t res = rb_data(rb_, vdest.data());
for (uint16_t i = 0; i < size(); i++) {
dest[i] = static_cast<T *>(vdest.at(i));
}
return res;
}
bool contains(T *p) const
{
std::vector<T *> elts(size());
data(elts.data());
return std::find(elts.begin(), elts.end(), p) != elts.end();
}
bool ok() const { return rb_ != nullptr; }
private:
RingBuffer *rb_;
};
TEST(RingBuffer, EmptyBufferReportsEmpty)
{
TypedRingBuffer<int *> rb(10);
ASSERT_TRUE(rb.ok());
EXPECT_TRUE(rb.empty());
}
TEST(RingBuffer, EmptyBufferReportsNotFull)
{
TypedRingBuffer<int *> rb(10);
ASSERT_TRUE(rb.ok());
EXPECT_FALSE(rb.full());
}
TEST(RingBuffer, ZeroSizedRingBufferIsBothEmptyAndFull)
{
TypedRingBuffer<int *> rb(0);
ASSERT_TRUE(rb.ok());
EXPECT_TRUE(rb.empty());
EXPECT_TRUE(rb.full());
}
TEST(RingBuffer, WritingMakesBufferNotEmpty)
{
TypedRingBuffer<int *> rb(2);
ASSERT_TRUE(rb.ok());
int value0 = 123;
rb.write(&value0);
EXPECT_FALSE(rb.empty());
}
TEST(RingBuffer, WritingOneElementMakesBufferNotFull)
{
TypedRingBuffer<int *> rb(2);
ASSERT_TRUE(rb.ok());
int value0 = 123;
rb.write(&value0);
EXPECT_FALSE(rb.full());
}
TEST(RingBuffer, WritingAllElementsMakesBufferFull)
{
TypedRingBuffer<int *> rb(2);
ASSERT_TRUE(rb.ok());
int value0 = 123;
int value1 = 231;
rb.write(&value0);
rb.write(&value1);
EXPECT_TRUE(rb.full());
}
TEST(RingBuffer, ReadingElementFromFullBufferMakesItNotFull)
{
TypedRingBuffer<int *> rb(2);
ASSERT_TRUE(rb.ok());
int value0 = 123;
int value1 = 231;
rb.write(&value0);
rb.write(&value1);
EXPECT_TRUE(rb.full());
int *retrieved;
// Reading deletes the element.
EXPECT_TRUE(rb.read(&retrieved));
EXPECT_FALSE(rb.full());
}
TEST(RingBuffer, ZeroSizeBufferCanBeWrittenToOnce)
{
TypedRingBuffer<int *> rb(0);
ASSERT_TRUE(rb.ok());
int value0 = 123;
// Strange behaviour: we can write one element to a 0-size buffer.
EXPECT_EQ(nullptr, rb.write(&value0));
EXPECT_EQ(&value0, rb.write(&value0));
int *retrieved = nullptr;
// But then we can't read it.
EXPECT_FALSE(rb.read(&retrieved));
EXPECT_EQ(nullptr, retrieved);
}
TEST(RingBuffer, ReadingFromEmptyBufferFails)
{
TypedRingBuffer<int *> rb(2);
ASSERT_TRUE(rb.ok());
int *retrieved;
EXPECT_FALSE(rb.read(&retrieved));
}
TEST(RingBuffer, WritingToBufferWhenFullOverwritesBeginning)
{
TypedRingBuffer<int *> rb(2);
ASSERT_TRUE(rb.ok());
int value0 = 123;
int value1 = 231;
int value2 = 312;
int value3 = 432;
EXPECT_EQ(nullptr, rb.write(&value0));
EXPECT_EQ(nullptr, rb.write(&value1));
EXPECT_TRUE(rb.contains(&value0));
EXPECT_TRUE(rb.contains(&value1));
// Adding another element evicts the first element.
EXPECT_EQ(&value0, rb.write(&value2));
EXPECT_FALSE(rb.contains(&value0));
EXPECT_TRUE(rb.contains(&value2));
// Adding another evicts the second.
EXPECT_EQ(&value1, rb.write(&value3));
EXPECT_FALSE(rb.contains(&value1));
EXPECT_TRUE(rb.contains(&value3));
}
TEST(RingBuffer, SizeIsNumberOfElementsInBuffer)
{
TypedRingBuffer<int *> rb(10);
ASSERT_TRUE(rb.ok());
int value0 = 123;
EXPECT_EQ(rb.size(), 0);
rb.write(&value0);
EXPECT_EQ(rb.size(), 1);
rb.write(&value0);
EXPECT_EQ(rb.size(), 2);
rb.write(&value0);
EXPECT_EQ(rb.size(), 3);
rb.write(&value0);
EXPECT_EQ(rb.size(), 4);
int *retrieved;
rb.read(&retrieved);
EXPECT_EQ(rb.size(), 3);
rb.read(&retrieved);
EXPECT_EQ(rb.size(), 2);
rb.read(&retrieved);
EXPECT_EQ(rb.size(), 1);
rb.read(&retrieved);
EXPECT_EQ(rb.size(), 0);
}
TEST(RingBuffer, SizeIsLimitedByMaxSize)
{
TypedRingBuffer<int *> rb(4);
ASSERT_TRUE(rb.ok());
int value0 = 123;
rb.write(&value0);
rb.write(&value0);
rb.write(&value0);
rb.write(&value0);
EXPECT_EQ(rb.size(), 4);
// Add one more.
rb.write(&value0);
// Still size is 4.
EXPECT_EQ(rb.size(), 4);
}
} // namespace

871
toxav/rtp.c Normal file
View File

@ -0,0 +1,871 @@
/* SPDX-License-Identifier: GPL-3.0-or-later
* Copyright © 2016-2018 The TokTok team.
* Copyright © 2013-2015 Tox project.
*/
#include "rtp.h"
#include <assert.h>
#include <errno.h>
#include <stdlib.h>
#include <string.h>
#include "bwcontroller.h"
#include "../toxcore/Messenger.h"
#include "../toxcore/ccompat.h"
#include "../toxcore/logger.h"
#include "../toxcore/mono_time.h"
#include "../toxcore/util.h"
/**
* The number of milliseconds we want to keep a keyframe in the buffer for,
* even though there are no free slots for incoming frames.
*/
#define VIDEO_KEEP_KEYFRAME_IN_BUFFER_FOR_MS 15
/**
* return -1 on failure, 0 on success
*
*/
static int rtp_send_custom_lossy_packet(Tox *tox, int32_t friendnumber, const uint8_t *data, uint32_t length)
{
Tox_Err_Friend_Custom_Packet error;
tox_friend_send_lossy_packet(tox, friendnumber, data, (size_t)length, &error);
if (error == TOX_ERR_FRIEND_CUSTOM_PACKET_OK) {
return 0;
}
return -1;
}
// allocate_len is NOT including header!
static struct RTPMessage *new_message(const struct RTPHeader *header, size_t allocate_len, const uint8_t *data,
uint16_t data_length)
{
assert(allocate_len >= data_length);
struct RTPMessage *msg = (struct RTPMessage *)calloc(1, sizeof(struct RTPMessage) + allocate_len);
if (msg == nullptr) {
return nullptr;
}
msg->len = data_length; // result without header
msg->header = *header;
memcpy(msg->data, data, msg->len);
return msg;
}
/**
* Instruct the caller to clear slot 0.
*/
#define GET_SLOT_RESULT_DROP_OLDEST_SLOT (-1)
/**
* Instruct the caller to drop the incoming packet.
*/
#define GET_SLOT_RESULT_DROP_INCOMING (-2)
/**
* Find the next free slot in work_buffer for the incoming data packet.
*
* - If the data packet belongs to a frame that's already in the work_buffer then
* use that slot.
* - If there is no free slot return GET_SLOT_RESULT_DROP_OLDEST_SLOT.
* - If the data packet is too old return GET_SLOT_RESULT_DROP_INCOMING.
*
* If there is a keyframe being assembled in slot 0, keep it a bit longer and
* do not kick it out right away if all slots are full instead kick out the new
* incoming interframe.
*/
static int8_t get_slot(const Logger *log, struct RTPWorkBufferList *wkbl, bool is_keyframe,
const struct RTPHeader *header, bool is_multipart)
{
if (is_multipart) {
// This RTP message is part of a multipart frame, so we try to find an
// existing slot with the previous parts of the frame in it.
for (uint8_t i = 0; i < wkbl->next_free_entry; ++i) {
const struct RTPWorkBuffer *slot = &wkbl->work_buffer[i];
if ((slot->buf->header.sequnum == header->sequnum) && (slot->buf->header.timestamp == header->timestamp)) {
// Sequence number and timestamp match, so this slot belongs to
// the same frame.
//
// In reality, these will almost certainly either both match or
// both not match. Only if somehow there were 65535 frames
// between, the timestamp will matter.
return i;
}
}
}
// The message may or may not be part of a multipart frame.
//
// If it is part of a multipart frame, then this is an entirely new frame
// for which we did not have a slot *or* the frame is so old that its slot
// has been evicted by now.
//
// |----------- time ----------->
// _________________
// slot 0 | |
// -----------------
// _________________
// slot 1 | |
// -----------------
// ____________
// slot 2 | | -> frame too old, drop
// ------------
//
//
//
// |----------- time ----------->
// _________________
// slot 0 | |
// -----------------
// _________________
// slot 1 | |
// -----------------
// ____________
// slot 2 | | -> ok, start filling in a new slot
// ------------
// If there is a free slot:
if (wkbl->next_free_entry < USED_RTP_WORKBUFFER_COUNT) {
// If there is at least one filled slot:
if (wkbl->next_free_entry > 0) {
// Get the most recently filled slot.
const struct RTPWorkBuffer *slot = &wkbl->work_buffer[wkbl->next_free_entry - 1];
// If the incoming packet is older than our newest slot, drop it.
// This is the first situation in the above diagram.
if (slot->buf->header.timestamp > header->timestamp) {
LOGGER_DEBUG(log, "workbuffer:2:timestamp too old");
return GET_SLOT_RESULT_DROP_INCOMING;
}
}
// Not all slots are filled, and the packet is newer than our most
// recent slot, so it's a new frame we want to start assembling. This is
// the second situation in the above diagram.
return wkbl->next_free_entry;
}
// If the incoming frame is a key frame, then stop assembling the oldest
// slot, regardless of whether there was a keyframe in that or not.
if (is_keyframe) {
return GET_SLOT_RESULT_DROP_OLDEST_SLOT;
}
// The incoming slot is not a key frame, so we look at slot 0 to see what to
// do next.
const struct RTPWorkBuffer *slot = &wkbl->work_buffer[0];
// The incoming frame is not a key frame, but the existing slot 0 is also
// not a keyframe, so we stop assembling the existing frame and make space
// for the new one.
if (!slot->is_keyframe) {
return GET_SLOT_RESULT_DROP_OLDEST_SLOT;
}
// If this key frame is fully received, we also stop assembling and clear
// slot 0. This also means sending the frame to the decoder.
if (slot->received_len == slot->buf->header.data_length_full) {
return GET_SLOT_RESULT_DROP_OLDEST_SLOT;
}
// This is a key frame, not fully received yet, but it's already much older
// than the incoming frame, so we stop assembling it and send whatever part
// we did receive to the decoder.
if (slot->buf->header.timestamp + VIDEO_KEEP_KEYFRAME_IN_BUFFER_FOR_MS <= header->timestamp) {
return GET_SLOT_RESULT_DROP_OLDEST_SLOT;
}
// This is a key frame, it's not too old yet, so we keep it in its slot for
// a little longer.
LOGGER_INFO(log, "keep KEYFRAME in workbuffer");
return GET_SLOT_RESULT_DROP_INCOMING;
}
/**
* Returns an assembled frame (as much data as we currently have for this frame,
* some pieces may be missing)
*
* If there are no frames ready, we return NULL. If this function returns
* non-NULL, it transfers ownership of the message to the caller, i.e. the
* caller is responsible for storing it elsewhere or calling `free()`.
*/
static struct RTPMessage *process_frame(const Logger *log, struct RTPWorkBufferList *wkbl, uint8_t slot_id)
{
assert(wkbl->next_free_entry >= 0);
if (wkbl->next_free_entry == 0) {
// There are no frames in any slot.
return nullptr;
}
// Slot 0 contains a key frame, slot_id points at an interframe that is
// relative to that key frame, so we don't use it yet.
if (wkbl->work_buffer[0].is_keyframe && slot_id != 0) {
LOGGER_DEBUG(log, "process_frame:KEYFRAME waiting in slot 0");
return nullptr;
}
// Either slot_id is 0 and slot 0 is a key frame, or there is no key frame
// in slot 0 (and slot_id is anything).
struct RTPWorkBuffer *const slot = &wkbl->work_buffer[slot_id];
// Move ownership of the frame out of the slot into m_new.
struct RTPMessage *const m_new = slot->buf;
slot->buf = nullptr;
assert(wkbl->next_free_entry >= 1 && wkbl->next_free_entry <= USED_RTP_WORKBUFFER_COUNT);
if (slot_id != wkbl->next_free_entry - 1) {
// The slot is not the last slot, so we created a gap. We move all the
// entries after it one step up.
for (uint8_t i = slot_id; i < wkbl->next_free_entry - 1; ++i) {
// Move entry (i+1) into entry (i).
wkbl->work_buffer[i] = wkbl->work_buffer[i + 1];
}
}
// We now have a free entry at the end of the array.
--wkbl->next_free_entry;
// Clear the newly freed entry.
const struct RTPWorkBuffer empty = {0};
wkbl->work_buffer[wkbl->next_free_entry] = empty;
// Move ownership of the frame to the caller.
return m_new;
}
/**
* @param log A logger.
* @param wkbl The list of in-progress frames, i.e. all the slots.
* @param slot_id The slot we want to fill the data into.
* @param is_keyframe Whether the data is part of a key frame.
* @param header The RTP header from the incoming packet.
* @param incoming_data The pure payload without header.
* @param incoming_data_length The length in bytes of the incoming data payload.
*/
static bool fill_data_into_slot(const Logger *log, struct RTPWorkBufferList *wkbl, const uint8_t slot_id,
bool is_keyframe, const struct RTPHeader *header,
const uint8_t *incoming_data, uint16_t incoming_data_length)
{
// We're either filling the data into an existing slot, or in a new one that
// is the next free entry.
assert(slot_id <= wkbl->next_free_entry);
struct RTPWorkBuffer *const slot = &wkbl->work_buffer[slot_id];
assert(header != nullptr);
assert(is_keyframe == (bool)((header->flags & RTP_KEY_FRAME) != 0));
if (slot->received_len == 0) {
assert(slot->buf == nullptr);
// No data for this slot has been received, yet, so we create a new
// message for it with enough memory for the entire frame.
struct RTPMessage *msg = (struct RTPMessage *)calloc(1, sizeof(struct RTPMessage) + header->data_length_full);
if (msg == nullptr) {
LOGGER_ERROR(log, "Out of memory while trying to allocate for frame of size %u",
(unsigned)header->data_length_full);
// Out of memory: throw away the incoming data.
return false;
}
// Unused in the new video receiving code, as it's 16 bit and can't hold
// the full length of large frames. Instead, we use slot->received_len.
msg->len = 0;
msg->header = *header;
slot->buf = msg;
slot->is_keyframe = is_keyframe;
slot->received_len = 0;
assert(wkbl->next_free_entry < USED_RTP_WORKBUFFER_COUNT);
++wkbl->next_free_entry;
}
// We already checked this when we received the packet, but we rely on it
// here, so assert again.
assert(header->offset_full < header->data_length_full);
// Copy the incoming chunk of data into the correct position in the full
// frame data array.
memcpy(
slot->buf->data + header->offset_full,
incoming_data,
incoming_data_length
);
// Update the total received length of this slot.
slot->received_len += incoming_data_length;
// Update received length also in the header of the message, for later use.
slot->buf->header.received_length_full = slot->received_len;
return slot->received_len == header->data_length_full;
}
static void update_bwc_values(const Logger *log, RTPSession *session, const struct RTPMessage *msg)
{
if (session->first_packets_counter < DISMISS_FIRST_LOST_VIDEO_PACKET_COUNT) {
++session->first_packets_counter;
} else {
const uint32_t data_length_full = msg->header.data_length_full; // without header
const uint32_t received_length_full = msg->header.received_length_full; // without header
bwc_add_recv(session->bwc, data_length_full);
if (received_length_full < data_length_full) {
LOGGER_DEBUG(log, "BWC: full length=%u received length=%d", data_length_full, received_length_full);
bwc_add_lost(session->bwc, data_length_full - received_length_full);
}
}
}
/**
* Handle a single RTP video packet.
*
* The packet may or may not be part of a multipart frame. This function will
* find out and handle it appropriately.
*
* @param session The current RTP session with:
* <code>
* session->mcb == vc_queue_message() // this function is called from here
* session->mp == struct RTPMessage *
* session->cs == call->video.second // == VCSession created by vc_new() call
* </code>
* @param header The RTP header deserialised from the packet.
* @param incoming_data The packet data *not* header, i.e. this is the actual
* payload.
* @param incoming_data_length The packet length *not* including header, i.e.
* this is the actual payload length.
* @param log A logger.
*
* @retval -1 on error.
* @retval 0 on success.
*/
static int handle_video_packet(RTPSession *session, const struct RTPHeader *header,
const uint8_t *incoming_data, uint16_t incoming_data_length, const Logger *log)
{
// Full frame length in bytes. The frame may be split into multiple packets,
// but this value is the complete assembled frame size.
const uint32_t full_frame_length = header->data_length_full;
// Current offset in the frame. If this is the first packet of a multipart
// frame or it's not a multipart frame, then this value is 0.
const uint32_t offset = header->offset_full; // without header
// The sender tells us whether this is a key frame.
const bool is_keyframe = (header->flags & RTP_KEY_FRAME) != 0;
LOGGER_DEBUG(log, "-- handle_video_packet -- full lens=%u len=%u offset=%u is_keyframe=%s",
(unsigned)incoming_data_length, (unsigned)full_frame_length, (unsigned)offset, is_keyframe ? "K" : ".");
LOGGER_DEBUG(log, "wkbl->next_free_entry:003=%d", session->work_buffer_list->next_free_entry);
const bool is_multipart = full_frame_length != incoming_data_length;
/* The message was sent in single part */
int8_t slot_id = get_slot(log, session->work_buffer_list, is_keyframe, header, is_multipart);
LOGGER_DEBUG(log, "slot num=%d", slot_id);
// get_slot told us to drop the packet, so we ignore it.
if (slot_id == GET_SLOT_RESULT_DROP_INCOMING) {
return -1;
}
// get_slot said there is no free slot.
if (slot_id == GET_SLOT_RESULT_DROP_OLDEST_SLOT) {
LOGGER_DEBUG(log, "there was no free slot, so we process the oldest frame");
// We now own the frame.
struct RTPMessage *m_new = process_frame(log, session->work_buffer_list, 0);
// The process_frame function returns NULL if there is no slot 0, i.e.
// the work buffer list is completely empty. It can't be empty, because
// get_slot just told us it's full, so process_frame must return non-null.
assert(m_new != nullptr);
LOGGER_DEBUG(log, "-- handle_video_packet -- CALLBACK-001a b0=%d b1=%d", (int)m_new->data[0], (int)m_new->data[1]);
update_bwc_values(log, session, m_new);
// Pass ownership of m_new to the callback.
session->mcb(session->m->mono_time, session->cs, m_new);
// Now we no longer own m_new.
m_new = nullptr;
// Now we must have a free slot, so we either get that slot, i.e. >= 0,
// or get told to drop the incoming packet if it's too old.
slot_id = get_slot(log, session->work_buffer_list, is_keyframe, header, /* is_multipart */false);
if (slot_id == GET_SLOT_RESULT_DROP_INCOMING) {
// The incoming frame is too old, so we drop it.
return -1;
}
}
// We must have a valid slot here.
assert(slot_id >= 0);
LOGGER_DEBUG(log, "fill_data_into_slot.1");
// fill in this part into the slot buffer at the correct offset
if (!fill_data_into_slot(
log,
session->work_buffer_list,
slot_id,
is_keyframe,
header,
incoming_data,
incoming_data_length)) {
// Memory allocation failed. Return error.
return -1;
}
struct RTPMessage *m_new = process_frame(log, session->work_buffer_list, slot_id);
if (m_new != nullptr) {
LOGGER_DEBUG(log, "-- handle_video_packet -- CALLBACK-003a b0=%d b1=%d", (int)m_new->data[0], (int)m_new->data[1]);
update_bwc_values(log, session, m_new);
session->mcb(session->m->mono_time, session->cs, m_new);
m_new = nullptr;
}
return 0;
}
/**
* @retval -1 on error.
* @retval 0 on success.
*/
static int handle_rtp_packet(Messenger *m, uint32_t friendnumber, const uint8_t *data, uint16_t length, void *object)
{
RTPSession *session = (RTPSession *)object;
if (session == nullptr || length < RTP_HEADER_SIZE + 1) {
LOGGER_WARNING(m->log, "No session or invalid length of received buffer!");
return -1;
}
// Get the packet type.
const uint8_t packet_type = data[0];
++data;
--length;
// Unpack the header.
struct RTPHeader header;
rtp_header_unpack(data, &header);
if (header.pt != packet_type % 128) {
LOGGER_WARNING(m->log, "RTPHeader packet type and Tox protocol packet type did not agree: %d != %d",
header.pt, packet_type % 128);
return -1;
}
if (header.pt != session->payload_type % 128) {
LOGGER_WARNING(m->log, "RTPHeader packet type does not match this session's payload type: %d != %d",
header.pt, session->payload_type % 128);
return -1;
}
if ((header.flags & RTP_LARGE_FRAME) != 0 && header.offset_full >= header.data_length_full) {
LOGGER_ERROR(m->log, "Invalid video packet: frame offset (%u) >= full frame length (%u)",
(unsigned)header.offset_full, (unsigned)header.data_length_full);
return -1;
}
if (header.offset_lower >= header.data_length_lower) {
LOGGER_ERROR(m->log, "Invalid old protocol video packet: frame offset (%u) >= full frame length (%u)",
(unsigned)header.offset_lower, (unsigned)header.data_length_lower);
return -1;
}
LOGGER_DEBUG(m->log, "header.pt %d, video %d", (uint8_t)header.pt, RTP_TYPE_VIDEO % 128);
// The sender uses the new large-frame capable protocol and is sending a
// video packet.
if ((header.flags & RTP_LARGE_FRAME) != 0 && header.pt == (RTP_TYPE_VIDEO % 128)) {
return handle_video_packet(session, &header, data + RTP_HEADER_SIZE, length - RTP_HEADER_SIZE, m->log);
}
// everything below here is for the old 16 bit protocol ------------------
if (header.data_length_lower == length - RTP_HEADER_SIZE) {
/* The message is sent in single part */
/* Message is not late; pick up the latest parameters */
session->rsequnum = header.sequnum;
session->rtimestamp = header.timestamp;
bwc_add_recv(session->bwc, length);
/* Invoke processing of active multiparted message */
if (session->mp != nullptr) {
session->mcb(session->m->mono_time, session->cs, session->mp);
session->mp = nullptr;
}
/* The message came in the allowed time;
*/
return session->mcb(session->m->mono_time, session->cs, new_message(&header, length - RTP_HEADER_SIZE,
data + RTP_HEADER_SIZE, length - RTP_HEADER_SIZE));
}
/* The message is sent in multiple parts */
if (session->mp != nullptr) {
/* There are 2 possible situations in this case:
* 1) being that we got the part of already processing message.
* 2) being that we got the part of a new/old message.
*
* We handle them differently as we only allow a single multiparted
* processing message
*/
if (session->mp->header.sequnum == header.sequnum &&
session->mp->header.timestamp == header.timestamp) {
/* First case */
/* Make sure we have enough allocated memory */
if (session->mp->header.data_length_lower - session->mp->len < length - RTP_HEADER_SIZE ||
session->mp->header.data_length_lower <= header.offset_lower) {
/* There happened to be some corruption on the stream;
* continue wihtout this part
*/
return 0;
}
memcpy(session->mp->data + header.offset_lower, data + RTP_HEADER_SIZE,
length - RTP_HEADER_SIZE);
session->mp->len += length - RTP_HEADER_SIZE;
bwc_add_recv(session->bwc, length);
if (session->mp->len == session->mp->header.data_length_lower) {
/* Received a full message; now push it for the further
* processing.
*/
session->mcb(session->m->mono_time, session->cs, session->mp);
session->mp = nullptr;
}
} else {
/* Second case */
if (session->mp->header.timestamp > header.timestamp) {
/* The received message part is from the old message;
* discard it.
*/
return 0;
}
/* Push the previous message for processing */
session->mcb(session->m->mono_time, session->cs, session->mp);
session->mp = nullptr;
goto NEW_MULTIPARTED;
}
} else {
/* In this case threat the message as if it was received in order
*/
/* This is also a point for new multiparted messages */
NEW_MULTIPARTED:
/* Message is not late; pick up the latest parameters */
session->rsequnum = header.sequnum;
session->rtimestamp = header.timestamp;
bwc_add_recv(session->bwc, length);
/* Store message.
*/
session->mp = new_message(&header, header.data_length_lower, data + RTP_HEADER_SIZE, length - RTP_HEADER_SIZE);
if (session->mp != nullptr) {
memmove(session->mp->data + header.offset_lower, session->mp->data, session->mp->len);
} else {
LOGGER_WARNING(m->log, "new_message() returned a null pointer");
return -1;
}
}
return 0;
}
size_t rtp_header_pack(uint8_t *const rdata, const struct RTPHeader *header)
{
uint8_t *p = rdata;
*p = (header->ve & 3) << 6
| (header->pe & 1) << 5
| (header->xe & 1) << 4
| (header->cc & 0xf);
++p;
*p = (header->ma & 1) << 7
| (header->pt & 0x7f);
++p;
p += net_pack_u16(p, header->sequnum);
p += net_pack_u32(p, header->timestamp);
p += net_pack_u32(p, header->ssrc);
p += net_pack_u64(p, header->flags);
p += net_pack_u32(p, header->offset_full);
p += net_pack_u32(p, header->data_length_full);
p += net_pack_u32(p, header->received_length_full);
for (size_t i = 0; i < RTP_PADDING_FIELDS; ++i) {
p += net_pack_u32(p, 0);
}
p += net_pack_u16(p, header->offset_lower);
p += net_pack_u16(p, header->data_length_lower);
assert(p == rdata + RTP_HEADER_SIZE);
return p - rdata;
}
size_t rtp_header_unpack(const uint8_t *data, struct RTPHeader *header)
{
const uint8_t *p = data;
header->ve = (*p >> 6) & 3;
header->pe = (*p >> 5) & 1;
header->xe = (*p >> 4) & 1;
header->cc = *p & 0xf;
++p;
header->ma = (*p >> 7) & 1;
header->pt = *p & 0x7f;
++p;
p += net_unpack_u16(p, &header->sequnum);
p += net_unpack_u32(p, &header->timestamp);
p += net_unpack_u32(p, &header->ssrc);
p += net_unpack_u64(p, &header->flags);
p += net_unpack_u32(p, &header->offset_full);
p += net_unpack_u32(p, &header->data_length_full);
p += net_unpack_u32(p, &header->received_length_full);
p += sizeof(uint32_t) * RTP_PADDING_FIELDS;
p += net_unpack_u16(p, &header->offset_lower);
p += net_unpack_u16(p, &header->data_length_lower);
assert(p == data + RTP_HEADER_SIZE);
return p - data;
}
RTPSession *rtp_new(int payload_type, Messenger *m, Tox *tox, uint32_t friendnumber,
BWController *bwc, void *cs, rtp_m_cb *mcb)
{
assert(mcb != nullptr);
assert(cs != nullptr);
assert(m != nullptr);
RTPSession *session = (RTPSession *)calloc(1, sizeof(RTPSession));
if (session == nullptr) {
LOGGER_WARNING(m->log, "Alloc failed! Program might misbehave!");
return nullptr;
}
session->work_buffer_list = (struct RTPWorkBufferList *)calloc(1, sizeof(struct RTPWorkBufferList));
if (session->work_buffer_list == nullptr) {
LOGGER_ERROR(m->log, "out of memory while allocating work buffer list");
free(session);
return nullptr;
}
// First entry is free.
session->work_buffer_list->next_free_entry = 0;
session->ssrc = payload_type == RTP_TYPE_VIDEO ? 0 : random_u32(m->rng);
session->payload_type = payload_type;
session->m = m;
session->tox = tox;
session->friend_number = friendnumber;
// set NULL just in case
session->mp = nullptr;
session->first_packets_counter = 1;
/* Also set payload type as prefix */
session->bwc = bwc;
session->cs = cs;
session->mcb = mcb;
if (-1 == rtp_allow_receiving(session)) {
LOGGER_WARNING(m->log, "Failed to start rtp receiving mode");
free(session->work_buffer_list);
free(session);
return nullptr;
}
return session;
}
void rtp_kill(RTPSession *session)
{
if (session == nullptr) {
return;
}
LOGGER_DEBUG(session->m->log, "Terminated RTP session: %p", (void *)session);
rtp_stop_receiving(session);
LOGGER_DEBUG(session->m->log, "Terminated RTP session V3 work_buffer_list->next_free_entry: %d",
(int)session->work_buffer_list->next_free_entry);
for (int8_t i = 0; i < session->work_buffer_list->next_free_entry; ++i) {
free(session->work_buffer_list->work_buffer[i].buf);
}
free(session->work_buffer_list);
free(session);
}
int rtp_allow_receiving(RTPSession *session)
{
if (session == nullptr) {
return -1;
}
if (m_callback_rtp_packet(session->m, session->friend_number, session->payload_type,
handle_rtp_packet, session) == -1) {
LOGGER_WARNING(session->m->log, "Failed to register rtp receive handler");
return -1;
}
LOGGER_DEBUG(session->m->log, "Started receiving on session: %p", (void *)session);
return 0;
}
int rtp_stop_receiving(RTPSession *session)
{
if (session == nullptr) {
return -1;
}
m_callback_rtp_packet(session->m, session->friend_number, session->payload_type, nullptr, nullptr);
LOGGER_DEBUG(session->m->log, "Stopped receiving on session: %p", (void *)session);
return 0;
}
/**
* Send a frame of audio or video data, chunked in @ref RTPMessage instances.
*
* @param session The A/V session to send the data for.
* @param data A byte array of length @p length.
* @param length The number of bytes to send from @p data.
* @param is_keyframe Whether this video frame is a key frame. If it is an
* audio frame, this parameter is ignored.
*/
int rtp_send_data(RTPSession *session, const uint8_t *data, uint32_t length,
bool is_keyframe, const Logger *log)
{
if (session == nullptr) {
LOGGER_ERROR(log, "No session!");
return -1;
}
struct RTPHeader header = {0};
header.ve = 2; // this is unused in toxav
header.pe = 0;
header.xe = 0;
header.cc = 0;
header.ma = 0;
header.pt = session->payload_type % 128;
header.sequnum = session->sequnum;
header.timestamp = current_time_monotonic(session->m->mono_time);
header.ssrc = session->ssrc;
header.offset_lower = 0;
// here the highest bits gets stripped anyway, no need to do keyframe bit magic here!
header.data_length_lower = length;
if (session->payload_type == RTP_TYPE_VIDEO) {
header.flags = RTP_LARGE_FRAME;
}
uint16_t length_safe = (uint16_t)length;
if (length > UINT16_MAX) {
length_safe = UINT16_MAX;
}
header.data_length_lower = length_safe;
header.data_length_full = length; // without header
header.offset_lower = 0;
header.offset_full = 0;
if (is_keyframe) {
header.flags |= RTP_KEY_FRAME;
}
VLA(uint8_t, rdata, length + RTP_HEADER_SIZE + 1);
memset(rdata, 0, SIZEOF_VLA(rdata));
rdata[0] = session->payload_type; // packet id == payload_type
if (MAX_CRYPTO_DATA_SIZE > (length + RTP_HEADER_SIZE + 1)) {
/*
* The length is lesser than the maximum allowed length (including header)
* Send the packet in single piece.
*/
rtp_header_pack(rdata + 1, &header);
memcpy(rdata + 1 + RTP_HEADER_SIZE, data, length);
if (-1 == rtp_send_custom_lossy_packet(session->tox, session->friend_number, rdata, SIZEOF_VLA(rdata))) {
char *netstrerror = net_new_strerror(net_error());
LOGGER_WARNING(session->m->log, "RTP send failed (len: %u)! net error: %s",
(unsigned)SIZEOF_VLA(rdata), netstrerror);
net_kill_strerror(netstrerror);
}
} else {
/*
* The length is greater than the maximum allowed length (including header)
* Send the packet in multiple pieces.
*/
uint32_t sent = 0;
uint16_t piece = MAX_CRYPTO_DATA_SIZE - (RTP_HEADER_SIZE + 1);
while ((length - sent) + RTP_HEADER_SIZE + 1 > MAX_CRYPTO_DATA_SIZE) {
rtp_header_pack(rdata + 1, &header);
memcpy(rdata + 1 + RTP_HEADER_SIZE, data + sent, piece);
if (-1 == rtp_send_custom_lossy_packet(session->tox, session->friend_number,
rdata, piece + RTP_HEADER_SIZE + 1)) {
char *netstrerror = net_new_strerror(net_error());
LOGGER_WARNING(session->m->log, "RTP send failed (len: %d)! net error: %s",
piece + RTP_HEADER_SIZE + 1, netstrerror);
net_kill_strerror(netstrerror);
}
sent += piece;
header.offset_lower = sent;
header.offset_full = sent; // raw data offset, without any header
}
/* Send remaining */
piece = length - sent;
if (piece != 0) {
rtp_header_pack(rdata + 1, &header);
memcpy(rdata + 1 + RTP_HEADER_SIZE, data + sent, piece);
if (-1 == rtp_send_custom_lossy_packet(session->tox, session->friend_number, rdata,
piece + RTP_HEADER_SIZE + 1)) {
char *netstrerror = net_new_strerror(net_error());
LOGGER_WARNING(session->m->log, "RTP send failed (len: %d)! net error: %s",
piece + RTP_HEADER_SIZE + 1, netstrerror);
net_kill_strerror(netstrerror);
}
}
}
++session->sequnum;
return 0;
}

210
toxav/rtp.h Normal file
View File

@ -0,0 +1,210 @@
/* SPDX-License-Identifier: GPL-3.0-or-later
* Copyright © 2016-2018 The TokTok team.
* Copyright © 2013-2015 Tox project.
*/
#ifndef C_TOXCORE_TOXAV_RTP_H
#define C_TOXCORE_TOXAV_RTP_H
#include <stdbool.h>
#include "bwcontroller.h"
#include "../toxcore/Messenger.h"
#include "../toxcore/logger.h"
#include "../toxcore/tox.h"
#ifdef __cplusplus
extern "C" {
#endif
/**
* RTPHeader serialised size in bytes.
*/
#define RTP_HEADER_SIZE 80
/**
* Number of 32 bit padding fields between @ref RTPHeader::offset_lower and
* everything before it.
*/
#define RTP_PADDING_FIELDS 11
/**
* Payload type identifier. Also used as rtp callback prefix.
*/
typedef enum RTP_Type {
RTP_TYPE_AUDIO = 192,
RTP_TYPE_VIDEO = 193,
} RTP_Type;
/**
* A bit mask (up to 64 bits) specifying features of the current frame affecting
* the behaviour of the decoder.
*/
typedef enum RTPFlags {
/**
* Support frames larger than 64KiB. The full 32 bit length and offset are
* set in @ref RTPHeader::data_length_full and @ref RTPHeader::offset_full.
*/
RTP_LARGE_FRAME = 1 << 0,
/**
* Whether the packet is part of a key frame.
*/
RTP_KEY_FRAME = 1 << 1,
} RTPFlags;
struct RTPHeader {
/* Standard RTP header */
unsigned ve: 2; /* Version has only 2 bits! */
unsigned pe: 1; /* Padding */
unsigned xe: 1; /* Extra header */
unsigned cc: 4; /* Contributing sources count */
unsigned ma: 1; /* Marker */
unsigned pt: 7; /* Payload type */
uint16_t sequnum;
uint32_t timestamp;
uint32_t ssrc;
/* Non-standard Tox-specific fields */
/**
* Bit mask of `RTPFlags` setting features of the current frame.
*/
uint64_t flags;
/**
* The full 32 bit data offset of the current data chunk. The @ref
* offset_lower data member contains the lower 16 bits of this value. For
* frames smaller than 64KiB, @ref offset_full and @ref offset_lower are
* equal.
*/
uint32_t offset_full;
/**
* The full 32 bit payload length without header and packet id.
*/
uint32_t data_length_full;
/**
* Only the receiver uses this field (why do we have this?).
*/
uint32_t received_length_full;
/**
* Data offset of the current part (lower bits).
*/
uint16_t offset_lower;
/**
* Total message length (lower bits).
*/
uint16_t data_length_lower;
};
struct RTPMessage {
/**
* This is used in the old code that doesn't deal with large frames, i.e.
* the audio code or receiving code for old 16 bit messages. We use it to
* record the number of bytes received so far in a multi-part message. The
* multi-part message in the old code is stored in `RTPSession::mp`.
*/
uint16_t len;
struct RTPHeader header;
uint8_t data[];
};
#define USED_RTP_WORKBUFFER_COUNT 3
/**
* One slot in the work buffer list. Represents one frame that is currently
* being assembled.
*/
struct RTPWorkBuffer {
/**
* Whether this slot contains a key frame. This is true iff
* `buf->header.flags & RTP_KEY_FRAME`.
*/
bool is_keyframe;
/**
* The number of bytes received so far, regardless of which pieces. I.e. we
* could have received the first 1000 bytes and the last 1000 bytes with
* 4000 bytes in the middle still to come, and this number would be 2000.
*/
uint32_t received_len;
/**
* The message currently being assembled.
*/
struct RTPMessage *buf;
};
struct RTPWorkBufferList {
int8_t next_free_entry;
struct RTPWorkBuffer work_buffer[USED_RTP_WORKBUFFER_COUNT];
};
#define DISMISS_FIRST_LOST_VIDEO_PACKET_COUNT 10
typedef int rtp_m_cb(Mono_Time *mono_time, void *cs, struct RTPMessage *msg);
/**
* RTP control session.
*/
typedef struct RTPSession {
uint8_t payload_type;
uint16_t sequnum; /* Sending sequence number */
uint16_t rsequnum; /* Receiving sequence number */
uint32_t rtimestamp;
uint32_t ssrc; // this seems to be unused!?
struct RTPMessage *mp; /* Expected parted message */
struct RTPWorkBufferList *work_buffer_list;
uint8_t first_packets_counter; /* dismiss first few lost video packets */
Messenger *m;
Tox *tox;
uint32_t friend_number;
BWController *bwc;
void *cs;
rtp_m_cb *mcb;
} RTPSession;
/**
* Serialise an RTPHeader to bytes to be sent over the network.
*
* @param rdata A byte array of length RTP_HEADER_SIZE. Does not need to be
* initialised. All RTP_HEADER_SIZE bytes will be initialised after a call
* to this function.
* @param header The RTPHeader to serialise.
*/
size_t rtp_header_pack(uint8_t *rdata, const struct RTPHeader *header);
/**
* Deserialise an RTPHeader from bytes received over the network.
*
* @param data A byte array of length RTP_HEADER_SIZE.
* @param header The RTPHeader to write the unpacked values to.
*/
size_t rtp_header_unpack(const uint8_t *data, struct RTPHeader *header);
RTPSession *rtp_new(int payload_type, Messenger *m, Tox *tox, uint32_t friendnumber,
BWController *bwc, void *cs, rtp_m_cb *mcb);
void rtp_kill(RTPSession *session);
int rtp_allow_receiving(RTPSession *session);
int rtp_stop_receiving(RTPSession *session);
/**
* Send a frame of audio or video data, chunked in @ref RTPMessage instances.
*
* @param session The A/V session to send the data for.
* @param data A byte array of length @p length.
* @param length The number of bytes to send from @p data.
* @param is_keyframe Whether this video frame is a key frame. If it is an
* audio frame, this parameter is ignored.
*/
int rtp_send_data(RTPSession *session, const uint8_t *data, uint32_t length,
bool is_keyframe, const Logger *log);
#ifdef __cplusplus
} // extern "C"
#endif
#endif // C_TOXCORE_TOXAV_RTP_H

81
toxav/rtp_test.cc Normal file
View File

@ -0,0 +1,81 @@
#include "rtp.h"
#include <gtest/gtest.h>
#include "../toxcore/crypto_core.h"
namespace {
RTPHeader random_header(const Random *rng)
{
return {
random_u16(rng),
random_u16(rng),
random_u16(rng),
random_u16(rng),
random_u16(rng),
random_u16(rng),
random_u16(rng),
random_u32(rng),
random_u32(rng),
random_u64(rng),
random_u32(rng),
random_u32(rng),
random_u32(rng),
random_u16(rng),
random_u16(rng),
};
}
TEST(Rtp, Deserialisation)
{
const Random *rng = system_random();
ASSERT_NE(rng, nullptr);
RTPHeader const header = random_header(rng);
uint8_t rdata[RTP_HEADER_SIZE];
EXPECT_EQ(rtp_header_pack(rdata, &header), RTP_HEADER_SIZE);
RTPHeader unpacked = {0};
EXPECT_EQ(rtp_header_unpack(rdata, &unpacked), RTP_HEADER_SIZE);
EXPECT_EQ(header.ve, unpacked.ve);
EXPECT_EQ(header.pe, unpacked.pe);
EXPECT_EQ(header.xe, unpacked.xe);
EXPECT_EQ(header.cc, unpacked.cc);
EXPECT_EQ(header.ma, unpacked.ma);
EXPECT_EQ(header.pt, unpacked.pt);
EXPECT_EQ(header.sequnum, unpacked.sequnum);
EXPECT_EQ(header.timestamp, unpacked.timestamp);
EXPECT_EQ(header.ssrc, unpacked.ssrc);
EXPECT_EQ(header.flags, unpacked.flags);
EXPECT_EQ(header.offset_full, unpacked.offset_full);
EXPECT_EQ(header.data_length_full, unpacked.data_length_full);
EXPECT_EQ(header.received_length_full, unpacked.received_length_full);
EXPECT_EQ(header.offset_lower, unpacked.offset_lower);
EXPECT_EQ(header.data_length_lower, unpacked.data_length_lower);
}
TEST(Rtp, SerialisingAllOnes)
{
RTPHeader header;
memset(&header, 0xff, sizeof header);
uint8_t rdata[RTP_HEADER_SIZE];
rtp_header_pack(rdata, &header);
EXPECT_EQ(std::string(reinterpret_cast<char const *>(rdata), sizeof rdata),
std::string("\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF"
"\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF"
"\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF"
"\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF"
"\x00\x00\x00\x00\x00\x00\x00\x00"
"\x00\x00\x00\x00\x00\x00\x00\x00"
"\x00\x00\x00\x00\x00\x00\x00\x00"
"\x00\x00\x00\x00\x00\x00\x00\x00"
"\x00\x00\x00\x00\x00\x00\x00\x00"
"\x00\x00\x00\x00\xFF\xFF\xFF\xFF",
RTP_HEADER_SIZE));
}
} // namespace

1503
toxav/toxav.c Normal file
View File

File diff suppressed because it is too large Load Diff

849
toxav/toxav.h Normal file
View File

@ -0,0 +1,849 @@
/* SPDX-License-Identifier: GPL-3.0-or-later
* Copyright © 2016-2018 The TokTok team.
* Copyright © 2013-2015 Tox project.
*/
/** @file
* @brief Public audio/video API for Tox clients.
*
* This API can handle multiple calls. Each call has its state, in very rare
* occasions the library can change the state of the call without apps knowledge.
*
* @section av_events Events and callbacks
*
* As in Core API, events are handled by callbacks. One callback can be
* registered per event. All events have a callback function type named
* `toxav_{event}_cb` and a function to register it named `toxav_callback_{event}`.
* Passing a NULL callback will result in no callback being registered for that
* event. Only one callback per event can be registered, so if a client needs
* multiple event listeners, it needs to implement the dispatch functionality
* itself. Unlike Core API, lack of some event handlers will cause the the
* library to drop calls before they are started. Hanging up call from a
* callback causes undefined behaviour.
*
* @section av_threading Threading implications
*
* Only toxav_iterate is thread-safe, all other functions must run from the
* tox thread.
*
* Important exceptions are the `*_iterate` and `*_iterate_interval`
* functions. You have to choose either the single thread or the multi thread
* functions and read their documentation.
*
* A common way to run ToxAV (multiple or single instance) is to have a thread,
* separate from tox instance thread, running a simple toxav_iterate loop,
* sleeping for `toxav_iteration_interval * milliseconds` on each iteration.
*
* An important thing to note is that events are triggered from both tox and
* toxav thread (see above). Audio and video receive frame events are triggered
* from toxav thread while all the other events are triggered from tox thread.
*
* Tox thread has priority with mutex mechanisms. Any api function can
* fail if mutexes are held by tox thread in which case they will set SYNC
* error code.
*
* @subsection av_multi_threading Separate audio and video threads
*
* ToxAV supports either a single thread for audio and video or decoding and
* encoding them in separate threads. You have to choose one mode and can not
* mix function calls to those different modes.
*
* For best results use the multi-threaded mode and run the audio thread with
* higher priority than the video thread. This prioritizes audio over video.
*/
#ifndef C_TOXCORE_TOXAV_TOXAV_H
#define C_TOXCORE_TOXAV_TOXAV_H
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
#endif
/**
* External Tox type.
*/
#ifndef TOX_DEFINED
#define TOX_DEFINED
typedef struct Tox Tox;
#endif /* TOX_DEFINED */
/**
* @brief The ToxAV instance type.
*
* Each ToxAV instance can be bound to only one Tox instance, and Tox instance
* can have only one ToxAV instance. One must make sure to close ToxAV instance
* prior closing Tox instance otherwise undefined behaviour occurs. Upon
* closing of ToxAV instance, all active calls will be forcibly terminated
* without notifying peers.
*/
typedef struct ToxAV ToxAV;
/** @{
* @brief Creation and destruction
*/
typedef enum Toxav_Err_New {
/**
* The function returned successfully.
*/
TOXAV_ERR_NEW_OK,
/**
* One of the arguments to the function was NULL when it was not expected.
*/
TOXAV_ERR_NEW_NULL,
/**
* Memory allocation failure while trying to allocate structures required for
* the A/V session.
*/
TOXAV_ERR_NEW_MALLOC,
/**
* Attempted to create a second session for the same Tox instance.
*/
TOXAV_ERR_NEW_MULTIPLE,
} Toxav_Err_New;
/**
* Start new A/V session. There can only be only one session per Tox instance.
*/
ToxAV *toxav_new(Tox *tox, Toxav_Err_New *error);
/**
* Releases all resources associated with the A/V session.
*
* If any calls were ongoing, these will be forcibly terminated without
* notifying peers. After calling this function, no other functions may be
* called and the av pointer becomes invalid.
*/
void toxav_kill(ToxAV *av);
/**
* Returns the Tox instance the A/V object was created for.
*/
Tox *toxav_get_tox(const ToxAV *av);
/** @} */
/** @{
* @brief A/V event loop, single thread
*/
/**
* Returns the interval in milliseconds when the next toxav_iterate call should
* be. If no call is active at the moment, this function returns 200.
* This function MUST be called from the same thread as toxav_iterate.
*/
uint32_t toxav_iteration_interval(const ToxAV *av);
/**
* Main loop for the session. This function needs to be called in intervals of
* `toxav_iteration_interval()` milliseconds. It is best called in the separate
* thread from tox_iterate.
*/
void toxav_iterate(ToxAV *av);
/** @} */
/** @{
* @brief A/V event loop, multiple threads
*/
/**
* Returns the interval in milliseconds when the next toxav_audio_iterate call
* should be. If no call is active at the moment, this function returns 200.
* This function MUST be called from the same thread as toxav_audio_iterate.
*/
uint32_t toxav_audio_iteration_interval(const ToxAV *av);
/**
* Main loop for the session. This function needs to be called in intervals of
* `toxav_audio_iteration_interval()` milliseconds. It is best called in a
* separate thread from tox_iterate and toxav_video_iterate. The thread calling
* this function should have higher priority than the one calling
* toxav_video_iterate to prioritize audio over video.
*/
void toxav_audio_iterate(ToxAV *av);
/**
* Returns the interval in milliseconds when the next toxav_video_iterate call
* should be. If no call is active at the moment, this function returns 200.
* This function MUST be called from the same thread as toxav_video_iterate.
*/
uint32_t toxav_video_iteration_interval(const ToxAV *av);
/**
* Main loop for the session. This function needs to be called in intervals of
* `toxav_video_iteration_interval()` milliseconds. It is best called in a
* separate thread from tox_iterate and toxav_audio_iterate. The thread calling
* this function should have lower priority than the one calling
* toxav_audio_iterate to prioritize audio over video.
*/
void toxav_video_iterate(ToxAV *av);
/** @} */
/** @{
* @brief Call setup
*/
typedef enum Toxav_Err_Call {
/**
* The function returned successfully.
*/
TOXAV_ERR_CALL_OK,
/**
* A resource allocation error occurred while trying to create the structures
* required for the call.
*/
TOXAV_ERR_CALL_MALLOC,
/**
* Synchronization error occurred.
*/
TOXAV_ERR_CALL_SYNC,
/**
* The friend number did not designate a valid friend.
*/
TOXAV_ERR_CALL_FRIEND_NOT_FOUND,
/**
* The friend was valid, but not currently connected.
*/
TOXAV_ERR_CALL_FRIEND_NOT_CONNECTED,
/**
* Attempted to call a friend while already in an audio or video call with
* them.
*/
TOXAV_ERR_CALL_FRIEND_ALREADY_IN_CALL,
/**
* Audio or video bit rate is invalid.
*/
TOXAV_ERR_CALL_INVALID_BIT_RATE,
} Toxav_Err_Call;
/**
* Call a friend. This will start ringing the friend.
*
* It is the client's responsibility to stop ringing after a certain timeout,
* if such behaviour is desired. If the client does not stop ringing, the
* library will not stop until the friend is disconnected. Audio and video
* receiving are both enabled by default.
*
* @param friend_number The friend number of the friend that should be called.
* @param audio_bit_rate Audio bit rate in Kb/sec. Set this to 0 to disable
* audio sending.
* @param video_bit_rate Video bit rate in Kb/sec. Set this to 0 to disable
* video sending.
*/
bool toxav_call(ToxAV *av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate,
Toxav_Err_Call *error);
/**
* The function type for the call callback.
*
* @param friend_number The friend number from which the call is incoming.
* @param audio_enabled True if friend is sending audio.
* @param video_enabled True if friend is sending video.
*/
typedef void toxav_call_cb(ToxAV *av, uint32_t friend_number, bool audio_enabled, bool video_enabled, void *user_data);
/**
* Set the callback for the `call` event. Pass NULL to unset.
*
*/
void toxav_callback_call(ToxAV *av, toxav_call_cb *callback, void *user_data);
typedef enum Toxav_Err_Answer {
/**
* The function returned successfully.
*/
TOXAV_ERR_ANSWER_OK,
/**
* Synchronization error occurred.
*/
TOXAV_ERR_ANSWER_SYNC,
/**
* Failed to initialize codecs for call session. Note that codec initiation
* will fail if there is no receive callback registered for either audio or
* video.
*/
TOXAV_ERR_ANSWER_CODEC_INITIALIZATION,
/**
* The friend number did not designate a valid friend.
*/
TOXAV_ERR_ANSWER_FRIEND_NOT_FOUND,
/**
* The friend was valid, but they are not currently trying to initiate a call.
* This is also returned if this client is already in a call with the friend.
*/
TOXAV_ERR_ANSWER_FRIEND_NOT_CALLING,
/**
* Audio or video bit rate is invalid.
*/
TOXAV_ERR_ANSWER_INVALID_BIT_RATE,
} Toxav_Err_Answer;
/**
* Accept an incoming call.
*
* If answering fails for any reason, the call will still be pending and it is
* possible to try and answer it later. Audio and video receiving are both
* enabled by default.
*
* @param friend_number The friend number of the friend that is calling.
* @param audio_bit_rate Audio bit rate in Kb/sec. Set this to 0 to disable
* audio sending.
* @param video_bit_rate Video bit rate in Kb/sec. Set this to 0 to disable
* video sending.
*/
bool toxav_answer(ToxAV *av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate,
Toxav_Err_Answer *error);
/** @} */
/** @{
* @brief Call state graph
*/
enum Toxav_Friend_Call_State {
/**
* The empty bit mask. None of the bits specified below are set.
*/
TOXAV_FRIEND_CALL_STATE_NONE = 0,
/**
* Set by the AV core if an error occurred on the remote end or if friend
* timed out. This is the final state after which no more state
* transitions can occur for the call. This call state will never be triggered
* in combination with other call states.
*/
TOXAV_FRIEND_CALL_STATE_ERROR = 1,
/**
* The call has finished. This is the final state after which no more state
* transitions can occur for the call. This call state will never be
* triggered in combination with other call states.
*/
TOXAV_FRIEND_CALL_STATE_FINISHED = 2,
/**
* The flag that marks that friend is sending audio.
*/
TOXAV_FRIEND_CALL_STATE_SENDING_A = 4,
/**
* The flag that marks that friend is sending video.
*/
TOXAV_FRIEND_CALL_STATE_SENDING_V = 8,
/**
* The flag that marks that friend is receiving audio.
*/
TOXAV_FRIEND_CALL_STATE_ACCEPTING_A = 16,
/**
* The flag that marks that friend is receiving video.
*/
TOXAV_FRIEND_CALL_STATE_ACCEPTING_V = 32,
};
/**
* The function type for the call_state callback.
*
* @param friend_number The friend number for which the call state changed.
* @param state The bitmask of the new call state which is guaranteed to be
* different than the previous state. The state is set to 0 when the call is
* paused. The bitmask represents all the activities currently performed by the
* friend.
*/
typedef void toxav_call_state_cb(ToxAV *av, uint32_t friend_number, uint32_t state, void *user_data);
/**
* Set the callback for the `call_state` event. Pass NULL to unset.
*
*/
void toxav_callback_call_state(ToxAV *av, toxav_call_state_cb *callback, void *user_data);
/** @} */
/** @{
* @brief Call control
*/
typedef enum Toxav_Call_Control {
/**
* Resume a previously paused call. Only valid if the pause was caused by this
* client, if not, this control is ignored. Not valid before the call is accepted.
*/
TOXAV_CALL_CONTROL_RESUME,
/**
* Put a call on hold. Not valid before the call is accepted.
*/
TOXAV_CALL_CONTROL_PAUSE,
/**
* Reject a call if it was not answered, yet. Cancel a call after it was
* answered.
*/
TOXAV_CALL_CONTROL_CANCEL,
/**
* Request that the friend stops sending audio. Regardless of the friend's
* compliance, this will cause the audio_receive_frame event to stop being
* triggered on receiving an audio frame from the friend.
*/
TOXAV_CALL_CONTROL_MUTE_AUDIO,
/**
* Calling this control will notify client to start sending audio again.
*/
TOXAV_CALL_CONTROL_UNMUTE_AUDIO,
/**
* Request that the friend stops sending video. Regardless of the friend's
* compliance, this will cause the video_receive_frame event to stop being
* triggered on receiving a video frame from the friend.
*/
TOXAV_CALL_CONTROL_HIDE_VIDEO,
/**
* Calling this control will notify client to start sending video again.
*/
TOXAV_CALL_CONTROL_SHOW_VIDEO,
} Toxav_Call_Control;
typedef enum Toxav_Err_Call_Control {
/**
* The function returned successfully.
*/
TOXAV_ERR_CALL_CONTROL_OK,
/**
* Synchronization error occurred.
*/
TOXAV_ERR_CALL_CONTROL_SYNC,
/**
* The friend_number passed did not designate a valid friend.
*/
TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_FOUND,
/**
* This client is currently not in a call with the friend. Before the call is
* answered, only CANCEL is a valid control.
*/
TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL,
/**
* Happens if user tried to pause an already paused call or if trying to
* resume a call that is not paused.
*/
TOXAV_ERR_CALL_CONTROL_INVALID_TRANSITION,
} Toxav_Err_Call_Control;
/**
* Sends a call control command to a friend.
*
* @param friend_number The friend number of the friend this client is in a call
* with.
* @param control The control command to send.
*
* @return true on success.
*/
bool toxav_call_control(ToxAV *av, uint32_t friend_number, Toxav_Call_Control control, Toxav_Err_Call_Control *error);
/** @} */
/** @{
* @brief Controlling bit rates
*/
typedef enum Toxav_Err_Bit_Rate_Set {
/**
* The function returned successfully.
*/
TOXAV_ERR_BIT_RATE_SET_OK,
/**
* Synchronization error occurred.
*/
TOXAV_ERR_BIT_RATE_SET_SYNC,
/**
* The bit rate passed was not one of the supported values.
*/
TOXAV_ERR_BIT_RATE_SET_INVALID_BIT_RATE,
/**
* The friend_number passed did not designate a valid friend.
*/
TOXAV_ERR_BIT_RATE_SET_FRIEND_NOT_FOUND,
/**
* This client is currently not in a call with the friend.
*/
TOXAV_ERR_BIT_RATE_SET_FRIEND_NOT_IN_CALL,
} Toxav_Err_Bit_Rate_Set;
/** @} */
/** @{
* @brief A/V sending
*/
typedef enum Toxav_Err_Send_Frame {
/**
* The function returned successfully.
*/
TOXAV_ERR_SEND_FRAME_OK,
/**
* In case of video, one of Y, U, or V was NULL. In case of audio, the samples
* data pointer was NULL.
*/
TOXAV_ERR_SEND_FRAME_NULL,
/**
* The friend_number passed did not designate a valid friend.
*/
TOXAV_ERR_SEND_FRAME_FRIEND_NOT_FOUND,
/**
* This client is currently not in a call with the friend.
*/
TOXAV_ERR_SEND_FRAME_FRIEND_NOT_IN_CALL,
/**
* Synchronization error occurred.
*/
TOXAV_ERR_SEND_FRAME_SYNC,
/**
* One of the frame parameters was invalid. E.g. the resolution may be too
* small or too large, or the audio sampling rate may be unsupported.
*/
TOXAV_ERR_SEND_FRAME_INVALID,
/**
* Either friend turned off audio or video receiving or we turned off sending
* for the said payload.
*/
TOXAV_ERR_SEND_FRAME_PAYLOAD_TYPE_DISABLED,
/**
* Failed to push frame through rtp interface.
*/
TOXAV_ERR_SEND_FRAME_RTP_FAILED,
} Toxav_Err_Send_Frame;
/**
* Send an audio frame to a friend.
*
* The expected format of the PCM data is: `[s1c1][s1c2][...][s2c1][s2c2][...]...`
* Meaning: sample 1 for channel 1, sample 1 for channel 2, ...
* For mono audio, this has no meaning, every sample is subsequent. For stereo,
* this means the expected format is LRLRLR... with samples for left and right
* alternating.
*
* @param friend_number The friend number of the friend to which to send an
* audio frame.
* @param pcm An array of audio samples. The size of this array must be
* `sample_count * channels`.
* @param sample_count Number of samples in this frame. Valid numbers here are
* `((sample rate) * (audio length) / 1000)`, where audio length can be
* 2.5, 5, 10, 20, 40 or 60 millseconds.
* @param channels Number of audio channels. Supported values are 1 and 2.
* @param sampling_rate Audio sampling rate used in this frame. Valid sampling
* rates are 8000, 12000, 16000, 24000, or 48000.
*/
bool toxav_audio_send_frame(ToxAV *av, uint32_t friend_number, const int16_t *pcm, size_t sample_count,
uint8_t channels, uint32_t sampling_rate, Toxav_Err_Send_Frame *error);
/**
* Set the bit rate to be used in subsequent video frames.
*
* @param friend_number The friend number of the friend for which to set the
* bit rate.
* @param bit_rate The new audio bit rate in Kb/sec. Set to 0 to disable.
*
* @return true on success.
*/
bool toxav_audio_set_bit_rate(ToxAV *av, uint32_t friend_number, uint32_t bit_rate, Toxav_Err_Bit_Rate_Set *error);
/**
* The function type for the audio_bit_rate callback. The event is triggered
* when the network becomes too saturated for current bit rates at which
* point core suggests new bit rates.
*
* @param friend_number The friend number of the friend for which to set the
* bit rate.
* @param audio_bit_rate Suggested maximum audio bit rate in Kb/sec.
*/
typedef void toxav_audio_bit_rate_cb(ToxAV *av, uint32_t friend_number, uint32_t audio_bit_rate, void *user_data);
/**
* Set the callback for the `audio_bit_rate` event. Pass NULL to unset.
*
*/
void toxav_callback_audio_bit_rate(ToxAV *av, toxav_audio_bit_rate_cb *callback, void *user_data);
/**
* Send a video frame to a friend.
*
* Y - plane should be of size: `height * width`
* U - plane should be of size: `(height/2) * (width/2)`
* V - plane should be of size: `(height/2) * (width/2)`
*
* @param friend_number The friend number of the friend to which to send a video
* frame.
* @param width Width of the frame in pixels.
* @param height Height of the frame in pixels.
* @param y Y (Luminance) plane data.
* @param u U (Chroma) plane data.
* @param v V (Chroma) plane data.
*/
bool toxav_video_send_frame(ToxAV *av, uint32_t friend_number, uint16_t width, uint16_t height, const uint8_t *y,
const uint8_t *u, const uint8_t *v, Toxav_Err_Send_Frame *error);
/**
* Set the bit rate to be used in subsequent video frames.
*
* @param friend_number The friend number of the friend for which to set the
* bit rate.
* @param bit_rate The new video bit rate in Kb/sec. Set to 0 to disable.
*
* @return true on success.
*/
bool toxav_video_set_bit_rate(ToxAV *av, uint32_t friend_number, uint32_t bit_rate, Toxav_Err_Bit_Rate_Set *error);
/**
* The function type for the video_bit_rate callback. The event is triggered
* when the network becomes too saturated for current bit rates at which
* point core suggests new bit rates.
*
* @param friend_number The friend number of the friend for which to set the
* bit rate.
* @param video_bit_rate Suggested maximum video bit rate in Kb/sec.
*/
typedef void toxav_video_bit_rate_cb(ToxAV *av, uint32_t friend_number, uint32_t video_bit_rate, void *user_data);
/**
* Set the callback for the `video_bit_rate` event. Pass NULL to unset.
*
*/
void toxav_callback_video_bit_rate(ToxAV *av, toxav_video_bit_rate_cb *callback, void *user_data);
/** @} */
/** @{
* @brief A/V receiving
*/
/**
* The function type for the audio_receive_frame callback. The callback can be
* called multiple times per single iteration depending on the amount of queued
* frames in the buffer. The received format is the same as in send function.
*
* @param friend_number The friend number of the friend who sent an audio frame.
* @param pcm An array of audio samples (`sample_count * channels` elements).
* @param sample_count The number of audio samples per channel in the PCM array.
* @param channels Number of audio channels.
* @param sampling_rate Sampling rate used in this frame.
*
*/
typedef void toxav_audio_receive_frame_cb(ToxAV *av, uint32_t friend_number, const int16_t *pcm, size_t sample_count,
uint8_t channels, uint32_t sampling_rate, void *user_data);
/**
* Set the callback for the `audio_receive_frame` event. Pass NULL to unset.
*
*/
void toxav_callback_audio_receive_frame(ToxAV *av, toxav_audio_receive_frame_cb *callback, void *user_data);
/**
* The function type for the video_receive_frame callback.
*
* The size of plane data is derived from width and height as documented
* below.
*
* Strides represent padding for each plane that may or may not be present.
* You must handle strides in your image processing code. Strides are
* negative if the image is bottom-up hence why you MUST `abs()` it when
* calculating plane buffer size.
*
* @param friend_number The friend number of the friend who sent a video frame.
* @param width Width of the frame in pixels.
* @param height Height of the frame in pixels.
* @param y Luminosity plane. `Size = MAX(width, abs(ystride)) * height`.
* @param u U chroma plane. `Size = MAX(width/2, abs(ustride)) * (height/2)`.
* @param v V chroma plane. `Size = MAX(width/2, abs(vstride)) * (height/2)`.
*
* @param ystride Luminosity plane stride.
* @param ustride U chroma plane stride.
* @param vstride V chroma plane stride.
*/
typedef void toxav_video_receive_frame_cb(ToxAV *av, uint32_t friend_number, uint16_t width, uint16_t height,
const uint8_t *y, const uint8_t *u, const uint8_t *v, int32_t ystride, int32_t ustride, int32_t vstride,
void *user_data);
/**
* Set the callback for the `video_receive_frame` event. Pass NULL to unset.
*
*/
void toxav_callback_video_receive_frame(ToxAV *av, toxav_video_receive_frame_cb *callback, void *user_data);
/***
* NOTE Compatibility with old toxav group calls. TODO(iphydf): remove
*
* TODO(iphydf): Use proper new API guidelines for these. E.g. don't use inline
* function types, don't have per-callback userdata, especially don't have one
* userdata per group.
*/
// TODO(iphydf): Use this better typed one instead of the void-pointer one below.
typedef void toxav_group_audio_cb(Tox *tox, uint32_t groupnumber, uint32_t peernumber, const int16_t *pcm,
uint32_t samples, uint8_t channels, uint32_t sample_rate, void *user_data);
typedef void toxav_audio_data_cb(void *tox, uint32_t groupnumber, uint32_t peernumber, const int16_t *pcm,
uint32_t samples, uint8_t channels, uint32_t sample_rate, void *userdata);
/** @brief Create a new toxav group.
*
* @return group number on success.
* @retval -1 on failure.
*
* Note that total size of pcm in bytes is equal to `samples * channels * sizeof(int16_t)`.
*/
int toxav_add_av_groupchat(Tox *tox, toxav_audio_data_cb *audio_callback, void *userdata);
/** @brief Join a AV group (you need to have been invited first).
*
* @return group number on success.
* @retval -1 on failure.
*
* Note that total size of pcm in bytes is equal to `samples * channels * sizeof(int16_t)`.
*/
int toxav_join_av_groupchat(Tox *tox, uint32_t friendnumber, const uint8_t *data, uint16_t length,
toxav_audio_data_cb *audio_callback, void *userdata);
/** @brief Send audio to the group chat.
*
* @retval 0 on success.
* @retval -1 on failure.
*
* Note that total size of pcm in bytes is equal to `samples * channels * sizeof(int16_t)`.
*
* Valid number of samples are `(sample rate) * (audio length) / 1000`
* (Valid values for audio length are: 2.5, 5, 10, 20, 40 or 60 ms)
* Valid number of channels are 1 or 2.
* Valid sample rates are 8000, 12000, 16000, 24000, or 48000.
*
* Recommended values are: samples = 960, channels = 1, sample_rate = 48000
*/
int toxav_group_send_audio(Tox *tox, uint32_t groupnumber, const int16_t *pcm, unsigned int samples, uint8_t channels,
uint32_t sample_rate);
/** @brief Enable A/V in a groupchat.
*
* A/V must be enabled on a groupchat for audio to be sent to it and for
* received audio to be handled.
*
* An A/V group created with `toxav_add_av_groupchat` or `toxav_join_av_groupchat`
* will start with A/V enabled.
*
* An A/V group loaded from a savefile will start with A/V disabled.
*
* @retval 0 on success.
* @retval -1 on failure.
*
* Note that total size of pcm in bytes is equal to `samples * channels * sizeof(int16_t)`.
*/
int toxav_groupchat_enable_av(Tox *tox, uint32_t groupnumber,
toxav_audio_data_cb *audio_callback, void *userdata);
/** @brief Disable A/V in a groupchat.
*
* @retval 0 on success.
* @retval -1 on failure.
*/
int toxav_groupchat_disable_av(Tox *tox, uint32_t groupnumber);
/** @brief Return whether A/V is enabled in the groupchat. */
bool toxav_groupchat_av_enabled(Tox *tox, uint32_t groupnumber);
/** @} */
#ifdef __cplusplus
}
#endif
//!TOKSTYLE-
#ifndef DOXYGEN_IGNORE
typedef Toxav_Err_Call TOXAV_ERR_CALL;
typedef Toxav_Err_New TOXAV_ERR_NEW;
typedef Toxav_Err_Answer TOXAV_ERR_ANSWER;
typedef Toxav_Err_Call_Control TOXAV_ERR_CALL_CONTROL;
typedef Toxav_Err_Bit_Rate_Set TOXAV_ERR_BIT_RATE_SET;
typedef Toxav_Err_Send_Frame TOXAV_ERR_SEND_FRAME;
typedef Toxav_Call_Control TOXAV_CALL_CONTROL;
typedef enum Toxav_Friend_Call_State TOXAV_FRIEND_CALL_STATE;
#endif
//!TOKSTYLE+
#endif // C_TOXCORE_TOXAV_TOXAV_H

86
toxav/toxav_old.c Normal file
View File

@ -0,0 +1,86 @@
/* SPDX-License-Identifier: GPL-3.0-or-later
* Copyright © 2016-2018 The TokTok team.
* Copyright © 2013-2015 Tox project.
*/
/**
* This file contains the group chats code for the backwards compatibility.
*/
#include "toxav.h"
#include "../toxcore/tox_struct.h"
#include "groupav.h"
int toxav_add_av_groupchat(Tox *tox, audio_data_cb *audio_callback, void *userdata)
{
return add_av_groupchat(tox->m->log, tox, tox->m->conferences_object, audio_callback, userdata);
}
/** @brief Join a AV group (you need to have been invited first).
*
* @return group number on success.
* @retval -1 on failure.
*
* Note that total size of pcm in bytes is equal to `samples * channels * sizeof(int16_t)`.
*/
int toxav_join_av_groupchat(Tox *tox, uint32_t friendnumber, const uint8_t *data, uint16_t length,
audio_data_cb *audio_callback, void *userdata)
{
return join_av_groupchat(tox->m->log, tox, tox->m->conferences_object, friendnumber, data, length, audio_callback, userdata);
}
/** @brief Send audio to the group chat.
*
* @retval 0 on success.
* @retval -1 on failure.
*
* Note that total size of pcm in bytes is equal to `samples * channels * sizeof(int16_t)`.
*
* Valid number of samples are `(sample rate) * (audio length) / 1000`
* (Valid values for audio length are: 2.5, 5, 10, 20, 40 or 60 ms)
* Valid number of channels are 1 or 2.
* Valid sample rates are 8000, 12000, 16000, 24000, or 48000.
*
* Recommended values are: samples = 960, channels = 1, sample_rate = 48000
*/
int toxav_group_send_audio(Tox *tox, uint32_t groupnumber, const int16_t *pcm, unsigned int samples, uint8_t channels,
uint32_t sample_rate)
{
return group_send_audio(tox->m->conferences_object, groupnumber, pcm, samples, channels, sample_rate);
}
/** @brief Enable A/V in a groupchat.
*
* A/V must be enabled on a groupchat for audio to be sent to it and for
* received audio to be handled.
*
* An A/V group created with `toxav_add_av_groupchat` or `toxav_join_av_groupchat`
* will start with A/V enabled.
*
* An A/V group loaded from a savefile will start with A/V disabled.
*
* @retval 0 on success.
* @retval -1 on failure.
*
* Note that total size of pcm in bytes is equal to `samples * channels * sizeof(int16_t)`.
*/
int toxav_groupchat_enable_av(Tox *tox, uint32_t groupnumber, audio_data_cb *audio_callback, void *userdata)
{
return groupchat_enable_av(tox->m->log, tox, tox->m->conferences_object, groupnumber, audio_callback, userdata);
}
/** @brief Disable A/V in a groupchat.
*
* @retval 0 on success.
* @retval -1 on failure.
*/
int toxav_groupchat_disable_av(Tox *tox, uint32_t groupnumber)
{
return groupchat_disable_av(tox->m->conferences_object, groupnumber);
}
/** @brief Return whether A/V is enabled in the groupchat. */
bool toxav_groupchat_av_enabled(Tox *tox, uint32_t groupnumber)
{
return groupchat_av_enabled(tox->m->conferences_object, groupnumber);
}

446
toxav/video.c Normal file
View File

@ -0,0 +1,446 @@
/* SPDX-License-Identifier: GPL-3.0-or-later
* Copyright © 2016-2018 The TokTok team.
* Copyright © 2013-2015 Tox project.
*/
#include "video.h"
#include <assert.h>
#include <stdlib.h>
#include <string.h>
#include "msi.h"
#include "ring_buffer.h"
#include "rtp.h"
#include "../toxcore/ccompat.h"
#include "../toxcore/logger.h"
#include "../toxcore/mono_time.h"
#include "../toxcore/network.h"
/**
* Soft deadline the decoder should attempt to meet, in "us" (microseconds).
* Set to zero for unlimited.
*
* By convention, the value 1 is used to mean "return as fast as possible."
*/
// TODO(zoff99): don't hardcode this, let the application choose it
#define WANTED_MAX_DECODER_FPS 40
/**
* VPX_DL_REALTIME (1)
* deadline parameter analogous to VPx REALTIME mode.
*
* VPX_DL_GOOD_QUALITY (1000000)
* deadline parameter analogous to VPx GOOD QUALITY mode.
*
* VPX_DL_BEST_QUALITY (0)
* deadline parameter analogous to VPx BEST QUALITY mode.
*/
#define MAX_DECODE_TIME_US (1000000 / WANTED_MAX_DECODER_FPS) // to allow x fps
/**
* Codec control function to set encoder internal speed settings. Changes in
* this value influences, among others, the encoder's selection of motion
* estimation methods. Values greater than 0 will increase encoder speed at the
* expense of quality.
*
* Note Valid range for VP8: `-16..16`
*/
#define VP8E_SET_CPUUSED_VALUE 16
/**
* Initialize encoder with this value.
*
* Target bandwidth to use for this stream, in kilobits per second.
*/
#define VIDEO_BITRATE_INITIAL_VALUE 5000
#define VIDEO_DECODE_BUFFER_SIZE 5 // this buffer has normally max. 1 entry
static vpx_codec_iface_t *video_codec_decoder_interface(void)
{
return vpx_codec_vp8_dx();
}
static vpx_codec_iface_t *video_codec_encoder_interface(void)
{
return vpx_codec_vp8_cx();
}
#define VIDEO_CODEC_DECODER_MAX_WIDTH 800 // its a dummy value, because the struct needs a value there
#define VIDEO_CODEC_DECODER_MAX_HEIGHT 600 // its a dummy value, because the struct needs a value there
#define VPX_MAX_DIST_START 40
#define VPX_MAX_ENCODER_THREADS 4
#define VPX_MAX_DECODER_THREADS 4
#define VIDEO_VP8_DECODER_POST_PROCESSING_ENABLED 0
static void vc_init_encoder_cfg(const Logger *log, vpx_codec_enc_cfg_t *cfg, int16_t kf_max_dist)
{
const vpx_codec_err_t rc = vpx_codec_enc_config_default(video_codec_encoder_interface(), cfg, 0);
if (rc != VPX_CODEC_OK) {
LOGGER_ERROR(log, "vc_init_encoder_cfg:Failed to get config: %s", vpx_codec_err_to_string(rc));
}
/* Target bandwidth to use for this stream, in kilobits per second */
cfg->rc_target_bitrate = VIDEO_BITRATE_INITIAL_VALUE;
cfg->g_w = VIDEO_CODEC_DECODER_MAX_WIDTH;
cfg->g_h = VIDEO_CODEC_DECODER_MAX_HEIGHT;
cfg->g_pass = VPX_RC_ONE_PASS;
cfg->g_error_resilient = VPX_ERROR_RESILIENT_DEFAULT | VPX_ERROR_RESILIENT_PARTITIONS;
cfg->g_lag_in_frames = 0;
/* Allow lagged encoding
*
* If set, this value allows the encoder to consume a number of input
* frames before producing output frames. This allows the encoder to
* base decisions for the current frame on future frames. This does
* increase the latency of the encoding pipeline, so it is not appropriate
* in all situations (ex: realtime encoding).
*
* Note that this is a maximum value -- the encoder may produce frames
* sooner than the given limit. Set this value to 0 to disable this
* feature.
*/
cfg->kf_min_dist = 0;
cfg->kf_mode = VPX_KF_AUTO; // Encoder determines optimal placement automatically
cfg->rc_end_usage = VPX_VBR; // what quality mode?
/*
* VPX_VBR Variable Bit Rate (VBR) mode
* VPX_CBR Constant Bit Rate (CBR) mode
* VPX_CQ Constrained Quality (CQ) mode -> give codec a hint that we may be on low bandwidth connection
* VPX_Q Constant Quality (Q) mode
*/
if (kf_max_dist > 1) {
cfg->kf_max_dist = kf_max_dist; // a full frame every x frames minimum (can be more often, codec decides automatically)
LOGGER_DEBUG(log, "kf_max_dist=%d (1)", cfg->kf_max_dist);
} else {
cfg->kf_max_dist = VPX_MAX_DIST_START;
LOGGER_DEBUG(log, "kf_max_dist=%d (2)", cfg->kf_max_dist);
}
cfg->g_threads = VPX_MAX_ENCODER_THREADS; // Maximum number of threads to use
/* TODO: set these to something reasonable */
// cfg->g_timebase.num = 1;
// cfg->g_timebase.den = 60; // 60 fps
cfg->rc_resize_allowed = 1; // allow encoder to resize to smaller resolution
cfg->rc_resize_up_thresh = 40;
cfg->rc_resize_down_thresh = 5;
/* TODO: make quality setting an API call, but start with normal quality */
#if 0
/* Highest-resolution encoder settings */
cfg->rc_dropframe_thresh = 0;
cfg->rc_resize_allowed = 0;
cfg->rc_min_quantizer = 2;
cfg->rc_max_quantizer = 56;
cfg->rc_undershoot_pct = 100;
cfg->rc_overshoot_pct = 15;
cfg->rc_buf_initial_sz = 500;
cfg->rc_buf_optimal_sz = 600;
cfg->rc_buf_sz = 1000;
#endif
}
VCSession *vc_new(Mono_Time *mono_time, const Logger *log, ToxAV *av, uint32_t friend_number,
toxav_video_receive_frame_cb *cb, void *cb_data)
{
VCSession *vc = (VCSession *)calloc(1, sizeof(VCSession));
vpx_codec_err_t rc;
if (vc == nullptr) {
LOGGER_WARNING(log, "Allocation failed! Application might misbehave!");
return nullptr;
}
if (create_recursive_mutex(vc->queue_mutex) != 0) {
LOGGER_WARNING(log, "Failed to create recursive mutex!");
free(vc);
return nullptr;
}
const int cpu_used_value = VP8E_SET_CPUUSED_VALUE;
vc->vbuf_raw = rb_new(VIDEO_DECODE_BUFFER_SIZE);
if (vc->vbuf_raw == nullptr) {
goto BASE_CLEANUP;
}
/*
* VPX_CODEC_USE_FRAME_THREADING
* Enable frame-based multi-threading
*
* VPX_CODEC_USE_ERROR_CONCEALMENT
* Conceal errors in decoded frames
*/
vpx_codec_dec_cfg_t dec_cfg;
dec_cfg.threads = VPX_MAX_DECODER_THREADS; // Maximum number of threads to use
dec_cfg.w = VIDEO_CODEC_DECODER_MAX_WIDTH;
dec_cfg.h = VIDEO_CODEC_DECODER_MAX_HEIGHT;
LOGGER_DEBUG(log, "Using VP8 codec for decoder (0)");
rc = vpx_codec_dec_init(vc->decoder, video_codec_decoder_interface(), &dec_cfg,
VPX_CODEC_USE_FRAME_THREADING | VPX_CODEC_USE_POSTPROC);
if (rc == VPX_CODEC_INCAPABLE) {
LOGGER_WARNING(log, "Postproc not supported by this decoder (0)");
rc = vpx_codec_dec_init(vc->decoder, video_codec_decoder_interface(), &dec_cfg, VPX_CODEC_USE_FRAME_THREADING);
}
if (rc != VPX_CODEC_OK) {
LOGGER_ERROR(log, "Init video_decoder failed: %s", vpx_codec_err_to_string(rc));
goto BASE_CLEANUP;
}
if (VIDEO_VP8_DECODER_POST_PROCESSING_ENABLED == 1) {
vp8_postproc_cfg_t pp = {VP8_DEBLOCK, 1, 0};
const vpx_codec_err_t cc_res = vpx_codec_control(vc->decoder, VP8_SET_POSTPROC, &pp);
if (cc_res != VPX_CODEC_OK) {
LOGGER_WARNING(log, "Failed to turn on postproc");
} else {
LOGGER_DEBUG(log, "turn on postproc: OK");
}
} else {
vp8_postproc_cfg_t pp = {0, 0, 0};
vpx_codec_err_t cc_res = vpx_codec_control(vc->decoder, VP8_SET_POSTPROC, &pp);
if (cc_res != VPX_CODEC_OK) {
LOGGER_WARNING(log, "Failed to turn OFF postproc");
} else {
LOGGER_DEBUG(log, "Disable postproc: OK");
}
}
/* Set encoder to some initial values
*/
vpx_codec_enc_cfg_t cfg;
vc_init_encoder_cfg(log, &cfg, 1);
LOGGER_DEBUG(log, "Using VP8 codec for encoder (0.1)");
rc = vpx_codec_enc_init(vc->encoder, video_codec_encoder_interface(), &cfg, VPX_CODEC_USE_FRAME_THREADING);
if (rc != VPX_CODEC_OK) {
LOGGER_ERROR(log, "Failed to initialize encoder: %s", vpx_codec_err_to_string(rc));
goto BASE_CLEANUP_1;
}
rc = vpx_codec_control(vc->encoder, VP8E_SET_CPUUSED, cpu_used_value);
if (rc != VPX_CODEC_OK) {
LOGGER_ERROR(log, "Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc));
vpx_codec_destroy(vc->encoder);
goto BASE_CLEANUP_1;
}
/*
* VPX_CTRL_USE_TYPE(VP8E_SET_NOISE_SENSITIVITY, unsigned int)
* control function to set noise sensitivity
* 0: off, 1: OnYOnly, 2: OnYUV, 3: OnYUVAggressive, 4: Adaptive
*/
#if 0
rc = vpx_codec_control(vc->encoder, VP8E_SET_NOISE_SENSITIVITY, 2);
if (rc != VPX_CODEC_OK) {
LOGGER_ERROR(log, "Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc));
vpx_codec_destroy(vc->encoder);
goto BASE_CLEANUP_1;
}
#endif
vc->linfts = current_time_monotonic(mono_time);
vc->lcfd = 60;
vc->vcb = cb;
vc->vcb_user_data = cb_data;
vc->friend_number = friend_number;
vc->av = av;
vc->log = log;
return vc;
BASE_CLEANUP_1:
vpx_codec_destroy(vc->decoder);
BASE_CLEANUP:
pthread_mutex_destroy(vc->queue_mutex);
rb_kill(vc->vbuf_raw);
free(vc);
return nullptr;
}
void vc_kill(VCSession *vc)
{
if (vc == nullptr) {
return;
}
vpx_codec_destroy(vc->encoder);
vpx_codec_destroy(vc->decoder);
void *p;
while (rb_read(vc->vbuf_raw, &p)) {
free(p);
}
rb_kill(vc->vbuf_raw);
pthread_mutex_destroy(vc->queue_mutex);
LOGGER_DEBUG(vc->log, "Terminated video handler: %p", (void *)vc);
free(vc);
}
void vc_iterate(VCSession *vc)
{
if (vc == nullptr) {
return;
}
pthread_mutex_lock(vc->queue_mutex);
struct RTPMessage *p;
if (!rb_read(vc->vbuf_raw, (void **)&p)) {
LOGGER_TRACE(vc->log, "no Video frame data available");
pthread_mutex_unlock(vc->queue_mutex);
return;
}
const uint16_t log_rb_size = rb_size(vc->vbuf_raw);
pthread_mutex_unlock(vc->queue_mutex);
const struct RTPHeader *const header = &p->header;
uint32_t full_data_len;
if ((header->flags & RTP_LARGE_FRAME) != 0) {
full_data_len = header->data_length_full;
LOGGER_DEBUG(vc->log, "vc_iterate:001:full_data_len=%d", (int)full_data_len);
} else {
full_data_len = p->len;
LOGGER_DEBUG(vc->log, "vc_iterate:002");
}
LOGGER_DEBUG(vc->log, "vc_iterate: rb_read p->len=%d p->header.xe=%d", (int)full_data_len, p->header.xe);
LOGGER_DEBUG(vc->log, "vc_iterate: rb_read rb size=%d", (int)log_rb_size);
const vpx_codec_err_t rc = vpx_codec_decode(vc->decoder, p->data, full_data_len, nullptr, MAX_DECODE_TIME_US);
free(p);
if (rc != VPX_CODEC_OK) {
LOGGER_ERROR(vc->log, "Error decoding video: %d %s", (int)rc, vpx_codec_err_to_string(rc));
return;
}
/* Play decoded images */
vpx_codec_iter_t iter = nullptr;
for (vpx_image_t *dest = vpx_codec_get_frame(vc->decoder, &iter);
dest != nullptr;
dest = vpx_codec_get_frame(vc->decoder, &iter)) {
if (vc->vcb != nullptr) {
vc->vcb(vc->av, vc->friend_number, dest->d_w, dest->d_h,
dest->planes[0], dest->planes[1], dest->planes[2],
dest->stride[0], dest->stride[1], dest->stride[2], vc->vcb_user_data);
}
vpx_img_free(dest); // is this needed? none of the VPx examples show that
}
}
int vc_queue_message(Mono_Time *mono_time, void *vcp, struct RTPMessage *msg)
{
/* This function is called with complete messages
* they have already been assembled.
* this function gets called from handle_rtp_packet() and handle_rtp_packet_v3()
*/
if (vcp == nullptr || msg == nullptr) {
free(msg);
return -1;
}
VCSession *vc = (VCSession *)vcp;
const struct RTPHeader *const header = &msg->header;
if (msg->header.pt == (RTP_TYPE_VIDEO + 2) % 128) {
LOGGER_WARNING(vc->log, "Got dummy!");
free(msg);
return 0;
}
if (msg->header.pt != RTP_TYPE_VIDEO % 128) {
LOGGER_WARNING(vc->log, "Invalid payload type! pt=%d", (int)msg->header.pt);
free(msg);
return -1;
}
pthread_mutex_lock(vc->queue_mutex);
if ((header->flags & RTP_LARGE_FRAME) != 0 && header->pt == RTP_TYPE_VIDEO % 128) {
LOGGER_DEBUG(vc->log, "rb_write msg->len=%d b0=%d b1=%d", (int)msg->len, (int)msg->data[0], (int)msg->data[1]);
}
free(rb_write(vc->vbuf_raw, msg));
/* Calculate time it took for peer to send us this frame */
const uint32_t t_lcfd = current_time_monotonic(mono_time) - vc->linfts;
vc->lcfd = t_lcfd > 100 ? vc->lcfd : t_lcfd;
vc->linfts = current_time_monotonic(mono_time);
pthread_mutex_unlock(vc->queue_mutex);
return 0;
}
int vc_reconfigure_encoder(VCSession *vc, uint32_t bit_rate, uint16_t width, uint16_t height, int16_t kf_max_dist)
{
if (vc == nullptr) {
return -1;
}
vpx_codec_enc_cfg_t cfg2 = *vc->encoder->config.enc;
if (cfg2.rc_target_bitrate == bit_rate && cfg2.g_w == width && cfg2.g_h == height && kf_max_dist == -1) {
return 0; /* Nothing changed */
}
if (cfg2.g_w == width && cfg2.g_h == height && kf_max_dist == -1) {
/* Only bit rate changed */
LOGGER_INFO(vc->log, "bitrate change from: %u to: %u", (uint32_t)cfg2.rc_target_bitrate, (uint32_t)bit_rate);
cfg2.rc_target_bitrate = bit_rate;
const vpx_codec_err_t rc = vpx_codec_enc_config_set(vc->encoder, &cfg2);
if (rc != VPX_CODEC_OK) {
LOGGER_ERROR(vc->log, "Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc));
return -1;
}
} else {
/* Resolution is changed, must reinitialize encoder since libvpx v1.4 doesn't support
* reconfiguring encoder to use resolutions greater than initially set.
*/
LOGGER_DEBUG(vc->log, "Have to reinitialize vpx encoder on session %p", (void *)vc);
vpx_codec_ctx_t new_c;
vpx_codec_enc_cfg_t cfg;
vc_init_encoder_cfg(vc->log, &cfg, kf_max_dist);
cfg.rc_target_bitrate = bit_rate;
cfg.g_w = width;
cfg.g_h = height;
LOGGER_DEBUG(vc->log, "Using VP8 codec for encoder");
vpx_codec_err_t rc = vpx_codec_enc_init(&new_c, video_codec_encoder_interface(), &cfg, VPX_CODEC_USE_FRAME_THREADING);
if (rc != VPX_CODEC_OK) {
LOGGER_ERROR(vc->log, "Failed to initialize encoder: %s", vpx_codec_err_to_string(rc));
return -1;
}
const int cpu_used_value = VP8E_SET_CPUUSED_VALUE;
rc = vpx_codec_control(&new_c, VP8E_SET_CPUUSED, cpu_used_value);
if (rc != VPX_CODEC_OK) {
LOGGER_ERROR(vc->log, "Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc));
vpx_codec_destroy(&new_c);
return -1;
}
vpx_codec_destroy(vc->encoder);
memcpy(vc->encoder, &new_c, sizeof(new_c));
}
return 0;
}

54
toxav/video.h Normal file
View File

@ -0,0 +1,54 @@
/* SPDX-License-Identifier: GPL-3.0-or-later
* Copyright © 2016-2018 The TokTok team.
* Copyright © 2013-2015 Tox project.
*/
#ifndef C_TOXCORE_TOXAV_VIDEO_H
#define C_TOXCORE_TOXAV_VIDEO_H
#include <vpx/vpx_decoder.h>
#include <vpx/vpx_encoder.h>
#include <vpx/vpx_image.h>
#include <vpx/vp8cx.h>
#include <vpx/vp8dx.h>
#include <pthread.h>
#include "toxav.h"
#include "../toxcore/logger.h"
#include "../toxcore/util.h"
#include "ring_buffer.h"
#include "rtp.h"
typedef struct VCSession {
/* encoding */
vpx_codec_ctx_t encoder[1];
uint32_t frame_counter;
/* decoding */
vpx_codec_ctx_t decoder[1];
struct RingBuffer *vbuf_raw; /* Un-decoded data */
uint64_t linfts; /* Last received frame time stamp */
uint32_t lcfd; /* Last calculated frame duration for incoming video payload */
const Logger *log;
ToxAV *av;
uint32_t friend_number;
/* Video frame receive callback */
toxav_video_receive_frame_cb *vcb;
void *vcb_user_data;
pthread_mutex_t queue_mutex[1];
} VCSession;
VCSession *vc_new(Mono_Time *mono_time, const Logger *log, ToxAV *av, uint32_t friend_number,
toxav_video_receive_frame_cb *cb, void *cb_data);
void vc_kill(VCSession *vc);
void vc_iterate(VCSession *vc);
int vc_queue_message(Mono_Time *mono_time, void *vcp, struct RTPMessage *msg);
int vc_reconfigure_encoder(VCSession *vc, uint32_t bit_rate, uint16_t width, uint16_t height, int16_t kf_max_dist);
#endif // C_TOXCORE_TOXAV_VIDEO_H