From 59cdb2638ff15b601c80f4f5b8c098b891eb56b9 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Fri, 27 Sep 2024 22:37:06 +0200 Subject: [PATCH] improve toxav module --- src/main_screen.cpp | 13 ++++ src/tox_av.cpp | 169 ++++++++++++++++++++++++++++++++++++++++++++ src/tox_av.hpp | 101 +++++++++++++++++++++++++- 3 files changed, 282 insertions(+), 1 deletion(-) diff --git a/src/main_screen.cpp b/src/main_screen.cpp index 3253f37d..16d0c3d3 100644 --- a/src/main_screen.cpp +++ b/src/main_screen.cpp @@ -518,6 +518,12 @@ Screen* MainScreen::tick(float time_delta, bool& quit) { quit = !tc.iterate(time_delta); // compute +#if TOMATO_TOX_AV + tav.toxavIterate(); + // HACK: pow by 1.18 to increase 200 -> ~500 + const float av_interval = std::pow(tav.toxavIterationInterval(), 1.18)/1000.f; +#endif + tcm.iterate(time_delta); // compute const float fo_interval = tffom.tick(time_delta); @@ -556,6 +562,13 @@ Screen* MainScreen::tick(float time_delta, bool& quit) { fo_interval ); +#if TOMATO_TOX_AV + _min_tick_interval = std::min( + _min_tick_interval, + av_interval + ); +#endif + //std::cout << "MS: min tick interval: " << _min_tick_interval << "\n"; switch (_compute_perf_mode) { diff --git a/src/tox_av.cpp b/src/tox_av.cpp index eeefb2b8..695744c3 100644 --- a/src/tox_av.cpp +++ b/src/tox_av.cpp @@ -2,14 +2,85 @@ #include +#include +#include + // https://almogfx.bandcamp.com/track/crushed-w-cassade +struct ToxAVFriendCallState final { + const uint32_t state {TOXAV_FRIEND_CALL_STATE_NONE}; + + [[nodiscard]] bool is_error(void) const { return state & TOXAV_FRIEND_CALL_STATE_ERROR; } + [[nodiscard]] bool is_finished(void) const { return state & TOXAV_FRIEND_CALL_STATE_FINISHED; } + [[nodiscard]] bool is_sending_a(void) const { return state & TOXAV_FRIEND_CALL_STATE_SENDING_A; } + [[nodiscard]] bool is_sending_v(void) const { return state & TOXAV_FRIEND_CALL_STATE_SENDING_V; } + [[nodiscard]] bool is_accepting_a(void) const { return state & TOXAV_FRIEND_CALL_STATE_ACCEPTING_A; } + [[nodiscard]] bool is_accepting_v(void) const { return state & TOXAV_FRIEND_CALL_STATE_ACCEPTING_V; } +}; + ToxAV::ToxAV(Tox* tox) : _tox(tox) { Toxav_Err_New err_new {TOXAV_ERR_NEW_OK}; _tox_av = toxav_new(_tox, &err_new); // TODO: throw assert(err_new == TOXAV_ERR_NEW_OK); + + toxav_callback_call( + _tox_av, + +[](ToxAV*, uint32_t friend_number, bool audio_enabled, bool video_enabled, void *user_data) { + assert(user_data != nullptr); + static_cast(user_data)->cb_call(friend_number, audio_enabled, video_enabled); + }, + this + ); + toxav_callback_call_state( + _tox_av, + +[](ToxAV*, uint32_t friend_number, uint32_t state, void *user_data) { + assert(user_data != nullptr); + static_cast(user_data)->cb_call_state(friend_number, state); + }, + this + ); + toxav_callback_audio_bit_rate( + _tox_av, + +[](ToxAV*, uint32_t friend_number, uint32_t audio_bit_rate, void *user_data) { + assert(user_data != nullptr); + static_cast(user_data)->cb_audio_bit_rate(friend_number, audio_bit_rate); + }, + this + ); + toxav_callback_video_bit_rate( + _tox_av, + +[](ToxAV*, uint32_t friend_number, uint32_t video_bit_rate, void *user_data) { + assert(user_data != nullptr); + static_cast(user_data)->cb_video_bit_rate(friend_number, video_bit_rate); + }, + this + ); + toxav_callback_audio_receive_frame( + _tox_av, + +[](ToxAV*, uint32_t friend_number, const int16_t pcm[], size_t sample_count, uint8_t channels, uint32_t sampling_rate, void *user_data) { + assert(user_data != nullptr); + static_cast(user_data)->cb_audio_receive_frame(friend_number, pcm, sample_count, channels, sampling_rate); + }, + this + ); + toxav_callback_video_receive_frame( + _tox_av, + +[](ToxAV*, uint32_t friend_number, + uint16_t width, uint16_t height, + const uint8_t y[/*! max(width, abs(ystride)) * height */], + const uint8_t u[/*! max(width/2, abs(ustride)) * (height/2) */], + const uint8_t v[/*! max(width/2, abs(vstride)) * (height/2) */], + int32_t ystride, int32_t ustride, int32_t vstride, + void *user_data + ) { + assert(user_data != nullptr); + static_cast(user_data)->cb_video_receive_frame(friend_number, width, height, y, u, v, ystride, ustride, vstride); + }, + this + ); } + ToxAV::~ToxAV(void) { toxav_kill(_tox_av); } @@ -80,3 +151,101 @@ Toxav_Err_Bit_Rate_Set ToxAV::toxavVideoSetBitRate(uint32_t friend_number, uint3 return err; } +void ToxAV::cb_call(uint32_t friend_number, bool audio_enabled, bool video_enabled) { + std::cerr << "TOXAV: receiving call f:" << friend_number << " a:" << audio_enabled << " v:" << video_enabled << "\n"; + //Toxav_Err_Answer err_answer { TOXAV_ERR_ANSWER_OK }; + //toxav_answer(_tox_av, friend_number, 0, 0, &err_answer); + //if (err_answer != TOXAV_ERR_ANSWER_OK) { + // std::cerr << "!!!!!!!! answer failed " << err_answer << "\n"; + //} + + dispatch( + ToxAV_Event::friend_call, + Events::FriendCall{ + friend_number, + audio_enabled, + video_enabled, + } + ); +} + +void ToxAV::cb_call_state(uint32_t friend_number, uint32_t state) { + //ToxAVFriendCallState w_state{state}; + + //w_state.is_error(); + + std::cerr << "TOXAV: call state f:" << friend_number << " s:" << state << "\n"; + + dispatch( + ToxAV_Event::friend_call_state, + Events::FriendCallState{ + friend_number, + state, + } + ); +} + +void ToxAV::cb_audio_bit_rate(uint32_t friend_number, uint32_t audio_bit_rate) { + std::cerr << "TOXAV: audio bitrate f:" << friend_number << " abr:" << audio_bit_rate << "\n"; + + dispatch( + ToxAV_Event::friend_audio_bitrate, + Events::FriendAudioBitrate{ + friend_number, + audio_bit_rate, + } + ); +} + +void ToxAV::cb_video_bit_rate(uint32_t friend_number, uint32_t video_bit_rate) { + std::cerr << "TOXAV: video bitrate f:" << friend_number << " vbr:" << video_bit_rate << "\n"; + + dispatch( + ToxAV_Event::friend_video_bitrate, + Events::FriendVideoBitrate{ + friend_number, + video_bit_rate, + } + ); +} + +void ToxAV::cb_audio_receive_frame(uint32_t friend_number, const int16_t pcm[], size_t sample_count, uint8_t channels, uint32_t sampling_rate) { + //std::cerr << "TOXAV: audio frame f:" << friend_number << " sc:" << sample_count << " ch:" << (int)channels << " sr:" << sampling_rate << "\n"; + + dispatch( + ToxAV_Event::friend_audio_frame, + Events::FriendAudioFrame{ + friend_number, + Span(pcm, sample_count*channels), // TODO: is sample count *ch or /ch? + channels, + sampling_rate, + } + ); +} + +void ToxAV::cb_video_receive_frame( + uint32_t friend_number, + uint16_t width, uint16_t height, + const uint8_t y[/*! max(width, abs(ystride)) * height */], + const uint8_t u[/*! max(width/2, abs(ustride)) * (height/2) */], + const uint8_t v[/*! max(width/2, abs(vstride)) * (height/2) */], + int32_t ystride, int32_t ustride, int32_t vstride +) { + //std::cerr << "TOXAV: video frame f:" << friend_number << " w:" << width << " h:" << height << "\n"; + + dispatch( + ToxAV_Event::friend_video_frame, + Events::FriendVideoFrame{ + friend_number, + width, + height, + Span(y, std::max(width, std::abs(ystride)) * height), + Span(u, std::max(width/2, std::abs(ustride)) * (height/2)), + Span(v, std::max(width/2, std::abs(vstride)) * (height/2)), + ystride, + ustride, + vstride, + } + ); +} + diff --git a/src/tox_av.hpp b/src/tox_av.hpp index c116cc56..bc00ce0f 100644 --- a/src/tox_av.hpp +++ b/src/tox_av.hpp @@ -1,15 +1,98 @@ #pragma once +#include +#include + #include -struct ToxAV { +namespace /*toxav*/ Events { + + struct FriendCall { + uint32_t friend_number; + bool audio_enabled; + bool video_enabled; + }; + + struct FriendCallState { + uint32_t friend_number; + uint32_t state; + }; + + struct FriendAudioBitrate { + uint32_t friend_number; + uint32_t audio_bit_rate; + }; + + struct FriendVideoBitrate { + uint32_t friend_number; + uint32_t video_bit_rate; + }; + + struct FriendAudioFrame { + uint32_t friend_number; + + Span pcm; + //size_t sample_count; + uint8_t channels; + uint32_t sampling_rate; + }; + + struct FriendVideoFrame { + uint32_t friend_number; + + uint16_t width; + uint16_t height; + //const uint8_t y[[>! max(width, abs(ystride)) * height <]]; + //const uint8_t u[[>! max(width/2, abs(ustride)) * (height/2) <]]; + //const uint8_t v[[>! max(width/2, abs(vstride)) * (height/2) <]]; + // mdspan would be nice here + // bc of the stride, span might be larger than the actual data it contains + Span y; + Span u; + Span v; + int32_t ystride; + int32_t ustride; + int32_t vstride; + }; + +} // Event + +enum class ToxAV_Event : uint32_t { + friend_call, + friend_call_state, + friend_audio_bitrate, + friend_video_bitrate, + friend_audio_frame, + friend_video_frame, + + MAX +}; + +struct ToxAVEventI { + using enumType = ToxAV_Event; + + virtual ~ToxAVEventI(void) {} + + virtual bool onEvent(const Events::FriendCall&) { return false; } + virtual bool onEvent(const Events::FriendCallState&) { return false; } + virtual bool onEvent(const Events::FriendAudioBitrate&) { return false; } + virtual bool onEvent(const Events::FriendVideoBitrate&) { return false; } + virtual bool onEvent(const Events::FriendAudioFrame&) { return false; } + virtual bool onEvent(const Events::FriendVideoFrame&) { return false; } +}; +using ToxAVEventProviderI = EventProviderI; + +struct ToxAV : public ToxAVEventProviderI{ Tox* _tox = nullptr; ToxAV* _tox_av = nullptr; + static constexpr const char* version {"0"}; + ToxAV(Tox* tox); virtual ~ToxAV(void); // interface + // if iterate is called on a different thread, it will fire events there uint32_t toxavIterationInterval(void) const; void toxavIterate(void); @@ -33,5 +116,21 @@ struct ToxAV { //int32_t toxav_groupchat_disable_av(Tox *tox, uint32_t groupnumber); //bool toxav_groupchat_av_enabled(Tox *tox, uint32_t groupnumber); + + + // toxav callbacks + void cb_call(uint32_t friend_number, bool audio_enabled, bool video_enabled); + void cb_call_state(uint32_t friend_number, uint32_t state); + void cb_audio_bit_rate(uint32_t friend_number, uint32_t audio_bit_rate); + void cb_video_bit_rate(uint32_t friend_number, uint32_t video_bit_rate); + void cb_audio_receive_frame(uint32_t friend_number, const int16_t pcm[], size_t sample_count, uint8_t channels, uint32_t sampling_rate); + void cb_video_receive_frame( + uint32_t friend_number, + uint16_t width, uint16_t height, + const uint8_t y[/*! max(width, abs(ystride)) * height */], + const uint8_t u[/*! max(width/2, abs(ustride)) * (height/2) */], + const uint8_t v[/*! max(width/2, abs(vstride)) * (height/2) */], + int32_t ystride, int32_t ustride, int32_t vstride + ); };