#include "./tox_av_voip_model.hpp" #include #include #include "./frame_streams/stream_manager.hpp" #include "./frame_streams/audio_stream2.hpp" #include "./frame_streams/locked_frame_stream.hpp" #include "./frame_streams/multi_source.hpp" #include "./frame_streams/audio_stream_pop_reframer.hpp" #include "./frame_streams/sdl/video.hpp" #include #include // fwd namespace Message { uint64_t getTimeMS(void); } // Message namespace Components { struct ToxAVIncomingAV { bool incoming_audio {false}; bool incoming_video {false}; }; struct ToxAVAudioSink { ObjectHandle o; // ptr? }; struct ToxAVVideoSink { ObjectHandle o; }; struct ToxAVAudioSource { ObjectHandle o; }; struct ToxAVVideoSource { ObjectHandle o; }; } // Components struct ToxAVCallAudioSink : public FrameStream2SinkI { ToxAVI& _toxav; // bitrate for enabled state uint32_t _audio_bitrate {32}; uint32_t _fid; std::shared_ptr>> _writer; ToxAVCallAudioSink(ToxAVI& toxav, uint32_t fid) : _toxav(toxav), _fid(fid) {} ~ToxAVCallAudioSink(void) { if (_writer) { _writer = nullptr; _toxav.toxavAudioSetBitRate(_fid, 0); } } // sink std::shared_ptr> subscribe(void) override { if (_writer) { // max 1 (exclusive for now) return nullptr; } auto err = _toxav.toxavAudioSetBitRate(_fid, _audio_bitrate); if (err != TOXAV_ERR_BIT_RATE_SET_OK) { return nullptr; } // 20ms for now, 10ms would work too, further investigate stutters at 5ms (probably too slow interval rate) _writer = std::make_shared>>(20); return _writer; } bool unsubscribe(const std::shared_ptr>& sub) override { if (!sub || !_writer) { // nah return false; } if (sub == _writer) { _writer = nullptr; /*auto err = */_toxav.toxavAudioSetBitRate(_fid, 0); // print warning? on error? return true; } // what return false; } }; void ToxAVVoIPModel::addAudioSource(ObjectHandle session, uint32_t friend_number) { auto& stream_source = session.get_or_emplace().streams; ObjectHandle incoming_audio {_os.registry(), _os.registry().create()}; auto new_asrc = std::make_unique>(); incoming_audio.emplace*>(new_asrc.get()); incoming_audio.emplace>(std::move(new_asrc)); incoming_audio.emplace(Components::StreamSource::create("ToxAV Friend Call Incoming Audio")); if ( const auto* defaults = session.try_get(); defaults != nullptr && defaults->incoming_audio ) { incoming_audio.emplace(); // depends on what was specified in enter() } stream_source.push_back(incoming_audio); session.emplace(incoming_audio); // TODO: tie session to stream _audio_sources[friend_number] = incoming_audio; _os.throwEventConstruct(incoming_audio); } void ToxAVVoIPModel::addAudioSink(ObjectHandle session, uint32_t friend_number) { auto& stream_sinks = session.get_or_emplace().streams; ObjectHandle outgoing_audio {_os.registry(), _os.registry().create()}; auto new_asink = std::make_unique(_av, friend_number); outgoing_audio.emplace(new_asink.get()); outgoing_audio.emplace>(std::move(new_asink)); outgoing_audio.emplace(Components::StreamSink::create("ToxAV Friend Call Outgoing Audio")); if ( const auto* defaults = session.try_get(); defaults != nullptr && defaults->outgoing_audio ) { outgoing_audio.emplace(); // depends on what was specified in enter() } stream_sinks.push_back(outgoing_audio); session.emplace(outgoing_audio); // TODO: tie session to stream _os.throwEventConstruct(outgoing_audio); } void ToxAVVoIPModel::addVideoSource(ObjectHandle session, uint32_t friend_number) { auto& stream_source = session.get_or_emplace().streams; ObjectHandle incoming_video {_os.registry(), _os.registry().create()}; auto new_vsrc = std::make_unique>(); incoming_video.emplace*>(new_vsrc.get()); incoming_video.emplace>(std::move(new_vsrc)); incoming_video.emplace(Components::StreamSource::create("ToxAV Friend Call Incoming Video")); if ( const auto* defaults = session.try_get(); defaults != nullptr && defaults->incoming_audio ) { incoming_video.emplace(); // depends on what was specified in enter() } stream_source.push_back(incoming_video); session.emplace(incoming_video); // TODO: tie session to stream _video_sources[friend_number] = incoming_video; _os.throwEventConstruct(incoming_video); } void ToxAVVoIPModel::addVideoSink(ObjectHandle session, uint32_t friend_number) { } void ToxAVVoIPModel::destroySession(ObjectHandle session) { if (!static_cast(session)) { return; } // remove lookup if (session.all_of()) { auto it_asrc = std::find_if( _audio_sources.cbegin(), _audio_sources.cend(), [o = session.get().o](const auto& it) { return it.second == o; } ); if (it_asrc != _audio_sources.cend()) { _audio_sources.erase(it_asrc); } } if (session.all_of()) { auto it_vsrc = std::find_if( _video_sources.cbegin(), _video_sources.cend(), [o = session.get().o](const auto& it) { return it.second == o; } ); if (it_vsrc != _video_sources.cend()) { _video_sources.erase(it_vsrc); } } // destory sources if (auto* ss = session.try_get(); ss != nullptr) { for (const auto ssov : ss->streams) { _os.throwEventDestroy(ssov); _os.registry().destroy(ssov); } } // destory sinks if (auto* ss = session.try_get(); ss != nullptr) { for (const auto ssov : ss->streams) { _os.throwEventDestroy(ssov); _os.registry().destroy(ssov); } } // destory session _os.throwEventDestroy(session); _os.registry().destroy(session); } ToxAVVoIPModel::ToxAVVoIPModel(ObjectStore2& os, ToxAVI& av, Contact3Registry& cr, ToxContactModel2& tcm) : _os(os), _av(av), _cr(cr), _tcm(tcm) { _av.subscribe(this, ToxAV_Event::friend_call); _av.subscribe(this, ToxAV_Event::friend_call_state); _av.subscribe(this, ToxAV_Event::friend_audio_bitrate); _av.subscribe(this, ToxAV_Event::friend_video_bitrate); _av.subscribe(this, ToxAV_Event::friend_audio_frame); _av.subscribe(this, ToxAV_Event::friend_video_frame); // attach to all tox friend contacts for (const auto& [cv, _] : _cr.view().each()) { _cr.emplace(cv, this); } // TODO: events } ToxAVVoIPModel::~ToxAVVoIPModel(void) { for (const auto& [ov, voipmodel] : _os.registry().view().each()) { if (voipmodel == this) { destroySession(_os.objectHandle(ov)); } } } void ToxAVVoIPModel::tick(void) { for (const auto& [oc, asink] : _os.registry().view().each()) { if (!asink->_writer) { continue; } for (size_t i = 0; i < 100; i++) { auto new_frame_opt = asink->_writer->pop(); if (!new_frame_opt.has_value()) { break; } const auto& new_frame = new_frame_opt.value(); //* @param sample_count Number of samples in this frame. Valid numbers here are //* `((sample rate) * (audio length) / 1000)`, where audio length can be //* 2.5, 5, 10, 20, 40 or 60 milliseconds. // we likely needs to subdivide/repackage // frame size should be an option exposed to the user // with 10ms as a default ? // the larger the frame size, the less overhead but the more delay auto err = _av.toxavAudioSendFrame( asink->_fid, new_frame.getSpan().ptr, new_frame.getSpan().size / new_frame.channels, new_frame.channels, new_frame.sample_rate ); if (err != TOXAV_ERR_SEND_FRAME_OK) { std::cerr << "DTC: failed to send audio frame " << err << "\n"; } } } } ObjectHandle ToxAVVoIPModel::enter(const Contact3 c, const Components::VoIP::DefaultConfig& defaults) { if (!_cr.all_of(c)) { return {}; } const auto friend_number = _cr.get(c).friend_number; const auto err = _av.toxavCall(friend_number, 0, 0); if (err != TOXAV_ERR_CALL_OK) { std::cerr << "TAVVOIP error: failed to start call: " << err << "\n"; return {}; } ObjectHandle new_session {_os.registry(), _os.registry().create()}; new_session.emplace(this); new_session.emplace(); // ?? new_session.emplace(c); new_session.emplace().state = Components::VoIP::SessionState::State::RINGING; new_session.emplace(defaults); _os.throwEventConstruct(new_session); return new_session; } bool ToxAVVoIPModel::accept(ObjectHandle session, const Components::VoIP::DefaultConfig& defaults) { if (!static_cast(session)) { return false; } if (!session.all_of< Components::VoIP::TagVoIPSession, VoIPModelI*, Components::VoIP::SessionContact, Components::VoIP::Incoming >()) { return false; } // check if self if (session.get() != this) { return false; } const auto session_contact = session.get().c; if (!_cr.all_of(session_contact)) { return false; } const auto friend_number = _cr.get(session_contact).friend_number; auto err = _av.toxavAnswer(friend_number, 0, 0); if (err != TOXAV_ERR_ANSWER_OK) { std::cerr << "TOXAVVOIP error: ansering call failed: " << err << "\n"; // we simply let it be for now, it apears we can try ansering later again // we also get an error here when the call is already in progress (: return false; } session.emplace(defaults); // answer defaults to enabled receiving audio and video // TODO: think about how we should handle this // set to disabled? and enable on src connection? // we already default disabled send and enabled on sink connection //_av.toxavCallControl(friend_number, TOXAV_CALL_CONTROL_HIDE_VIDEO); //_av.toxavCallControl(friend_number, TOXAV_CALL_CONTROL_MUTE_AUDIO); // how do we know the other side is accepting audio // bitrate cb or what? assert(!session.all_of()); addAudioSink(session, friend_number); assert(!session.all_of()); addVideoSink(session, friend_number); if (const auto* i_av = session.try_get(); i_av != nullptr) { // create audio src if (i_av->incoming_audio) { assert(!session.all_of()); addAudioSource(session, friend_number); } // create video src if (i_av->incoming_video) { assert(!session.all_of()); addVideoSource(session, friend_number); } } session.get_or_emplace().state = Components::VoIP::SessionState::State::CONNECTED; _os.throwEventUpdate(session); return true; } bool ToxAVVoIPModel::leave(ObjectHandle session) { // rename to end? if (!static_cast(session)) { return false; } if (!session.all_of< Components::VoIP::TagVoIPSession, VoIPModelI*, Components::VoIP::SessionContact >()) { return false; } // check if self if (session.get() != this) { return false; } const auto session_contact = session.get().c; if (!_cr.all_of(session_contact)) { return false; } const auto friend_number = _cr.get(session_contact).friend_number; // check error? (we delete anyway) _av.toxavCallControl(friend_number, Toxav_Call_Control::TOXAV_CALL_CONTROL_CANCEL); destroySession(session); return true; } bool ToxAVVoIPModel::onEvent(const Events::FriendCall& e) { // new incoming call, create voip session, ready to be accepted // (or rejected...) const auto session_contact = _tcm.getContactFriend(e.friend_number); if (!_cr.valid(session_contact)) { return false; } ObjectHandle new_session {_os.registry(), _os.registry().create()}; new_session.emplace(this); new_session.emplace(); // ?? new_session.emplace(session_contact); // in 1on1 its always the same contact, might leave blank new_session.emplace(session_contact); new_session.emplace().state = Components::VoIP::SessionState::State::RINGING; new_session.emplace(e.audio_enabled, e.video_enabled); _os.throwEventConstruct(new_session); return true; } bool ToxAVVoIPModel::onEvent(const Events::FriendCallState& e) { const auto session_contact = _tcm.getContactFriend(e.friend_number); if (!_cr.valid(session_contact)) { return false; } ToxAVFriendCallState s{e.state}; // find session(s?) // TODO: keep lookup table for (const auto& [ov, voipmodel] : _os.registry().view().each()) { if (voipmodel == this) { auto o = _os.objectHandle(ov); if (!o.all_of()) { continue; } if (session_contact != o.get().c) { continue; } if (s.is_error() || s.is_finished()) { // destroy call destroySession(o); } else { // remote accepted our call, or av send/recv conditions changed? o.get().state = Components::VoIP::SessionState::State::CONNECTED; // set to in call ?? if (s.is_accepting_a() && !o.all_of()) { addAudioSink(o, e.friend_number); } else if (!s.is_accepting_a() && o.all_of()) { // remove asink? } // video if (s.is_accepting_v() && !o.all_of()) { addVideoSink(o, e.friend_number); } else if (!s.is_accepting_v() && o.all_of()) { // remove vsink? } // add/update sources // audio if (s.is_sending_a() && !o.all_of()) { addAudioSource(o, e.friend_number); } else if (!s.is_sending_a() && o.all_of()) { // remove asrc? } // video if (s.is_sending_v() && !o.all_of()) { addVideoSource(o, e.friend_number); } else if (!s.is_sending_v() && o.all_of()) { // remove vsrc? } } } } return true; } bool ToxAVVoIPModel::onEvent(const Events::FriendAudioBitrate&) { return false; } bool ToxAVVoIPModel::onEvent(const Events::FriendVideoBitrate&) { return false; } bool ToxAVVoIPModel::onEvent(const Events::FriendAudioFrame& e) { auto asrc_it = _audio_sources.find(e.friend_number); if (asrc_it == _audio_sources.cend()) { // missing src from lookup table return false; } auto asrc = asrc_it->second; if (!static_cast(asrc)) { // missing src to put frame into ?? return false; } assert(asrc.all_of*>()); assert(asrc.all_of>()); asrc.get*>()->push(AudioFrame2{ e.sampling_rate, e.channels, std::vector(e.pcm.begin(), e.pcm.end()) // copy }); return true; } bool ToxAVVoIPModel::onEvent(const Events::FriendVideoFrame& e) { auto vsrc_it = _video_sources.find(e.friend_number); if (vsrc_it == _video_sources.cend()) { // missing src from lookup table return false; } auto vsrc = vsrc_it->second; if (!static_cast(vsrc)) { // missing src to put frame into ?? return false; } assert(vsrc.all_of*>()); assert(vsrc.all_of>()); auto* new_surf = SDL_CreateSurface(e.width, e.height, SDL_PIXELFORMAT_IYUV); assert(new_surf); if (SDL_LockSurface(new_surf)) { // copy the data // we know how the implementation works, its y u v consecutivlely // y for (size_t y = 0; y < e.height; y++) { std::memcpy( //static_cast(new_surf->pixels) + new_surf->pitch*y, static_cast(new_surf->pixels) + e.width*y, e.y.ptr + e.ystride*y, e.width ); } // u for (size_t y = 0; y < e.height/2; y++) { std::memcpy( static_cast(new_surf->pixels) + (e.width*e.height) + (e.width/2)*y, e.u.ptr + e.ustride*y, e.width/2 ); } // v for (size_t y = 0; y < e.height/2; y++) { std::memcpy( static_cast(new_surf->pixels) + (e.width*e.height) + ((e.width/2)*(e.height/2)) + (e.width/2)*y, e.v.ptr + e.vstride*y, e.width/2 ); } SDL_UnlockSurface(new_surf); } vsrc.get*>()->push({ // ms -> us Message::getTimeMS() * 1000, // TODO: make more precise new_surf }); SDL_DestroySurface(new_surf); return false; }