From 51ec99a42f8b6351776213e17738b3977bdebc64 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Wed, 2 Oct 2024 11:28:36 +0200 Subject: [PATCH] add toxav incoming video (sdl video) --- src/tox_av_voip_model.cpp | 136 ++++++++++++++++++++++++++++++++++++-- src/tox_av_voip_model.hpp | 4 +- 2 files changed, 133 insertions(+), 7 deletions(-) diff --git a/src/tox_av_voip_model.cpp b/src/tox_av_voip_model.cpp index 9da602a1..249f8b6c 100644 --- a/src/tox_av_voip_model.cpp +++ b/src/tox_av_voip_model.cpp @@ -9,8 +9,17 @@ #include "./frame_streams/multi_source.hpp" #include "./frame_streams/audio_stream_pop_reframer.hpp" +#include "./frame_streams/sdl/video.hpp" + +#include + #include +// fwd +namespace Message { +uint64_t getTimeMS(void); +} // Message + namespace Components { struct ToxAVIncomingAV { bool incoming_audio {false}; @@ -21,12 +30,15 @@ namespace Components { ObjectHandle o; // ptr? }; - // vid + struct ToxAVVideoSink { + ObjectHandle o; + }; struct ToxAVAudioSource { ObjectHandle o; - // ptr? }; - // vid + struct ToxAVVideoSource { + ObjectHandle o; + }; } // Components struct ToxAVCallAudioSink : public FrameStream2SinkI { @@ -94,13 +106,11 @@ void ToxAVVoIPModel::addAudioSource(ObjectHandle session, uint32_t friend_number incoming_audio.emplace>(std::move(new_asrc)); incoming_audio.emplace(Components::StreamSource::create("ToxAV Friend Call Incoming Audio")); - std::cout << "new incoming audio\n"; if ( const auto* defaults = session.try_get(); defaults != nullptr && defaults->incoming_audio ) { incoming_audio.emplace(); // depends on what was specified in enter() - std::cout << "with default\n"; } stream_source.push_back(incoming_audio); @@ -135,6 +145,35 @@ void ToxAVVoIPModel::addAudioSink(ObjectHandle session, uint32_t friend_number) _os.throwEventConstruct(outgoing_audio); } +void ToxAVVoIPModel::addVideoSource(ObjectHandle session, uint32_t friend_number) { + auto& stream_source = session.get_or_emplace().streams; + + ObjectHandle incoming_video {_os.registry(), _os.registry().create()}; + + auto new_vsrc = std::make_unique>(); + incoming_video.emplace*>(new_vsrc.get()); + incoming_video.emplace>(std::move(new_vsrc)); + incoming_video.emplace(Components::StreamSource::create("ToxAV Friend Call Incoming Video")); + + if ( + const auto* defaults = session.try_get(); + defaults != nullptr && defaults->incoming_audio + ) { + incoming_video.emplace(); // depends on what was specified in enter() + } + + stream_source.push_back(incoming_video); + session.emplace(incoming_video); + // TODO: tie session to stream + + _video_sources[friend_number] = incoming_video; + + _os.throwEventConstruct(incoming_video); +} + +void ToxAVVoIPModel::addVideoSink(ObjectHandle session, uint32_t friend_number) { +} + void ToxAVVoIPModel::destroySession(ObjectHandle session) { if (!static_cast(session)) { return; @@ -153,6 +192,18 @@ void ToxAVVoIPModel::destroySession(ObjectHandle session) { _audio_sources.erase(it_asrc); } } + if (session.all_of()) { + auto it_vsrc = std::find_if( + _video_sources.cbegin(), _video_sources.cend(), + [o = session.get().o](const auto& it) { + return it.second == o; + } + ); + + if (it_vsrc != _video_sources.cend()) { + _video_sources.erase(it_vsrc); + } + } // destory sources if (auto* ss = session.try_get(); ss != nullptr) { @@ -311,6 +362,8 @@ bool ToxAVVoIPModel::accept(ObjectHandle session, const Components::VoIP::Defaul // bitrate cb or what? assert(!session.all_of()); addAudioSink(session, friend_number); + assert(!session.all_of()); + addVideoSink(session, friend_number); if (const auto* i_av = session.try_get(); i_av != nullptr) { // create audio src @@ -321,6 +374,8 @@ bool ToxAVVoIPModel::accept(ObjectHandle session, const Components::VoIP::Defaul // create video src if (i_av->incoming_video) { + assert(!session.all_of()); + addVideoSource(session, friend_number); } } @@ -419,6 +474,11 @@ bool ToxAVVoIPModel::onEvent(const Events::FriendCallState& e) { } // video + if (s.is_accepting_v() && !o.all_of()) { + addVideoSink(o, e.friend_number); + } else if (!s.is_accepting_v() && o.all_of()) { + // remove vsink? + } // add/update sources // audio @@ -429,6 +489,11 @@ bool ToxAVVoIPModel::onEvent(const Events::FriendCallState& e) { } // video + if (s.is_sending_v() && !o.all_of()) { + addVideoSource(o, e.friend_number); + } else if (!s.is_sending_v() && o.all_of()) { + // remove vsrc? + } } } } @@ -470,7 +535,66 @@ bool ToxAVVoIPModel::onEvent(const Events::FriendAudioFrame& e) { return true; } -bool ToxAVVoIPModel::onEvent(const Events::FriendVideoFrame&) { +bool ToxAVVoIPModel::onEvent(const Events::FriendVideoFrame& e) { + auto vsrc_it = _video_sources.find(e.friend_number); + if (vsrc_it == _video_sources.cend()) { + // missing src from lookup table + return false; + } + + auto vsrc = vsrc_it->second; + + if (!static_cast(vsrc)) { + // missing src to put frame into ?? + return false; + } + + assert(vsrc.all_of*>()); + assert(vsrc.all_of>()); + + auto* new_surf = SDL_CreateSurface(e.width, e.height, SDL_PIXELFORMAT_IYUV); + assert(new_surf); + if (SDL_LockSurface(new_surf)) { + // copy the data + // we know how the implementation works, its y u v consecutivlely + // y + for (size_t y = 0; y < e.height; y++) { + std::memcpy( + //static_cast(new_surf->pixels) + new_surf->pitch*y, + static_cast(new_surf->pixels) + e.width*y, + e.y.ptr + e.ystride*y, + e.width + ); + } + + // u + for (size_t y = 0; y < e.height/2; y++) { + std::memcpy( + static_cast(new_surf->pixels) + (e.width*e.height) + (e.width/2)*y, + e.u.ptr + e.ustride*y, + e.width/2 + ); + } + + // v + for (size_t y = 0; y < e.height/2; y++) { + std::memcpy( + static_cast(new_surf->pixels) + (e.width*e.height) + ((e.width/2)*(e.height/2)) + (e.width/2)*y, + e.v.ptr + e.vstride*y, + e.width/2 + ); + } + + SDL_UnlockSurface(new_surf); + } + + vsrc.get*>()->push({ + // ms -> us + Message::getTimeMS() * 1000, // TODO: make more precise + new_surf + }); + + SDL_DestroySurface(new_surf); return false; } diff --git a/src/tox_av_voip_model.hpp b/src/tox_av_voip_model.hpp index 743e5952..277588a5 100644 --- a/src/tox_av_voip_model.hpp +++ b/src/tox_av_voip_model.hpp @@ -16,11 +16,13 @@ class ToxAVVoIPModel : protected ToxAVEventI, public VoIPModelI { // for faster lookup std::unordered_map _audio_sources; + std::unordered_map _video_sources; // TODO: virtual? strategy? protected? virtual void addAudioSource(ObjectHandle session, uint32_t friend_number); virtual void addAudioSink(ObjectHandle session, uint32_t friend_number); - // TODO: video + virtual void addVideoSource(ObjectHandle session, uint32_t friend_number); + virtual void addVideoSink(ObjectHandle session, uint32_t friend_number); void destroySession(ObjectHandle session);