diff --git a/src/content/frame_stream2.hpp b/src/content/frame_stream2.hpp index 494c8ac..b2addc0 100644 --- a/src/content/frame_stream2.hpp +++ b/src/content/frame_stream2.hpp @@ -104,11 +104,7 @@ struct FrameStream2MultiSource : public FrameStream2SourceI, public F virtual ~FrameStream2MultiSource(void) {} - //// TODO: forward args instead - //SubStreamType* aquireSubStream(size_t queue_size = 10, bool lossy = true) { - // std::lock_guard lg{_sub_stream_lock}; - // return _sub_streams.emplace_back(std::make_unique(queue_size, lossy)).get(); - //} + // TODO: forward args instead std::shared_ptr> subscribe(void) override { // TODO: args??? size_t queue_size = 10; @@ -118,15 +114,6 @@ struct FrameStream2MultiSource : public FrameStream2SourceI, public F return _sub_streams.emplace_back(std::make_unique(queue_size, lossy)); } - //void releaseSubStream(SubStreamType* sub) { - // std::lock_guard lg{_sub_stream_lock}; - // for (auto it = _sub_streams.begin(); it != _sub_streams.end(); it++) { - // if (it->get() == sub) { - // _sub_streams.erase(it); - // break; - // } - // } - //} bool unsubscribe(const std::shared_ptr>& sub) override { std::lock_guard lg{_sub_stream_lock}; for (auto it = _sub_streams.begin(); it != _sub_streams.end(); it++) { diff --git a/src/content/sdl_audio_frame_stream2.cpp b/src/content/sdl_audio_frame_stream2.cpp index a6d6999..7c4a705 100644 --- a/src/content/sdl_audio_frame_stream2.cpp +++ b/src/content/sdl_audio_frame_stream2.cpp @@ -100,33 +100,39 @@ bool SDLAudioOutputDeviceDefaultInstance::push(const AudioFrame& value) { (value.isF32() && _last_format != SDL_AUDIO_F32) || (value.isS16() && _last_format != SDL_AUDIO_S16) ) { - const auto device_id = SDL_GetAudioStreamDevice(_stream.get()); - SDL_FlushAudioStream(_stream.get()); - const SDL_AudioSpec spec = { static_cast((value.isF32() ? SDL_AUDIO_F32 : SDL_AUDIO_S16)), static_cast(value.channels), static_cast(value.sample_rate) }; - _stream = { - SDL_OpenAudioDeviceStream(device_id, &spec, nullptr, nullptr), - &SDL_DestroyAudioStream - }; + SDL_SetAudioStreamFormat(_stream.get(), &spec, nullptr); + + std::cerr << "SDLAOD: audio format changed\n"; } - // HACK - assert(value.isS16()); + if (value.isS16()) { + auto data = value.getSpan(); - auto data = value.getSpan(); + if (data.size == 0) { + std::cerr << "empty audio frame??\n"; + } - if (data.size == 0) { - std::cerr << "empty audio frame??\n"; - } + if (!SDL_PutAudioStreamData(_stream.get(), data.ptr, data.size * sizeof(int16_t))) { + std::cerr << "put data error\n"; + return false; // return true? + } + } else if (value.isF32()) { + auto data = value.getSpan(); - if (!SDL_PutAudioStreamData(_stream.get(), data.ptr, data.size * sizeof(int16_t))) { - std::cerr << "put data error\n"; - return false; // return true? + if (data.size == 0) { + std::cerr << "empty audio frame??\n"; + } + + if (!SDL_PutAudioStreamData(_stream.get(), data.ptr, data.size * sizeof(float))) { + std::cerr << "put data error\n"; + return false; // return true? + } } _last_sample_rate = value.sample_rate; diff --git a/src/debug_tox_call.cpp b/src/debug_tox_call.cpp index 42ac85c..5ee0a89 100644 --- a/src/debug_tox_call.cpp +++ b/src/debug_tox_call.cpp @@ -2,6 +2,7 @@ #include "./stream_manager.hpp" #include "./content/sdl_video_frame_stream2.hpp" +#include "./content/sdl_audio_frame_stream2.hpp" #include @@ -18,20 +19,6 @@ namespace Message { uint64_t getTimeMS(); } -namespace Components { - struct ToxAVFriendAudioSource { - }; - - struct ToxAVFriendAudioSink { - }; - - struct ToxAVFriendVideoSource { - }; - - struct ToxAVFriendVideoSink { - }; -} - static bool isFormatPlanar(SDL_PixelFormat f) { return f == SDL_PIXELFORMAT_YV12 || @@ -166,20 +153,34 @@ struct PushConversionQueuedVideoStream : public QueuedFrameStream2 { + ToxAV& _toxav; + + // bitrate for enabled state + uint32_t _video_bitrate {2}; + uint32_t _fid; std::shared_ptr _writer; - ToxAVCallVideoSink(uint32_t fid) : _fid(fid) {} - ~ToxAVCallVideoSink(void) {} + ToxAVCallVideoSink(ToxAV& toxav, uint32_t fid) : _toxav(toxav), _fid(fid) {} + ~ToxAVCallVideoSink(void) { + if (_writer) { + _writer = nullptr; + _toxav.toxavVideoSetBitRate(_fid, 0); + } + } // sink std::shared_ptr> subscribe(void) override { if (_writer) { - // max 1 (exclusive) + // max 1 (exclusive, composite video somewhere else) + return nullptr; + } + + auto err = _toxav.toxavVideoSetBitRate(_fid, _video_bitrate); + if (err != TOXAV_ERR_BIT_RATE_SET_OK) { return nullptr; } - // TODO: enable video here _writer = std::make_shared(10, true); return _writer; @@ -192,8 +193,11 @@ struct ToxAVCallVideoSink : public FrameStream2SinkI { } if (sub == _writer) { - // TODO: disable video here _writer = nullptr; + + /*auto err = */_toxav.toxavVideoSetBitRate(_fid, 0); + // print warning? on error? + return true; } @@ -211,9 +215,28 @@ DebugToxCall::DebugToxCall(ObjectStore2& os, ToxAV& toxav, TextureUploaderI& tu) _toxav.subscribe(this, ToxAV_Event::friend_video_frame); } +DebugToxCall::~DebugToxCall(void) { + // destroy all calls/connections/sources/sinks here + + for (auto& [fid, call] : _calls) { + if (static_cast(call.incoming_vsrc)) { + call.incoming_vsrc.destroy(); + } + if (static_cast(call.incoming_asrc)) { + call.incoming_asrc.destroy(); + } + if (static_cast(call.outgoing_vsink)) { + call.outgoing_vsink.destroy(); + } + if (static_cast(call.outgoing_asink)) { + call.outgoing_asink.destroy(); + } + } +} + void DebugToxCall::tick(float) { - // pump sink to tox - // TODO: own thread or direct on push + // pump sinks to tox + // TODO: own thread or direct on push (requires thread save toxcore) // TODO: pump at double the frame rate for (const auto& [oc, vsink] : _os.registry().view().each()) { if (!vsink->_writer) { @@ -258,8 +281,9 @@ float DebugToxCall::render(void) { if (call.incoming) { ImGui::SameLine(); if (ImGui::SmallButton("answer")) { + const auto ret = _toxav.toxavAnswer(fid, 0, 0); //const auto ret = _toxav.toxavAnswer(fid, 0, 1); // 1mbit/s - const auto ret = _toxav.toxavAnswer(fid, 0, 2); // 2mbit/s + //const auto ret = _toxav.toxavAnswer(fid, 0, 2); // 2mbit/s //const auto ret = _toxav.toxavAnswer(fid, 0, 100); // 100mbit/s //const auto ret = _toxav.toxavAnswer(fid, 0, 2500); // 2500mbit/s if (ret == TOXAV_ERR_ANSWER_OK) { @@ -268,7 +292,7 @@ float DebugToxCall::render(void) { // create sinks call.outgoing_vsink = {_os.registry(), _os.registry().create()}; { - auto new_vsink = std::make_unique(fid); + auto new_vsink = std::make_unique(_toxav, fid); call.outgoing_vsink.emplace(new_vsink.get()); call.outgoing_vsink.emplace>(std::move(new_vsink)); call.outgoing_vsink.emplace("ToxAV friend call video", std::string{entt::type_name::value()}); @@ -284,6 +308,15 @@ float DebugToxCall::render(void) { call.incoming_vsrc.emplace("ToxAV friend call video", std::string{entt::type_name::value()}); } } + if (call.incoming_a) { + call.incoming_asrc = {_os.registry(), _os.registry().create()}; + { + auto new_asrc = std::make_unique(); + call.incoming_asrc.emplace(new_asrc.get()); + call.incoming_asrc.emplace>(std::move(new_asrc)); + call.incoming_asrc.emplace("ToxAV friend call audio", std::string{entt::type_name::value()}); + } + } } } } else if (call.state != TOXAV_FRIEND_CALL_STATE_FINISHED) { @@ -297,11 +330,17 @@ float DebugToxCall::render(void) { call.state = TOXAV_FRIEND_CALL_STATE_FINISHED; // TODO: stream manager disconnectAll() + if (static_cast(call.incoming_vsrc)) { + call.incoming_vsrc.destroy(); + } + if (static_cast(call.incoming_asrc)) { + call.incoming_asrc.destroy(); + } if (static_cast(call.outgoing_vsink)) { call.outgoing_vsink.destroy(); } - if (static_cast(call.incoming_vsrc)) { - call.incoming_vsrc.destroy(); + if (static_cast(call.outgoing_asink)) { + call.outgoing_asink.destroy(); } } } @@ -341,11 +380,17 @@ bool DebugToxCall::onEvent(const Events::FriendCallState& e) { (call.state & TOXAV_FRIEND_CALL_STATE_FINISHED) != 0 || (call.state & TOXAV_FRIEND_CALL_STATE_ERROR) != 0 ) { + if (static_cast(call.incoming_vsrc)) { + call.incoming_vsrc.destroy(); + } + if (static_cast(call.incoming_asrc)) { + call.incoming_asrc.destroy(); + } if (static_cast(call.outgoing_vsink)) { call.outgoing_vsink.destroy(); } - if (static_cast(call.incoming_vsrc)) { - call.incoming_vsrc.destroy(); + if (static_cast(call.outgoing_asink)) { + call.outgoing_asink.destroy(); } } @@ -362,8 +407,25 @@ bool DebugToxCall::onEvent(const Events::FriendVideoBitrate&) { bool DebugToxCall::onEvent(const Events::FriendAudioFrame& e) { auto& call = _calls[e.friend_number]; + + if (!static_cast(call.incoming_asrc)) { + // missing src to put frame into ?? + return false; + } + + assert(call.incoming_asrc.all_of()); + assert(call.incoming_asrc.all_of>()); + call.num_a_frames++; - return false; + + call.incoming_asrc.get()->push(AudioFrame{ + 0, //seq + e.sampling_rate, + e.channels, + std::vector(e.pcm.begin(), e.pcm.end()) // copy + }); + + return true; } bool DebugToxCall::onEvent(const Events::FriendVideoFrame& e) { diff --git a/src/debug_tox_call.hpp b/src/debug_tox_call.hpp index 369de2e..bb6f6f4 100644 --- a/src/debug_tox_call.hpp +++ b/src/debug_tox_call.hpp @@ -37,7 +37,7 @@ class DebugToxCall : public ToxAVEventI { public: DebugToxCall(ObjectStore2& os, ToxAV& toxav, TextureUploaderI& tu); - ~DebugToxCall(void) {} + ~DebugToxCall(void); void tick(float time_delta); float render(void); diff --git a/src/stream_manager_ui.cpp b/src/stream_manager_ui.cpp index 03d221e..11a4a4c 100644 --- a/src/stream_manager_ui.cpp +++ b/src/stream_manager_ui.cpp @@ -1,6 +1,7 @@ #include "./stream_manager_ui.hpp" #include "./content/sdl_video_frame_stream2.hpp" +#include "./content/audio_stream.hpp" #include @@ -58,6 +59,31 @@ void StreamManagerUI::render(void) { ImGui::EndMenu(); } + + if (ImGui::BeginMenu("connect audio", ss.frame_type_name == entt::type_name::value())) { + for (const auto& [oc_src, s_src] : _os.registry().view().each()) { + if (s_src.frame_type_name != ss.frame_type_name) { + continue; + } + + ImGui::PushID(entt::to_integral(oc_src)); + + std::string source_label {"src "}; + source_label += std::to_string(entt::to_integral(entt::to_entity(oc_src))); + source_label += " ("; + source_label += s_src.name; + source_label += ")["; + source_label += s_src.frame_type_name; + source_label += "]"; + if (ImGui::MenuItem(source_label.c_str())) { + _sm.connect(oc_src, oc); + } + + ImGui::PopID(); + } + + ImGui::EndMenu(); + } ImGui::EndPopup(); } ImGui::PopID(); diff --git a/src/tox_av.cpp b/src/tox_av.cpp index 4d3d7dd..c856f53 100644 --- a/src/tox_av.cpp +++ b/src/tox_av.cpp @@ -216,7 +216,7 @@ void ToxAV::cb_audio_receive_frame(uint32_t friend_number, const int16_t pcm[], ToxAV_Event::friend_audio_frame, Events::FriendAudioFrame{ friend_number, - Span(pcm, sample_count), // TODO: is sample count *ch or /ch? + Span(pcm, sample_count*channels), // TODO: is sample count *ch or /ch? channels, sampling_rate, }