toxav incomming audio src and fix sdl audio output sink

src/content/frame_stream2.hpp

@@ -104,11 +104,7 @@ struct FrameStream2MultiSource : public FrameStream2SourceI<FrameType>, public F
 
 	virtual ~FrameStream2MultiSource(void) {}
 
-	//// TODO: forward args instead
+	// TODO: forward args instead
-	//SubStreamType* aquireSubStream(size_t queue_size = 10, bool lossy = true) {
-	//    std::lock_guard lg{_sub_stream_lock};
-	//    return _sub_streams.emplace_back(std::make_unique<SubStreamType>(queue_size, lossy)).get();
-	//}
 	std::shared_ptr<FrameStream2I<FrameType>> subscribe(void) override {
 		// TODO: args???
 		size_t queue_size = 10;
@@ -118,15 +114,6 @@ struct FrameStream2MultiSource : public FrameStream2SourceI<FrameType>, public F
 		return _sub_streams.emplace_back(std::make_unique<SubStreamType>(queue_size, lossy));
 	}
 
-	//void releaseSubStream(SubStreamType* sub) {
-	//    std::lock_guard lg{_sub_stream_lock};
-	//    for (auto it = _sub_streams.begin(); it != _sub_streams.end(); it++) {
-	//        if (it->get() == sub) {
-	//            _sub_streams.erase(it);
-	//            break;
-	//        }
-	//    }
-	//}
 	bool unsubscribe(const std::shared_ptr<FrameStream2I<FrameType>>& sub) override {
 		std::lock_guard lg{_sub_stream_lock};
 		for (auto it = _sub_streams.begin(); it != _sub_streams.end(); it++) {

src/content/sdl_audio_frame_stream2.cpp

@@ -100,33 +100,39 @@ bool SDLAudioOutputDeviceDefaultInstance::push(const AudioFrame& value) {
 		(value.isF32() && _last_format != SDL_AUDIO_F32) ||
 		(value.isS16() && _last_format != SDL_AUDIO_S16)
 	) {
-		const auto device_id = SDL_GetAudioStreamDevice(_stream.get());
-		SDL_FlushAudioStream(_stream.get());
-
 		const SDL_AudioSpec spec = {
 			static_cast<SDL_AudioFormat>((value.isF32() ? SDL_AUDIO_F32 :  SDL_AUDIO_S16)),
 			static_cast<int>(value.channels),
 			static_cast<int>(value.sample_rate)
 		};
 
-		_stream = {
+		SDL_SetAudioStreamFormat(_stream.get(), &spec, nullptr);
-			SDL_OpenAudioDeviceStream(device_id, &spec, nullptr, nullptr),
+
-			&SDL_DestroyAudioStream
+		std::cerr << "SDLAOD: audio format changed\n";
-		};
 	}
 
-	// HACK
+	if (value.isS16()) {
-	assert(value.isS16());
+		auto data = value.getSpan<int16_t>();
 
-	auto data = value.getSpan<int16_t>();
+		if (data.size == 0) {
+			std::cerr << "empty audio frame??\n";
+		}
 
-	if (data.size == 0) {
+		if (!SDL_PutAudioStreamData(_stream.get(), data.ptr, data.size * sizeof(int16_t))) {
-		std::cerr << "empty audio frame??\n";
+			std::cerr << "put data error\n";
-	}
+			return false; // return true?
+		}
+	} else if (value.isF32()) {
+		auto data = value.getSpan<float>();
 
-	if (!SDL_PutAudioStreamData(_stream.get(), data.ptr, data.size * sizeof(int16_t))) {
+		if (data.size == 0) {
-		std::cerr << "put data error\n";
+			std::cerr << "empty audio frame??\n";
-		return false; // return true?
+		}
+
+		if (!SDL_PutAudioStreamData(_stream.get(), data.ptr, data.size * sizeof(float))) {
+			std::cerr << "put data error\n";
+			return false; // return true?
+		}
 	}
 
 	_last_sample_rate = value.sample_rate;

src/debug_tox_call.cpp

@@ -2,6 +2,7 @@
 
 #include "./stream_manager.hpp"
 #include "./content/sdl_video_frame_stream2.hpp"
+#include "./content/sdl_audio_frame_stream2.hpp"
 
 #include <SDL3/SDL.h>
 
@@ -18,20 +19,6 @@ namespace Message {
 	uint64_t getTimeMS();
 }
 
-namespace Components {
-	struct ToxAVFriendAudioSource {
-	};
-
-	struct ToxAVFriendAudioSink {
-	};
-
-	struct ToxAVFriendVideoSource {
-	};
-
-	struct ToxAVFriendVideoSink {
-	};
-}
-
 static bool isFormatPlanar(SDL_PixelFormat f) {
 	return
 		f == SDL_PIXELFORMAT_YV12 ||
@@ -166,20 +153,34 @@ struct PushConversionQueuedVideoStream : public QueuedFrameStream2<SDLVideoFrame
 // exlusive
 // TODO: replace with something better than a queue
 struct ToxAVCallVideoSink : public FrameStream2SinkI<SDLVideoFrame> {
+	ToxAV& _toxav;
+
+	// bitrate for enabled state
+	uint32_t _video_bitrate {2};
+
 	uint32_t _fid;
 	std::shared_ptr<PushConversionQueuedVideoStream> _writer;
 
-	ToxAVCallVideoSink(uint32_t fid) : _fid(fid) {}
+	ToxAVCallVideoSink(ToxAV& toxav, uint32_t fid) : _toxav(toxav), _fid(fid) {}
-	~ToxAVCallVideoSink(void) {}
+	~ToxAVCallVideoSink(void) {
+		if (_writer) {
+			_writer = nullptr;
+			_toxav.toxavVideoSetBitRate(_fid, 0);
+		}
+	}
 
 	// sink
 	std::shared_ptr<FrameStream2I<SDLVideoFrame>> subscribe(void) override {
 		if (_writer) {
-			// max 1 (exclusive)
+			// max 1 (exclusive, composite video somewhere else)
+			return nullptr;
+		}
+
+		auto err = _toxav.toxavVideoSetBitRate(_fid, _video_bitrate);
+		if (err != TOXAV_ERR_BIT_RATE_SET_OK) {
 			return nullptr;
 		}
 
-		// TODO: enable video here
 		_writer = std::make_shared<PushConversionQueuedVideoStream>(10, true);
 
 		return _writer;
@@ -192,8 +193,11 @@ struct ToxAVCallVideoSink : public FrameStream2SinkI<SDLVideoFrame> {
 		}
 
 		if (sub == _writer) {
-			// TODO: disable video here
 			_writer = nullptr;
+
+			/*auto err = */_toxav.toxavVideoSetBitRate(_fid, 0);
+			// print warning? on error?
+
 			return true;
 		}
 
@@ -211,9 +215,28 @@ DebugToxCall::DebugToxCall(ObjectStore2& os, ToxAV& toxav, TextureUploaderI& tu)
 	_toxav.subscribe(this, ToxAV_Event::friend_video_frame);
 }
 
+DebugToxCall::~DebugToxCall(void) {
+	// destroy all calls/connections/sources/sinks here
+
+	for (auto& [fid, call] : _calls) {
+		if (static_cast<bool>(call.incoming_vsrc)) {
+			call.incoming_vsrc.destroy();
+		}
+		if (static_cast<bool>(call.incoming_asrc)) {
+			call.incoming_asrc.destroy();
+		}
+		if (static_cast<bool>(call.outgoing_vsink)) {
+			call.outgoing_vsink.destroy();
+		}
+		if (static_cast<bool>(call.outgoing_asink)) {
+			call.outgoing_asink.destroy();
+		}
+	}
+}
+
 void DebugToxCall::tick(float) {
-	// pump sink to tox
+	// pump sinks to tox
-	// TODO: own thread or direct on push
+	// TODO: own thread or direct on push (requires thread save toxcore)
 	// TODO: pump at double the frame rate
 	for (const auto& [oc, vsink] : _os.registry().view<ToxAVCallVideoSink*>().each()) {
 		if (!vsink->_writer) {
@@ -258,8 +281,9 @@ float DebugToxCall::render(void) {
 			if (call.incoming) {
 				ImGui::SameLine();
 				if (ImGui::SmallButton("answer")) {
+					const auto ret = _toxav.toxavAnswer(fid, 0, 0);
 					//const auto ret = _toxav.toxavAnswer(fid, 0, 1); // 1mbit/s
-					const auto ret = _toxav.toxavAnswer(fid, 0, 2); // 2mbit/s
+					//const auto ret = _toxav.toxavAnswer(fid, 0, 2); // 2mbit/s
 					//const auto ret = _toxav.toxavAnswer(fid, 0, 100); // 100mbit/s
 					//const auto ret = _toxav.toxavAnswer(fid, 0, 2500); // 2500mbit/s
 					if (ret == TOXAV_ERR_ANSWER_OK) {
@@ -268,7 +292,7 @@ float DebugToxCall::render(void) {
 						// create sinks
 						call.outgoing_vsink = {_os.registry(), _os.registry().create()};
 						{
-							auto new_vsink = std::make_unique<ToxAVCallVideoSink>(fid);
+							auto new_vsink = std::make_unique<ToxAVCallVideoSink>(_toxav, fid);
 							call.outgoing_vsink.emplace<ToxAVCallVideoSink*>(new_vsink.get());
 							call.outgoing_vsink.emplace<Components::FrameStream2Sink<SDLVideoFrame>>(std::move(new_vsink));
 							call.outgoing_vsink.emplace<Components::StreamSink>("ToxAV friend call video", std::string{entt::type_name<SDLVideoFrame>::value()});
@@ -284,6 +308,15 @@ float DebugToxCall::render(void) {
 								call.incoming_vsrc.emplace<Components::StreamSource>("ToxAV friend call video", std::string{entt::type_name<SDLVideoFrame>::value()});
 							}
 						}
+						if (call.incoming_a) {
+							call.incoming_asrc = {_os.registry(), _os.registry().create()};
+							{
+								auto new_asrc = std::make_unique<AudioFrameStream2MultiSource>();
+								call.incoming_asrc.emplace<AudioFrameStream2MultiSource*>(new_asrc.get());
+								call.incoming_asrc.emplace<Components::FrameStream2Source<AudioFrame>>(std::move(new_asrc));
+								call.incoming_asrc.emplace<Components::StreamSource>("ToxAV friend call audio", std::string{entt::type_name<AudioFrame>::value()});
+							}
+						}
 					}
 				}
 			} else if (call.state != TOXAV_FRIEND_CALL_STATE_FINISHED) {
@@ -297,11 +330,17 @@ float DebugToxCall::render(void) {
 						call.state = TOXAV_FRIEND_CALL_STATE_FINISHED;
 
 						// TODO: stream manager disconnectAll()
+						if (static_cast<bool>(call.incoming_vsrc)) {
+							call.incoming_vsrc.destroy();
+						}
+						if (static_cast<bool>(call.incoming_asrc)) {
+							call.incoming_asrc.destroy();
+						}
 						if (static_cast<bool>(call.outgoing_vsink)) {
 							call.outgoing_vsink.destroy();
 						}
-						if (static_cast<bool>(call.incoming_vsrc)) {
+						if (static_cast<bool>(call.outgoing_asink)) {
-							call.incoming_vsrc.destroy();
+							call.outgoing_asink.destroy();
 						}
 					}
 				}
@@ -341,11 +380,17 @@ bool DebugToxCall::onEvent(const Events::FriendCallState& e)  {
 		(call.state & TOXAV_FRIEND_CALL_STATE_FINISHED) != 0 ||
 		(call.state & TOXAV_FRIEND_CALL_STATE_ERROR) != 0
 	) {
+		if (static_cast<bool>(call.incoming_vsrc)) {
+			call.incoming_vsrc.destroy();
+		}
+		if (static_cast<bool>(call.incoming_asrc)) {
+			call.incoming_asrc.destroy();
+		}
 		if (static_cast<bool>(call.outgoing_vsink)) {
 			call.outgoing_vsink.destroy();
 		}
-		if (static_cast<bool>(call.incoming_vsrc)) {
+		if (static_cast<bool>(call.outgoing_asink)) {
-			call.incoming_vsrc.destroy();
+			call.outgoing_asink.destroy();
 		}
 	}
 
@@ -362,8 +407,25 @@ bool DebugToxCall::onEvent(const Events::FriendVideoBitrate&)  {
 
 bool DebugToxCall::onEvent(const Events::FriendAudioFrame& e)  {
 	auto& call = _calls[e.friend_number];
+
+	if (!static_cast<bool>(call.incoming_asrc)) {
+		// missing src to put frame into ??
+		return false;
+	}
+
+	assert(call.incoming_asrc.all_of<AudioFrameStream2MultiSource*>());
+	assert(call.incoming_asrc.all_of<Components::FrameStream2Source<AudioFrame>>());
+
 	call.num_a_frames++;
-	return false;
+
+	call.incoming_asrc.get<AudioFrameStream2MultiSource*>()->push(AudioFrame{
+		0, //seq
+		e.sampling_rate,
+		e.channels,
+		std::vector<int16_t>(e.pcm.begin(), e.pcm.end()) // copy
+	});
+
+	return true;
 }
 
 bool DebugToxCall::onEvent(const Events::FriendVideoFrame& e)  {

src/debug_tox_call.hpp

@@ -37,7 +37,7 @@ class DebugToxCall : public ToxAVEventI {
 
 	public:
 		DebugToxCall(ObjectStore2& os, ToxAV& toxav, TextureUploaderI& tu);
-		~DebugToxCall(void) {}
+		~DebugToxCall(void);
 
 		void tick(float time_delta);
 		float render(void);

src/stream_manager_ui.cpp

@@ -1,6 +1,7 @@
 #include "./stream_manager_ui.hpp"
 
 #include "./content/sdl_video_frame_stream2.hpp"
+#include "./content/audio_stream.hpp"
 
 #include <solanaceae/object_store/object_store.hpp>
 
@@ -58,6 +59,31 @@ void StreamManagerUI::render(void) {
 
 					ImGui::EndMenu();
 				}
+
+				if (ImGui::BeginMenu("connect audio", ss.frame_type_name == entt::type_name<AudioFrame>::value())) {
+					for (const auto& [oc_src, s_src] : _os.registry().view<Components::StreamSource>().each()) {
+						if (s_src.frame_type_name != ss.frame_type_name) {
+							continue;
+						}
+
+						ImGui::PushID(entt::to_integral(oc_src));
+
+						std::string source_label {"src "};
+						source_label += std::to_string(entt::to_integral(entt::to_entity(oc_src)));
+						source_label += " (";
+						source_label += s_src.name;
+						source_label += ")[";
+						source_label += s_src.frame_type_name;
+						source_label += "]";
+						if (ImGui::MenuItem(source_label.c_str())) {
+							_sm.connect<AudioFrame>(oc_src, oc);
+						}
+
+						ImGui::PopID();
+					}
+
+					ImGui::EndMenu();
+				}
 				ImGui::EndPopup();
 			}
 			ImGui::PopID();

src/tox_av.cpp

@@ -216,7 +216,7 @@ void ToxAV::cb_audio_receive_frame(uint32_t friend_number, const int16_t pcm[],
 		ToxAV_Event::friend_audio_frame,
 		Events::FriendAudioFrame{
 			friend_number,
-			Span<int16_t>(pcm, sample_count), // TODO: is sample count *ch or /ch?
+			Span<int16_t>(pcm, sample_count*channels), // TODO: is sample count *ch or /ch?
 			channels,
 			sampling_rate,
 		}