rewrite audio input, always open? but thin stream wrapper

src/content/audio_stream.hpp

@@ -67,3 +67,6 @@ struct AudioFrame {
 
 using AudioFrameStream2I = FrameStream2I<AudioFrame>;
 
+using AudioFrameStream2MultiSource = FrameStream2MultiSource<AudioFrame>;
+using AudioFrameStream2 = AudioFrameStream2MultiSource::sub_stream_type_t; // just use the default for now
+

src/content/sdl_audio_frame_stream2.cpp

@@ -1,83 +1,171 @@
 #include "./sdl_audio_frame_stream2.hpp"
 
 #include <iostream>
-#include <vector>
 
-SDLAudioInputDeviceDefault::SDLAudioInputDeviceDefault(void) : _stream{nullptr, &SDL_DestroyAudioStream} {
+// "thin" wrapper around sdl audio streams
-	constexpr SDL_AudioSpec spec = { SDL_AUDIO_S16, 1, 48000 };
+// we dont needs to get fance, as they already provide everything we need
+struct SDLAudioStreamReader : public AudioFrameStream2I {
+	std::unique_ptr<SDL_AudioStream, decltype(&SDL_DestroyAudioStream)> _stream;
 
-	_stream = {
+	uint32_t _seq_counter {0};
-		SDL_OpenAudioDeviceStream(SDL_AUDIO_DEVICE_DEFAULT_RECORDING, &spec, nullptr, nullptr),
-		&SDL_DestroyAudioStream
-	};
 
-	if (!static_cast<bool>(_stream)) {
+	uint32_t _sample_rate {48'000};
-		std::cerr << "SDL open audio device failed!\n";
+	size_t _channels {0};
+	SDL_AudioFormat _format {SDL_AUDIO_S16};
+
+	std::vector<int16_t> _buffer;
+
+	SDLAudioStreamReader(void) : _stream(nullptr, nullptr) {}
+	SDLAudioStreamReader(SDLAudioStreamReader&& other) :
+		_stream(std::move(other._stream)),
+		_sample_rate(other._sample_rate),
+		_channels(other._channels),
+		_format(other._format)
+	{
+		static const size_t buffer_size {960*_channels};
+		_buffer.resize(buffer_size);
 	}
 
-	const auto audio_device_id = SDL_GetAudioStreamDevice(_stream.get());
+	~SDLAudioStreamReader(void) {
-	SDL_ResumeAudioDevice(audio_device_id);
+		if (_stream) {
+			SDL_UnbindAudioStream(_stream.get());
+		}
+	}
 
-	static constexpr size_t buffer_size {512*2}; // in samples
+	int32_t size(void) override {
-	const auto interval_ms {(buffer_size * 1000) / spec.freq};
+		//assert(_stream);
+		// returns bytes
+		//SDL_GetAudioStreamAvailable(_stream.get());
+		return -1;
+	}
 
-	_thread = std::thread([this, interval_ms, spec](void) {
+	std::optional<AudioFrame> pop(void) override {
-		while (!_thread_should_quit) {
+		assert(_stream);
-			//static std::vector<int16_t> buffer(buffer_size);
+		assert(_format == SDL_AUDIO_S16);
-			static AudioFrame tmp_frame {
-				0, // TODO: seq
-				spec.freq, spec.channels,
-				std::vector<int16_t>(buffer_size)
-			};
 
-			auto& buffer = std::get<std::vector<int16_t>>(tmp_frame.buffer);
+		static const size_t buffer_size {960*_channels};
-			buffer.resize(buffer_size);
+		_buffer.resize(buffer_size); // noop?
 
 		const auto read_bytes = SDL_GetAudioStreamData(
 			_stream.get(),
-				buffer.data(),
+			_buffer.data(),
-				buffer.size()*sizeof(int16_t)
+			_buffer.size()*sizeof(int16_t)
 		);
-			//if (read_bytes != 0) {
-				//std::cerr << "read " << read_bytes << "/" << buffer.size()*sizeof(int16_t) << " audio bytes\n";
-			//}
 
 		// no new frame yet, or error
 		if (read_bytes <= 0) {
-				// only sleep 1/5, we expected a frame
+			return std::nullopt;
-				std::this_thread::sleep_for(std::chrono::milliseconds(int64_t(interval_ms/5)));
-				continue;
 		}
 
-			buffer.resize(read_bytes/sizeof(int16_t)); // this might be costly?
+		return AudioFrame {
-
+			_seq_counter++,
-			bool someone_listening {false};
+			_sample_rate, _channels,
-			someone_listening = push(tmp_frame);
+			Span<int16_t>(_buffer.data(), read_bytes/sizeof(int16_t)),
-
+		};
-			if (someone_listening) {
-				// double the interval on acquire
-				std::this_thread::sleep_for(std::chrono::milliseconds(int64_t(interval_ms/2)));
-			} else {
-				std::cerr << "i guess no one is listening\n";
-				std::cerr << "first value: " << buffer.front() << "\n";
-				// we just sleep 32x as long, bc no one is listening
-				// with the hardcoded settings, this is ~320ms
-				// TODO: just hardcode something like 500ms?
-				// TODO: suspend
-				// TODO: this is not gonna cut it, since playback slows down dramatically and will be very behind
-				std::this_thread::sleep_for(std::chrono::milliseconds(int64_t(interval_ms*32)));
-			}
-		}
-	});
 	}
 
-SDLAudioInputDeviceDefault::~SDLAudioInputDeviceDefault(void) {
+	bool push(const AudioFrame&) override {
-	// TODO: pause audio device?
+		// TODO: make universal sdl stream wrapper (combine with SDLAudioOutputDeviceDefaultInstance)
-	_thread_should_quit = true;
+		assert(false && "read only");
-	_thread.join();
+		return false;
-	// TODO: what to do if readers are still present?
+	}
+};
+
+SDLAudioInputDevice::SDLAudioInputDevice(void) : SDLAudioInputDevice(SDL_AUDIO_DEVICE_DEFAULT_RECORDING) {
 }
 
+SDLAudioInputDevice::SDLAudioInputDevice(SDL_AudioDeviceID device_id) {
+	// this spec is more like a hint to the hardware
+	SDL_AudioSpec spec {
+		SDL_AUDIO_S16,
+		1, // configurable?
+		48'000,
+	};
+	_device_id = SDL_OpenAudioDevice(device_id, &spec);
+
+	if (_device_id == 0) {
+		// TODO: proper error handling
+		throw int(1);
+	}
+}
+
+SDLAudioInputDevice::~SDLAudioInputDevice(void) {
+	_streams.clear();
+
+	SDL_CloseAudioDevice(_device_id);
+}
+
+std::shared_ptr<FrameStream2I<AudioFrame>> SDLAudioInputDevice::subscribe(void) {
+	SDL_AudioSpec spec {
+		SDL_AUDIO_S16,
+		1,
+		48'000,
+	};
+
+	SDL_AudioSpec device_spec {
+		SDL_AUDIO_S16,
+		1,
+		48'000,
+	};
+	// TODO: error check
+	SDL_GetAudioDeviceFormat(_device_id, &device_spec, nullptr);
+
+	// error check
+	auto* sdl_stream = SDL_CreateAudioStream(&device_spec, &spec);
+
+	// error check
+	SDL_BindAudioStream(_device_id, sdl_stream);
+
+	auto new_stream = std::make_shared<SDLAudioStreamReader>();
+	// TODO: move to ctr
+	new_stream->_stream = {sdl_stream, &SDL_DestroyAudioStream};
+	new_stream->_sample_rate = spec.freq;
+	new_stream->_channels = spec.channels;
+	new_stream->_format = spec.format;
+
+	_streams.emplace_back(new_stream);
+
+	return new_stream;
+}
+
+bool SDLAudioInputDevice::unsubscribe(const std::shared_ptr<FrameStream2I<AudioFrame>>& sub) {
+	for (auto it = _streams.cbegin(); it != _streams.cend(); it++) {
+		if (*it == sub) {
+			_streams.erase(it);
+			return true;
+		}
+	}
+
+	return false;
+}
+
+// does not need to be visible in the header
+struct SDLAudioOutputDeviceDefaultInstance : public AudioFrameStream2I {
+	std::unique_ptr<SDL_AudioStream, decltype(&SDL_DestroyAudioStream)> _stream;
+
+	uint32_t _last_sample_rate {48'000};
+	size_t _last_channels {0};
+	SDL_AudioFormat _last_format {SDL_AUDIO_S16};
+
+	// TODO: audio device
+	SDLAudioOutputDeviceDefaultInstance(void);
+	SDLAudioOutputDeviceDefaultInstance(SDLAudioOutputDeviceDefaultInstance&& other);
+
+	~SDLAudioOutputDeviceDefaultInstance(void);
+
+	int32_t size(void) override;
+	std::optional<AudioFrame> pop(void) override;
+	bool push(const AudioFrame& value) override;
+};
+
+SDLAudioOutputDeviceDefaultInstance::SDLAudioOutputDeviceDefaultInstance(void) : _stream(nullptr, nullptr) {
+}
+
+SDLAudioOutputDeviceDefaultInstance::SDLAudioOutputDeviceDefaultInstance(SDLAudioOutputDeviceDefaultInstance&& other) : _stream(std::move(other._stream)) {
+}
+
+SDLAudioOutputDeviceDefaultInstance::~SDLAudioOutputDeviceDefaultInstance(void) {
+}
 int32_t SDLAudioOutputDeviceDefaultInstance::size(void) {
 	return -1;
 }
@@ -144,15 +232,6 @@ bool SDLAudioOutputDeviceDefaultInstance::push(const AudioFrame& value) {
 	return true;
 }
 
-SDLAudioOutputDeviceDefaultInstance::SDLAudioOutputDeviceDefaultInstance(void) : _stream(nullptr, nullptr) {
-}
-
-SDLAudioOutputDeviceDefaultInstance::SDLAudioOutputDeviceDefaultInstance(SDLAudioOutputDeviceDefaultInstance&& other) : _stream(std::move(other._stream)) {
-}
-
-SDLAudioOutputDeviceDefaultInstance::~SDLAudioOutputDeviceDefaultInstance(void) {
-}
-
 
 SDLAudioOutputDeviceDefaultSink::~SDLAudioOutputDeviceDefaultSink(void) {
 	// TODO: pause and close device?

src/content/sdl_audio_frame_stream2.hpp

@@ -6,52 +6,30 @@
 #include <SDL3/SDL.h>
 
 #include <cstdint>
-#include <variant>
 #include <vector>
-#include <thread>
 
-// we dont have to multicast ourself, because sdl streams and virtual devices already do this, but we do it anyway
+// we dont have to multicast ourself, because sdl streams and virtual devices already do this
-using AudioFrameStream2MultiSource = FrameStream2MultiSource<AudioFrame>;
-using AudioFrameStream2 = AudioFrameStream2MultiSource::sub_stream_type_t; // just use the default for now
-
-// object components?
 
 // source
-struct SDLAudioInputDeviceDefault : public AudioFrameStream2MultiSource {
+// opens device
-	std::unique_ptr<SDL_AudioStream, decltype(&SDL_DestroyAudioStream)> _stream;
+// creates a sdl audio stream for each subscribed reader stream
+struct SDLAudioInputDevice : public FrameStream2SourceI<AudioFrame> {
+	// held by instances
+	using sdl_stream_type = std::unique_ptr<SDL_AudioStream, decltype(&SDL_DestroyAudioStream)>;
 
-	std::atomic<bool> _thread_should_quit {false};
+	SDL_AudioDeviceID _device_id {0};
-	std::thread _thread;
 
-	// construct source and start thread
+	std::vector<std::shared_ptr<FrameStream2I<AudioFrame>>> _streams;
-	// TODO: optimize so the thread is not always running
-	SDLAudioInputDeviceDefault(void);
 
-	// stops the thread and closes the device?
+	SDLAudioInputDevice(void);
-	~SDLAudioInputDeviceDefault(void);
+	SDLAudioInputDevice(SDL_AudioDeviceID device_id);
+	~SDLAudioInputDevice(void);
 
-	using AudioFrameStream2MultiSource::subscribe;
+	std::shared_ptr<FrameStream2I<AudioFrame>> subscribe(void) override;
-	using AudioFrameStream2MultiSource::unsubscribe;
+	bool unsubscribe(const std::shared_ptr<FrameStream2I<AudioFrame>>& sub) override;
 };
 
 // sink
-struct SDLAudioOutputDeviceDefaultInstance : public AudioFrameStream2I {
-	std::unique_ptr<SDL_AudioStream, decltype(&SDL_DestroyAudioStream)> _stream;
-
-	uint32_t _last_sample_rate {48'000};
-	size_t _last_channels {0};
-	SDL_AudioFormat _last_format {SDL_AUDIO_S16};
-
-	SDLAudioOutputDeviceDefaultInstance(void);
-	SDLAudioOutputDeviceDefaultInstance(SDLAudioOutputDeviceDefaultInstance&& other);
-
-	~SDLAudioOutputDeviceDefaultInstance(void);
-
-	int32_t size(void) override;
-	std::optional<AudioFrame> pop(void) override;
-	bool push(const AudioFrame& value) override;
-};
-
 // constructs entirely new streams, since sdl handles sync and mixing for us (or should)
 struct SDLAudioOutputDeviceDefaultSink : public FrameStream2SinkI<AudioFrame> {
 	// TODO: pause device?

src/debug_tox_call.cpp

@@ -1,8 +1,8 @@
 #include "./debug_tox_call.hpp"
 
 #include "./stream_manager.hpp"
+#include "./content/audio_stream.hpp"
 #include "./content/sdl_video_frame_stream2.hpp"
-#include "./content/sdl_audio_frame_stream2.hpp"
 
 #include <SDL3/SDL.h>
 
@@ -341,14 +341,25 @@ void DebugToxCall::tick(float) {
 			const auto& new_frame = new_frame_opt.value();
 			assert(new_frame.isS16());
 
-			// TODO: error code
+//* @param sample_count Number of samples in this frame. Valid numbers here are
-			_toxav.toxavAudioSendFrame(
+//*   `((sample rate) * (audio length) / 1000)`, where audio length can be
+//*   2.5, 5, 10, 20, 40 or 60 milliseconds.
+
+			// we likely needs to subdivide/repackage
+			// frame size should be an option exposed to the user
+			// with 10ms as a default ?
+			// the larger the frame size, the less overhead but the more delay
+
+			auto err = _toxav.toxavAudioSendFrame(
 				asink->_fid,
 				new_frame.getSpan<int16_t>().ptr,
 				new_frame.getSpan<int16_t>().size / new_frame.channels,
 				new_frame.channels,
 				new_frame.sample_rate
 			);
+			if (err != TOXAV_ERR_SEND_FRAME_OK) {
+				std::cerr << "DTC: failed to send audio frame " << err << "\n";
+			}
 		}
 	}
 }

src/main.cpp

@@ -79,8 +79,8 @@ int main(int argc, char** argv) {
 	if (!SDL_Init(SDL_INIT_AUDIO)) {
 		std::cerr << "SDL_Init AUDIO failed (" << SDL_GetError() << ")\n";
 	} else if (false) {
-		SDLAudioInputDeviceDefault aidd;
+		SDLAudioInputDevice aid;
-		auto reader = aidd.subscribe();
+		auto reader = aid.subscribe();
 
 		auto writer = SDLAudioOutputDeviceDefaultSink{}.subscribe();
 
@@ -95,7 +95,7 @@ int main(int argc, char** argv) {
 			}
 		}
 
-		aidd.unsubscribe(reader);
+		aid.unsubscribe(reader);
 	}
 
 	if (!SDL_Init(SDL_INIT_CAMERA)) {

src/main_screen.cpp

@@ -162,11 +162,11 @@ MainScreen::MainScreen(SimpleConfigModel&& conf_, SDL_Renderer* renderer_, Theme
 			}
 		}
 
-		if (false) { // audio in
+		if (true) { // audio in
 			ObjectHandle asrc {os.registry(), os.registry().create()};
 			try {
 				asrc.emplace<Components::FrameStream2Source<AudioFrame>>(
-					std::make_unique<SDLAudioInputDeviceDefault>()
+					std::make_unique<SDLAudioInputDevice>()
 				);
 
 				asrc.emplace<Components::StreamSource>(Components::StreamSource::create<AudioFrame>("SDL Audio Default Recording Device"));