From 5b3e0e2a0b047232be0d36c378861fc0b778d005 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Wed, 25 Sep 2024 14:36:51 +0200 Subject: [PATCH] rewrite audio input, always open? but thin stream wrapper --- src/content/audio_stream.hpp | 3 + src/content/sdl_audio_frame_stream2.cpp | 231 ++++++++++++++++-------- src/content/sdl_audio_frame_stream2.hpp | 48 ++--- src/debug_tox_call.cpp | 17 +- src/main.cpp | 6 +- src/main_screen.cpp | 4 +- 6 files changed, 190 insertions(+), 119 deletions(-) diff --git a/src/content/audio_stream.hpp b/src/content/audio_stream.hpp index 9098a3c..82d3f89 100644 --- a/src/content/audio_stream.hpp +++ b/src/content/audio_stream.hpp @@ -67,3 +67,6 @@ struct AudioFrame { using AudioFrameStream2I = FrameStream2I; +using AudioFrameStream2MultiSource = FrameStream2MultiSource; +using AudioFrameStream2 = AudioFrameStream2MultiSource::sub_stream_type_t; // just use the default for now + diff --git a/src/content/sdl_audio_frame_stream2.cpp b/src/content/sdl_audio_frame_stream2.cpp index a04a5f3..31b4378 100644 --- a/src/content/sdl_audio_frame_stream2.cpp +++ b/src/content/sdl_audio_frame_stream2.cpp @@ -1,83 +1,171 @@ #include "./sdl_audio_frame_stream2.hpp" #include -#include -SDLAudioInputDeviceDefault::SDLAudioInputDeviceDefault(void) : _stream{nullptr, &SDL_DestroyAudioStream} { - constexpr SDL_AudioSpec spec = { SDL_AUDIO_S16, 1, 48000 }; +// "thin" wrapper around sdl audio streams +// we dont needs to get fance, as they already provide everything we need +struct SDLAudioStreamReader : public AudioFrameStream2I { + std::unique_ptr _stream; - _stream = { - SDL_OpenAudioDeviceStream(SDL_AUDIO_DEVICE_DEFAULT_RECORDING, &spec, nullptr, nullptr), - &SDL_DestroyAudioStream - }; + uint32_t _seq_counter {0}; - if (!static_cast(_stream)) { - std::cerr << "SDL open audio device failed!\n"; + uint32_t _sample_rate {48'000}; + size_t _channels {0}; + SDL_AudioFormat _format {SDL_AUDIO_S16}; + + std::vector _buffer; + + SDLAudioStreamReader(void) : _stream(nullptr, nullptr) {} + SDLAudioStreamReader(SDLAudioStreamReader&& other) : + _stream(std::move(other._stream)), + _sample_rate(other._sample_rate), + _channels(other._channels), + _format(other._format) + { + static const size_t buffer_size {960*_channels}; + _buffer.resize(buffer_size); } - const auto audio_device_id = SDL_GetAudioStreamDevice(_stream.get()); - SDL_ResumeAudioDevice(audio_device_id); - - static constexpr size_t buffer_size {512*2}; // in samples - const auto interval_ms {(buffer_size * 1000) / spec.freq}; - - _thread = std::thread([this, interval_ms, spec](void) { - while (!_thread_should_quit) { - //static std::vector buffer(buffer_size); - static AudioFrame tmp_frame { - 0, // TODO: seq - spec.freq, spec.channels, - std::vector(buffer_size) - }; - - auto& buffer = std::get>(tmp_frame.buffer); - buffer.resize(buffer_size); - - const auto read_bytes = SDL_GetAudioStreamData( - _stream.get(), - buffer.data(), - buffer.size()*sizeof(int16_t) - ); - //if (read_bytes != 0) { - //std::cerr << "read " << read_bytes << "/" << buffer.size()*sizeof(int16_t) << " audio bytes\n"; - //} - - // no new frame yet, or error - if (read_bytes <= 0) { - // only sleep 1/5, we expected a frame - std::this_thread::sleep_for(std::chrono::milliseconds(int64_t(interval_ms/5))); - continue; - } - - buffer.resize(read_bytes/sizeof(int16_t)); // this might be costly? - - bool someone_listening {false}; - someone_listening = push(tmp_frame); - - if (someone_listening) { - // double the interval on acquire - std::this_thread::sleep_for(std::chrono::milliseconds(int64_t(interval_ms/2))); - } else { - std::cerr << "i guess no one is listening\n"; - std::cerr << "first value: " << buffer.front() << "\n"; - // we just sleep 32x as long, bc no one is listening - // with the hardcoded settings, this is ~320ms - // TODO: just hardcode something like 500ms? - // TODO: suspend - // TODO: this is not gonna cut it, since playback slows down dramatically and will be very behind - std::this_thread::sleep_for(std::chrono::milliseconds(int64_t(interval_ms*32))); - } + ~SDLAudioStreamReader(void) { + if (_stream) { + SDL_UnbindAudioStream(_stream.get()); } - }); + } + + int32_t size(void) override { + //assert(_stream); + // returns bytes + //SDL_GetAudioStreamAvailable(_stream.get()); + return -1; + } + + std::optional pop(void) override { + assert(_stream); + assert(_format == SDL_AUDIO_S16); + + static const size_t buffer_size {960*_channels}; + _buffer.resize(buffer_size); // noop? + + const auto read_bytes = SDL_GetAudioStreamData( + _stream.get(), + _buffer.data(), + _buffer.size()*sizeof(int16_t) + ); + + // no new frame yet, or error + if (read_bytes <= 0) { + return std::nullopt; + } + + return AudioFrame { + _seq_counter++, + _sample_rate, _channels, + Span(_buffer.data(), read_bytes/sizeof(int16_t)), + }; + } + + bool push(const AudioFrame&) override { + // TODO: make universal sdl stream wrapper (combine with SDLAudioOutputDeviceDefaultInstance) + assert(false && "read only"); + return false; + } +}; + +SDLAudioInputDevice::SDLAudioInputDevice(void) : SDLAudioInputDevice(SDL_AUDIO_DEVICE_DEFAULT_RECORDING) { } -SDLAudioInputDeviceDefault::~SDLAudioInputDeviceDefault(void) { - // TODO: pause audio device? - _thread_should_quit = true; - _thread.join(); - // TODO: what to do if readers are still present? +SDLAudioInputDevice::SDLAudioInputDevice(SDL_AudioDeviceID device_id) { + // this spec is more like a hint to the hardware + SDL_AudioSpec spec { + SDL_AUDIO_S16, + 1, // configurable? + 48'000, + }; + _device_id = SDL_OpenAudioDevice(device_id, &spec); + + if (_device_id == 0) { + // TODO: proper error handling + throw int(1); + } } +SDLAudioInputDevice::~SDLAudioInputDevice(void) { + _streams.clear(); + + SDL_CloseAudioDevice(_device_id); +} + +std::shared_ptr> SDLAudioInputDevice::subscribe(void) { + SDL_AudioSpec spec { + SDL_AUDIO_S16, + 1, + 48'000, + }; + + SDL_AudioSpec device_spec { + SDL_AUDIO_S16, + 1, + 48'000, + }; + // TODO: error check + SDL_GetAudioDeviceFormat(_device_id, &device_spec, nullptr); + + // error check + auto* sdl_stream = SDL_CreateAudioStream(&device_spec, &spec); + + // error check + SDL_BindAudioStream(_device_id, sdl_stream); + + auto new_stream = std::make_shared(); + // TODO: move to ctr + new_stream->_stream = {sdl_stream, &SDL_DestroyAudioStream}; + new_stream->_sample_rate = spec.freq; + new_stream->_channels = spec.channels; + new_stream->_format = spec.format; + + _streams.emplace_back(new_stream); + + return new_stream; +} + +bool SDLAudioInputDevice::unsubscribe(const std::shared_ptr>& sub) { + for (auto it = _streams.cbegin(); it != _streams.cend(); it++) { + if (*it == sub) { + _streams.erase(it); + return true; + } + } + + return false; +} + +// does not need to be visible in the header +struct SDLAudioOutputDeviceDefaultInstance : public AudioFrameStream2I { + std::unique_ptr _stream; + + uint32_t _last_sample_rate {48'000}; + size_t _last_channels {0}; + SDL_AudioFormat _last_format {SDL_AUDIO_S16}; + + // TODO: audio device + SDLAudioOutputDeviceDefaultInstance(void); + SDLAudioOutputDeviceDefaultInstance(SDLAudioOutputDeviceDefaultInstance&& other); + + ~SDLAudioOutputDeviceDefaultInstance(void); + + int32_t size(void) override; + std::optional pop(void) override; + bool push(const AudioFrame& value) override; +}; + +SDLAudioOutputDeviceDefaultInstance::SDLAudioOutputDeviceDefaultInstance(void) : _stream(nullptr, nullptr) { +} + +SDLAudioOutputDeviceDefaultInstance::SDLAudioOutputDeviceDefaultInstance(SDLAudioOutputDeviceDefaultInstance&& other) : _stream(std::move(other._stream)) { +} + +SDLAudioOutputDeviceDefaultInstance::~SDLAudioOutputDeviceDefaultInstance(void) { +} int32_t SDLAudioOutputDeviceDefaultInstance::size(void) { return -1; } @@ -144,15 +232,6 @@ bool SDLAudioOutputDeviceDefaultInstance::push(const AudioFrame& value) { return true; } -SDLAudioOutputDeviceDefaultInstance::SDLAudioOutputDeviceDefaultInstance(void) : _stream(nullptr, nullptr) { -} - -SDLAudioOutputDeviceDefaultInstance::SDLAudioOutputDeviceDefaultInstance(SDLAudioOutputDeviceDefaultInstance&& other) : _stream(std::move(other._stream)) { -} - -SDLAudioOutputDeviceDefaultInstance::~SDLAudioOutputDeviceDefaultInstance(void) { -} - SDLAudioOutputDeviceDefaultSink::~SDLAudioOutputDeviceDefaultSink(void) { // TODO: pause and close device? diff --git a/src/content/sdl_audio_frame_stream2.hpp b/src/content/sdl_audio_frame_stream2.hpp index a506346..0b6c2bb 100644 --- a/src/content/sdl_audio_frame_stream2.hpp +++ b/src/content/sdl_audio_frame_stream2.hpp @@ -6,52 +6,30 @@ #include #include -#include #include -#include -// we dont have to multicast ourself, because sdl streams and virtual devices already do this, but we do it anyway -using AudioFrameStream2MultiSource = FrameStream2MultiSource; -using AudioFrameStream2 = AudioFrameStream2MultiSource::sub_stream_type_t; // just use the default for now - -// object components? +// we dont have to multicast ourself, because sdl streams and virtual devices already do this // source -struct SDLAudioInputDeviceDefault : public AudioFrameStream2MultiSource { - std::unique_ptr _stream; +// opens device +// creates a sdl audio stream for each subscribed reader stream +struct SDLAudioInputDevice : public FrameStream2SourceI { + // held by instances + using sdl_stream_type = std::unique_ptr; - std::atomic _thread_should_quit {false}; - std::thread _thread; + SDL_AudioDeviceID _device_id {0}; - // construct source and start thread - // TODO: optimize so the thread is not always running - SDLAudioInputDeviceDefault(void); + std::vector>> _streams; - // stops the thread and closes the device? - ~SDLAudioInputDeviceDefault(void); + SDLAudioInputDevice(void); + SDLAudioInputDevice(SDL_AudioDeviceID device_id); + ~SDLAudioInputDevice(void); - using AudioFrameStream2MultiSource::subscribe; - using AudioFrameStream2MultiSource::unsubscribe; + std::shared_ptr> subscribe(void) override; + bool unsubscribe(const std::shared_ptr>& sub) override; }; // sink -struct SDLAudioOutputDeviceDefaultInstance : public AudioFrameStream2I { - std::unique_ptr _stream; - - uint32_t _last_sample_rate {48'000}; - size_t _last_channels {0}; - SDL_AudioFormat _last_format {SDL_AUDIO_S16}; - - SDLAudioOutputDeviceDefaultInstance(void); - SDLAudioOutputDeviceDefaultInstance(SDLAudioOutputDeviceDefaultInstance&& other); - - ~SDLAudioOutputDeviceDefaultInstance(void); - - int32_t size(void) override; - std::optional pop(void) override; - bool push(const AudioFrame& value) override; -}; - // constructs entirely new streams, since sdl handles sync and mixing for us (or should) struct SDLAudioOutputDeviceDefaultSink : public FrameStream2SinkI { // TODO: pause device? diff --git a/src/debug_tox_call.cpp b/src/debug_tox_call.cpp index c8ba558..c6b8a38 100644 --- a/src/debug_tox_call.cpp +++ b/src/debug_tox_call.cpp @@ -1,8 +1,8 @@ #include "./debug_tox_call.hpp" #include "./stream_manager.hpp" +#include "./content/audio_stream.hpp" #include "./content/sdl_video_frame_stream2.hpp" -#include "./content/sdl_audio_frame_stream2.hpp" #include @@ -341,14 +341,25 @@ void DebugToxCall::tick(float) { const auto& new_frame = new_frame_opt.value(); assert(new_frame.isS16()); - // TODO: error code - _toxav.toxavAudioSendFrame( +//* @param sample_count Number of samples in this frame. Valid numbers here are +//* `((sample rate) * (audio length) / 1000)`, where audio length can be +//* 2.5, 5, 10, 20, 40 or 60 milliseconds. + + // we likely needs to subdivide/repackage + // frame size should be an option exposed to the user + // with 10ms as a default ? + // the larger the frame size, the less overhead but the more delay + + auto err = _toxav.toxavAudioSendFrame( asink->_fid, new_frame.getSpan().ptr, new_frame.getSpan().size / new_frame.channels, new_frame.channels, new_frame.sample_rate ); + if (err != TOXAV_ERR_SEND_FRAME_OK) { + std::cerr << "DTC: failed to send audio frame " << err << "\n"; + } } } } diff --git a/src/main.cpp b/src/main.cpp index 29c54a8..988fa92 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -79,8 +79,8 @@ int main(int argc, char** argv) { if (!SDL_Init(SDL_INIT_AUDIO)) { std::cerr << "SDL_Init AUDIO failed (" << SDL_GetError() << ")\n"; } else if (false) { - SDLAudioInputDeviceDefault aidd; - auto reader = aidd.subscribe(); + SDLAudioInputDevice aid; + auto reader = aid.subscribe(); auto writer = SDLAudioOutputDeviceDefaultSink{}.subscribe(); @@ -95,7 +95,7 @@ int main(int argc, char** argv) { } } - aidd.unsubscribe(reader); + aid.unsubscribe(reader); } if (!SDL_Init(SDL_INIT_CAMERA)) { diff --git a/src/main_screen.cpp b/src/main_screen.cpp index b468f22..d848df1 100644 --- a/src/main_screen.cpp +++ b/src/main_screen.cpp @@ -162,11 +162,11 @@ MainScreen::MainScreen(SimpleConfigModel&& conf_, SDL_Renderer* renderer_, Theme } } - if (false) { // audio in + if (true) { // audio in ObjectHandle asrc {os.registry(), os.registry().create()}; try { asrc.emplace>( - std::make_unique() + std::make_unique() ); asrc.emplace(Components::StreamSource::create("SDL Audio Default Recording Device"));