From 803bce93fb43c178950e59477204621a93002d69 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Thu, 2 May 2024 12:10:31 +0200 Subject: [PATCH] frame refactoring, add audio frame and sdl default device input --- src/CMakeLists.txt | 8 +-- src/content/audio_stream.hpp | 69 +++++++++++++++++++++ src/content/frame_stream2.hpp | 10 ++-- src/content/sdl_audio_frame_stream2.cpp | 79 +++++++++++++++++++++++++ src/content/sdl_audio_frame_stream2.hpp | 37 ++++++++++++ src/content/sdl_video_frame_stream2.hpp | 2 +- src/main.cpp | 17 ++++++ 7 files changed, 211 insertions(+), 11 deletions(-) create mode 100644 src/content/audio_stream.hpp create mode 100644 src/content/sdl_audio_frame_stream2.cpp create mode 100644 src/content/sdl_audio_frame_stream2.hpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6d34b46d..4d986c84 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -80,14 +80,12 @@ add_executable(tomato ./chat_gui4.cpp ./content/content.hpp - #./content/stream_reader.hpp - #./content/stream_reader_sdl_audio.hpp - #./content/stream_reader_sdl_audio.cpp - #./content/stream_reader_sdl_video.hpp - #./content/stream_reader_sdl_video.cpp ./content/frame_stream2.hpp ./content/sdl_video_frame_stream2.hpp ./content/sdl_video_frame_stream2.cpp + ./content/audio_stream.hpp + ./content/sdl_audio_frame_stream2.hpp + ./content/sdl_audio_frame_stream2.cpp ) target_compile_features(tomato PUBLIC cxx_std_17) diff --git a/src/content/audio_stream.hpp b/src/content/audio_stream.hpp new file mode 100644 index 00000000..9098a3c5 --- /dev/null +++ b/src/content/audio_stream.hpp @@ -0,0 +1,69 @@ +#pragma once + +#include "./frame_stream2.hpp" + +#include + +#include +#include +#include + +// raw audio +// channels make samples interleaved, +// planar channels are not supported +struct AudioFrame { + // sequence number, to detect gaps + uint32_t seq {0}; + // TODO: maybe use ts instead to discard old? + // since buffer size is variable, some timestamp would be needed to estimate the lost time + + // samples per second + uint32_t sample_rate {48'000}; + + size_t channels {0}; + std::variant< + std::vector, // S16, platform endianess + Span, // non owning variant, for direct consumption + + std::vector, // f32 + Span // non owning variant, for direct consumption + > buffer; + + // helpers + + bool isS16(void) const { + return + std::holds_alternative>(buffer) || + std::holds_alternative>(buffer) + ; + } + bool isF32(void) const { + return + std::holds_alternative>(buffer) || + std::holds_alternative>(buffer) + ; + } + template + Span getSpan(void) const { + static_assert(std::is_same_v || std::is_same_v); + if constexpr (std::is_same_v) { + assert(isS16()); + if (std::holds_alternative>(buffer)) { + return Span{std::get>(buffer)}; + } else { + return std::get>(buffer); + } + } else if constexpr (std::is_same_v) { + assert(isF32()); + if (std::holds_alternative>(buffer)) { + return Span{std::get>(buffer)}; + } else { + return std::get>(buffer); + } + } + return {}; + } +}; + +using AudioFrameStream2I = FrameStream2I; + diff --git a/src/content/frame_stream2.hpp b/src/content/frame_stream2.hpp index 85c55231..c6fa3b26 100644 --- a/src/content/frame_stream2.hpp +++ b/src/content/frame_stream2.hpp @@ -18,7 +18,7 @@ //}; template -struct FrameStream2 { +struct FrameStream2I { // get number of available frames [[nodiscard]] virtual int32_t size(void) = 0; @@ -34,7 +34,7 @@ struct FrameStream2 { // needs count frames queue size // having ~1-2sec buffer size is often sufficent template -struct QueuedFrameStream2 : public FrameStream2 { +struct QueuedFrameStream2 : public FrameStream2I { using frame_type = FrameType; rigtorp::SPSCQueue _queue; @@ -74,9 +74,9 @@ struct QueuedFrameStream2 : public FrameStream2 { } }; -template -struct QueuedFrameStream2Multiplexer : public FrameStream2 { - using ReaderType = QueuedFrameStream2; +template> +struct QueuedFrameStream2Multiplexer : public FrameStream2I { + using reader_type_t = ReaderType; // pointer stability std::vector> _readers; diff --git a/src/content/sdl_audio_frame_stream2.cpp b/src/content/sdl_audio_frame_stream2.cpp new file mode 100644 index 00000000..d79b003e --- /dev/null +++ b/src/content/sdl_audio_frame_stream2.cpp @@ -0,0 +1,79 @@ +#include "./sdl_audio_frame_stream2.hpp" +#include "SDL_audio.h" + +#include +#include + +SDLAudioInputDevice::SDLAudioInputDevice(void) : _stream{nullptr, &SDL_DestroyAudioStream} { + constexpr SDL_AudioSpec spec = { SDL_AUDIO_S16, 1, 48000 }; + + _stream = { + SDL_OpenAudioDeviceStream(SDL_AUDIO_DEVICE_DEFAULT_CAPTURE, &spec, nullptr, nullptr), + &SDL_DestroyAudioStream + }; + + if (!static_cast(_stream)) { + std::cerr << "SDL open audio device failed!\n"; + } + + const auto audio_device_id = SDL_GetAudioStreamDevice(_stream.get()); + SDL_ResumeAudioDevice(audio_device_id); + + static constexpr size_t buffer_size {512}; // in samples + const auto interval_ms {buffer_size/(spec.freq * 1000)}; + + _thread = std::thread([this, interval_ms, spec](void) { + while (!_thread_should_quit) { + //static std::vector buffer(buffer_size); + static AudioFrame tmp_frame { + 0, // TODO: seq + spec.freq, spec.channels, + std::vector(buffer_size) + }; + + auto& buffer = std::get>(tmp_frame.buffer); + buffer.resize(buffer_size); + + const auto read_bytes = SDL_GetAudioStreamData( + _stream.get(), + buffer.data(), + buffer.size()*sizeof(int16_t) + ); + //if (read_bytes != 0) { + //std::cerr << "read " << read_bytes << "/" << buffer.size()*sizeof(int16_t) << " audio bytes\n"; + //} + + // no new frame yet, or error + if (read_bytes <= 0) { + // only sleep 1/5, we expected a frame + std::this_thread::sleep_for(std::chrono::milliseconds(int64_t(interval_ms/5))); + continue; + } + + buffer.resize(read_bytes/sizeof(int16_t)); // this might be costly? + + bool someone_listening {false}; + someone_listening = push(tmp_frame); + + if (someone_listening) { + // double the interval on acquire + std::this_thread::sleep_for(std::chrono::milliseconds(int64_t(interval_ms/2))); + } else { + std::cerr << "i guess no one is listening\n"; + // we just sleep 32x as long, bc no one is listening + // with the hardcoded settings, this is ~320ms + // TODO: just hardcode something like 500ms? + // TODO: suspend + std::this_thread::sleep_for(std::chrono::milliseconds(int64_t(interval_ms*32))); + } + } + }); +} + +SDLAudioInputDevice::~SDLAudioInputDevice(void) { + // TODO: pause audio device? + _thread_should_quit = true; + _thread.join(); + // TODO: what to do if readers are still present? +} + diff --git a/src/content/sdl_audio_frame_stream2.hpp b/src/content/sdl_audio_frame_stream2.hpp new file mode 100644 index 00000000..d27d8293 --- /dev/null +++ b/src/content/sdl_audio_frame_stream2.hpp @@ -0,0 +1,37 @@ +#pragma once + +#include "./frame_stream2.hpp" +#include "./audio_stream.hpp" + +#include + +#include +#include +#include +#include + +// we dont have to multiplex ourself, because sdl streams and virtual devices already do this, but we do it anyway +using SDLAudioInputFrameStream2Multiplexer = QueuedFrameStream2Multiplexer; +using SDLAudioInputFrameStream2 = SDLAudioInputFrameStream2Multiplexer::reader_type_t; // just use the default for now + +// object components? +struct SDLAudioInputDevice : protected SDLAudioInputFrameStream2Multiplexer { + std::unique_ptr _stream; + + std::atomic _thread_should_quit {false}; + std::thread _thread; + + // construct source and start thread + // TODO: optimize so the thread is not always running + SDLAudioInputDevice(void); + + // stops the thread and closes the device? + ~SDLAudioInputDevice(void); + + using SDLAudioInputFrameStream2Multiplexer::aquireReader; + using SDLAudioInputFrameStream2Multiplexer::releaseReader; +}; + +struct SDLAudioOutputDevice { +}; + diff --git a/src/content/sdl_video_frame_stream2.hpp b/src/content/sdl_video_frame_stream2.hpp index 56a62e57..64480367 100644 --- a/src/content/sdl_video_frame_stream2.hpp +++ b/src/content/sdl_video_frame_stream2.hpp @@ -43,7 +43,7 @@ struct SDLVideoFrame { }; using SDLVideoFrameStream2Multiplexer = QueuedFrameStream2Multiplexer; -using SDLVideoFrameStream2 = SDLVideoFrameStream2Multiplexer::ReaderType; +using SDLVideoFrameStream2 = SDLVideoFrameStream2Multiplexer::reader_type_t; // just use the default for now struct SDLVideoCameraContent : protected SDLVideoFrameStream2Multiplexer { // meh, empty default diff --git a/src/main.cpp b/src/main.cpp index bb04459c..bcf6c397 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -9,6 +9,7 @@ #include "./chat_gui/theme.hpp" #include "./start_screen.hpp" +#include "./content/sdl_audio_frame_stream2.hpp" #include "./content/sdl_video_frame_stream2.hpp" #include @@ -62,7 +63,23 @@ int main(int argc, char** argv) { // optionally init audio and camera if (SDL_Init(SDL_INIT_AUDIO) < 0) { std::cerr << "SDL_Init AUDIO failed (" << SDL_GetError() << ")\n"; + } else { + SDLAudioInputDevice aid; + auto* reader = aid.aquireReader(); + + for (size_t i = 0; i < 20; i++) { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + auto new_frame_opt = reader->pop(); + if (new_frame_opt.has_value()) { + std::cout << "audio frame was seq:" << new_frame_opt.value().seq << " sr:" << new_frame_opt.value().sample_rate << " " << (new_frame_opt.value().isS16()?"S16":"F32") << " l:" << (new_frame_opt.value().isS16()?new_frame_opt.value().getSpan().size:new_frame_opt.value().getSpan().size) << "\n"; + } else { + std::cout << "no audio frame\n"; + } + } + + aid.releaseReader(reader); } + if (SDL_Init(SDL_INIT_CAMERA) < 0) { std::cerr << "SDL_Init CAMERA failed (" << SDL_GetError() << ")\n"; } else { // HACK