frame refactoring, add audio frame and sdl default device input

This commit is contained in:
Green Sky 2024-05-02 12:10:31 +02:00
parent 31905b5468
commit 803bce93fb
No known key found for this signature in database
7 changed files with 211 additions and 11 deletions

View File

@ -80,14 +80,12 @@ add_executable(tomato
./chat_gui4.cpp ./chat_gui4.cpp
./content/content.hpp ./content/content.hpp
#./content/stream_reader.hpp
#./content/stream_reader_sdl_audio.hpp
#./content/stream_reader_sdl_audio.cpp
#./content/stream_reader_sdl_video.hpp
#./content/stream_reader_sdl_video.cpp
./content/frame_stream2.hpp ./content/frame_stream2.hpp
./content/sdl_video_frame_stream2.hpp ./content/sdl_video_frame_stream2.hpp
./content/sdl_video_frame_stream2.cpp ./content/sdl_video_frame_stream2.cpp
./content/audio_stream.hpp
./content/sdl_audio_frame_stream2.hpp
./content/sdl_audio_frame_stream2.cpp
) )
target_compile_features(tomato PUBLIC cxx_std_17) target_compile_features(tomato PUBLIC cxx_std_17)

View File

@ -0,0 +1,69 @@
#pragma once
#include "./frame_stream2.hpp"
#include <solanaceae/util/span.hpp>
#include <cstdint>
#include <variant>
#include <vector>
// raw audio
// channels make samples interleaved,
// planar channels are not supported
struct AudioFrame {
// sequence number, to detect gaps
uint32_t seq {0};
// TODO: maybe use ts instead to discard old?
// since buffer size is variable, some timestamp would be needed to estimate the lost time
// samples per second
uint32_t sample_rate {48'000};
size_t channels {0};
std::variant<
std::vector<int16_t>, // S16, platform endianess
Span<int16_t>, // non owning variant, for direct consumption
std::vector<float>, // f32
Span<float> // non owning variant, for direct consumption
> buffer;
// helpers
bool isS16(void) const {
return
std::holds_alternative<std::vector<int16_t>>(buffer) ||
std::holds_alternative<Span<int16_t>>(buffer)
;
}
bool isF32(void) const {
return
std::holds_alternative<std::vector<float>>(buffer) ||
std::holds_alternative<Span<float>>(buffer)
;
}
template<typename T>
Span<T> getSpan(void) const {
static_assert(std::is_same_v<int16_t, T> || std::is_same_v<float, T>);
if constexpr (std::is_same_v<int16_t, T>) {
assert(isS16());
if (std::holds_alternative<std::vector<int16_t>>(buffer)) {
return Span<int16_t>{std::get<std::vector<int16_t>>(buffer)};
} else {
return std::get<Span<int16_t>>(buffer);
}
} else if constexpr (std::is_same_v<float, T>) {
assert(isF32());
if (std::holds_alternative<std::vector<float>>(buffer)) {
return Span<float>{std::get<std::vector<float>>(buffer)};
} else {
return std::get<Span<float>>(buffer);
}
}
return {};
}
};
using AudioFrameStream2I = FrameStream2I<AudioFrame>;

View File

@ -18,7 +18,7 @@
//}; //};
template<typename FrameType> template<typename FrameType>
struct FrameStream2 { struct FrameStream2I {
// get number of available frames // get number of available frames
[[nodiscard]] virtual int32_t size(void) = 0; [[nodiscard]] virtual int32_t size(void) = 0;
@ -34,7 +34,7 @@ struct FrameStream2 {
// needs count frames queue size // needs count frames queue size
// having ~1-2sec buffer size is often sufficent // having ~1-2sec buffer size is often sufficent
template<typename FrameType> template<typename FrameType>
struct QueuedFrameStream2 : public FrameStream2<FrameType> { struct QueuedFrameStream2 : public FrameStream2I<FrameType> {
using frame_type = FrameType; using frame_type = FrameType;
rigtorp::SPSCQueue<FrameType> _queue; rigtorp::SPSCQueue<FrameType> _queue;
@ -74,9 +74,9 @@ struct QueuedFrameStream2 : public FrameStream2<FrameType> {
} }
}; };
template<typename FrameType> template<typename FrameType, typename ReaderType = QueuedFrameStream2<FrameType>>
struct QueuedFrameStream2Multiplexer : public FrameStream2<FrameType> { struct QueuedFrameStream2Multiplexer : public FrameStream2I<FrameType> {
using ReaderType = QueuedFrameStream2<FrameType>; using reader_type_t = ReaderType;
// pointer stability // pointer stability
std::vector<std::unique_ptr<ReaderType>> _readers; std::vector<std::unique_ptr<ReaderType>> _readers;

View File

@ -0,0 +1,79 @@
#include "./sdl_audio_frame_stream2.hpp"
#include "SDL_audio.h"
#include <iostream>
#include <vector>
SDLAudioInputDevice::SDLAudioInputDevice(void) : _stream{nullptr, &SDL_DestroyAudioStream} {
constexpr SDL_AudioSpec spec = { SDL_AUDIO_S16, 1, 48000 };
_stream = {
SDL_OpenAudioDeviceStream(SDL_AUDIO_DEVICE_DEFAULT_CAPTURE, &spec, nullptr, nullptr),
&SDL_DestroyAudioStream
};
if (!static_cast<bool>(_stream)) {
std::cerr << "SDL open audio device failed!\n";
}
const auto audio_device_id = SDL_GetAudioStreamDevice(_stream.get());
SDL_ResumeAudioDevice(audio_device_id);
static constexpr size_t buffer_size {512}; // in samples
const auto interval_ms {buffer_size/(spec.freq * 1000)};
_thread = std::thread([this, interval_ms, spec](void) {
while (!_thread_should_quit) {
//static std::vector<int16_t> buffer(buffer_size);
static AudioFrame tmp_frame {
0, // TODO: seq
spec.freq, spec.channels,
std::vector<int16_t>(buffer_size)
};
auto& buffer = std::get<std::vector<int16_t>>(tmp_frame.buffer);
buffer.resize(buffer_size);
const auto read_bytes = SDL_GetAudioStreamData(
_stream.get(),
buffer.data(),
buffer.size()*sizeof(int16_t)
);
//if (read_bytes != 0) {
//std::cerr << "read " << read_bytes << "/" << buffer.size()*sizeof(int16_t) << " audio bytes\n";
//}
// no new frame yet, or error
if (read_bytes <= 0) {
// only sleep 1/5, we expected a frame
std::this_thread::sleep_for(std::chrono::milliseconds(int64_t(interval_ms/5)));
continue;
}
buffer.resize(read_bytes/sizeof(int16_t)); // this might be costly?
bool someone_listening {false};
someone_listening = push(tmp_frame);
if (someone_listening) {
// double the interval on acquire
std::this_thread::sleep_for(std::chrono::milliseconds(int64_t(interval_ms/2)));
} else {
std::cerr << "i guess no one is listening\n";
// we just sleep 32x as long, bc no one is listening
// with the hardcoded settings, this is ~320ms
// TODO: just hardcode something like 500ms?
// TODO: suspend
std::this_thread::sleep_for(std::chrono::milliseconds(int64_t(interval_ms*32)));
}
}
});
}
SDLAudioInputDevice::~SDLAudioInputDevice(void) {
// TODO: pause audio device?
_thread_should_quit = true;
_thread.join();
// TODO: what to do if readers are still present?
}

View File

@ -0,0 +1,37 @@
#pragma once
#include "./frame_stream2.hpp"
#include "./audio_stream.hpp"
#include <SDL3/SDL.h>
#include <cstdint>
#include <variant>
#include <vector>
#include <thread>
// we dont have to multiplex ourself, because sdl streams and virtual devices already do this, but we do it anyway
using SDLAudioInputFrameStream2Multiplexer = QueuedFrameStream2Multiplexer<AudioFrame>;
using SDLAudioInputFrameStream2 = SDLAudioInputFrameStream2Multiplexer::reader_type_t; // just use the default for now
// object components?
struct SDLAudioInputDevice : protected SDLAudioInputFrameStream2Multiplexer {
std::unique_ptr<SDL_AudioStream, decltype(&SDL_DestroyAudioStream)> _stream;
std::atomic<bool> _thread_should_quit {false};
std::thread _thread;
// construct source and start thread
// TODO: optimize so the thread is not always running
SDLAudioInputDevice(void);
// stops the thread and closes the device?
~SDLAudioInputDevice(void);
using SDLAudioInputFrameStream2Multiplexer::aquireReader;
using SDLAudioInputFrameStream2Multiplexer::releaseReader;
};
struct SDLAudioOutputDevice {
};

View File

@ -43,7 +43,7 @@ struct SDLVideoFrame {
}; };
using SDLVideoFrameStream2Multiplexer = QueuedFrameStream2Multiplexer<SDLVideoFrame>; using SDLVideoFrameStream2Multiplexer = QueuedFrameStream2Multiplexer<SDLVideoFrame>;
using SDLVideoFrameStream2 = SDLVideoFrameStream2Multiplexer::ReaderType; using SDLVideoFrameStream2 = SDLVideoFrameStream2Multiplexer::reader_type_t; // just use the default for now
struct SDLVideoCameraContent : protected SDLVideoFrameStream2Multiplexer { struct SDLVideoCameraContent : protected SDLVideoFrameStream2Multiplexer {
// meh, empty default // meh, empty default

View File

@ -9,6 +9,7 @@
#include "./chat_gui/theme.hpp" #include "./chat_gui/theme.hpp"
#include "./start_screen.hpp" #include "./start_screen.hpp"
#include "./content/sdl_audio_frame_stream2.hpp"
#include "./content/sdl_video_frame_stream2.hpp" #include "./content/sdl_video_frame_stream2.hpp"
#include <memory> #include <memory>
@ -62,7 +63,23 @@ int main(int argc, char** argv) {
// optionally init audio and camera // optionally init audio and camera
if (SDL_Init(SDL_INIT_AUDIO) < 0) { if (SDL_Init(SDL_INIT_AUDIO) < 0) {
std::cerr << "SDL_Init AUDIO failed (" << SDL_GetError() << ")\n"; std::cerr << "SDL_Init AUDIO failed (" << SDL_GetError() << ")\n";
} else {
SDLAudioInputDevice aid;
auto* reader = aid.aquireReader();
for (size_t i = 0; i < 20; i++) {
std::this_thread::sleep_for(std::chrono::milliseconds(10));
auto new_frame_opt = reader->pop();
if (new_frame_opt.has_value()) {
std::cout << "audio frame was seq:" << new_frame_opt.value().seq << " sr:" << new_frame_opt.value().sample_rate << " " << (new_frame_opt.value().isS16()?"S16":"F32") << " l:" << (new_frame_opt.value().isS16()?new_frame_opt.value().getSpan<int16_t>().size:new_frame_opt.value().getSpan<float>().size) << "\n";
} else {
std::cout << "no audio frame\n";
} }
}
aid.releaseReader(reader);
}
if (SDL_Init(SDL_INIT_CAMERA) < 0) { if (SDL_Init(SDL_INIT_CAMERA) < 0) {
std::cerr << "SDL_Init CAMERA failed (" << SDL_GetError() << ")\n"; std::cerr << "SDL_Init CAMERA failed (" << SDL_GetError() << ")\n";
} else { // HACK } else { // HACK