Green-Sky/tomato
Green-Sky/tomato
1
0
Fork 1
Code Issues Pull Requests 1 Actions Packages Projects Releases 1 Wiki Activity

rewrite audio input, always open? but thin stream wrapper

Browse Source
This commit is contained in:
Green Sky 2024-09-25 14:36:51 +02:00
parent 5eca1a99e0
commit 5b3e0e2a0b
No known key found for this signature in database
6 changed files with 190 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/content/audio_stream.hpp
View File

@ -67,3 +67,6 @@ struct AudioFrame {
using AudioFrameStream2I = FrameStream2I<AudioFrame>;
using AudioFrameStream2MultiSource = FrameStream2MultiSource<AudioFrame>;
using AudioFrameStream2 = AudioFrameStream2MultiSource::sub_stream_type_t; // just use the default for now

209
src/content/sdl_audio_frame_stream2.cpp
View File

@ -1,83 +1,171 @@
#include "./sdl_audio_frame_stream2.hpp"
#include <iostream>
#include <vector>
SDLAudioInputDeviceDefault::SDLAudioInputDeviceDefault(void) : _stream{nullptr, &SDL_DestroyAudioStream} {
constexpr SDL_AudioSpec spec = { SDL_AUDIO_S16, 1, 48000 };
// "thin" wrapper around sdl audio streams
// we dont needs to get fance, as they already provide everything we need
struct SDLAudioStreamReader : public AudioFrameStream2I {
std::unique_ptr<SDL_AudioStream, decltype(&SDL_DestroyAudioStream)> _stream;
_stream = {
SDL_OpenAudioDeviceStream(SDL_AUDIO_DEVICE_DEFAULT_RECORDING, &spec, nullptr, nullptr),
&SDL_DestroyAudioStream
};
uint32_t _seq_counter {0};
if (!static_cast<bool>(_stream)) {
std::cerr << "SDL open audio device failed!\n";
uint32_t _sample_rate {48'000};
size_t _channels {0};
SDL_AudioFormat _format {SDL_AUDIO_S16};
std::vector<int16_t> _buffer;
SDLAudioStreamReader(void) : _stream(nullptr, nullptr) {}
SDLAudioStreamReader(SDLAudioStreamReader&& other) :
_stream(std::move(other._stream)),
_sample_rate(other._sample_rate),
_channels(other._channels),
_format(other._format)
{
static const size_t buffer_size {960*_channels};
_buffer.resize(buffer_size);
}
const auto audio_device_id = SDL_GetAudioStreamDevice(_stream.get());
SDL_ResumeAudioDevice(audio_device_id);
~SDLAudioStreamReader(void) {
if (_stream) {
SDL_UnbindAudioStream(_stream.get());
}
}
static constexpr size_t buffer_size {512*2}; // in samples
const auto interval_ms {(buffer_size * 1000) / spec.freq};
int32_t size(void) override {
//assert(_stream);
// returns bytes
//SDL_GetAudioStreamAvailable(_stream.get());
return -1;
}
_thread = std::thread([this, interval_ms, spec](void) {
while (!_thread_should_quit) {
//static std::vector<int16_t> buffer(buffer_size);
static AudioFrame tmp_frame {
0, // TODO: seq
spec.freq, spec.channels,
std::vector<int16_t>(buffer_size)
};
std::optional<AudioFrame> pop(void) override {
assert(_stream);
assert(_format == SDL_AUDIO_S16);
auto& buffer = std::get<std::vector<int16_t>>(tmp_frame.buffer);
buffer.resize(buffer_size);
static const size_t buffer_size {960*_channels};
_buffer.resize(buffer_size); // noop?
const auto read_bytes = SDL_GetAudioStreamData(
_stream.get(),
buffer.data(),
buffer.size()*sizeof(int16_t)
_buffer.data(),
_buffer.size()*sizeof(int16_t)
);
//if (read_bytes != 0) {
//std::cerr << "read " << read_bytes << "/" << buffer.size()*sizeof(int16_t) << " audio bytes\n";
//}
// no new frame yet, or error
if (read_bytes <= 0) {
// only sleep 1/5, we expected a frame
std::this_thread::sleep_for(std::chrono::milliseconds(int64_t(interval_ms/5)));
continue;
return std::nullopt;
}
buffer.resize(read_bytes/sizeof(int16_t)); // this might be costly?
bool someone_listening {false};
someone_listening = push(tmp_frame);
if (someone_listening) {
// double the interval on acquire
std::this_thread::sleep_for(std::chrono::milliseconds(int64_t(interval_ms/2)));
} else {
std::cerr << "i guess no one is listening\n";
std::cerr << "first value: " << buffer.front() << "\n";
// we just sleep 32x as long, bc no one is listening
// with the hardcoded settings, this is ~320ms
// TODO: just hardcode something like 500ms?
// TODO: suspend
// TODO: this is not gonna cut it, since playback slows down dramatically and will be very behind
std::this_thread::sleep_for(std::chrono::milliseconds(int64_t(interval_ms*32)));
}
}
});
return AudioFrame {
_seq_counter++,
_sample_rate, _channels,
Span<int16_t>(_buffer.data(), read_bytes/sizeof(int16_t)),
};
}
SDLAudioInputDeviceDefault::~SDLAudioInputDeviceDefault(void) {
// TODO: pause audio device?
_thread_should_quit = true;
_thread.join();
// TODO: what to do if readers are still present?
bool push(const AudioFrame&) override {
// TODO: make universal sdl stream wrapper (combine with SDLAudioOutputDeviceDefaultInstance)
assert(false && "read only");
return false;
}
};
SDLAudioInputDevice::SDLAudioInputDevice(void) : SDLAudioInputDevice(SDL_AUDIO_DEVICE_DEFAULT_RECORDING) {
}
SDLAudioInputDevice::SDLAudioInputDevice(SDL_AudioDeviceID device_id) {
// this spec is more like a hint to the hardware
SDL_AudioSpec spec {
SDL_AUDIO_S16,
1, // configurable?
48'000,
};
_device_id = SDL_OpenAudioDevice(device_id, &spec);
if (_device_id == 0) {
// TODO: proper error handling
throw int(1);
}
}
SDLAudioInputDevice::~SDLAudioInputDevice(void) {
_streams.clear();
SDL_CloseAudioDevice(_device_id);
}
std::shared_ptr<FrameStream2I<AudioFrame>> SDLAudioInputDevice::subscribe(void) {
SDL_AudioSpec spec {
SDL_AUDIO_S16,
1,
48'000,
};
SDL_AudioSpec device_spec {
SDL_AUDIO_S16,
1,
48'000,
};
// TODO: error check
SDL_GetAudioDeviceFormat(_device_id, &device_spec, nullptr);
// error check
auto* sdl_stream = SDL_CreateAudioStream(&device_spec, &spec);
// error check
SDL_BindAudioStream(_device_id, sdl_stream);
auto new_stream = std::make_shared<SDLAudioStreamReader>();
// TODO: move to ctr
new_stream->_stream = {sdl_stream, &SDL_DestroyAudioStream};
new_stream->_sample_rate = spec.freq;
new_stream->_channels = spec.channels;
new_stream->_format = spec.format;
_streams.emplace_back(new_stream);
return new_stream;
}
bool SDLAudioInputDevice::unsubscribe(const std::shared_ptr<FrameStream2I<AudioFrame>>& sub) {
for (auto it = _streams.cbegin(); it != _streams.cend(); it++) {
if (*it == sub) {
_streams.erase(it);
return true;
}
}
return false;
}
// does not need to be visible in the header
struct SDLAudioOutputDeviceDefaultInstance : public AudioFrameStream2I {
std::unique_ptr<SDL_AudioStream, decltype(&SDL_DestroyAudioStream)> _stream;
uint32_t _last_sample_rate {48'000};
size_t _last_channels {0};
SDL_AudioFormat _last_format {SDL_AUDIO_S16};
// TODO: audio device
SDLAudioOutputDeviceDefaultInstance(void);
SDLAudioOutputDeviceDefaultInstance(SDLAudioOutputDeviceDefaultInstance&& other);
~SDLAudioOutputDeviceDefaultInstance(void);
int32_t size(void) override;
std::optional<AudioFrame> pop(void) override;
bool push(const AudioFrame& value) override;
};
SDLAudioOutputDeviceDefaultInstance::SDLAudioOutputDeviceDefaultInstance(void) : _stream(nullptr, nullptr) {
}
SDLAudioOutputDeviceDefaultInstance::SDLAudioOutputDeviceDefaultInstance(SDLAudioOutputDeviceDefaultInstance&& other) : _stream(std::move(other._stream)) {
}
SDLAudioOutputDeviceDefaultInstance::~SDLAudioOutputDeviceDefaultInstance(void) {
}
int32_t SDLAudioOutputDeviceDefaultInstance::size(void) {
return -1;
}
@ -144,15 +232,6 @@ bool SDLAudioOutputDeviceDefaultInstance::push(const AudioFrame& value) {
return true;
}
SDLAudioOutputDeviceDefaultInstance::SDLAudioOutputDeviceDefaultInstance(void) : _stream(nullptr, nullptr) {
}
SDLAudioOutputDeviceDefaultInstance::SDLAudioOutputDeviceDefaultInstance(SDLAudioOutputDeviceDefaultInstance&& other) : _stream(std::move(other._stream)) {
}
SDLAudioOutputDeviceDefaultInstance::~SDLAudioOutputDeviceDefaultInstance(void) {
}
SDLAudioOutputDeviceDefaultSink::~SDLAudioOutputDeviceDefaultSink(void) {
// TODO: pause and close device?

48
src/content/sdl_audio_frame_stream2.hpp
View File

@ -6,52 +6,30 @@
#include <SDL3/SDL.h>
#include <cstdint>
#include <variant>
#include <vector>
#include <thread>
// we dont have to multicast ourself, because sdl streams and virtual devices already do this, but we do it anyway
using AudioFrameStream2MultiSource = FrameStream2MultiSource<AudioFrame>;
using AudioFrameStream2 = AudioFrameStream2MultiSource::sub_stream_type_t; // just use the default for now
// object components?
// we dont have to multicast ourself, because sdl streams and virtual devices already do this
// source
struct SDLAudioInputDeviceDefault : public AudioFrameStream2MultiSource {
std::unique_ptr<SDL_AudioStream, decltype(&SDL_DestroyAudioStream)> _stream;
// opens device
// creates a sdl audio stream for each subscribed reader stream
struct SDLAudioInputDevice : public FrameStream2SourceI<AudioFrame> {
// held by instances
using sdl_stream_type = std::unique_ptr<SDL_AudioStream, decltype(&SDL_DestroyAudioStream)>;
std::atomic<bool> _thread_should_quit {false};
std::thread _thread;
SDL_AudioDeviceID _device_id {0};
// construct source and start thread
// TODO: optimize so the thread is not always running
SDLAudioInputDeviceDefault(void);
std::vector<std::shared_ptr<FrameStream2I<AudioFrame>>> _streams;
// stops the thread and closes the device?
~SDLAudioInputDeviceDefault(void);
SDLAudioInputDevice(void);
SDLAudioInputDevice(SDL_AudioDeviceID device_id);
~SDLAudioInputDevice(void);
using AudioFrameStream2MultiSource::subscribe;
using AudioFrameStream2MultiSource::unsubscribe;
std::shared_ptr<FrameStream2I<AudioFrame>> subscribe(void) override;
bool unsubscribe(const std::shared_ptr<FrameStream2I<AudioFrame>>& sub) override;
};
// sink
struct SDLAudioOutputDeviceDefaultInstance : public AudioFrameStream2I {
std::unique_ptr<SDL_AudioStream, decltype(&SDL_DestroyAudioStream)> _stream;
uint32_t _last_sample_rate {48'000};
size_t _last_channels {0};
SDL_AudioFormat _last_format {SDL_AUDIO_S16};
SDLAudioOutputDeviceDefaultInstance(void);
SDLAudioOutputDeviceDefaultInstance(SDLAudioOutputDeviceDefaultInstance&& other);
~SDLAudioOutputDeviceDefaultInstance(void);
int32_t size(void) override;
std::optional<AudioFrame> pop(void) override;
bool push(const AudioFrame& value) override;
};
// constructs entirely new streams, since sdl handles sync and mixing for us (or should)
struct SDLAudioOutputDeviceDefaultSink : public FrameStream2SinkI<AudioFrame> {
// TODO: pause device?

17
src/debug_tox_call.cpp
View File

@ -1,8 +1,8 @@
#include "./debug_tox_call.hpp"
#include "./stream_manager.hpp"
#include "./content/audio_stream.hpp"
#include "./content/sdl_video_frame_stream2.hpp"
#include "./content/sdl_audio_frame_stream2.hpp"
#include <SDL3/SDL.h>
@ -341,14 +341,25 @@ void DebugToxCall::tick(float) {
const auto& new_frame = new_frame_opt.value();
assert(new_frame.isS16());
// TODO: error code
_toxav.toxavAudioSendFrame(
//* @param sample_count Number of samples in this frame. Valid numbers here are
//* `((sample rate) * (audio length) / 1000)`, where audio length can be
//* 2.5, 5, 10, 20, 40 or 60 milliseconds.
// we likely needs to subdivide/repackage
// frame size should be an option exposed to the user
// with 10ms as a default ?
// the larger the frame size, the less overhead but the more delay
auto err = _toxav.toxavAudioSendFrame(
asink->_fid,
new_frame.getSpan<int16_t>().ptr,
new_frame.getSpan<int16_t>().size / new_frame.channels,
new_frame.channels,
new_frame.sample_rate
);
if (err != TOXAV_ERR_SEND_FRAME_OK) {
std::cerr << "DTC: failed to send audio frame " << err << "\n";
}
}
}
}

6
src/main.cpp
View File

@ -79,8 +79,8 @@ int main(int argc, char** argv) {
if (!SDL_Init(SDL_INIT_AUDIO)) {
std::cerr << "SDL_Init AUDIO failed (" << SDL_GetError() << ")\n";
} else if (false) {
SDLAudioInputDeviceDefault aidd;
auto reader = aidd.subscribe();
SDLAudioInputDevice aid;
auto reader = aid.subscribe();
auto writer = SDLAudioOutputDeviceDefaultSink{}.subscribe();
@ -95,7 +95,7 @@ int main(int argc, char** argv) {
}
}
aidd.unsubscribe(reader);
aid.unsubscribe(reader);
}
if (!SDL_Init(SDL_INIT_CAMERA)) {

4
src/main_screen.cpp
View File

@ -162,11 +162,11 @@ MainScreen::MainScreen(SimpleConfigModel&& conf_, SDL_Renderer* renderer_, Theme
}
}
if (false) { // audio in
if (true) { // audio in
ObjectHandle asrc {os.registry(), os.registry().create()};
try {
asrc.emplace<Components::FrameStream2Source<AudioFrame>>(
std::make_unique<SDLAudioInputDeviceDefault>()
std::make_unique<SDLAudioInputDevice>()
);
asrc.emplace<Components::StreamSource>(Components::StreamSource::create<AudioFrame>("SDL Audio Default Recording Device"));