forked from Green-Sky/tomato
frame refactoring, add audio frame and sdl default device input
This commit is contained in:
parent
31905b5468
commit
803bce93fb
@ -80,14 +80,12 @@ add_executable(tomato
|
|||||||
./chat_gui4.cpp
|
./chat_gui4.cpp
|
||||||
|
|
||||||
./content/content.hpp
|
./content/content.hpp
|
||||||
#./content/stream_reader.hpp
|
|
||||||
#./content/stream_reader_sdl_audio.hpp
|
|
||||||
#./content/stream_reader_sdl_audio.cpp
|
|
||||||
#./content/stream_reader_sdl_video.hpp
|
|
||||||
#./content/stream_reader_sdl_video.cpp
|
|
||||||
./content/frame_stream2.hpp
|
./content/frame_stream2.hpp
|
||||||
./content/sdl_video_frame_stream2.hpp
|
./content/sdl_video_frame_stream2.hpp
|
||||||
./content/sdl_video_frame_stream2.cpp
|
./content/sdl_video_frame_stream2.cpp
|
||||||
|
./content/audio_stream.hpp
|
||||||
|
./content/sdl_audio_frame_stream2.hpp
|
||||||
|
./content/sdl_audio_frame_stream2.cpp
|
||||||
)
|
)
|
||||||
|
|
||||||
target_compile_features(tomato PUBLIC cxx_std_17)
|
target_compile_features(tomato PUBLIC cxx_std_17)
|
||||||
|
69
src/content/audio_stream.hpp
Normal file
69
src/content/audio_stream.hpp
Normal file
@ -0,0 +1,69 @@
|
|||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include "./frame_stream2.hpp"
|
||||||
|
|
||||||
|
#include <solanaceae/util/span.hpp>
|
||||||
|
|
||||||
|
#include <cstdint>
|
||||||
|
#include <variant>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
// raw audio
|
||||||
|
// channels make samples interleaved,
|
||||||
|
// planar channels are not supported
|
||||||
|
struct AudioFrame {
|
||||||
|
// sequence number, to detect gaps
|
||||||
|
uint32_t seq {0};
|
||||||
|
// TODO: maybe use ts instead to discard old?
|
||||||
|
// since buffer size is variable, some timestamp would be needed to estimate the lost time
|
||||||
|
|
||||||
|
// samples per second
|
||||||
|
uint32_t sample_rate {48'000};
|
||||||
|
|
||||||
|
size_t channels {0};
|
||||||
|
std::variant<
|
||||||
|
std::vector<int16_t>, // S16, platform endianess
|
||||||
|
Span<int16_t>, // non owning variant, for direct consumption
|
||||||
|
|
||||||
|
std::vector<float>, // f32
|
||||||
|
Span<float> // non owning variant, for direct consumption
|
||||||
|
> buffer;
|
||||||
|
|
||||||
|
// helpers
|
||||||
|
|
||||||
|
bool isS16(void) const {
|
||||||
|
return
|
||||||
|
std::holds_alternative<std::vector<int16_t>>(buffer) ||
|
||||||
|
std::holds_alternative<Span<int16_t>>(buffer)
|
||||||
|
;
|
||||||
|
}
|
||||||
|
bool isF32(void) const {
|
||||||
|
return
|
||||||
|
std::holds_alternative<std::vector<float>>(buffer) ||
|
||||||
|
std::holds_alternative<Span<float>>(buffer)
|
||||||
|
;
|
||||||
|
}
|
||||||
|
template<typename T>
|
||||||
|
Span<T> getSpan(void) const {
|
||||||
|
static_assert(std::is_same_v<int16_t, T> || std::is_same_v<float, T>);
|
||||||
|
if constexpr (std::is_same_v<int16_t, T>) {
|
||||||
|
assert(isS16());
|
||||||
|
if (std::holds_alternative<std::vector<int16_t>>(buffer)) {
|
||||||
|
return Span<int16_t>{std::get<std::vector<int16_t>>(buffer)};
|
||||||
|
} else {
|
||||||
|
return std::get<Span<int16_t>>(buffer);
|
||||||
|
}
|
||||||
|
} else if constexpr (std::is_same_v<float, T>) {
|
||||||
|
assert(isF32());
|
||||||
|
if (std::holds_alternative<std::vector<float>>(buffer)) {
|
||||||
|
return Span<float>{std::get<std::vector<float>>(buffer)};
|
||||||
|
} else {
|
||||||
|
return std::get<Span<float>>(buffer);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
using AudioFrameStream2I = FrameStream2I<AudioFrame>;
|
||||||
|
|
@ -18,7 +18,7 @@
|
|||||||
//};
|
//};
|
||||||
|
|
||||||
template<typename FrameType>
|
template<typename FrameType>
|
||||||
struct FrameStream2 {
|
struct FrameStream2I {
|
||||||
// get number of available frames
|
// get number of available frames
|
||||||
[[nodiscard]] virtual int32_t size(void) = 0;
|
[[nodiscard]] virtual int32_t size(void) = 0;
|
||||||
|
|
||||||
@ -34,7 +34,7 @@ struct FrameStream2 {
|
|||||||
// needs count frames queue size
|
// needs count frames queue size
|
||||||
// having ~1-2sec buffer size is often sufficent
|
// having ~1-2sec buffer size is often sufficent
|
||||||
template<typename FrameType>
|
template<typename FrameType>
|
||||||
struct QueuedFrameStream2 : public FrameStream2<FrameType> {
|
struct QueuedFrameStream2 : public FrameStream2I<FrameType> {
|
||||||
using frame_type = FrameType;
|
using frame_type = FrameType;
|
||||||
|
|
||||||
rigtorp::SPSCQueue<FrameType> _queue;
|
rigtorp::SPSCQueue<FrameType> _queue;
|
||||||
@ -74,9 +74,9 @@ struct QueuedFrameStream2 : public FrameStream2<FrameType> {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
template<typename FrameType>
|
template<typename FrameType, typename ReaderType = QueuedFrameStream2<FrameType>>
|
||||||
struct QueuedFrameStream2Multiplexer : public FrameStream2<FrameType> {
|
struct QueuedFrameStream2Multiplexer : public FrameStream2I<FrameType> {
|
||||||
using ReaderType = QueuedFrameStream2<FrameType>;
|
using reader_type_t = ReaderType;
|
||||||
|
|
||||||
// pointer stability
|
// pointer stability
|
||||||
std::vector<std::unique_ptr<ReaderType>> _readers;
|
std::vector<std::unique_ptr<ReaderType>> _readers;
|
||||||
|
79
src/content/sdl_audio_frame_stream2.cpp
Normal file
79
src/content/sdl_audio_frame_stream2.cpp
Normal file
@ -0,0 +1,79 @@
|
|||||||
|
#include "./sdl_audio_frame_stream2.hpp"
|
||||||
|
#include "SDL_audio.h"
|
||||||
|
|
||||||
|
#include <iostream>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
SDLAudioInputDevice::SDLAudioInputDevice(void) : _stream{nullptr, &SDL_DestroyAudioStream} {
|
||||||
|
constexpr SDL_AudioSpec spec = { SDL_AUDIO_S16, 1, 48000 };
|
||||||
|
|
||||||
|
_stream = {
|
||||||
|
SDL_OpenAudioDeviceStream(SDL_AUDIO_DEVICE_DEFAULT_CAPTURE, &spec, nullptr, nullptr),
|
||||||
|
&SDL_DestroyAudioStream
|
||||||
|
};
|
||||||
|
|
||||||
|
if (!static_cast<bool>(_stream)) {
|
||||||
|
std::cerr << "SDL open audio device failed!\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
const auto audio_device_id = SDL_GetAudioStreamDevice(_stream.get());
|
||||||
|
SDL_ResumeAudioDevice(audio_device_id);
|
||||||
|
|
||||||
|
static constexpr size_t buffer_size {512}; // in samples
|
||||||
|
const auto interval_ms {buffer_size/(spec.freq * 1000)};
|
||||||
|
|
||||||
|
_thread = std::thread([this, interval_ms, spec](void) {
|
||||||
|
while (!_thread_should_quit) {
|
||||||
|
//static std::vector<int16_t> buffer(buffer_size);
|
||||||
|
static AudioFrame tmp_frame {
|
||||||
|
0, // TODO: seq
|
||||||
|
spec.freq, spec.channels,
|
||||||
|
std::vector<int16_t>(buffer_size)
|
||||||
|
};
|
||||||
|
|
||||||
|
auto& buffer = std::get<std::vector<int16_t>>(tmp_frame.buffer);
|
||||||
|
buffer.resize(buffer_size);
|
||||||
|
|
||||||
|
const auto read_bytes = SDL_GetAudioStreamData(
|
||||||
|
_stream.get(),
|
||||||
|
buffer.data(),
|
||||||
|
buffer.size()*sizeof(int16_t)
|
||||||
|
);
|
||||||
|
//if (read_bytes != 0) {
|
||||||
|
//std::cerr << "read " << read_bytes << "/" << buffer.size()*sizeof(int16_t) << " audio bytes\n";
|
||||||
|
//}
|
||||||
|
|
||||||
|
// no new frame yet, or error
|
||||||
|
if (read_bytes <= 0) {
|
||||||
|
// only sleep 1/5, we expected a frame
|
||||||
|
std::this_thread::sleep_for(std::chrono::milliseconds(int64_t(interval_ms/5)));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
buffer.resize(read_bytes/sizeof(int16_t)); // this might be costly?
|
||||||
|
|
||||||
|
bool someone_listening {false};
|
||||||
|
someone_listening = push(tmp_frame);
|
||||||
|
|
||||||
|
if (someone_listening) {
|
||||||
|
// double the interval on acquire
|
||||||
|
std::this_thread::sleep_for(std::chrono::milliseconds(int64_t(interval_ms/2)));
|
||||||
|
} else {
|
||||||
|
std::cerr << "i guess no one is listening\n";
|
||||||
|
// we just sleep 32x as long, bc no one is listening
|
||||||
|
// with the hardcoded settings, this is ~320ms
|
||||||
|
// TODO: just hardcode something like 500ms?
|
||||||
|
// TODO: suspend
|
||||||
|
std::this_thread::sleep_for(std::chrono::milliseconds(int64_t(interval_ms*32)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
SDLAudioInputDevice::~SDLAudioInputDevice(void) {
|
||||||
|
// TODO: pause audio device?
|
||||||
|
_thread_should_quit = true;
|
||||||
|
_thread.join();
|
||||||
|
// TODO: what to do if readers are still present?
|
||||||
|
}
|
||||||
|
|
37
src/content/sdl_audio_frame_stream2.hpp
Normal file
37
src/content/sdl_audio_frame_stream2.hpp
Normal file
@ -0,0 +1,37 @@
|
|||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include "./frame_stream2.hpp"
|
||||||
|
#include "./audio_stream.hpp"
|
||||||
|
|
||||||
|
#include <SDL3/SDL.h>
|
||||||
|
|
||||||
|
#include <cstdint>
|
||||||
|
#include <variant>
|
||||||
|
#include <vector>
|
||||||
|
#include <thread>
|
||||||
|
|
||||||
|
// we dont have to multiplex ourself, because sdl streams and virtual devices already do this, but we do it anyway
|
||||||
|
using SDLAudioInputFrameStream2Multiplexer = QueuedFrameStream2Multiplexer<AudioFrame>;
|
||||||
|
using SDLAudioInputFrameStream2 = SDLAudioInputFrameStream2Multiplexer::reader_type_t; // just use the default for now
|
||||||
|
|
||||||
|
// object components?
|
||||||
|
struct SDLAudioInputDevice : protected SDLAudioInputFrameStream2Multiplexer {
|
||||||
|
std::unique_ptr<SDL_AudioStream, decltype(&SDL_DestroyAudioStream)> _stream;
|
||||||
|
|
||||||
|
std::atomic<bool> _thread_should_quit {false};
|
||||||
|
std::thread _thread;
|
||||||
|
|
||||||
|
// construct source and start thread
|
||||||
|
// TODO: optimize so the thread is not always running
|
||||||
|
SDLAudioInputDevice(void);
|
||||||
|
|
||||||
|
// stops the thread and closes the device?
|
||||||
|
~SDLAudioInputDevice(void);
|
||||||
|
|
||||||
|
using SDLAudioInputFrameStream2Multiplexer::aquireReader;
|
||||||
|
using SDLAudioInputFrameStream2Multiplexer::releaseReader;
|
||||||
|
};
|
||||||
|
|
||||||
|
struct SDLAudioOutputDevice {
|
||||||
|
};
|
||||||
|
|
@ -43,7 +43,7 @@ struct SDLVideoFrame {
|
|||||||
};
|
};
|
||||||
|
|
||||||
using SDLVideoFrameStream2Multiplexer = QueuedFrameStream2Multiplexer<SDLVideoFrame>;
|
using SDLVideoFrameStream2Multiplexer = QueuedFrameStream2Multiplexer<SDLVideoFrame>;
|
||||||
using SDLVideoFrameStream2 = SDLVideoFrameStream2Multiplexer::ReaderType;
|
using SDLVideoFrameStream2 = SDLVideoFrameStream2Multiplexer::reader_type_t; // just use the default for now
|
||||||
|
|
||||||
struct SDLVideoCameraContent : protected SDLVideoFrameStream2Multiplexer {
|
struct SDLVideoCameraContent : protected SDLVideoFrameStream2Multiplexer {
|
||||||
// meh, empty default
|
// meh, empty default
|
||||||
|
17
src/main.cpp
17
src/main.cpp
@ -9,6 +9,7 @@
|
|||||||
#include "./chat_gui/theme.hpp"
|
#include "./chat_gui/theme.hpp"
|
||||||
|
|
||||||
#include "./start_screen.hpp"
|
#include "./start_screen.hpp"
|
||||||
|
#include "./content/sdl_audio_frame_stream2.hpp"
|
||||||
#include "./content/sdl_video_frame_stream2.hpp"
|
#include "./content/sdl_video_frame_stream2.hpp"
|
||||||
|
|
||||||
#include <memory>
|
#include <memory>
|
||||||
@ -62,7 +63,23 @@ int main(int argc, char** argv) {
|
|||||||
// optionally init audio and camera
|
// optionally init audio and camera
|
||||||
if (SDL_Init(SDL_INIT_AUDIO) < 0) {
|
if (SDL_Init(SDL_INIT_AUDIO) < 0) {
|
||||||
std::cerr << "SDL_Init AUDIO failed (" << SDL_GetError() << ")\n";
|
std::cerr << "SDL_Init AUDIO failed (" << SDL_GetError() << ")\n";
|
||||||
|
} else {
|
||||||
|
SDLAudioInputDevice aid;
|
||||||
|
auto* reader = aid.aquireReader();
|
||||||
|
|
||||||
|
for (size_t i = 0; i < 20; i++) {
|
||||||
|
std::this_thread::sleep_for(std::chrono::milliseconds(10));
|
||||||
|
auto new_frame_opt = reader->pop();
|
||||||
|
if (new_frame_opt.has_value()) {
|
||||||
|
std::cout << "audio frame was seq:" << new_frame_opt.value().seq << " sr:" << new_frame_opt.value().sample_rate << " " << (new_frame_opt.value().isS16()?"S16":"F32") << " l:" << (new_frame_opt.value().isS16()?new_frame_opt.value().getSpan<int16_t>().size:new_frame_opt.value().getSpan<float>().size) << "\n";
|
||||||
|
} else {
|
||||||
|
std::cout << "no audio frame\n";
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
aid.releaseReader(reader);
|
||||||
|
}
|
||||||
|
|
||||||
if (SDL_Init(SDL_INIT_CAMERA) < 0) {
|
if (SDL_Init(SDL_INIT_CAMERA) < 0) {
|
||||||
std::cerr << "SDL_Init CAMERA failed (" << SDL_GetError() << ")\n";
|
std::cerr << "SDL_Init CAMERA failed (" << SDL_GetError() << ")\n";
|
||||||
} else { // HACK
|
} else { // HACK
|
||||||
|
Loading…
Reference in New Issue
Block a user