Green-Sky/tomato
Green-Sky/tomato
1
0
Fork 1
Code Issues Pull Requests 1 Actions Packages Projects Releases 1 Wiki Activity

toxav incomming audio src and fix sdl audio output sink

Browse Source
This commit is contained in:
Green Sky 2024-09-18 16:40:44 +02:00
parent 36e75c0fab
commit 42c7ab8571
No known key found for this signature in database
6 changed files with 141 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
src/content/frame_stream2.hpp
View File

@ -104,11 +104,7 @@ struct FrameStream2MultiSource : public FrameStream2SourceI<FrameType>, public F
virtual ~FrameStream2MultiSource(void) {}
//// TODO: forward args instead
//SubStreamType* aquireSubStream(size_t queue_size = 10, bool lossy = true) {
// std::lock_guard lg{_sub_stream_lock};
// return _sub_streams.emplace_back(std::make_unique<SubStreamType>(queue_size, lossy)).get();
//}
// TODO: forward args instead
std::shared_ptr<FrameStream2I<FrameType>> subscribe(void) override {
// TODO: args???
size_t queue_size = 10;
@ -118,15 +114,6 @@ struct FrameStream2MultiSource : public FrameStream2SourceI<FrameType>, public F
return _sub_streams.emplace_back(std::make_unique<SubStreamType>(queue_size, lossy));
}
//void releaseSubStream(SubStreamType* sub) {
// std::lock_guard lg{_sub_stream_lock};
// for (auto it = _sub_streams.begin(); it != _sub_streams.end(); it++) {
// if (it->get() == sub) {
// _sub_streams.erase(it);
// break;
// }
// }
//}
bool unsubscribe(const std::shared_ptr<FrameStream2I<FrameType>>& sub) override {
std::lock_guard lg{_sub_stream_lock};
for (auto it = _sub_streams.begin(); it != _sub_streams.end(); it++) {

26
src/content/sdl_audio_frame_stream2.cpp
View File

@ -100,24 +100,18 @@ bool SDLAudioOutputDeviceDefaultInstance::push(const AudioFrame& value) {
(value.isF32() && _last_format != SDL_AUDIO_F32) ||
(value.isS16() && _last_format != SDL_AUDIO_S16)
) {
const auto device_id = SDL_GetAudioStreamDevice(_stream.get());
SDL_FlushAudioStream(_stream.get());
const SDL_AudioSpec spec = {
static_cast<SDL_AudioFormat>((value.isF32() ? SDL_AUDIO_F32 : SDL_AUDIO_S16)),
static_cast<int>(value.channels),
static_cast<int>(value.sample_rate)
};
_stream = {
SDL_OpenAudioDeviceStream(device_id, &spec, nullptr, nullptr),
&SDL_DestroyAudioStream
};
SDL_SetAudioStreamFormat(_stream.get(), &spec, nullptr);
std::cerr << "SDLAOD: audio format changed\n";
}
// HACK
assert(value.isS16());
if (value.isS16()) {
auto data = value.getSpan<int16_t>();
if (data.size == 0) {
@ -128,6 +122,18 @@ bool SDLAudioOutputDeviceDefaultInstance::push(const AudioFrame& value) {
std::cerr << "put data error\n";
return false; // return true?
}
} else if (value.isF32()) {
auto data = value.getSpan<float>();
if (data.size == 0) {
std::cerr << "empty audio frame??\n";
}
if (!SDL_PutAudioStreamData(_stream.get(), data.ptr, data.size * sizeof(float))) {
std::cerr << "put data error\n";
return false; // return true?
}
}
_last_sample_rate = value.sample_rate;
_last_channels = value.channels;

118
src/debug_tox_call.cpp
View File

@ -2,6 +2,7 @@
#include "./stream_manager.hpp"
#include "./content/sdl_video_frame_stream2.hpp"
#include "./content/sdl_audio_frame_stream2.hpp"
#include <SDL3/SDL.h>
@ -18,20 +19,6 @@ namespace Message {
uint64_t getTimeMS();
}
namespace Components {
struct ToxAVFriendAudioSource {
};
struct ToxAVFriendAudioSink {
};
struct ToxAVFriendVideoSource {
};
struct ToxAVFriendVideoSink {
};
}
static bool isFormatPlanar(SDL_PixelFormat f) {
return
f == SDL_PIXELFORMAT_YV12 ||
@ -166,20 +153,34 @@ struct PushConversionQueuedVideoStream : public QueuedFrameStream2<SDLVideoFrame
// exlusive
// TODO: replace with something better than a queue
struct ToxAVCallVideoSink : public FrameStream2SinkI<SDLVideoFrame> {
ToxAV& _toxav;
// bitrate for enabled state
uint32_t _video_bitrate {2};
uint32_t _fid;
std::shared_ptr<PushConversionQueuedVideoStream> _writer;
ToxAVCallVideoSink(uint32_t fid) : _fid(fid) {}
~ToxAVCallVideoSink(void) {}
ToxAVCallVideoSink(ToxAV& toxav, uint32_t fid) : _toxav(toxav), _fid(fid) {}
~ToxAVCallVideoSink(void) {
if (_writer) {
_writer = nullptr;
_toxav.toxavVideoSetBitRate(_fid, 0);
}
}
// sink
std::shared_ptr<FrameStream2I<SDLVideoFrame>> subscribe(void) override {
if (_writer) {
// max 1 (exclusive)
// max 1 (exclusive, composite video somewhere else)
return nullptr;
}
auto err = _toxav.toxavVideoSetBitRate(_fid, _video_bitrate);
if (err != TOXAV_ERR_BIT_RATE_SET_OK) {
return nullptr;
}
// TODO: enable video here
_writer = std::make_shared<PushConversionQueuedVideoStream>(10, true);
return _writer;
@ -192,8 +193,11 @@ struct ToxAVCallVideoSink : public FrameStream2SinkI<SDLVideoFrame> {
}
if (sub == _writer) {
// TODO: disable video here
_writer = nullptr;
/*auto err = */_toxav.toxavVideoSetBitRate(_fid, 0);
// print warning? on error?
return true;
}
@ -211,9 +215,28 @@ DebugToxCall::DebugToxCall(ObjectStore2& os, ToxAV& toxav, TextureUploaderI& tu)
_toxav.subscribe(this, ToxAV_Event::friend_video_frame);
}
DebugToxCall::~DebugToxCall(void) {
// destroy all calls/connections/sources/sinks here
for (auto& [fid, call] : _calls) {
if (static_cast<bool>(call.incoming_vsrc)) {
call.incoming_vsrc.destroy();
}
if (static_cast<bool>(call.incoming_asrc)) {
call.incoming_asrc.destroy();
}
if (static_cast<bool>(call.outgoing_vsink)) {
call.outgoing_vsink.destroy();
}
if (static_cast<bool>(call.outgoing_asink)) {
call.outgoing_asink.destroy();
}
}
}
void DebugToxCall::tick(float) {
// pump sink to tox
// TODO: own thread or direct on push
// pump sinks to tox
// TODO: own thread or direct on push (requires thread save toxcore)
// TODO: pump at double the frame rate
for (const auto& [oc, vsink] : _os.registry().view<ToxAVCallVideoSink*>().each()) {
if (!vsink->_writer) {
@ -258,8 +281,9 @@ float DebugToxCall::render(void) {
if (call.incoming) {
ImGui::SameLine();
if (ImGui::SmallButton("answer")) {
const auto ret = _toxav.toxavAnswer(fid, 0, 0);
//const auto ret = _toxav.toxavAnswer(fid, 0, 1); // 1mbit/s
const auto ret = _toxav.toxavAnswer(fid, 0, 2); // 2mbit/s
//const auto ret = _toxav.toxavAnswer(fid, 0, 2); // 2mbit/s
//const auto ret = _toxav.toxavAnswer(fid, 0, 100); // 100mbit/s
//const auto ret = _toxav.toxavAnswer(fid, 0, 2500); // 2500mbit/s
if (ret == TOXAV_ERR_ANSWER_OK) {
@ -268,7 +292,7 @@ float DebugToxCall::render(void) {
// create sinks
call.outgoing_vsink = {_os.registry(), _os.registry().create()};
{
auto new_vsink = std::make_unique<ToxAVCallVideoSink>(fid);
auto new_vsink = std::make_unique<ToxAVCallVideoSink>(_toxav, fid);
call.outgoing_vsink.emplace<ToxAVCallVideoSink*>(new_vsink.get());
call.outgoing_vsink.emplace<Components::FrameStream2Sink<SDLVideoFrame>>(std::move(new_vsink));
call.outgoing_vsink.emplace<Components::StreamSink>("ToxAV friend call video", std::string{entt::type_name<SDLVideoFrame>::value()});
@ -284,6 +308,15 @@ float DebugToxCall::render(void) {
call.incoming_vsrc.emplace<Components::StreamSource>("ToxAV friend call video", std::string{entt::type_name<SDLVideoFrame>::value()});
}
}
if (call.incoming_a) {
call.incoming_asrc = {_os.registry(), _os.registry().create()};
{
auto new_asrc = std::make_unique<AudioFrameStream2MultiSource>();
call.incoming_asrc.emplace<AudioFrameStream2MultiSource*>(new_asrc.get());
call.incoming_asrc.emplace<Components::FrameStream2Source<AudioFrame>>(std::move(new_asrc));
call.incoming_asrc.emplace<Components::StreamSource>("ToxAV friend call audio", std::string{entt::type_name<AudioFrame>::value()});
}
}
}
}
} else if (call.state != TOXAV_FRIEND_CALL_STATE_FINISHED) {
@ -297,11 +330,17 @@ float DebugToxCall::render(void) {
call.state = TOXAV_FRIEND_CALL_STATE_FINISHED;
// TODO: stream manager disconnectAll()
if (static_cast<bool>(call.incoming_vsrc)) {
call.incoming_vsrc.destroy();
}
if (static_cast<bool>(call.incoming_asrc)) {
call.incoming_asrc.destroy();
}
if (static_cast<bool>(call.outgoing_vsink)) {
call.outgoing_vsink.destroy();
}
if (static_cast<bool>(call.incoming_vsrc)) {
call.incoming_vsrc.destroy();
if (static_cast<bool>(call.outgoing_asink)) {
call.outgoing_asink.destroy();
}
}
}
@ -341,11 +380,17 @@ bool DebugToxCall::onEvent(const Events::FriendCallState& e) {
(call.state & TOXAV_FRIEND_CALL_STATE_FINISHED) != 0 ||
(call.state & TOXAV_FRIEND_CALL_STATE_ERROR) != 0
) {
if (static_cast<bool>(call.incoming_vsrc)) {
call.incoming_vsrc.destroy();
}
if (static_cast<bool>(call.incoming_asrc)) {
call.incoming_asrc.destroy();
}
if (static_cast<bool>(call.outgoing_vsink)) {
call.outgoing_vsink.destroy();
}
if (static_cast<bool>(call.incoming_vsrc)) {
call.incoming_vsrc.destroy();
if (static_cast<bool>(call.outgoing_asink)) {
call.outgoing_asink.destroy();
}
}
@ -362,10 +407,27 @@ bool DebugToxCall::onEvent(const Events::FriendVideoBitrate&) {
bool DebugToxCall::onEvent(const Events::FriendAudioFrame& e) {
auto& call = _calls[e.friend_number];
call.num_a_frames++;
if (!static_cast<bool>(call.incoming_asrc)) {
// missing src to put frame into ??
return false;
}
assert(call.incoming_asrc.all_of<AudioFrameStream2MultiSource*>());
assert(call.incoming_asrc.all_of<Components::FrameStream2Source<AudioFrame>>());
call.num_a_frames++;
call.incoming_asrc.get<AudioFrameStream2MultiSource*>()->push(AudioFrame{
0, //seq
e.sampling_rate,
e.channels,
std::vector<int16_t>(e.pcm.begin(), e.pcm.end()) // copy
});
return true;
}
bool DebugToxCall::onEvent(const Events::FriendVideoFrame& e) {
// TODO: skip if we dont know about this call
auto& call = _calls[e.friend_number];

2
src/debug_tox_call.hpp
View File

@ -37,7 +37,7 @@ class DebugToxCall : public ToxAVEventI {
public:
DebugToxCall(ObjectStore2& os, ToxAV& toxav, TextureUploaderI& tu);
~DebugToxCall(void) {}
~DebugToxCall(void);
void tick(float time_delta);
float render(void);

26
src/stream_manager_ui.cpp
View File

@ -1,6 +1,7 @@
#include "./stream_manager_ui.hpp"
#include "./content/sdl_video_frame_stream2.hpp"
#include "./content/audio_stream.hpp"
#include <solanaceae/object_store/object_store.hpp>
@ -58,6 +59,31 @@ void StreamManagerUI::render(void) {
ImGui::EndMenu();
}
if (ImGui::BeginMenu("connect audio", ss.frame_type_name == entt::type_name<AudioFrame>::value())) {
for (const auto& [oc_src, s_src] : _os.registry().view<Components::StreamSource>().each()) {
if (s_src.frame_type_name != ss.frame_type_name) {
continue;
}
ImGui::PushID(entt::to_integral(oc_src));
std::string source_label {"src "};
source_label += std::to_string(entt::to_integral(entt::to_entity(oc_src)));
source_label += " (";
source_label += s_src.name;
source_label += ")[";
source_label += s_src.frame_type_name;
source_label += "]";
if (ImGui::MenuItem(source_label.c_str())) {
_sm.connect<AudioFrame>(oc_src, oc);
}
ImGui::PopID();
}
ImGui::EndMenu();
}
ImGui::EndPopup();
}
ImGui::PopID();

2
src/tox_av.cpp
View File

@ -216,7 +216,7 @@ void ToxAV::cb_audio_receive_frame(uint32_t friend_number, const int16_t pcm[],
ToxAV_Event::friend_audio_frame,
Events::FriendAudioFrame{
friend_number,
Span<int16_t>(pcm, sample_count), // TODO: is sample count *ch or /ch?
Span<int16_t>(pcm, sample_count*channels), // TODO: is sample count *ch or /ch?
channels,
sampling_rate,
}