Compare commits

..

2 Commits

Author SHA1 Message Date
54a57896b6
sdl video push conversion stream and toxav video sink
Some checks are pending
ContinuousDelivery / linux-ubuntu (push) Waiting to run
ContinuousDelivery / android (map[ndk_abi:arm64-v8a vcpkg_toolkit:arm64-android]) (push) Waiting to run
ContinuousDelivery / android (map[ndk_abi:x86_64 vcpkg_toolkit:x64-android]) (push) Waiting to run
ContinuousDelivery / windows (push) Waiting to run
ContinuousDelivery / windows-asan (push) Waiting to run
ContinuousDelivery / release (push) Blocked by required conditions
ContinuousIntegration / linux (push) Waiting to run
ContinuousIntegration / android (map[ndk_abi:arm64-v8a vcpkg_toolkit:arm64-android]) (push) Waiting to run
ContinuousIntegration / android (map[ndk_abi:x86_64 vcpkg_toolkit:x64-android]) (push) Waiting to run
ContinuousIntegration / macos (push) Waiting to run
ContinuousIntegration / windows (push) Waiting to run
2024-10-02 12:42:17 +02:00
51ec99a42f
add toxav incoming video (sdl video) 2024-10-02 11:45:06 +02:00
5 changed files with 393 additions and 7 deletions

View File

@ -115,6 +115,8 @@ target_sources(tomato PUBLIC
./frame_streams/sdl/sdl_audio2_frame_stream2.hpp ./frame_streams/sdl/sdl_audio2_frame_stream2.hpp
./frame_streams/sdl/sdl_audio2_frame_stream2.cpp ./frame_streams/sdl/sdl_audio2_frame_stream2.cpp
./frame_streams/sdl/video.hpp ./frame_streams/sdl/video.hpp
./frame_streams/sdl/video_push_converter.hpp
./frame_streams/sdl/video_push_converter.cpp
./stream_manager_ui.hpp ./stream_manager_ui.hpp
./stream_manager_ui.cpp ./stream_manager_ui.cpp

View File

@ -0,0 +1,58 @@
#include "./video_push_converter.hpp"
SDL_Surface* convertYUY2_IYUV(SDL_Surface* surf) {
if (surf->format != SDL_PIXELFORMAT_YUY2) {
return nullptr;
}
if ((surf->w % 2) != 0) {
SDL_SetError("YUY2->IYUV does not support odd widths");
// hmmm, we dont handle odd widths
return nullptr;
}
SDL_LockSurface(surf);
SDL_Surface* conv_surf = SDL_CreateSurface(surf->w, surf->h, SDL_PIXELFORMAT_IYUV);
SDL_LockSurface(conv_surf);
// YUY2 is 4:2:2 packed
// Y is simple, we just copy it over
// U V are double the resolution (vertically), so we avg both
// Packed mode: Y0+U0+Y1+V0 (1 plane)
uint8_t* y_plane = static_cast<uint8_t*>(conv_surf->pixels);
uint8_t* u_plane = static_cast<uint8_t*>(conv_surf->pixels) + conv_surf->w*conv_surf->h;
uint8_t* v_plane = static_cast<uint8_t*>(conv_surf->pixels) + conv_surf->w*conv_surf->h + (conv_surf->w/2)*(conv_surf->h/2);
const uint8_t* yuy2_data = static_cast<const uint8_t*>(surf->pixels);
for (int y = 0; y < surf->h; y++) {
for (int x = 0; x < surf->w; x += 2) {
// every pixel uses 2 bytes
const uint8_t* yuy2_curser = yuy2_data + y*surf->w*2 + x*2;
uint8_t src_y0 = yuy2_curser[0];
uint8_t src_u = yuy2_curser[1];
uint8_t src_y1 = yuy2_curser[2];
uint8_t src_v = yuy2_curser[3];
y_plane[y*conv_surf->w + x] = src_y0;
y_plane[y*conv_surf->w + x+1] = src_y1;
size_t uv_index = (y/2) * (conv_surf->w/2) + x/2;
if (y % 2 == 0) {
// first write
u_plane[uv_index] = src_u;
v_plane[uv_index] = src_v;
} else {
// second write, mix with existing value
u_plane[uv_index] = (int(u_plane[uv_index]) + int(src_u)) / 2;
v_plane[uv_index] = (int(v_plane[uv_index]) + int(src_v)) / 2;
}
}
}
SDL_UnlockSurface(conv_surf);
SDL_UnlockSurface(surf);
return conv_surf;
}

View File

@ -0,0 +1,89 @@
#pragma once
#include "./video.hpp"
#include "../frame_stream2.hpp"
#include <cassert>
#include <iostream> // meh
static bool isFormatYUV(SDL_PixelFormat f) {
return
f == SDL_PIXELFORMAT_YV12 ||
f == SDL_PIXELFORMAT_IYUV ||
f == SDL_PIXELFORMAT_YUY2 ||
f == SDL_PIXELFORMAT_UYVY ||
f == SDL_PIXELFORMAT_YVYU ||
f == SDL_PIXELFORMAT_NV12 ||
f == SDL_PIXELFORMAT_NV21 ||
f == SDL_PIXELFORMAT_P010
;
}
SDL_Surface* convertYUY2_IYUV(SDL_Surface* surf);
template<typename RealStream>
struct PushConversionVideoStream : public RealStream {
SDL_PixelFormat _forced_format {SDL_PIXELFORMAT_IYUV};
template<typename... Args>
PushConversionVideoStream(SDL_PixelFormat forced_format, Args&&... args) : RealStream(std::forward<Args>(args)...), _forced_format(forced_format) {}
~PushConversionVideoStream(void) {}
bool push(const SDLVideoFrame& value) override {
SDL_Surface* surf = value.surface.get();
if (surf->format != _forced_format) {
//std::cerr << "DTC: need to convert from " << SDL_GetPixelFormatName(converted_surf->format) << " to SDL_PIXELFORMAT_IYUV\n";
if (surf->format == SDL_PIXELFORMAT_YUY2 && _forced_format == SDL_PIXELFORMAT_IYUV) {
// optimized custom impl
//auto start = Message::getTimeMS();
surf = convertYUY2_IYUV(surf);
//auto end = Message::getTimeMS();
// 3ms
//std::cerr << "DTC: timing " << SDL_GetPixelFormatName(converted_surf->format) << "->SDL_PIXELFORMAT_IYUV: " << end-start << "ms\n";
} else if (isFormatYUV(surf->format)) {
// TODO: fix sdl rgb->yuv conversion resulting in too dark (colorspace) issues
// https://github.com/libsdl-org/SDL/issues/10877
// meh, need to convert to rgb as a stopgap
//auto start = Message::getTimeMS();
SDL_Surface* tmp_conv_surf = SDL_ConvertSurfaceAndColorspace(surf, SDL_PIXELFORMAT_RGB24, nullptr, SDL_COLORSPACE_RGB_DEFAULT, 0);
//auto end = Message::getTimeMS();
// 1ms
//std::cerr << "DTC: timing " << SDL_GetPixelFormatName(converted_surf->format) << "->SDL_PIXELFORMAT_RGB24: " << end-start << "ms\n";
//start = Message::getTimeMS();
surf = SDL_ConvertSurfaceAndColorspace(tmp_conv_surf, _forced_format, nullptr, SDL_COLORSPACE_YUV_DEFAULT, 0);
//end = Message::getTimeMS();
// 60ms
//std::cerr << "DTC: timing SDL_PIXELFORMAT_RGB24->" << SDL_GetPixelFormatName(_forced_format) << ": " << end-start << "ms\n";
SDL_DestroySurface(tmp_conv_surf);
} else {
surf = SDL_ConvertSurface(surf, _forced_format);
}
if (surf == nullptr) {
// oh god
std::cerr << "DTC error: failed to convert surface to IYUV: " << SDL_GetError() << "\n";
return false;
}
}
assert(surf != nullptr);
if (surf != value.surface.get()) {
// TODO: add ctr with uptr
SDLVideoFrame new_value{value.timestampUS, nullptr};
new_value.surface = {
surf,
&SDL_DestroySurface
};
return RealStream::push(std::move(new_value));
} else {
return RealStream::push(value);
}
}
};

View File

@ -9,8 +9,18 @@
#include "./frame_streams/multi_source.hpp" #include "./frame_streams/multi_source.hpp"
#include "./frame_streams/audio_stream_pop_reframer.hpp" #include "./frame_streams/audio_stream_pop_reframer.hpp"
#include "./frame_streams/sdl/video.hpp"
#include "./frame_streams/sdl/video_push_converter.hpp"
#include <cstring>
#include <iostream> #include <iostream>
// fwd
namespace Message {
uint64_t getTimeMS(void);
} // Message
namespace Components { namespace Components {
struct ToxAVIncomingAV { struct ToxAVIncomingAV {
bool incoming_audio {false}; bool incoming_audio {false};
@ -21,12 +31,15 @@ namespace Components {
ObjectHandle o; ObjectHandle o;
// ptr? // ptr?
}; };
// vid struct ToxAVVideoSink {
ObjectHandle o;
};
struct ToxAVAudioSource { struct ToxAVAudioSource {
ObjectHandle o; ObjectHandle o;
// ptr?
}; };
// vid struct ToxAVVideoSource {
ObjectHandle o;
};
} // Components } // Components
struct ToxAVCallAudioSink : public FrameStream2SinkI<AudioFrame2> { struct ToxAVCallAudioSink : public FrameStream2SinkI<AudioFrame2> {
@ -84,6 +97,63 @@ struct ToxAVCallAudioSink : public FrameStream2SinkI<AudioFrame2> {
} }
}; };
// exlusive
struct ToxAVCallVideoSink : public FrameStream2SinkI<SDLVideoFrame> {
using stream_type = PushConversionVideoStream<LockedFrameStream2<SDLVideoFrame>>;
ToxAVI& _toxav;
// bitrate for enabled state
uint32_t _video_bitrate {2}; // HACK: hardcode to 2mbits (toxap wrongly multiplies internally by 1000)
uint32_t _fid;
std::shared_ptr<stream_type> _writer;
ToxAVCallVideoSink(ToxAVI& toxav, uint32_t fid) : _toxav(toxav), _fid(fid) {}
~ToxAVCallVideoSink(void) {
if (_writer) {
_writer = nullptr;
_toxav.toxavVideoSetBitRate(_fid, 0);
}
}
// sink
std::shared_ptr<FrameStream2I<SDLVideoFrame>> subscribe(void) override {
if (_writer) {
// max 1 (exclusive, composite video somewhere else)
return nullptr;
}
auto err = _toxav.toxavVideoSetBitRate(_fid, _video_bitrate);
if (err != TOXAV_ERR_BIT_RATE_SET_OK) {
return nullptr;
}
// toxav needs I420
_writer = std::make_shared<stream_type>(SDL_PIXELFORMAT_IYUV);
return _writer;
}
bool unsubscribe(const std::shared_ptr<FrameStream2I<SDLVideoFrame>>& sub) override {
if (!sub || !_writer) {
// nah
return false;
}
if (sub == _writer) {
_writer = nullptr;
/*auto err = */_toxav.toxavVideoSetBitRate(_fid, 0);
// print warning? on error?
return true;
}
// what
return false;
}
};
void ToxAVVoIPModel::addAudioSource(ObjectHandle session, uint32_t friend_number) { void ToxAVVoIPModel::addAudioSource(ObjectHandle session, uint32_t friend_number) {
auto& stream_source = session.get_or_emplace<Components::VoIP::StreamSources>().streams; auto& stream_source = session.get_or_emplace<Components::VoIP::StreamSources>().streams;
@ -94,13 +164,11 @@ void ToxAVVoIPModel::addAudioSource(ObjectHandle session, uint32_t friend_number
incoming_audio.emplace<Components::FrameStream2Source<AudioFrame2>>(std::move(new_asrc)); incoming_audio.emplace<Components::FrameStream2Source<AudioFrame2>>(std::move(new_asrc));
incoming_audio.emplace<Components::StreamSource>(Components::StreamSource::create<AudioFrame2>("ToxAV Friend Call Incoming Audio")); incoming_audio.emplace<Components::StreamSource>(Components::StreamSource::create<AudioFrame2>("ToxAV Friend Call Incoming Audio"));
std::cout << "new incoming audio\n";
if ( if (
const auto* defaults = session.try_get<Components::VoIP::DefaultConfig>(); const auto* defaults = session.try_get<Components::VoIP::DefaultConfig>();
defaults != nullptr && defaults->incoming_audio defaults != nullptr && defaults->incoming_audio
) { ) {
incoming_audio.emplace<Components::TagConnectToDefault>(); // depends on what was specified in enter() incoming_audio.emplace<Components::TagConnectToDefault>(); // depends on what was specified in enter()
std::cout << "with default\n";
} }
stream_source.push_back(incoming_audio); stream_source.push_back(incoming_audio);
@ -135,6 +203,55 @@ void ToxAVVoIPModel::addAudioSink(ObjectHandle session, uint32_t friend_number)
_os.throwEventConstruct(outgoing_audio); _os.throwEventConstruct(outgoing_audio);
} }
void ToxAVVoIPModel::addVideoSource(ObjectHandle session, uint32_t friend_number) {
auto& stream_source = session.get_or_emplace<Components::VoIP::StreamSources>().streams;
ObjectHandle incoming_video {_os.registry(), _os.registry().create()};
auto new_vsrc = std::make_unique<FrameStream2MultiSource<SDLVideoFrame>>();
incoming_video.emplace<FrameStream2MultiSource<SDLVideoFrame>*>(new_vsrc.get());
incoming_video.emplace<Components::FrameStream2Source<SDLVideoFrame>>(std::move(new_vsrc));
incoming_video.emplace<Components::StreamSource>(Components::StreamSource::create<SDLVideoFrame>("ToxAV Friend Call Incoming Video"));
if (
const auto* defaults = session.try_get<Components::VoIP::DefaultConfig>();
defaults != nullptr && defaults->incoming_video
) {
incoming_video.emplace<Components::TagConnectToDefault>(); // depends on what was specified in enter()
}
stream_source.push_back(incoming_video);
session.emplace<Components::ToxAVVideoSource>(incoming_video);
// TODO: tie session to stream
_video_sources[friend_number] = incoming_video;
_os.throwEventConstruct(incoming_video);
}
void ToxAVVoIPModel::addVideoSink(ObjectHandle session, uint32_t friend_number) {
auto& stream_sinks = session.get_or_emplace<Components::VoIP::StreamSinks>().streams;
ObjectHandle outgoing_video {_os.registry(), _os.registry().create()};
auto new_vsink = std::make_unique<ToxAVCallVideoSink>(_av, friend_number);
outgoing_video.emplace<ToxAVCallVideoSink*>(new_vsink.get());
outgoing_video.emplace<Components::FrameStream2Sink<SDLVideoFrame>>(std::move(new_vsink));
outgoing_video.emplace<Components::StreamSink>(Components::StreamSink::create<SDLVideoFrame>("ToxAV Friend Call Outgoing Video"));
if (
const auto* defaults = session.try_get<Components::VoIP::DefaultConfig>();
defaults != nullptr && defaults->outgoing_video
) {
outgoing_video.emplace<Components::TagConnectToDefault>(); // depends on what was specified in enter()
}
stream_sinks.push_back(outgoing_video);
session.emplace<Components::ToxAVVideoSink>(outgoing_video);
// TODO: tie session to stream
_os.throwEventConstruct(outgoing_video);
}
void ToxAVVoIPModel::destroySession(ObjectHandle session) { void ToxAVVoIPModel::destroySession(ObjectHandle session) {
if (!static_cast<bool>(session)) { if (!static_cast<bool>(session)) {
return; return;
@ -153,6 +270,18 @@ void ToxAVVoIPModel::destroySession(ObjectHandle session) {
_audio_sources.erase(it_asrc); _audio_sources.erase(it_asrc);
} }
} }
if (session.all_of<Components::ToxAVVideoSource>()) {
auto it_vsrc = std::find_if(
_video_sources.cbegin(), _video_sources.cend(),
[o = session.get<Components::ToxAVVideoSource>().o](const auto& it) {
return it.second == o;
}
);
if (it_vsrc != _video_sources.cend()) {
_video_sources.erase(it_vsrc);
}
}
// destory sources // destory sources
if (auto* ss = session.try_get<Components::VoIP::StreamSources>(); ss != nullptr) { if (auto* ss = session.try_get<Components::VoIP::StreamSources>(); ss != nullptr) {
@ -237,6 +366,39 @@ void ToxAVVoIPModel::tick(void) {
} }
} }
} }
for (const auto& [oc, vsink] : _os.registry().view<ToxAVCallVideoSink*>().each()) {
if (!vsink->_writer) {
continue;
}
for (size_t i = 0; i < 10; i++) {
auto new_frame_opt = vsink->_writer->pop();
if (!new_frame_opt.has_value()) {
break;
}
const auto& new_frame = new_frame_opt.value();
if (!new_frame.surface) {
// wtf?
continue;
}
// conversion is done in the sink's stream
SDL_Surface* surf = new_frame.surface.get();
assert(surf != nullptr);
SDL_LockSurface(surf);
_av.toxavVideoSendFrame(
vsink->_fid,
surf->w, surf->h,
static_cast<const uint8_t*>(surf->pixels),
static_cast<const uint8_t*>(surf->pixels) + surf->w * surf->h,
static_cast<const uint8_t*>(surf->pixels) + surf->w * surf->h + (surf->w/2) * (surf->h/2)
);
SDL_UnlockSurface(surf);
}
}
} }
ObjectHandle ToxAVVoIPModel::enter(const Contact3 c, const Components::VoIP::DefaultConfig& defaults) { ObjectHandle ToxAVVoIPModel::enter(const Contact3 c, const Components::VoIP::DefaultConfig& defaults) {
@ -311,6 +473,8 @@ bool ToxAVVoIPModel::accept(ObjectHandle session, const Components::VoIP::Defaul
// bitrate cb or what? // bitrate cb or what?
assert(!session.all_of<Components::ToxAVAudioSink>()); assert(!session.all_of<Components::ToxAVAudioSink>());
addAudioSink(session, friend_number); addAudioSink(session, friend_number);
assert(!session.all_of<Components::ToxAVVideoSink>());
addVideoSink(session, friend_number);
if (const auto* i_av = session.try_get<Components::ToxAVIncomingAV>(); i_av != nullptr) { if (const auto* i_av = session.try_get<Components::ToxAVIncomingAV>(); i_av != nullptr) {
// create audio src // create audio src
@ -321,6 +485,8 @@ bool ToxAVVoIPModel::accept(ObjectHandle session, const Components::VoIP::Defaul
// create video src // create video src
if (i_av->incoming_video) { if (i_av->incoming_video) {
assert(!session.all_of<Components::ToxAVVideoSource>());
addVideoSource(session, friend_number);
} }
} }
@ -419,6 +585,11 @@ bool ToxAVVoIPModel::onEvent(const Events::FriendCallState& e) {
} }
// video // video
if (s.is_accepting_v() && !o.all_of<Components::ToxAVVideoSink>()) {
addVideoSink(o, e.friend_number);
} else if (!s.is_accepting_v() && o.all_of<Components::ToxAVVideoSink>()) {
// remove vsink?
}
// add/update sources // add/update sources
// audio // audio
@ -429,6 +600,11 @@ bool ToxAVVoIPModel::onEvent(const Events::FriendCallState& e) {
} }
// video // video
if (s.is_sending_v() && !o.all_of<Components::ToxAVVideoSource>()) {
addVideoSource(o, e.friend_number);
} else if (!s.is_sending_v() && o.all_of<Components::ToxAVVideoSource>()) {
// remove vsrc?
}
} }
} }
} }
@ -470,7 +646,66 @@ bool ToxAVVoIPModel::onEvent(const Events::FriendAudioFrame& e) {
return true; return true;
} }
bool ToxAVVoIPModel::onEvent(const Events::FriendVideoFrame&) { bool ToxAVVoIPModel::onEvent(const Events::FriendVideoFrame& e) {
auto vsrc_it = _video_sources.find(e.friend_number);
if (vsrc_it == _video_sources.cend()) {
// missing src from lookup table
return false;
}
auto vsrc = vsrc_it->second;
if (!static_cast<bool>(vsrc)) {
// missing src to put frame into ??
return false;
}
assert(vsrc.all_of<FrameStream2MultiSource<SDLVideoFrame>*>());
assert(vsrc.all_of<Components::FrameStream2Source<SDLVideoFrame>>());
auto* new_surf = SDL_CreateSurface(e.width, e.height, SDL_PIXELFORMAT_IYUV);
assert(new_surf);
if (SDL_LockSurface(new_surf)) {
// copy the data
// we know how the implementation works, its y u v consecutivlely
// y
for (size_t y = 0; y < e.height; y++) {
std::memcpy(
//static_cast<uint8_t*>(new_surf->pixels) + new_surf->pitch*y,
static_cast<uint8_t*>(new_surf->pixels) + e.width*y,
e.y.ptr + e.ystride*y,
e.width
);
}
// u
for (size_t y = 0; y < e.height/2; y++) {
std::memcpy(
static_cast<uint8_t*>(new_surf->pixels) + (e.width*e.height) + (e.width/2)*y,
e.u.ptr + e.ustride*y,
e.width/2
);
}
// v
for (size_t y = 0; y < e.height/2; y++) {
std::memcpy(
static_cast<uint8_t*>(new_surf->pixels) + (e.width*e.height) + ((e.width/2)*(e.height/2)) + (e.width/2)*y,
e.v.ptr + e.vstride*y,
e.width/2
);
}
SDL_UnlockSurface(new_surf);
}
vsrc.get<FrameStream2MultiSource<SDLVideoFrame>*>()->push({
// ms -> us
Message::getTimeMS() * 1000, // TODO: make more precise
new_surf
});
SDL_DestroySurface(new_surf);
return false; return false;
} }

View File

@ -16,11 +16,13 @@ class ToxAVVoIPModel : protected ToxAVEventI, public VoIPModelI {
// for faster lookup // for faster lookup
std::unordered_map<uint32_t, ObjectHandle> _audio_sources; std::unordered_map<uint32_t, ObjectHandle> _audio_sources;
std::unordered_map<uint32_t, ObjectHandle> _video_sources;
// TODO: virtual? strategy? protected? // TODO: virtual? strategy? protected?
virtual void addAudioSource(ObjectHandle session, uint32_t friend_number); virtual void addAudioSource(ObjectHandle session, uint32_t friend_number);
virtual void addAudioSink(ObjectHandle session, uint32_t friend_number); virtual void addAudioSink(ObjectHandle session, uint32_t friend_number);
// TODO: video virtual void addVideoSource(ObjectHandle session, uint32_t friend_number);
virtual void addVideoSink(ObjectHandle session, uint32_t friend_number);
void destroySession(ObjectHandle session); void destroySession(ObjectHandle session);