add video frame type and debug viewer and debug test source
Some checks are pending
ContinuousDelivery / linux-ubuntu (push) Waiting to run
ContinuousDelivery / android (map[ndk_abi:arm64-v8a vcpkg_toolkit:arm64-android]) (push) Waiting to run
ContinuousDelivery / android (map[ndk_abi:x86_64 vcpkg_toolkit:x64-android]) (push) Waiting to run
ContinuousDelivery / windows (push) Waiting to run
ContinuousDelivery / windows-asan (push) Waiting to run
ContinuousDelivery / release (push) Blocked by required conditions
ContinuousIntegration / linux (push) Waiting to run
ContinuousIntegration / android (map[ndk_abi:arm64-v8a vcpkg_toolkit:arm64-android]) (push) Waiting to run
ContinuousIntegration / android (map[ndk_abi:x86_64 vcpkg_toolkit:x64-android]) (push) Waiting to run
ContinuousIntegration / macos (push) Waiting to run
ContinuousIntegration / windows (push) Waiting to run

the test source thread will always exist for now
the debug view will open a window for each connection
This commit is contained in:
Green Sky 2024-09-28 11:56:47 +02:00
parent 59cdb2638f
commit 248b00dafb
No known key found for this signature in database
6 changed files with 370 additions and 1 deletions

View File

@ -110,9 +110,13 @@ target_sources(tomato PUBLIC
./frame_streams/sdl/sdl_audio2_frame_stream2.hpp ./frame_streams/sdl/sdl_audio2_frame_stream2.hpp
./frame_streams/sdl/sdl_audio2_frame_stream2.cpp ./frame_streams/sdl/sdl_audio2_frame_stream2.cpp
./frame_streams/sdl/video.hpp
./stream_manager_ui.hpp ./stream_manager_ui.hpp
./stream_manager_ui.cpp ./stream_manager_ui.cpp
./debug_video_tap.hpp
./debug_video_tap.cpp
) )
if (TOMATO_TOX_AV) if (TOMATO_TOX_AV)

295
src/debug_video_tap.cpp Normal file
View File

@ -0,0 +1,295 @@
#include "./debug_video_tap.hpp"
#include <solanaceae/object_store/object_store.hpp>
#include <entt/entity/entity.hpp>
#include <SDL3/SDL.h>
#include <imgui/imgui.h>
#include "./frame_streams/sdl/video.hpp"
#include "./frame_streams/frame_stream2.hpp"
#include <string>
#include <memory>
#include <mutex>
#include <deque>
#include <thread>
#include <chrono>
#include <atomic>
#include <iostream>
// fwd
namespace Message {
uint64_t getTimeMS(void);
}
// threadsafe queue frame stream
// protected by a simple mutex lock
template<typename FrameType>
struct LockedFrameStream2 : public FrameStream2I<FrameType> {
std::mutex _lock;
std::deque<FrameType> _frames;
~LockedFrameStream2(void) {}
int32_t size(void) { return -1; }
std::optional<FrameType> pop(void) {
std::lock_guard lg{_lock};
if (_frames.empty()) {
return std::nullopt;
}
FrameType new_frame = std::move(_frames.front());
_frames.pop_front();
return std::move(new_frame);
}
bool push(const FrameType& value) {
std::lock_guard lg{_lock};
_frames.push_back(value);
return true;
}
};
struct DebugVideoTapSink : public FrameStream2SinkI<SDLVideoFrame> {
TextureUploaderI& _tu;
uint32_t _id_counter {0};
struct Writer {
struct View {
uint32_t _id {0}; // for stable imgui ids
uint64_t _tex {0};
uint32_t _tex_w {0};
uint32_t _tex_h {0};
bool _mirror {false}; // flip horizontally
uint64_t _v_last_ts {0}; // us
float _v_interval_avg {0.f}; // s
} view;
std::shared_ptr<LockedFrameStream2<SDLVideoFrame>> stream;
};
std::vector<Writer> _writers;
DebugVideoTapSink(TextureUploaderI& tu) : _tu(tu) {}
~DebugVideoTapSink(void) {}
// sink
std::shared_ptr<FrameStream2I<SDLVideoFrame>> subscribe(void) override {
_writers.emplace_back(Writer{
Writer::View{_id_counter++},
std::make_shared<LockedFrameStream2<SDLVideoFrame>>()
});
return _writers.back().stream;
}
bool unsubscribe(const std::shared_ptr<FrameStream2I<SDLVideoFrame>>& sub) override {
if (!sub || _writers.empty()) {
// nah
return false;
}
for (auto it = _writers.cbegin(); it != _writers.cend(); it++) {
if (it->stream == sub) {
_tu.destroy(it->view._tex);
_writers.erase(it);
return true;
}
}
// what
return false;
}
};
struct DebugVideoTestSource : public FrameStream2SourceI<SDLVideoFrame> {
std::vector<std::shared_ptr<LockedFrameStream2<SDLVideoFrame>>> _readers;
std::atomic_bool _stop {false};
std::thread _thread;
DebugVideoTestSource(void) {
std::cout << "DVTS: starting new test video source\n";
_thread = std::thread([this](void) {
while (!_stop) {
if (!_readers.empty()) {
auto* surf = SDL_CreateSurface(960, 720, SDL_PIXELFORMAT_ARGB32);
// color
static auto start_time = Message::getTimeMS();
const float time = (Message::getTimeMS() - start_time)/1000.f;
SDL_ClearSurface(surf, std::sin(time), std::cos(time), 0.5f, 1.f);
SDLVideoFrame frame{ // non-owning
Message::getTimeMS()*1000,
surf,
};
for (auto& stream : _readers) {
stream->push(frame); // copy
}
SDL_DestroySurface(surf);
}
std::this_thread::sleep_for(std::chrono::milliseconds(50));
}
});
}
~DebugVideoTestSource(void) {
_stop = true;
_thread.join();
}
std::shared_ptr<FrameStream2I<SDLVideoFrame>> subscribe(void) override {
return _readers.emplace_back(std::make_shared<LockedFrameStream2<SDLVideoFrame>>());
}
bool unsubscribe(const std::shared_ptr<FrameStream2I<SDLVideoFrame>>& sub) override {
for (auto it = _readers.cbegin(); it != _readers.cend(); it++) {
if (it->get() == sub.get()) {
_readers.erase(it);
return true;
}
}
return false;
}
};
DebugVideoTap::DebugVideoTap(ObjectStore2& os, StreamManager& sm, TextureUploaderI& tu) : _os(os), _sm(sm), _tu(tu) {
// post self as video sink
_tap = {_os.registry(), _os.registry().create()};
try {
auto dvts = std::make_unique<DebugVideoTapSink>(_tu);
_tap.emplace<DebugVideoTapSink*>(dvts.get()); // to get our data back
_tap.emplace<Components::FrameStream2Sink<SDLVideoFrame>>(
std::move(dvts)
);
_tap.emplace<Components::StreamSink>(Components::StreamSink::create<SDLVideoFrame>("DebugVideoTap"));
_os.throwEventConstruct(_tap);
} catch (...) {
_os.registry().destroy(_tap);
}
_src = {_os.registry(), _os.registry().create()};
try {
auto dvts = std::make_unique<DebugVideoTestSource>();
_src.emplace<DebugVideoTestSource*>(dvts.get());
_src.emplace<Components::FrameStream2Source<SDLVideoFrame>>(
std::move(dvts)
);
_src.emplace<Components::StreamSource>(Components::StreamSource::create<SDLVideoFrame>("DebugVideoTest"));
_os.throwEventConstruct(_src);
} catch (...) {
_os.registry().destroy(_src);
}
}
DebugVideoTap::~DebugVideoTap(void) {
if (static_cast<bool>(_tap)) {
_os.registry().destroy(_tap);
}
if (static_cast<bool>(_src)) {
_os.registry().destroy(_src);
}
}
float DebugVideoTap::render(void) {
float min_interval {2.f};
auto& dvtsw = _tap.get<DebugVideoTapSink*>()->_writers;
for (auto& [view, stream] : dvtsw) {
std::string window_title {"DebugVideoTap #"};
window_title += std::to_string(view._id);
ImGui::SetNextWindowSize({250, 250}, ImGuiCond_Appearing);
if (ImGui::Begin(window_title.c_str())) {
while (auto new_frame_opt = stream->pop()) {
// timing
if (view._v_last_ts == 0) {
view._v_last_ts = new_frame_opt.value().timestampUS;
} else {
auto delta = int64_t(new_frame_opt.value().timestampUS) - int64_t(view._v_last_ts);
view._v_last_ts = new_frame_opt.value().timestampUS;
if (view._v_interval_avg == 0) {
view._v_interval_avg = delta/1'000'000.f;
} else {
const float r = 0.2f;
view._v_interval_avg = view._v_interval_avg * (1.f-r) + (delta/1'000'000.f) * r;
}
}
SDL_Surface* new_frame_surf = new_frame_opt.value().surface.get();
SDL_Surface* converted_surf = new_frame_surf;
if (new_frame_surf->format != SDL_PIXELFORMAT_RGBA32) {
// we need to convert
//std::cerr << "DVT: need to convert\n";
converted_surf = SDL_ConvertSurfaceAndColorspace(new_frame_surf, SDL_PIXELFORMAT_RGBA32, nullptr, SDL_COLORSPACE_RGB_DEFAULT, 0);
assert(converted_surf->format == SDL_PIXELFORMAT_RGBA32);
}
SDL_LockSurface(converted_surf);
if (view._tex == 0 || (int)view._tex_w != converted_surf->w || (int)view._tex_h != converted_surf->h) {
_tu.destroy(view._tex);
view._tex = _tu.uploadRGBA(
static_cast<const uint8_t*>(converted_surf->pixels),
converted_surf->w,
converted_surf->h,
TextureUploaderI::LINEAR,
TextureUploaderI::STREAMING
);
view._tex_w = converted_surf->w;
view._tex_h = converted_surf->h;
} else {
_tu.updateRGBA(view._tex, static_cast<const uint8_t*>(converted_surf->pixels), converted_surf->w * converted_surf->h * 4);
}
SDL_UnlockSurface(converted_surf);
if (new_frame_surf != converted_surf) {
// clean up temp
SDL_DestroySurface(converted_surf);
}
}
ImGui::Checkbox("mirror", &view._mirror);
// img here
if (view._tex != 0) {
ImGui::SameLine();
ImGui::Text("moving avg interval: %f", view._v_interval_avg);
const float img_w = ImGui::GetContentRegionAvail().x;
ImGui::Image(
reinterpret_cast<ImTextureID>(view._tex),
ImVec2{img_w, img_w * float(view._tex_h)/view._tex_w},
ImVec2{view._mirror?1.f:0.f, 0},
ImVec2{view._mirror?0.f:1.f, 1}
);
}
}
ImGui::End();
}
return min_interval;
}

23
src/debug_video_tap.hpp Normal file
View File

@ -0,0 +1,23 @@
#pragma once
#include <solanaceae/object_store/fwd.hpp>
#include "./frame_streams/stream_manager.hpp"
#include "./texture_uploader.hpp"
// provides a sink and a small window displaying a SDLVideoFrame
// HACK: provides a test video source
class DebugVideoTap {
ObjectStore2& _os;
StreamManager& _sm;
TextureUploaderI& _tu;
ObjectHandle _tap;
ObjectHandle _src;
public:
DebugVideoTap(ObjectStore2& os, StreamManager& sm, TextureUploaderI& tu);
~DebugVideoTap(void);
float render(void);
};

View File

@ -0,0 +1,41 @@
#pragma once
#include <SDL3/SDL.h>
#include <cstdint>
#include <memory>
// https://youtu.be/71Iw4Q74OaE
inline void nopSurfaceDestructor(SDL_Surface*) {}
// this is very sdl specific
// but allows us to autoconvert between formats (to a degree)
struct SDLVideoFrame {
// micro seconds (nano is way too much)
uint64_t timestampUS {0};
std::unique_ptr<SDL_Surface, decltype(&SDL_DestroySurface)> surface {nullptr, &SDL_DestroySurface};
// special non-owning constructor
SDLVideoFrame(
uint64_t ts,
SDL_Surface* surf
) {
timestampUS = ts;
surface = {surf, &nopSurfaceDestructor};
}
SDLVideoFrame(SDLVideoFrame&& other) = default;
// copy
SDLVideoFrame(const SDLVideoFrame& other) {
timestampUS = other.timestampUS;
if (static_cast<bool>(other.surface)) {
surface = {
SDL_DuplicateSurface(other.surface.get()),
&SDL_DestroySurface
};
}
}
SDLVideoFrame& operator=(const SDLVideoFrame& other) = delete;
};

View File

@ -45,7 +45,8 @@ MainScreen::MainScreen(SimpleConfigModel&& conf_, SDL_Renderer* renderer_, Theme
osui(os), osui(os),
tuiu(tc, conf), tuiu(tc, conf),
tdch(tpi), tdch(tpi),
smui(os, sm) smui(os, sm),
dvt(os, sm, sdlrtu)
{ {
tel.subscribeAll(tc); tel.subscribeAll(tc);
@ -300,6 +301,7 @@ Screen* MainScreen::render(float time_delta, bool&) {
tuiu.render(); // render tuiu.render(); // render
tdch.render(); // render tdch.render(); // render
smui.render(); smui.render();
const float dvt_interval = dvt.render();
{ // main window menubar injection { // main window menubar injection
if (ImGui::Begin("tomato")) { if (ImGui::Begin("tomato")) {
@ -482,6 +484,7 @@ Screen* MainScreen::render(float time_delta, bool&) {
if (!_window_hidden && _time_since_event < curr_profile.low_delay_window) { if (!_window_hidden && _time_since_event < curr_profile.low_delay_window) {
_render_interval = std::min<float>(_render_interval, ctc_interval); _render_interval = std::min<float>(_render_interval, ctc_interval);
_render_interval = std::min<float>(_render_interval, msgtc_interval); _render_interval = std::min<float>(_render_interval, msgtc_interval);
_render_interval = std::min<float>(_render_interval, dvt_interval);
_render_interval = std::clamp( _render_interval = std::clamp(
_render_interval, _render_interval,
@ -492,6 +495,7 @@ Screen* MainScreen::render(float time_delta, bool&) {
} else if (!_window_hidden && _time_since_event < curr_profile.mid_delay_window) { } else if (!_window_hidden && _time_since_event < curr_profile.mid_delay_window) {
_render_interval = std::min<float>(_render_interval, ctc_interval); _render_interval = std::min<float>(_render_interval, ctc_interval);
_render_interval = std::min<float>(_render_interval, msgtc_interval); _render_interval = std::min<float>(_render_interval, msgtc_interval);
_render_interval = std::min<float>(_render_interval, dvt_interval);
_render_interval = std::clamp( _render_interval = std::clamp(
_render_interval, _render_interval,

View File

@ -35,6 +35,7 @@
#include "./tox_dht_cap_histo.hpp" #include "./tox_dht_cap_histo.hpp"
#include "./tox_friend_faux_offline_messaging.hpp" #include "./tox_friend_faux_offline_messaging.hpp"
#include "./stream_manager_ui.hpp" #include "./stream_manager_ui.hpp"
#include "./debug_video_tap.hpp"
#if TOMATO_TOX_AV #if TOMATO_TOX_AV
#include "./tox_av.hpp" #include "./tox_av.hpp"
@ -93,6 +94,7 @@ struct MainScreen final : public Screen {
ToxUIUtils tuiu; ToxUIUtils tuiu;
ToxDHTCapHisto tdch; ToxDHTCapHisto tdch;
StreamManagerUI smui; StreamManagerUI smui;
DebugVideoTap dvt;
PluginManager pm; // last, so it gets destroyed first PluginManager pm; // last, so it gets destroyed first