From 248b00dafb88f9c04a649d2d6b297677de110a2a Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sat, 28 Sep 2024 11:56:47 +0200 Subject: [PATCH] add video frame type and debug viewer and debug test source the test source thread will always exist for now the debug view will open a window for each connection --- src/CMakeLists.txt | 4 + src/debug_video_tap.cpp | 295 ++++++++++++++++++++++++++++++++ src/debug_video_tap.hpp | 23 +++ src/frame_streams/sdl/video.hpp | 41 +++++ src/main_screen.cpp | 6 +- src/main_screen.hpp | 2 + 6 files changed, 370 insertions(+), 1 deletion(-) create mode 100644 src/debug_video_tap.cpp create mode 100644 src/debug_video_tap.hpp create mode 100644 src/frame_streams/sdl/video.hpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ab9eb9b3..1bfe5e64 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -110,9 +110,13 @@ target_sources(tomato PUBLIC ./frame_streams/sdl/sdl_audio2_frame_stream2.hpp ./frame_streams/sdl/sdl_audio2_frame_stream2.cpp + ./frame_streams/sdl/video.hpp ./stream_manager_ui.hpp ./stream_manager_ui.cpp + + ./debug_video_tap.hpp + ./debug_video_tap.cpp ) if (TOMATO_TOX_AV) diff --git a/src/debug_video_tap.cpp b/src/debug_video_tap.cpp new file mode 100644 index 00000000..3f0d3ae7 --- /dev/null +++ b/src/debug_video_tap.cpp @@ -0,0 +1,295 @@ +#include "./debug_video_tap.hpp" + +#include + +#include + +#include + +#include + +#include "./frame_streams/sdl/video.hpp" +#include "./frame_streams/frame_stream2.hpp" + +#include +#include +#include +#include + +#include +#include +#include + +#include + +// fwd +namespace Message { +uint64_t getTimeMS(void); +} + +// threadsafe queue frame stream +// protected by a simple mutex lock +template +struct LockedFrameStream2 : public FrameStream2I { + std::mutex _lock; + + std::deque _frames; + + ~LockedFrameStream2(void) {} + + int32_t size(void) { return -1; } + + std::optional pop(void) { + std::lock_guard lg{_lock}; + + if (_frames.empty()) { + return std::nullopt; + } + + FrameType new_frame = std::move(_frames.front()); + _frames.pop_front(); + + return std::move(new_frame); + } + + bool push(const FrameType& value) { + std::lock_guard lg{_lock}; + + _frames.push_back(value); + + return true; + } +}; + +struct DebugVideoTapSink : public FrameStream2SinkI { + TextureUploaderI& _tu; + + uint32_t _id_counter {0}; + + struct Writer { + struct View { + uint32_t _id {0}; // for stable imgui ids + + uint64_t _tex {0}; + uint32_t _tex_w {0}; + uint32_t _tex_h {0}; + + bool _mirror {false}; // flip horizontally + + uint64_t _v_last_ts {0}; // us + float _v_interval_avg {0.f}; // s + } view; + + std::shared_ptr> stream; + }; + std::vector _writers; + + DebugVideoTapSink(TextureUploaderI& tu) : _tu(tu) {} + ~DebugVideoTapSink(void) {} + + // sink + std::shared_ptr> subscribe(void) override { + _writers.emplace_back(Writer{ + Writer::View{_id_counter++}, + std::make_shared>() + }); + + return _writers.back().stream; + } + + bool unsubscribe(const std::shared_ptr>& sub) override { + if (!sub || _writers.empty()) { + // nah + return false; + } + + for (auto it = _writers.cbegin(); it != _writers.cend(); it++) { + if (it->stream == sub) { + _tu.destroy(it->view._tex); + _writers.erase(it); + return true; + } + } + + // what + return false; + } +}; + +struct DebugVideoTestSource : public FrameStream2SourceI { + std::vector>> _readers; + + std::atomic_bool _stop {false}; + std::thread _thread; + + DebugVideoTestSource(void) { + std::cout << "DVTS: starting new test video source\n"; + _thread = std::thread([this](void) { + while (!_stop) { + if (!_readers.empty()) { + auto* surf = SDL_CreateSurface(960, 720, SDL_PIXELFORMAT_ARGB32); + + // color + static auto start_time = Message::getTimeMS(); + const float time = (Message::getTimeMS() - start_time)/1000.f; + SDL_ClearSurface(surf, std::sin(time), std::cos(time), 0.5f, 1.f); + + SDLVideoFrame frame{ // non-owning + Message::getTimeMS()*1000, + surf, + }; + + for (auto& stream : _readers) { + stream->push(frame); // copy + } + + SDL_DestroySurface(surf); + } + + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + }); + } + ~DebugVideoTestSource(void) { + _stop = true; + _thread.join(); + } + + std::shared_ptr> subscribe(void) override { + return _readers.emplace_back(std::make_shared>()); + } + + bool unsubscribe(const std::shared_ptr>& sub) override { + for (auto it = _readers.cbegin(); it != _readers.cend(); it++) { + if (it->get() == sub.get()) { + _readers.erase(it); + return true; + } + } + + return false; + } +}; + +DebugVideoTap::DebugVideoTap(ObjectStore2& os, StreamManager& sm, TextureUploaderI& tu) : _os(os), _sm(sm), _tu(tu) { + // post self as video sink + _tap = {_os.registry(), _os.registry().create()}; + try { + auto dvts = std::make_unique(_tu); + _tap.emplace(dvts.get()); // to get our data back + _tap.emplace>( + std::move(dvts) + ); + + _tap.emplace(Components::StreamSink::create("DebugVideoTap")); + + _os.throwEventConstruct(_tap); + } catch (...) { + _os.registry().destroy(_tap); + } + + _src = {_os.registry(), _os.registry().create()}; + try { + auto dvts = std::make_unique(); + _src.emplace(dvts.get()); + _src.emplace>( + std::move(dvts) + ); + + _src.emplace(Components::StreamSource::create("DebugVideoTest")); + + _os.throwEventConstruct(_src); + } catch (...) { + _os.registry().destroy(_src); + } +} + +DebugVideoTap::~DebugVideoTap(void) { + if (static_cast(_tap)) { + _os.registry().destroy(_tap); + } + if (static_cast(_src)) { + _os.registry().destroy(_src); + } +} + +float DebugVideoTap::render(void) { + float min_interval {2.f}; + auto& dvtsw = _tap.get()->_writers; + for (auto& [view, stream] : dvtsw) { + std::string window_title {"DebugVideoTap #"}; + window_title += std::to_string(view._id); + ImGui::SetNextWindowSize({250, 250}, ImGuiCond_Appearing); + if (ImGui::Begin(window_title.c_str())) { + while (auto new_frame_opt = stream->pop()) { + // timing + if (view._v_last_ts == 0) { + view._v_last_ts = new_frame_opt.value().timestampUS; + } else { + auto delta = int64_t(new_frame_opt.value().timestampUS) - int64_t(view._v_last_ts); + view._v_last_ts = new_frame_opt.value().timestampUS; + + if (view._v_interval_avg == 0) { + view._v_interval_avg = delta/1'000'000.f; + } else { + const float r = 0.2f; + view._v_interval_avg = view._v_interval_avg * (1.f-r) + (delta/1'000'000.f) * r; + } + } + + SDL_Surface* new_frame_surf = new_frame_opt.value().surface.get(); + + SDL_Surface* converted_surf = new_frame_surf; + if (new_frame_surf->format != SDL_PIXELFORMAT_RGBA32) { + // we need to convert + //std::cerr << "DVT: need to convert\n"; + converted_surf = SDL_ConvertSurfaceAndColorspace(new_frame_surf, SDL_PIXELFORMAT_RGBA32, nullptr, SDL_COLORSPACE_RGB_DEFAULT, 0); + assert(converted_surf->format == SDL_PIXELFORMAT_RGBA32); + } + + SDL_LockSurface(converted_surf); + if (view._tex == 0 || (int)view._tex_w != converted_surf->w || (int)view._tex_h != converted_surf->h) { + _tu.destroy(view._tex); + view._tex = _tu.uploadRGBA( + static_cast(converted_surf->pixels), + converted_surf->w, + converted_surf->h, + TextureUploaderI::LINEAR, + TextureUploaderI::STREAMING + ); + + view._tex_w = converted_surf->w; + view._tex_h = converted_surf->h; + } else { + _tu.updateRGBA(view._tex, static_cast(converted_surf->pixels), converted_surf->w * converted_surf->h * 4); + } + SDL_UnlockSurface(converted_surf); + + if (new_frame_surf != converted_surf) { + // clean up temp + SDL_DestroySurface(converted_surf); + } + } + + ImGui::Checkbox("mirror", &view._mirror); + + // img here + if (view._tex != 0) { + ImGui::SameLine(); + ImGui::Text("moving avg interval: %f", view._v_interval_avg); + const float img_w = ImGui::GetContentRegionAvail().x; + ImGui::Image( + reinterpret_cast(view._tex), + ImVec2{img_w, img_w * float(view._tex_h)/view._tex_w}, + ImVec2{view._mirror?1.f:0.f, 0}, + ImVec2{view._mirror?0.f:1.f, 1} + ); + } + + } + ImGui::End(); + } + + return min_interval; +} + diff --git a/src/debug_video_tap.hpp b/src/debug_video_tap.hpp new file mode 100644 index 00000000..47f7356a --- /dev/null +++ b/src/debug_video_tap.hpp @@ -0,0 +1,23 @@ +#pragma once + +#include +#include "./frame_streams/stream_manager.hpp" +#include "./texture_uploader.hpp" + +// provides a sink and a small window displaying a SDLVideoFrame +// HACK: provides a test video source +class DebugVideoTap { + ObjectStore2& _os; + StreamManager& _sm; + TextureUploaderI& _tu; + + ObjectHandle _tap; + ObjectHandle _src; + + public: + DebugVideoTap(ObjectStore2& os, StreamManager& sm, TextureUploaderI& tu); + ~DebugVideoTap(void); + + float render(void); +}; + diff --git a/src/frame_streams/sdl/video.hpp b/src/frame_streams/sdl/video.hpp new file mode 100644 index 00000000..b2c1b115 --- /dev/null +++ b/src/frame_streams/sdl/video.hpp @@ -0,0 +1,41 @@ +#pragma once + +#include + +#include +#include + +// https://youtu.be/71Iw4Q74OaE + +inline void nopSurfaceDestructor(SDL_Surface*) {} + +// this is very sdl specific +// but allows us to autoconvert between formats (to a degree) +struct SDLVideoFrame { + // micro seconds (nano is way too much) + uint64_t timestampUS {0}; + + std::unique_ptr surface {nullptr, &SDL_DestroySurface}; + + // special non-owning constructor + SDLVideoFrame( + uint64_t ts, + SDL_Surface* surf + ) { + timestampUS = ts; + surface = {surf, &nopSurfaceDestructor}; + } + SDLVideoFrame(SDLVideoFrame&& other) = default; + // copy + SDLVideoFrame(const SDLVideoFrame& other) { + timestampUS = other.timestampUS; + if (static_cast(other.surface)) { + surface = { + SDL_DuplicateSurface(other.surface.get()), + &SDL_DestroySurface + }; + } + } + SDLVideoFrame& operator=(const SDLVideoFrame& other) = delete; +}; + diff --git a/src/main_screen.cpp b/src/main_screen.cpp index 16d0c3d3..6871b1b6 100644 --- a/src/main_screen.cpp +++ b/src/main_screen.cpp @@ -45,7 +45,8 @@ MainScreen::MainScreen(SimpleConfigModel&& conf_, SDL_Renderer* renderer_, Theme osui(os), tuiu(tc, conf), tdch(tpi), - smui(os, sm) + smui(os, sm), + dvt(os, sm, sdlrtu) { tel.subscribeAll(tc); @@ -300,6 +301,7 @@ Screen* MainScreen::render(float time_delta, bool&) { tuiu.render(); // render tdch.render(); // render smui.render(); + const float dvt_interval = dvt.render(); { // main window menubar injection if (ImGui::Begin("tomato")) { @@ -482,6 +484,7 @@ Screen* MainScreen::render(float time_delta, bool&) { if (!_window_hidden && _time_since_event < curr_profile.low_delay_window) { _render_interval = std::min(_render_interval, ctc_interval); _render_interval = std::min(_render_interval, msgtc_interval); + _render_interval = std::min(_render_interval, dvt_interval); _render_interval = std::clamp( _render_interval, @@ -492,6 +495,7 @@ Screen* MainScreen::render(float time_delta, bool&) { } else if (!_window_hidden && _time_since_event < curr_profile.mid_delay_window) { _render_interval = std::min(_render_interval, ctc_interval); _render_interval = std::min(_render_interval, msgtc_interval); + _render_interval = std::min(_render_interval, dvt_interval); _render_interval = std::clamp( _render_interval, diff --git a/src/main_screen.hpp b/src/main_screen.hpp index 42f11254..cd985a9a 100644 --- a/src/main_screen.hpp +++ b/src/main_screen.hpp @@ -35,6 +35,7 @@ #include "./tox_dht_cap_histo.hpp" #include "./tox_friend_faux_offline_messaging.hpp" #include "./stream_manager_ui.hpp" +#include "./debug_video_tap.hpp" #if TOMATO_TOX_AV #include "./tox_av.hpp" @@ -93,6 +94,7 @@ struct MainScreen final : public Screen { ToxUIUtils tuiu; ToxDHTCapHisto tdch; StreamManagerUI smui; + DebugVideoTap dvt; PluginManager pm; // last, so it gets destroyed first