add video frame type and debug viewer and debug test source

the test source thread will always exist for now
the debug view will open a window for each connection

src/CMakeLists.txt

@@ -110,9 +110,13 @@ target_sources(tomato PUBLIC
 
 	./frame_streams/sdl/sdl_audio2_frame_stream2.hpp
 	./frame_streams/sdl/sdl_audio2_frame_stream2.cpp
+	./frame_streams/sdl/video.hpp
 
 	./stream_manager_ui.hpp
 	./stream_manager_ui.cpp
+
+	./debug_video_tap.hpp
+	./debug_video_tap.cpp
 )
 
 if (TOMATO_TOX_AV)

src/debug_video_tap.cpp

@@ -0,0 +1,295 @@
+#include "./debug_video_tap.hpp"
+
+#include <solanaceae/object_store/object_store.hpp>
+
+#include <entt/entity/entity.hpp>
+
+#include <SDL3/SDL.h>
+
+#include <imgui/imgui.h>
+
+#include "./frame_streams/sdl/video.hpp"
+#include "./frame_streams/frame_stream2.hpp"
+
+#include <string>
+#include <memory>
+#include <mutex>
+#include <deque>
+
+#include <thread>
+#include <chrono>
+#include <atomic>
+
+#include <iostream>
+
+// fwd
+namespace Message {
+uint64_t getTimeMS(void);
+}
+
+// threadsafe queue frame stream
+// protected by a simple mutex lock
+template<typename FrameType>
+struct LockedFrameStream2 : public FrameStream2I<FrameType> {
+	std::mutex _lock;
+
+	std::deque<FrameType> _frames;
+
+	~LockedFrameStream2(void) {}
+
+	int32_t size(void) { return -1; }
+
+	std::optional<FrameType> pop(void) {
+		std::lock_guard lg{_lock};
+
+		if (_frames.empty()) {
+			return std::nullopt;
+		}
+
+		FrameType new_frame = std::move(_frames.front());
+		_frames.pop_front();
+
+		return std::move(new_frame);
+	}
+
+	bool push(const FrameType& value) {
+		std::lock_guard lg{_lock};
+
+		_frames.push_back(value);
+
+		return true;
+	}
+};
+
+struct DebugVideoTapSink : public FrameStream2SinkI<SDLVideoFrame> {
+	TextureUploaderI& _tu;
+
+	uint32_t _id_counter {0};
+
+	struct Writer {
+		struct View {
+			uint32_t _id {0}; // for stable imgui ids
+
+			uint64_t _tex {0};
+			uint32_t _tex_w {0};
+			uint32_t _tex_h {0};
+
+			bool _mirror {false}; // flip horizontally
+
+			uint64_t _v_last_ts {0}; // us
+			float _v_interval_avg {0.f}; // s
+		} view;
+
+		std::shared_ptr<LockedFrameStream2<SDLVideoFrame>> stream;
+	};
+	std::vector<Writer> _writers;
+
+	DebugVideoTapSink(TextureUploaderI& tu) : _tu(tu) {}
+	~DebugVideoTapSink(void) {}
+
+	// sink
+	std::shared_ptr<FrameStream2I<SDLVideoFrame>> subscribe(void) override {
+		_writers.emplace_back(Writer{
+			Writer::View{_id_counter++},
+			std::make_shared<LockedFrameStream2<SDLVideoFrame>>()
+		});
+
+		return _writers.back().stream;
+	}
+
+	bool unsubscribe(const std::shared_ptr<FrameStream2I<SDLVideoFrame>>& sub) override {
+		if (!sub || _writers.empty()) {
+			// nah
+			return false;
+		}
+
+		for (auto it = _writers.cbegin(); it != _writers.cend(); it++) {
+			if (it->stream == sub) {
+				_tu.destroy(it->view._tex);
+				_writers.erase(it);
+				return true;
+			}
+		}
+
+		// what
+		return false;
+	}
+};
+
+struct DebugVideoTestSource : public FrameStream2SourceI<SDLVideoFrame> {
+	std::vector<std::shared_ptr<LockedFrameStream2<SDLVideoFrame>>> _readers;
+
+	std::atomic_bool _stop {false};
+	std::thread _thread;
+
+	DebugVideoTestSource(void) {
+		std::cout << "DVTS: starting new test video source\n";
+		_thread = std::thread([this](void) {
+			while (!_stop) {
+				if (!_readers.empty()) {
+					auto* surf = SDL_CreateSurface(960, 720, SDL_PIXELFORMAT_ARGB32);
+
+					// color
+					static auto start_time = Message::getTimeMS();
+					const float time = (Message::getTimeMS() - start_time)/1000.f;
+					SDL_ClearSurface(surf, std::sin(time), std::cos(time), 0.5f, 1.f);
+
+					SDLVideoFrame frame{ // non-owning
+						Message::getTimeMS()*1000,
+						surf,
+					};
+
+					for (auto& stream : _readers) {
+						stream->push(frame); // copy
+					}
+
+					SDL_DestroySurface(surf);
+				}
+
+				std::this_thread::sleep_for(std::chrono::milliseconds(50));
+			}
+		});
+	}
+	~DebugVideoTestSource(void) {
+		_stop = true;
+		_thread.join();
+	}
+
+	std::shared_ptr<FrameStream2I<SDLVideoFrame>> subscribe(void) override {
+		return _readers.emplace_back(std::make_shared<LockedFrameStream2<SDLVideoFrame>>());
+	}
+
+	bool unsubscribe(const std::shared_ptr<FrameStream2I<SDLVideoFrame>>& sub) override {
+		for (auto it = _readers.cbegin(); it != _readers.cend(); it++) {
+			if (it->get() == sub.get()) {
+				_readers.erase(it);
+				return true;
+			}
+		}
+
+		return false;
+	}
+};
+
+DebugVideoTap::DebugVideoTap(ObjectStore2& os, StreamManager& sm, TextureUploaderI& tu) : _os(os), _sm(sm), _tu(tu) {
+	// post self as video sink
+	_tap = {_os.registry(), _os.registry().create()};
+	try {
+		auto dvts = std::make_unique<DebugVideoTapSink>(_tu);
+		_tap.emplace<DebugVideoTapSink*>(dvts.get()); // to get our data back
+		_tap.emplace<Components::FrameStream2Sink<SDLVideoFrame>>(
+			std::move(dvts)
+		);
+
+		_tap.emplace<Components::StreamSink>(Components::StreamSink::create<SDLVideoFrame>("DebugVideoTap"));
+
+		_os.throwEventConstruct(_tap);
+	} catch (...) {
+		_os.registry().destroy(_tap);
+	}
+
+	_src = {_os.registry(), _os.registry().create()};
+	try {
+		auto dvts = std::make_unique<DebugVideoTestSource>();
+		_src.emplace<DebugVideoTestSource*>(dvts.get());
+		_src.emplace<Components::FrameStream2Source<SDLVideoFrame>>(
+			std::move(dvts)
+		);
+
+		_src.emplace<Components::StreamSource>(Components::StreamSource::create<SDLVideoFrame>("DebugVideoTest"));
+
+		_os.throwEventConstruct(_src);
+	} catch (...) {
+		_os.registry().destroy(_src);
+	}
+}
+
+DebugVideoTap::~DebugVideoTap(void) {
+	if (static_cast<bool>(_tap)) {
+		_os.registry().destroy(_tap);
+	}
+	if (static_cast<bool>(_src)) {
+		_os.registry().destroy(_src);
+	}
+}
+
+float DebugVideoTap::render(void) {
+	float min_interval {2.f};
+	auto& dvtsw = _tap.get<DebugVideoTapSink*>()->_writers;
+	for (auto& [view, stream] : dvtsw) {
+		std::string window_title {"DebugVideoTap #"};
+		window_title += std::to_string(view._id);
+		ImGui::SetNextWindowSize({250, 250}, ImGuiCond_Appearing);
+		if (ImGui::Begin(window_title.c_str())) {
+			while (auto new_frame_opt = stream->pop()) {
+				// timing
+				if (view._v_last_ts == 0) {
+					view._v_last_ts = new_frame_opt.value().timestampUS;
+				} else {
+					auto delta = int64_t(new_frame_opt.value().timestampUS) - int64_t(view._v_last_ts);
+					view._v_last_ts = new_frame_opt.value().timestampUS;
+
+					if (view._v_interval_avg == 0) {
+						view._v_interval_avg = delta/1'000'000.f;
+					} else {
+						const float r = 0.2f;
+						view._v_interval_avg = view._v_interval_avg * (1.f-r) + (delta/1'000'000.f) * r;
+					}
+				}
+
+				SDL_Surface* new_frame_surf = new_frame_opt.value().surface.get();
+
+				SDL_Surface* converted_surf = new_frame_surf;
+				if (new_frame_surf->format != SDL_PIXELFORMAT_RGBA32) {
+					// we need to convert
+					//std::cerr << "DVT: need to convert\n";
+					converted_surf = SDL_ConvertSurfaceAndColorspace(new_frame_surf, SDL_PIXELFORMAT_RGBA32, nullptr, SDL_COLORSPACE_RGB_DEFAULT, 0);
+					assert(converted_surf->format == SDL_PIXELFORMAT_RGBA32);
+				}
+
+				SDL_LockSurface(converted_surf);
+				if (view._tex == 0 || (int)view._tex_w != converted_surf->w || (int)view._tex_h != converted_surf->h) {
+					_tu.destroy(view._tex);
+					view._tex = _tu.uploadRGBA(
+						static_cast<const uint8_t*>(converted_surf->pixels),
+						converted_surf->w,
+						converted_surf->h,
+						TextureUploaderI::LINEAR,
+						TextureUploaderI::STREAMING
+					);
+
+					view._tex_w = converted_surf->w;
+					view._tex_h = converted_surf->h;
+				} else {
+					_tu.updateRGBA(view._tex, static_cast<const uint8_t*>(converted_surf->pixels), converted_surf->w * converted_surf->h * 4);
+				}
+				SDL_UnlockSurface(converted_surf);
+
+				if (new_frame_surf != converted_surf) {
+					// clean up temp
+					SDL_DestroySurface(converted_surf);
+				}
+			}
+
+			ImGui::Checkbox("mirror", &view._mirror);
+
+			// img here
+			if (view._tex != 0) {
+				ImGui::SameLine();
+				ImGui::Text("moving avg interval: %f", view._v_interval_avg);
+				const float img_w = ImGui::GetContentRegionAvail().x;
+				ImGui::Image(
+					reinterpret_cast<ImTextureID>(view._tex),
+					ImVec2{img_w, img_w * float(view._tex_h)/view._tex_w},
+					ImVec2{view._mirror?1.f:0.f, 0},
+					ImVec2{view._mirror?0.f:1.f, 1}
+				);
+			}
+
+		}
+		ImGui::End();
+	}
+
+	return min_interval;
+}
+

src/debug_video_tap.hpp

@@ -0,0 +1,23 @@
+#pragma once
+
+#include <solanaceae/object_store/fwd.hpp>
+#include "./frame_streams/stream_manager.hpp"
+#include "./texture_uploader.hpp"
+
+// provides a sink and a small window displaying a SDLVideoFrame
+// HACK: provides a test video source
+class DebugVideoTap {
+	ObjectStore2& _os;
+	StreamManager& _sm;
+	TextureUploaderI& _tu;
+
+	ObjectHandle _tap;
+	ObjectHandle _src;
+
+	public:
+		DebugVideoTap(ObjectStore2& os, StreamManager& sm, TextureUploaderI& tu);
+		~DebugVideoTap(void);
+
+		float render(void);
+};
+

src/frame_streams/sdl/video.hpp

@@ -0,0 +1,41 @@
+#pragma once
+
+#include <SDL3/SDL.h>
+
+#include <cstdint>
+#include <memory>
+
+// https://youtu.be/71Iw4Q74OaE
+
+inline void nopSurfaceDestructor(SDL_Surface*) {}
+
+// this is very sdl specific
+// but allows us to autoconvert between formats (to a degree)
+struct SDLVideoFrame {
+	// micro seconds (nano is way too much)
+	uint64_t timestampUS {0};
+
+	std::unique_ptr<SDL_Surface, decltype(&SDL_DestroySurface)> surface {nullptr, &SDL_DestroySurface};
+
+	// special non-owning constructor
+	SDLVideoFrame(
+		uint64_t ts,
+		SDL_Surface* surf
+	) {
+		timestampUS = ts;
+		surface = {surf, &nopSurfaceDestructor};
+	}
+	SDLVideoFrame(SDLVideoFrame&& other) = default;
+	// copy
+	SDLVideoFrame(const SDLVideoFrame& other) {
+		timestampUS = other.timestampUS;
+		if (static_cast<bool>(other.surface)) {
+			surface = {
+				SDL_DuplicateSurface(other.surface.get()),
+				&SDL_DestroySurface
+			};
+		}
+	}
+	SDLVideoFrame& operator=(const SDLVideoFrame& other) = delete;
+};
+

src/main_screen.cpp

@@ -45,7 +45,8 @@ MainScreen::MainScreen(SimpleConfigModel&& conf_, SDL_Renderer* renderer_, Theme
 	osui(os),
 	tuiu(tc, conf),
 	tdch(tpi),
-	smui(os, sm)
+	smui(os, sm),
+	dvt(os, sm, sdlrtu)
 {
 	tel.subscribeAll(tc);
 
@@ -300,6 +301,7 @@ Screen* MainScreen::render(float time_delta, bool&) {
 	tuiu.render(); // render
 	tdch.render(); // render
 	smui.render();
+	const float dvt_interval = dvt.render();
 
 	{ // main window menubar injection
 		if (ImGui::Begin("tomato")) {
@@ -482,6 +484,7 @@ Screen* MainScreen::render(float time_delta, bool&) {
 	if (!_window_hidden && _time_since_event < curr_profile.low_delay_window) {
 		_render_interval = std::min<float>(_render_interval, ctc_interval);
 		_render_interval = std::min<float>(_render_interval, msgtc_interval);
+		_render_interval = std::min<float>(_render_interval, dvt_interval);
 
 		_render_interval = std::clamp(
 			_render_interval,
@@ -492,6 +495,7 @@ Screen* MainScreen::render(float time_delta, bool&) {
 	} else if (!_window_hidden && _time_since_event < curr_profile.mid_delay_window) {
 		_render_interval = std::min<float>(_render_interval, ctc_interval);
 		_render_interval = std::min<float>(_render_interval, msgtc_interval);
+		_render_interval = std::min<float>(_render_interval, dvt_interval);
 
 		_render_interval = std::clamp(
 			_render_interval,

src/main_screen.hpp

@@ -35,6 +35,7 @@
 #include "./tox_dht_cap_histo.hpp"
 #include "./tox_friend_faux_offline_messaging.hpp"
 #include "./stream_manager_ui.hpp"
+#include "./debug_video_tap.hpp"
 
 #if TOMATO_TOX_AV
 #include "./tox_av.hpp"
@@ -93,6 +94,7 @@ struct MainScreen final : public Screen {
 	ToxUIUtils tuiu;
 	ToxDHTCapHisto tdch;
 	StreamManagerUI smui;
+	DebugVideoTap dvt;
 
 	PluginManager pm; // last, so it gets destroyed first