use sdl yuv to yuv conversion, and better fallbacks

display intended main screen intervals
actually report min interval in debug view and more

src/CMakeLists.txt

@@ -116,7 +116,6 @@ target_sources(tomato PUBLIC
 	./frame_streams/sdl/sdl_audio2_frame_stream2.cpp
 	./frame_streams/sdl/video.hpp
 	./frame_streams/sdl/video_push_converter.hpp
-	./frame_streams/sdl/video_push_converter.cpp
 	./frame_streams/sdl/sdl_video_frame_stream2.hpp
 	./frame_streams/sdl/sdl_video_frame_stream2.cpp
 

src/debug_video_tap.cpp

@@ -220,7 +220,7 @@ float DebugVideoTap::render(void) {
 	for (auto& [view, stream] : dvtsw) {
 		std::string window_title {"DebugVideoTap #"};
 		window_title += std::to_string(view._id);
-		ImGui::SetNextWindowSize({250, 250}, ImGuiCond_Appearing);
+		ImGui::SetNextWindowSize({400, 420}, ImGuiCond_Appearing);
 		if (ImGui::Begin(window_title.c_str())) {
 			while (auto new_frame_opt = stream->pop()) {
 				// timing
@@ -233,7 +233,7 @@ float DebugVideoTap::render(void) {
 					if (view._v_interval_avg == 0) {
 						view._v_interval_avg = delta/1'000'000.f;
 					} else {
-						const float r = 0.2f;
+						const float r = 0.05f;
 						view._v_interval_avg = view._v_interval_avg * (1.f-r) + (delta/1'000'000.f) * r;
 					}
 				}
@@ -277,7 +277,7 @@ float DebugVideoTap::render(void) {
 			// img here
 			if (view._tex != 0) {
 				ImGui::SameLine();
-				ImGui::Text("moving avg interval: %f", view._v_interval_avg);
+				ImGui::Text("%dx%d ~avg interval: %.0fms (%.2ffps)", view._tex_w, view._tex_h, view._v_interval_avg*1000.f, 1.f/view._v_interval_avg);
 				const float img_w = ImGui::GetContentRegionAvail().x;
 				ImGui::Image(
 					reinterpret_cast<ImTextureID>(view._tex),
@@ -287,6 +287,9 @@ float DebugVideoTap::render(void) {
 				);
 			}
 
+			if (view._v_interval_avg > 0) {
+				min_interval = std::min(min_interval, view._v_interval_avg);
+			}
 		}
 		ImGui::End();
 	}

src/frame_streams/sdl/video_push_converter.cpp

@@ -1,106 +0,0 @@
-#include "./video_push_converter.hpp"
-
-#include <cstring>
-
-SDL_Surface* convertYUY2_IYUV(SDL_Surface* surf) {
-	if (surf->format != SDL_PIXELFORMAT_YUY2) {
-		return nullptr;
-	}
-	if ((surf->w % 2) != 0) {
-		SDL_SetError("YUY2->IYUV does not support odd widths");
-		// hmmm, we dont handle odd widths
-		return nullptr;
-	}
-
-	SDL_LockSurface(surf);
-
-	SDL_Surface* conv_surf = SDL_CreateSurface(surf->w, surf->h, SDL_PIXELFORMAT_IYUV);
-	SDL_LockSurface(conv_surf);
-
-	// YUY2 is 4:2:2 packed
-	// Y is simple, we just copy it over
-	// U V are double the resolution (vertically), so we avg both
-	// Packed mode: Y0+U0+Y1+V0 (1 plane)
-
-	uint8_t* y_plane = static_cast<uint8_t*>(conv_surf->pixels);
-	uint8_t* u_plane = static_cast<uint8_t*>(conv_surf->pixels) + conv_surf->w*conv_surf->h;
-	uint8_t* v_plane = static_cast<uint8_t*>(conv_surf->pixels) + conv_surf->w*conv_surf->h + (conv_surf->w/2)*(conv_surf->h/2);
-
-	const uint8_t* yuy2_data = static_cast<const uint8_t*>(surf->pixels);
-
-	for (int y = 0; y < surf->h; y++) {
-		for (int x = 0; x < surf->w; x += 2) {
-			// every pixel uses 2 bytes
-			const uint8_t* yuy2_curser = yuy2_data + y*surf->w*2 + x*2;
-			uint8_t src_y0 = yuy2_curser[0];
-			uint8_t src_u = yuy2_curser[1];
-			uint8_t src_y1 = yuy2_curser[2];
-			uint8_t src_v = yuy2_curser[3];
-
-			y_plane[y*conv_surf->w + x] = src_y0;
-			y_plane[y*conv_surf->w + x+1] = src_y1;
-
-			size_t uv_index = (y/2) * (conv_surf->w/2) + x/2;
-			if (y % 2 == 0) {
-				// first write
-				u_plane[uv_index] = src_u;
-				v_plane[uv_index] = src_v;
-			} else {
-				// second write, mix with existing value
-				u_plane[uv_index] = (int(u_plane[uv_index]) + int(src_u)) / 2;
-				v_plane[uv_index] = (int(v_plane[uv_index]) + int(src_v)) / 2;
-			}
-		}
-	}
-
-	SDL_UnlockSurface(conv_surf);
-	SDL_UnlockSurface(surf);
-	return conv_surf;
-}
-
-SDL_Surface* convertNV12_IYUV(SDL_Surface* surf) {
-	if (surf->format != SDL_PIXELFORMAT_NV12) {
-		return nullptr;
-	}
-	if ((surf->w % 2) != 0) {
-		SDL_SetError("NV12->IYUV does not support odd widths");
-		// hmmm, we dont handle odd widths
-		return nullptr;
-	}
-
-	SDL_LockSurface(surf);
-
-	SDL_Surface* conv_surf = SDL_CreateSurface(surf->w, surf->h, SDL_PIXELFORMAT_IYUV);
-	SDL_LockSurface(conv_surf);
-
-	// NV12 is planar Y and interleaved UV in 4:2:0
-	// Y is simple, we just copy it over
-	// U V are ... packed?
-
-	uint8_t* y_plane = static_cast<uint8_t*>(conv_surf->pixels);
-	uint8_t* u_plane = static_cast<uint8_t*>(conv_surf->pixels) + conv_surf->w*conv_surf->h;
-	uint8_t* v_plane = static_cast<uint8_t*>(conv_surf->pixels) + conv_surf->w*conv_surf->h + (conv_surf->w/2)*(conv_surf->h/2);
-
-	const uint8_t* nv12_y_plane = static_cast<const uint8_t*>(surf->pixels);
-	const uint8_t* nv12_uv_data = static_cast<const uint8_t*>(surf->pixels) + surf->w*surf->h;
-
-	// y can be copied as is
-	std::memcpy(
-		y_plane,
-		nv12_y_plane,
-		surf->w*surf->h
-	);
-
-	// we operate at half res
-	for (int y = 0; y < surf->h/2; y++) {
-		for (int x = 0; x < surf->w/2; x++) {
-			u_plane[y*(surf->w/2)+x] = nv12_uv_data[y*(surf->w/2)*2+x*2];
-			v_plane[y*(surf->w/2)+x] = nv12_uv_data[y*(surf->w/2)*2+x*2+1];
-		}
-	}
-
-	SDL_UnlockSurface(conv_surf);
-	SDL_UnlockSurface(surf);
-	return conv_surf;
-}
-

src/frame_streams/sdl/video_push_converter.hpp

@@ -7,25 +7,10 @@
 
 #include <iostream> // meh
 
-static bool isFormatYUV(SDL_PixelFormat f) {
-	return
-		f == SDL_PIXELFORMAT_YV12 ||
-		f == SDL_PIXELFORMAT_IYUV ||
-		f == SDL_PIXELFORMAT_YUY2 ||
-		f == SDL_PIXELFORMAT_UYVY ||
-		f == SDL_PIXELFORMAT_YVYU ||
-		f == SDL_PIXELFORMAT_NV12 ||
-		f == SDL_PIXELFORMAT_NV21 ||
-		f == SDL_PIXELFORMAT_P010
-	;
-}
-
-SDL_Surface* convertYUY2_IYUV(SDL_Surface* surf);
-SDL_Surface* convertNV12_IYUV(SDL_Surface* surf);
-
 template<typename RealStream>
 struct PushConversionVideoStream : public RealStream {
 	SDL_PixelFormat _forced_format {SDL_PIXELFORMAT_IYUV};
+	// TODO: force colorspace?
 
 	template<typename... Args>
 	PushConversionVideoStream(SDL_PixelFormat forced_format, Args&&... args) : RealStream(std::forward<Args>(args)...), _forced_format(forced_format) {}
@@ -34,43 +19,33 @@ struct PushConversionVideoStream : public RealStream {
 	bool push(const SDLVideoFrame& value) override {
 		SDL_Surface* surf = value.surface.get();
 		if (surf->format != _forced_format) {
-			//std::cerr << "DTC: need to convert from " << SDL_GetPixelFormatName(converted_surf->format) << " to SDL_PIXELFORMAT_IYUV\n";
+			//std::cerr << "PCVS: need to convert from " << SDL_GetPixelFormatName(surf->format) << " to " << SDL_GetPixelFormatName(_forced_format) << "\n";
-			if (surf->format == SDL_PIXELFORMAT_YUY2 && _forced_format == SDL_PIXELFORMAT_IYUV) {
+			if ((surf = SDL_ConvertSurface(surf, _forced_format)) == nullptr) {
-				// optimized custom impl
+				surf = value.surface.get(); // reset ptr
+				//std::cerr << "PCVS warning: default conversion failed: " << SDL_GetError() << "\n";
 
-				//auto start = Message::getTimeMS();
+				// sdl hardcodes BT709_LIMITED
-				surf = convertYUY2_IYUV(surf);
+				if ((surf = SDL_ConvertSurfaceAndColorspace(surf, _forced_format, nullptr, SDL_GetSurfaceColorspace(surf), 0)) == nullptr) {
-				//auto end = Message::getTimeMS();
+				//if ((surf = SDL_ConvertSurfaceAndColorspace(surf, _forced_format, nullptr, SDL_COLORSPACE_BT709_LIMITED, 0)) == nullptr) {
-				// 3ms
+					surf = value.surface.get(); // reset ptr
-				//std::cerr << "DTC: timing " << SDL_GetPixelFormatName(converted_surf->format) << "->SDL_PIXELFORMAT_IYUV: " << end-start << "ms\n";
+					//std::cerr << "PCVS warning: default conversion with same colorspace failed: " << SDL_GetError() << "\n";
-			} else if (surf->format == SDL_PIXELFORMAT_NV12 && _forced_format == SDL_PIXELFORMAT_IYUV) {
+					// simple convert failed, fall back to ->rgb->yuv
-				surf = convertNV12_IYUV(surf);
+					//SDL_Surface* tmp_conv_surf = SDL_ConvertSurfaceAndColorspace(surf, SDL_PIXELFORMAT_RGB24, nullptr, SDL_COLORSPACE_RGB_DEFAULT, 0);
-			} else if (isFormatYUV(surf->format)) {
+					SDL_Surface* tmp_conv_surf = SDL_ConvertSurface(surf, SDL_PIXELFORMAT_RGB24);
-				// TODO: fix sdl rgb->yuv conversion resulting in too dark (colorspace) issues
+					if (tmp_conv_surf == nullptr) {
-				// https://github.com/libsdl-org/SDL/issues/10877
+						std::cerr << "PCVS error: conversion to RGB failed: " << SDL_GetError() << "\n";
-
+					} else {
-				// meh, need to convert to rgb as a stopgap
+						//surf = SDL_ConvertSurfaceAndColorspace(tmp_conv_surf, _forced_format, nullptr, SDL_COLORSPACE_YUV_DEFAULT, 0);
-
+						surf = SDL_ConvertSurface(tmp_conv_surf, _forced_format);
-				//auto start = Message::getTimeMS();
+						//surf = SDL_ConvertSurfaceAndColorspace(tmp_conv_surf, _forced_format, nullptr, SDL_COLORSPACE_BT601_LIMITED, 0);
-				SDL_Surface* tmp_conv_surf = SDL_ConvertSurfaceAndColorspace(surf, SDL_PIXELFORMAT_RGB24, nullptr, SDL_COLORSPACE_RGB_DEFAULT, 0);
+						SDL_DestroySurface(tmp_conv_surf);
-				//auto end = Message::getTimeMS();
+					}
-				// 1ms
+				}
-				//std::cerr << "DTC: timing " << SDL_GetPixelFormatName(converted_surf->format) << "->SDL_PIXELFORMAT_RGB24: " << end-start << "ms\n";
-
-				//start = Message::getTimeMS();
-				surf = SDL_ConvertSurfaceAndColorspace(tmp_conv_surf, _forced_format, nullptr, SDL_COLORSPACE_YUV_DEFAULT, 0);
-				//end = Message::getTimeMS();
-				// 60ms
-				//std::cerr << "DTC: timing SDL_PIXELFORMAT_RGB24->" << SDL_GetPixelFormatName(_forced_format) << ": " << end-start << "ms\n";
-
-				SDL_DestroySurface(tmp_conv_surf);
-			} else {
-				surf = SDL_ConvertSurface(surf, _forced_format);
 			}
 
 			if (surf == nullptr) {
 				// oh god
-				std::cerr << "DTC error: failed to convert surface to IYUV: " << SDL_GetError() << "\n";
+				std::cerr << "PCVS error: failed to convert surface to IYUV: " << SDL_GetError() << "\n";
 				return false;
 			}
 		}

src/frame_streams/stream_manager.hpp

@@ -194,7 +194,7 @@ bool StreamManager::connect(Object src, Object sink, bool threaded) {
 		std::move(our_data),
 		[](Connection& con) -> void {
 			// there might be more stored
-			for (size_t i = 0; i < 10; i++) {
+			for (size_t i = 0; i < 64; i++) {
 				auto new_frame_opt = static_cast<inlineData*>(con.data.get())->reader->pop();
 				// TODO: frame interval estimates
 				if (new_frame_opt.has_value()) {

src/main_screen.cpp

@@ -346,6 +346,11 @@ Screen* MainScreen::render(float time_delta, bool&) {
 						ImGui::SetItemTooltip("Limiting compute can slow down things like filetransfers!");
 					}
 
+					ImGui::Separator();
+
+					ImGui::Text("render interval: %.0fms (%.2ffps)", _render_interval*1000.f, 1.f/_render_interval);
+					ImGui::Text("tick   interval: %.0fms (%.2ftps)", _min_tick_interval*1000.f, 1.f/_min_tick_interval);
+
 					ImGui::EndMenu();
 				}
 				if (ImGui::BeginMenu("Settings")) {
@@ -504,12 +509,12 @@ Screen* MainScreen::render(float time_delta, bool&) {
 	// min over non animations in all cases
 	_render_interval = std::min<float>(pm_interval, cg_interval);
 	_render_interval = std::min<float>(_render_interval, tc_unfinished_queue_interval);
+	_render_interval = std::min<float>(_render_interval, dvt_interval);
 
 	// low delay time window
 	if (!_window_hidden && _time_since_event < curr_profile.low_delay_window) {
 		_render_interval = std::min<float>(_render_interval, ctc_interval);
 		_render_interval = std::min<float>(_render_interval, msgtc_interval);
-		_render_interval = std::min<float>(_render_interval, dvt_interval);
 
 		_render_interval = std::clamp(
 			_render_interval,
@@ -520,7 +525,6 @@ Screen* MainScreen::render(float time_delta, bool&) {
 	} else if (!_window_hidden && _time_since_event < curr_profile.mid_delay_window) {
 		_render_interval = std::min<float>(_render_interval, ctc_interval);
 		_render_interval = std::min<float>(_render_interval, msgtc_interval);
-		_render_interval = std::min<float>(_render_interval, dvt_interval);
 
 		_render_interval = std::clamp(
 			_render_interval,

src/tox_av_voip_model.cpp

@@ -701,7 +701,9 @@ bool ToxAVVoIPModel::onEvent(const Events::FriendVideoFrame& e) {
 
 	vsrc.get<FrameStream2MultiSource<SDLVideoFrame>*>()->push({
 		// ms -> us
-		Message::getTimeMS() * 1000, // TODO: make more precise
+		// would be nice if we had been giving this from toxcore
+		// TODO: make more precise
+		Message::getTimeMS() * 1000,
 		new_surf
 	});