support direct yuv texture upload (limited) and update dvt

src/debug_video_tap.cpp

@@ -59,7 +59,7 @@ struct DebugVideoTapSink : public FrameStream2SinkI<SDLVideoFrame> {
 	std::shared_ptr<FrameStream2I<SDLVideoFrame>> subscribe(void) override {
 		_writers.emplace_back(Writer{
 			Writer::View{_id_counter++},
-			std::make_shared<PushConversionVideoStream<LockedFrameStream2<SDLVideoFrame>>>(SDL_PIXELFORMAT_RGBA32)
+			std::make_shared<PushConversionVideoStream<LockedFrameStream2<SDLVideoFrame>>>(SDL_PIXELFORMAT_IYUV)
 		});
 
 		return _writers.back().stream;
@@ -189,7 +189,7 @@ float DebugVideoTap::render(void) {
 		std::string window_title {"DebugVideoTap #"};
 		window_title += std::to_string(view._id);
 		ImGui::SetNextWindowSize({400, 420}, ImGuiCond_Appearing);
-		if (ImGui::Begin(window_title.c_str())) {
+		if (ImGui::Begin(window_title.c_str(), nullptr, ImGuiWindowFlags_NoScrollbar)) {
 			while (auto new_frame_opt = stream->pop()) {
 				// timing
 				if (view._v_last_ts == 0) {
@@ -209,20 +209,21 @@ float DebugVideoTap::render(void) {
 				SDL_Surface* new_frame_surf = new_frame_opt.value().surface.get();
 
 				SDL_Surface* converted_surf = new_frame_surf;
-				if (new_frame_surf->format != SDL_PIXELFORMAT_RGBA32) {
+				//if (new_frame_surf->format != SDL_PIXELFORMAT_RGBA32) {
-					// we need to convert
+				//    // we need to convert
-					//std::cerr << "DVT: need to convert\n";
+				//    //std::cerr << "DVT: need to convert\n";
-					converted_surf = SDL_ConvertSurfaceAndColorspace(new_frame_surf, SDL_PIXELFORMAT_RGBA32, nullptr, SDL_COLORSPACE_RGB_DEFAULT, 0);
+				//    converted_surf = SDL_ConvertSurfaceAndColorspace(new_frame_surf, SDL_PIXELFORMAT_RGBA32, nullptr, SDL_COLORSPACE_RGB_DEFAULT, 0);
-					assert(converted_surf->format == SDL_PIXELFORMAT_RGBA32);
+				//    assert(converted_surf->format == SDL_PIXELFORMAT_RGBA32);
-				}
+				//}
 
 				SDL_LockSurface(converted_surf);
 				if (view._tex == 0 || (int)view._tex_w != converted_surf->w || (int)view._tex_h != converted_surf->h) {
 					_tu.destroy(view._tex);
-					view._tex = _tu.uploadRGBA(
+					view._tex = _tu.upload(
 						static_cast<const uint8_t*>(converted_surf->pixels),
 						converted_surf->w,
 						converted_surf->h,
+						TextureUploaderI::IYUV, // forced conversion
 						TextureUploaderI::LINEAR,
 						TextureUploaderI::STREAMING
 					);
@@ -230,22 +231,24 @@ float DebugVideoTap::render(void) {
 					view._tex_w = converted_surf->w;
 					view._tex_h = converted_surf->h;
 				} else {
-					_tu.updateRGBA(view._tex, static_cast<const uint8_t*>(converted_surf->pixels), converted_surf->w * converted_surf->h * 4);
+					//_tu.update(view._tex, static_cast<const uint8_t*>(converted_surf->pixels), converted_surf->w * converted_surf->h * 4);
+					_tu.update(view._tex, static_cast<const uint8_t*>(converted_surf->pixels), converted_surf->w * converted_surf->h * 3/2);
+					//_tu.updateRGBA(view._tex, static_cast<const uint8_t*>(converted_surf->pixels), converted_surf->w * converted_surf->h * 4);
 				}
 				SDL_UnlockSurface(converted_surf);
 
-				if (new_frame_surf != converted_surf) {
+				//if (new_frame_surf != converted_surf) {
-					// clean up temp
+				//    // clean up temp
-					SDL_DestroySurface(converted_surf);
+				//    SDL_DestroySurface(converted_surf);
-				}
+				//}
 			}
 
-			ImGui::Checkbox("mirror", &view._mirror);
+			ImGui::Checkbox("mirror ", &view._mirror);
 
 			// img here
 			if (view._tex != 0) {
 				ImGui::SameLine();
-				ImGui::Text("%dx%d ~avg interval: %.0fms (%.2ffps)", view._tex_w, view._tex_h, view._v_interval_avg*1000.f, 1.f/view._v_interval_avg);
+				ImGui::Text("%dx%d interval: ~%.0fms (%.2ffps)", view._tex_w, view._tex_h, view._v_interval_avg*1000.f, 1.f/view._v_interval_avg);
 				const float img_w = ImGui::GetContentRegionAvail().x;
 				ImGui::Image(
 					reinterpret_cast<ImTextureID>(view._tex),

src/frame_streams/sdl/video_push_converter.hpp

@@ -5,6 +5,7 @@
 
 #include <cassert>
 
+#include <chrono> // meh
 #include <iostream> // meh
 
 template<typename RealStream>
@@ -20,6 +21,7 @@ struct PushConversionVideoStream : public RealStream {
 		SDL_Surface* surf = value.surface.get();
 		if (surf->format != _forced_format) {
 			//std::cerr << "PCVS: need to convert from " << SDL_GetPixelFormatName(surf->format) << " to " << SDL_GetPixelFormatName(_forced_format) << "\n";
+			//const auto start = std::chrono::steady_clock::now();
 			if ((surf = SDL_ConvertSurface(surf, _forced_format)) == nullptr) {
 				surf = value.surface.get(); // reset ptr
 				//std::cerr << "PCVS warning: default conversion failed: " << SDL_GetError() << "\n";
@@ -42,15 +44,16 @@ struct PushConversionVideoStream : public RealStream {
 					}
 				}
 			}
+			//const auto end = std::chrono::steady_clock::now();
+			//std::cerr << "PCVS: conversion took " << std::chrono::duration_cast<std::chrono::milliseconds>(end-start).count() << "ms\n";
 
 			if (surf == nullptr) {
 				// oh god
-				std::cerr << "PCVS error: failed to convert surface to IYUV: " << SDL_GetError() << "\n";
+				std::cerr << "PCVS error: failed to convert surface to " << SDL_GetPixelFormatName(_forced_format) << ": " << SDL_GetError() << "\n";
 				return false;
 			}
-		}
 			assert(surf != nullptr);
-		if (surf != value.surface.get()) {
+
 			// TODO: add ctr with uptr
 			SDLVideoFrame new_value{value.timestampUS, nullptr};
 			new_value.surface = {

src/sdlrenderer_texture_uploader.cpp

@@ -3,23 +3,59 @@
 #include <cassert>
 #include <cstdint>
 #include <cstring>
+#include <memory>
 
 #include <iostream>
 
+static SDL_PixelFormat format2sdl(TextureUploaderI::Format format) {
+	switch (format) {
+		case TextureUploaderI::Format::RGBA: return SDL_PIXELFORMAT_RGBA32;
+		//case TextureUploaderI::Format::RGB: return SDL_PIXELFORMAT_RGB24;
+		case TextureUploaderI::Format::IYUV: return SDL_PIXELFORMAT_IYUV;
+		case TextureUploaderI::Format::YV12: return SDL_PIXELFORMAT_YV12;
+		case TextureUploaderI::Format::NV12: return SDL_PIXELFORMAT_NV12;
+		case TextureUploaderI::Format::NV21: return SDL_PIXELFORMAT_NV21;
+		case TextureUploaderI::Format::MAX: return SDL_PIXELFORMAT_UNKNOWN;
+	}
+	return SDL_PIXELFORMAT_UNKNOWN;
+}
+
 SDLRendererTextureUploader::SDLRendererTextureUploader(SDL_Renderer* renderer_) :
 	renderer(renderer_)
 {
 }
 
 uint64_t SDLRendererTextureUploader::uploadRGBA(const uint8_t* data, uint32_t width, uint32_t height, Filter filter, Access access) {
-	// TODO: test if pitch is 4 or 4*width
+	return upload(data, width, height, Format::RGBA, filter, access);
-	SDL_Surface* surf = SDL_CreateSurfaceFrom(
+}
+
+bool SDLRendererTextureUploader::updateRGBA(uint64_t tex_id, const uint8_t* data, size_t size) {
+	return update(tex_id, data, size);
+}
+
+uint64_t SDLRendererTextureUploader::upload(const uint8_t* data, uint32_t width, uint32_t height, Format format, Filter filter, Access access) {
+	const auto sdl_format = format2sdl(format);
+	if (sdl_format == SDL_PIXELFORMAT_UNKNOWN) {
+		std::cerr << "SDLRTU error: unsupported format '" << format << "'\n";
+		return 0;
+	}
+
+	// TODO: why do we even create a non owning surface here???
+	std::unique_ptr<SDL_Surface, decltype(&SDL_DestroySurface)> surf = {
+		SDL_CreateSurfaceFrom(
 			width, height,
-		SDL_PIXELFORMAT_RGBA32, // auto big/little
+			sdl_format,
 			(void*)data,
-		4*width
+			4*width // ???
-	);
+		),
-	assert(surf); // TODO: add error reporting
+		&SDL_DestroySurface
+	};
+
+	assert(surf);
+	if (!surf) {
+		std::cerr << "SDLRTU error: surf creation failed " << SDL_GetError() << "\n";
+		return 0;
+	}
 
 	SDL_Texture* tex = SDL_CreateTexture(
 		renderer,
@@ -27,12 +63,33 @@ uint64_t SDLRendererTextureUploader::uploadRGBA(const uint8_t* data, uint32_t wi
 		access == Access::STREAMING ? SDL_TEXTUREACCESS_STREAMING : SDL_TEXTUREACCESS_STATIC,
 		surf->w, surf->h
 	);
-	assert(tex); // TODO: add error reporting
+
-	// TODO: error reporting
+	assert(tex);
-	SDL_UpdateTexture(tex, nullptr, surf->pixels, surf->pitch);
+	if (tex == nullptr) {
+		std::cerr << "SDLRTU error: tex creation failed " << SDL_GetError() << "\n";
+		return 0;
+	}
+
+	bool need_to_lock = SDL_MUSTLOCK(surf);
+	if (need_to_lock) {
+		if (!SDL_LockSurface(surf.get())) {
+			std::cerr << "SDLRTU error: failed to lock surface " << SDL_GetError() << "\n";
+			SDL_DestroyTexture(tex);
+			return 0;
+		}
+	}
+
+	if (!SDL_UpdateTexture(tex, nullptr, surf->pixels, surf->pitch)) {
+		std::cerr << "SDLRTU error: tex update failed " << SDL_GetError() << "\n";
+	}
+
+	if (need_to_lock) {
+		// error check?
+		SDL_UnlockSurface(surf.get());
+	}
 
 	SDL_BlendMode surf_blend_mode = SDL_BLENDMODE_NONE;
-	if (SDL_GetSurfaceBlendMode(surf, &surf_blend_mode)) {
+	if (SDL_GetSurfaceBlendMode(surf.get(), &surf_blend_mode)) {
 		SDL_SetTextureBlendMode(tex, surf_blend_mode);
 	}
 
@@ -42,12 +99,10 @@ uint64_t SDLRendererTextureUploader::uploadRGBA(const uint8_t* data, uint32_t wi
 		SDL_SetTextureScaleMode(tex, SDL_SCALEMODE_LINEAR);
 	}
 
-	SDL_DestroySurface(surf);
-
 	return reinterpret_cast<uint64_t>(tex);
 }
 
-bool SDLRendererTextureUploader::updateRGBA(uint64_t tex_id, const uint8_t* data, size_t size) {
+bool SDLRendererTextureUploader::update(uint64_t tex_id, const uint8_t* data, size_t size) {
 	auto* texture = static_cast<SDL_Texture*>(reinterpret_cast<void*>(tex_id));
 	if (texture == nullptr) {
 		return false;

src/sdlrenderer_texture_uploader.hpp

@@ -12,6 +12,8 @@ struct SDLRendererTextureUploader : public TextureUploaderI {
 
 	uint64_t uploadRGBA(const uint8_t* data, uint32_t width, uint32_t height, Filter filter, Access access) override;
 	bool updateRGBA(uint64_t tex_id, const uint8_t* data, size_t size) override;
+	uint64_t upload(const uint8_t* data, uint32_t width, uint32_t height, Format format, Filter filter, Access access) override;
+	bool update(uint64_t tex_id, const uint8_t* data, size_t size) override;
 	void destroy(uint64_t tex_id) override;
 };
 

src/texture_uploader.hpp

@@ -4,7 +4,7 @@
 #include <cstddef>
 
 struct TextureUploaderI {
-	static constexpr const char* version {"2"};
+	static constexpr const char* version {"3"};
 
 	enum Filter {
 		NEAREST,
@@ -17,13 +17,30 @@ struct TextureUploaderI {
 		// target?
 	};
 
+	enum Format {
+		RGBA,
+		//RGB,
+
+		IYUV,
+		YV12,
+
+		NV12,
+		NV21,
+
+		MAX
+	};
+
 	virtual ~TextureUploaderI(void) {}
 
-	virtual uint64_t uploadRGBA(const uint8_t* data, uint32_t width, uint32_t height, Filter filter = LINEAR, Access access = STATIC) = 0;
+	[[deprecated]] virtual uint64_t uploadRGBA(const uint8_t* data, uint32_t width, uint32_t height, Filter filter = LINEAR, Access access = STATIC) = 0;
 
 	// keeps width height filter
 	// TODO: wh instead of size?
-	virtual bool updateRGBA(uint64_t tex_id, const uint8_t* data, size_t size) = 0;
+	[[deprecated]] virtual bool updateRGBA(uint64_t tex_id, const uint8_t* data, size_t size) = 0;
+
+	// use upload to create a texture, and update to update existing
+	virtual uint64_t upload(const uint8_t* data, uint32_t width, uint32_t height, Format format = RGBA, Filter filter = LINEAR, Access access = STATIC) = 0;
+	virtual bool update(uint64_t tex_id, const uint8_t* data, size_t size) = 0;
 
 	virtual void destroy(uint64_t tex_id) = 0;
 };