get sdl camera working

CMakeLists.txt

@@ -23,10 +23,17 @@ option(TOMATO_ASAN "Build tomato with asan (gcc/clang/msvc)" OFF)
 if (TOMATO_ASAN)
 	if (${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU" OR ${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang")
 		if (NOT WIN32) # exclude mingw
-			#link_libraries(-fsanitize=address)
+			add_compile_options(-fsanitize=address,undefined)
 			link_libraries(-fsanitize=address,undefined)
-			#link_libraries(-fsanitize=undefined)
+
+			#add_compile_options(-fsanitize=thread)
+			#link_libraries(-fsanitize=thread)
+
 			message("II enabled ASAN")
+			if (OFF) # TODO: switch for minimal runtime in deployed scenarios
+				add_compile_options(-fsanitize-minimal-runtime)
+				link_libraries(-fsanitize-minimal-runtime)
+			endif()
 		else()
 			message("!! can not enable ASAN on this platform (gcc/clang + win)")
 		endif()

flake.nix

@@ -74,6 +74,8 @@
           #(libsodium.override { stdenv = pkgs.pkgsStatic.stdenv; })
           #pkgsStatic.libsodium
           libsodium
+          libopus
+          libvpx
         ] ++ self.packages.${system}.default.dlopenBuildInputs;
 
         cmakeFlags = [

src/content/content.hpp

@@ -15,11 +15,17 @@ using ContentHandle = entt::basic_handle<ContentRegistry>;
 
 namespace Content::Components {
 
+	// TODO: design it as a tree?
+
 	// or something
 	struct TagFile {};
 	struct TagAudioStream {};
 	struct TagVideoStream {};
 
+	struct TimingTiedTo {
+		entt::dense_set<ContentHandle> ties;
+	};
+
 	// the associated messages, if any
 	// useful if you want to update progress on the message
 	struct Messages {

src/content/sdl_video_frame_stream2.cpp

@@ -7,7 +7,7 @@
 #include <thread>
 
 SDLVideoCameraContent::SDLVideoCameraContent(void) {
-	int devcount = 0;
+	int devcount {0};
 	SDL_CameraDeviceID *devices = SDL_GetCameraDevices(&devcount);
 
 	if (devices == nullptr || devcount < 1) {
@@ -17,17 +17,42 @@ SDLVideoCameraContent::SDLVideoCameraContent(void) {
 	std::cout << "### found cameras:\n";
 	for (int i = 0; i < devcount; i++) {
 		const SDL_CameraDeviceID device = devices[i];
+
 		char *name = SDL_GetCameraDeviceName(device);
-
 		std::cout << "  - Camera #" << i << ": " << name << "\n";
-
 		SDL_free(name);
+
+		int speccount {0};
+		SDL_CameraSpec* specs = SDL_GetCameraDeviceSupportedFormats(device, &speccount);
+		if (specs == nullptr) {
+			std::cout << "    - no supported spec\n";
+		} else {
+			for (int spec_i = 0; spec_i < speccount; spec_i++) {
+				std::cout << "    - " << specs[spec_i].width << "x" << specs[spec_i].height << "@" << float(specs[spec_i].interval_denominator)/specs[spec_i].interval_numerator << " " << SDL_GetPixelFormatName(specs[spec_i].format) << "\n";
+
+			}
+
+			SDL_free(specs);
+		}
 	}
 
-	_camera = {
-		SDL_OpenCameraDevice(devices[0], nullptr),
-		&SDL_CloseCamera
-	};
+	{
+		SDL_CameraSpec spec {
+			// FORCE a diffrent pixel format
+			SDL_PIXELFORMAT_RGBA8888,
+
+			//1280, 720,
+			//640, 360,
+			640, 480,
+
+			1, 30
+		};
+		_camera = {
+			SDL_OpenCameraDevice(devices[0], &spec),
+			&SDL_CloseCamera
+		};
+	}
+	SDL_free(devices);
 	if (!static_cast<bool>(_camera)) {
 		throw int(2);
 	}
@@ -40,6 +65,8 @@ SDLVideoCameraContent::SDLVideoCameraContent(void) {
 		// interval
 		interval = float(spec.interval_numerator)/float(spec.interval_denominator);
 		std::cout << "camera interval: " << interval*1000 << "ms\n";
+		auto* format_name = SDL_GetPixelFormatName(spec.format);
+		std::cout << "camera format: " << format_name << "\n";
 	}
 
 	_thread = std::thread([this, interval](void) {
@@ -54,19 +81,39 @@ SDLVideoCameraContent::SDLVideoCameraContent(void) {
 				continue;
 			}
 
-			SDLVideoFrame new_frame {
-				timestampNS,
-				sdl_frame_next
-			};
+#if 0
+			{ // test copy to trigger bug
+				SDL_Surface* test_surf = SDL_CreateSurface(
+					sdl_frame_next->w,
+					sdl_frame_next->h,
+					SDL_PIXELFORMAT_RGBA8888
+				);
 
-			bool someone_listening = pushValue(new_frame);
+				assert(test_surf != nullptr);
 
+				SDL_BlitSurface(sdl_frame_next, nullptr, test_surf, nullptr);
+
+				SDL_DestroySurface(test_surf);
+			}
+#endif
+
+			bool someone_listening {false};
+			{
+				SDLVideoFrame new_frame_non_owning {
+					timestampNS,
+					sdl_frame_next
+				};
+
+				// creates surface copies
+				someone_listening = pushValue(new_frame_non_owning);
+			}
 			SDL_ReleaseCameraFrame(_camera.get(), sdl_frame_next);
 
 			if (someone_listening) {
-				// TODO: maybe double the freq?
-				std::this_thread::sleep_for(std::chrono::milliseconds(int64_t(interval*1000)));
+				// double the interval on acquire
+				std::this_thread::sleep_for(std::chrono::milliseconds(int64_t(interval*1000*0.5)));
 			} else {
+				std::cerr << "i guess no one is listening\n";
 				// we just sleep 4x as long, bc no one is listening
 				std::this_thread::sleep_for(std::chrono::milliseconds(int64_t(interval*1000*4)));
 			}
@@ -78,4 +125,17 @@ SDLVideoCameraContent::~SDLVideoCameraContent(void) {
 	_thread_should_quit = true;
 	_thread.join();
 	// TODO: what to do if readers are still present?
+
+	// HACK: sdl is buggy and freezes otherwise. it is likely still possible, but rare to freeze here
+	// flush unused frames
+#if 1
+	while (true) {
+		SDL_Surface* sdl_frame_next = SDL_AcquireCameraFrame(_camera.get(), nullptr);
+		if (sdl_frame_next != nullptr) {
+			SDL_ReleaseCameraFrame(_camera.get(), sdl_frame_next);
+		} else {
+			break;
+		}
+	}
+#endif
 }