video src now picks the best <=1080p mode

display intended main screen intervals
actually report min interval in debug view and more

src/CMakeLists.txt

@@ -115,6 +115,9 @@ target_sources(tomato PUBLIC
 	./frame_streams/sdl/sdl_audio2_frame_stream2.hpp
 	./frame_streams/sdl/sdl_audio2_frame_stream2.cpp
 	./frame_streams/sdl/video.hpp
+	./frame_streams/sdl/video_push_converter.hpp
+	./frame_streams/sdl/sdl_video_frame_stream2.hpp
+	./frame_streams/sdl/sdl_video_frame_stream2.cpp
 
 	./stream_manager_ui.hpp
 	./stream_manager_ui.cpp

src/chat_gui4.cpp

@@ -297,7 +297,10 @@ float ChatGui4::render(float time_delta) {
 								acceptable_sessions.push_back(o);
 							}
 
-							static Components::VoIP::DefaultConfig g_default_connections{};
+							static Components::VoIP::DefaultConfig g_default_connections{
+								true, true,
+								true, false
+							};
 
 							if (ImGui::BeginMenu("default connections")) {
 								ImGui::MenuItem("incoming audio", nullptr, &g_default_connections.incoming_audio);

src/debug_video_tap.cpp

@@ -10,6 +10,8 @@
 
 #include "./frame_streams/sdl/video.hpp"
 #include "./frame_streams/frame_stream2.hpp"
+#include "./frame_streams/locked_frame_stream.hpp"
+#include "./frame_streams/sdl/video_push_converter.hpp"
 
 #include <string>
 #include <memory>
@@ -27,40 +29,6 @@ namespace Message {
 uint64_t getTimeMS(void);
 }
 
-// threadsafe queue frame stream
-// protected by a simple mutex lock
-template<typename FrameType>
-struct LockedFrameStream2 : public FrameStream2I<FrameType> {
-	std::mutex _lock;
-
-	std::deque<FrameType> _frames;
-
-	~LockedFrameStream2(void) {}
-
-	int32_t size(void) { return -1; }
-
-	std::optional<FrameType> pop(void) {
-		std::lock_guard lg{_lock};
-
-		if (_frames.empty()) {
-			return std::nullopt;
-		}
-
-		FrameType new_frame = std::move(_frames.front());
-		_frames.pop_front();
-
-		return std::move(new_frame);
-	}
-
-	bool push(const FrameType& value) {
-		std::lock_guard lg{_lock};
-
-		_frames.push_back(value);
-
-		return true;
-	}
-};
-
 struct DebugVideoTapSink : public FrameStream2SinkI<SDLVideoFrame> {
 	TextureUploaderI& _tu;
 
@@ -80,7 +48,7 @@ struct DebugVideoTapSink : public FrameStream2SinkI<SDLVideoFrame> {
 			float _v_interval_avg {0.f}; // s
 		} view;
 
-		std::shared_ptr<LockedFrameStream2<SDLVideoFrame>> stream;
+		std::shared_ptr<PushConversionVideoStream<LockedFrameStream2<SDLVideoFrame>>> stream;
 	};
 	std::vector<Writer> _writers;
 
@@ -91,7 +59,7 @@ struct DebugVideoTapSink : public FrameStream2SinkI<SDLVideoFrame> {
 	std::shared_ptr<FrameStream2I<SDLVideoFrame>> subscribe(void) override {
 		_writers.emplace_back(Writer{
 			Writer::View{_id_counter++},
-			std::make_shared<LockedFrameStream2<SDLVideoFrame>>()
+			std::make_shared<PushConversionVideoStream<LockedFrameStream2<SDLVideoFrame>>>(SDL_PIXELFORMAT_RGBA32)
 		});
 
 		return _writers.back().stream;
@@ -127,7 +95,7 @@ struct DebugVideoTestSource : public FrameStream2SourceI<SDLVideoFrame> {
 		_thread = std::thread([this](void) {
 			while (!_stop) {
 				if (!_readers.empty()) {
-					auto* surf = SDL_CreateSurface(960, 720, SDL_PIXELFORMAT_ARGB32);
+					auto* surf = SDL_CreateSurface(960, 720, SDL_PIXELFORMAT_RGBA32);
 
 					// color
 					static auto start_time = Message::getTimeMS();
@@ -220,7 +188,7 @@ float DebugVideoTap::render(void) {
 	for (auto& [view, stream] : dvtsw) {
 		std::string window_title {"DebugVideoTap #"};
 		window_title += std::to_string(view._id);
-		ImGui::SetNextWindowSize({250, 250}, ImGuiCond_Appearing);
+		ImGui::SetNextWindowSize({400, 420}, ImGuiCond_Appearing);
 		if (ImGui::Begin(window_title.c_str())) {
 			while (auto new_frame_opt = stream->pop()) {
 				// timing
@@ -233,7 +201,7 @@ float DebugVideoTap::render(void) {
 					if (view._v_interval_avg == 0) {
 						view._v_interval_avg = delta/1'000'000.f;
 					} else {
-						const float r = 0.2f;
+						const float r = 0.05f;
 						view._v_interval_avg = view._v_interval_avg * (1.f-r) + (delta/1'000'000.f) * r;
 					}
 				}
@@ -277,7 +245,7 @@ float DebugVideoTap::render(void) {
 			// img here
 			if (view._tex != 0) {
 				ImGui::SameLine();
-				ImGui::Text("moving avg interval: %f", view._v_interval_avg);
+				ImGui::Text("%dx%d ~avg interval: %.0fms (%.2ffps)", view._tex_w, view._tex_h, view._v_interval_avg*1000.f, 1.f/view._v_interval_avg);
 				const float img_w = ImGui::GetContentRegionAvail().x;
 				ImGui::Image(
 					reinterpret_cast<ImTextureID>(view._tex),
@@ -287,6 +255,9 @@ float DebugVideoTap::render(void) {
 				);
 			}
 
+			if (view._v_interval_avg > 0) {
+				min_interval = std::min(min_interval, view._v_interval_avg);
+			}
 		}
 		ImGui::End();
 	}

src/frame_streams/sdl/sdl_video_frame_stream2.cpp

@@ -0,0 +1,187 @@
+#include "./sdl_video_frame_stream2.hpp"
+
+#include <chrono>
+
+#include <iostream>
+
+SDLVideo2InputDevice::SDLVideo2InputDevice(void) {
+	int devcount {0};
+	SDL_CameraID *devices = SDL_GetCameras(&devcount);
+	std::cout << "SDLVID: SDL Camera Driver: " << SDL_GetCurrentCameraDriver() << "\n";
+
+	if (devices == nullptr || devcount < 1) {
+		throw int(2); // TODO: proper error code
+	}
+
+	std::cout << "SDLVID: found cameras:\n";
+	for (int i = 0; i < devcount; i++) {
+		const SDL_CameraID device = devices[i];
+
+		const char *name = SDL_GetCameraName(device);
+		std::cout << "  - Camera #" << i << ": " << name << "\n";
+
+		int speccount {0};
+		SDL_CameraSpec** specs = SDL_GetCameraSupportedFormats(device, &speccount);
+		if (specs == nullptr) {
+			std::cout << "    - no supported spec\n";
+		} else {
+			for (int spec_i = 0; spec_i < speccount; spec_i++) {
+				std::cout << "    - " << specs[spec_i]->width << "x" << specs[spec_i]->height << "@" << float(specs[spec_i]->framerate_numerator)/specs[spec_i]->framerate_denominator << "fps " << SDL_GetPixelFormatName(specs[spec_i]->format) << "\n";
+
+			}
+			SDL_free(specs);
+		}
+	}
+	SDL_free(devices);
+}
+
+SDLVideo2InputDevice::~SDLVideo2InputDevice(void) {
+}
+
+std::shared_ptr<FrameStream2I<SDLVideoFrame>> SDLVideo2InputDevice::subscribe(void) {
+	const int prev_ref = _ref++;
+	if (prev_ref == 0) {
+		// there was previously no stream, we assume no thread
+		// open device here? or on the thread?
+
+		int devcount {0};
+		SDL_CameraID *devices = SDL_GetCameras(&devcount);
+
+		if (devices == nullptr || devcount < 1) {
+			_ref--;
+			// error/no devices, should we do this in the constructor?
+			SDL_free(devices);
+			return nullptr;
+		}
+
+		auto device = devices[0];
+		//auto device = devices[devcount-1];
+
+		SDL_CameraSpec spec {
+			// FORCE a different pixel format
+			SDL_PIXELFORMAT_UNKNOWN,
+			//SDL_PIXELFORMAT_YUY2,
+			SDL_COLORSPACE_UNKNOWN,
+			//SDL_COLORSPACE_YUV_DEFAULT,
+
+			1280, 720,
+
+			60, 1
+		};
+
+		// choose a good spec, large res but <= 1080p
+		int speccount {0};
+		SDL_CameraSpec** specs = SDL_GetCameraSupportedFormats(device, &speccount);
+		if (specs != nullptr) {
+			spec = *specs[0];
+			for (int spec_i = 1; spec_i < speccount; spec_i++) {
+				if (specs[spec_i]->height > 1080) {
+					continue;
+				}
+				if (spec.height > specs[spec_i]->height) {
+					continue;
+				}
+				if (
+					float(spec.framerate_numerator)/float(spec.framerate_denominator)
+					>
+					float(specs[spec_i]->framerate_numerator)/float(specs[spec_i]->framerate_denominator)
+				) {
+					continue;
+				}
+
+				if (spec.format == SDL_PIXELFORMAT_NV12 && specs[spec_i]->format == SDL_PIXELFORMAT_YUY2) {
+					// HACK: prefer nv12 over yuy2
+					continue;
+				}
+
+				// seems to be better
+				spec = *specs[spec_i];
+			}
+			SDL_free(specs);
+		}
+
+		std::unique_ptr<SDL_Camera, decltype(&SDL_CloseCamera)> camera {nullptr, &SDL_CloseCamera};
+
+		camera = {
+			//SDL_OpenCamera(device, nullptr),
+			SDL_OpenCamera(device, &spec),
+			&SDL_CloseCamera
+		};
+		SDL_free(devices);
+
+		if (!camera) {
+			std::cerr << "SDLVID error: failed opening camera device\n";
+			_ref--;
+			return nullptr;
+		}
+
+		// seems like we need this before get format() ?
+		// TODO: best would be waiting in thread, but that obv does not work well
+		// TODO: sometimes if the device is/was in use it will stay 0 for ever
+		while (SDL_GetCameraPermissionState(camera.get()) == 0) {
+			//std::cerr << "permission for camera not granted\n";
+			std::this_thread::sleep_for(std::chrono::milliseconds(10));
+		}
+
+		if (SDL_GetCameraPermissionState(camera.get()) <= 0) {
+			std::cerr << "SDLVID error: user denied camera permission\n";
+			_ref--;
+			return nullptr;
+		}
+
+		float fps {1.f};
+		if (!SDL_GetCameraFormat(camera.get(), &spec)) {
+			// meh
+			_ref--;
+			return nullptr;
+		} else {
+			fps = float(spec.framerate_numerator)/float(spec.framerate_denominator);
+			std::cout << "SDLVID: camera fps: " << fps << "fps (" << spec.framerate_numerator << "/" << spec.framerate_denominator << ")\n";
+			auto* format_name = SDL_GetPixelFormatName(spec.format);
+			std::cout << "SDLVID: camera format: " << format_name << "\n";
+		}
+
+		_thread = std::thread([this, camera = std::move(camera), fps](void) {
+			while (_ref > 0) {
+				Uint64 timestampNS = 0;
+
+				// aquire frame
+				SDL_Surface* sdl_frame_next = SDL_AcquireCameraFrame(camera.get(), &timestampNS);
+
+				if (sdl_frame_next != nullptr) {
+					SDLVideoFrame new_frame_non_owning {
+						timestampNS/1000,
+						sdl_frame_next
+					};
+
+					// creates surface copies
+					push(new_frame_non_owning);
+
+					SDL_ReleaseCameraFrame(camera.get(), sdl_frame_next);
+				}
+
+				// sleep for interval
+				// TODO: do we really need half?
+				std::this_thread::sleep_for(std::chrono::milliseconds(int64_t((1000/fps)*0.5)));
+			}
+			// camera destructor closes device here
+		});
+		std::cout << "SDLVID: started new cam thread\n";
+	}
+
+	return FrameStream2MultiSource<SDLVideoFrame>::subscribe();
+}
+
+bool SDLVideo2InputDevice::unsubscribe(const std::shared_ptr<FrameStream2I<SDLVideoFrame>>& sub) {
+	if (FrameStream2MultiSource<SDLVideoFrame>::unsubscribe(sub)) {
+		if (--_ref == 0) {
+			// was last stream, close device and thread
+			_thread.join(); // TODO: defer to destructor or new thread?
+							// this might take a moment and lock up the main thread
+			std::cout << "SDLVID: ended cam thread\n";
+		}
+		return true;
+	}
+	return false;
+}
+

src/frame_streams/sdl/sdl_video_frame_stream2.hpp

@@ -0,0 +1,29 @@
+#pragma once
+
+#include "./video.hpp"
+#include "../frame_stream2.hpp"
+#include "../multi_source.hpp"
+
+#include <atomic>
+#include <thread>
+
+// tips: you can force a SDL vido driver by setting an env:
+// SDL_CAMERA_DRIVER=v4l2
+// SDL_CAMERA_DRIVER=pipewire
+// etc.
+
+// while a stream is subscribed, have the camera device open
+// and aquire and push frames from a thread
+struct SDLVideo2InputDevice : public FrameStream2MultiSource<SDLVideoFrame> {
+	std::atomic_uint _ref {0};
+	std::thread _thread;
+
+	// TODO: device id
+	SDLVideo2InputDevice(void);
+	virtual ~SDLVideo2InputDevice(void);
+
+	// we hook int multi source
+	std::shared_ptr<FrameStream2I<SDLVideoFrame>> subscribe(void) override;
+	bool unsubscribe(const std::shared_ptr<FrameStream2I<SDLVideoFrame>>& sub) override;
+};
+

src/frame_streams/sdl/video_push_converter.hpp

@@ -0,0 +1,67 @@
+#pragma once
+
+#include "./video.hpp"
+#include "../frame_stream2.hpp"
+
+#include <cassert>
+
+#include <iostream> // meh
+
+template<typename RealStream>
+struct PushConversionVideoStream : public RealStream {
+	SDL_PixelFormat _forced_format {SDL_PIXELFORMAT_IYUV};
+	// TODO: force colorspace?
+
+	template<typename... Args>
+	PushConversionVideoStream(SDL_PixelFormat forced_format, Args&&... args) : RealStream(std::forward<Args>(args)...), _forced_format(forced_format) {}
+	~PushConversionVideoStream(void) {}
+
+	bool push(const SDLVideoFrame& value) override {
+		SDL_Surface* surf = value.surface.get();
+		if (surf->format != _forced_format) {
+			//std::cerr << "PCVS: need to convert from " << SDL_GetPixelFormatName(surf->format) << " to " << SDL_GetPixelFormatName(_forced_format) << "\n";
+			if ((surf = SDL_ConvertSurface(surf, _forced_format)) == nullptr) {
+				surf = value.surface.get(); // reset ptr
+				//std::cerr << "PCVS warning: default conversion failed: " << SDL_GetError() << "\n";
+
+				// sdl hardcodes BT709_LIMITED
+				if ((surf = SDL_ConvertSurfaceAndColorspace(surf, _forced_format, nullptr, SDL_GetSurfaceColorspace(surf), 0)) == nullptr) {
+				//if ((surf = SDL_ConvertSurfaceAndColorspace(surf, _forced_format, nullptr, SDL_COLORSPACE_BT709_LIMITED, 0)) == nullptr) {
+					surf = value.surface.get(); // reset ptr
+					//std::cerr << "PCVS warning: default conversion with same colorspace failed: " << SDL_GetError() << "\n";
+					// simple convert failed, fall back to ->rgb->yuv
+					//SDL_Surface* tmp_conv_surf = SDL_ConvertSurfaceAndColorspace(surf, SDL_PIXELFORMAT_RGB24, nullptr, SDL_COLORSPACE_RGB_DEFAULT, 0);
+					SDL_Surface* tmp_conv_surf = SDL_ConvertSurface(surf, SDL_PIXELFORMAT_RGB24);
+					if (tmp_conv_surf == nullptr) {
+						std::cerr << "PCVS error: conversion to RGB failed: " << SDL_GetError() << "\n";
+					} else {
+						//surf = SDL_ConvertSurfaceAndColorspace(tmp_conv_surf, _forced_format, nullptr, SDL_COLORSPACE_YUV_DEFAULT, 0);
+						surf = SDL_ConvertSurface(tmp_conv_surf, _forced_format);
+						//surf = SDL_ConvertSurfaceAndColorspace(tmp_conv_surf, _forced_format, nullptr, SDL_COLORSPACE_BT601_LIMITED, 0);
+						SDL_DestroySurface(tmp_conv_surf);
+					}
+				}
+			}
+
+			if (surf == nullptr) {
+				// oh god
+				std::cerr << "PCVS error: failed to convert surface to IYUV: " << SDL_GetError() << "\n";
+				return false;
+			}
+		}
+		assert(surf != nullptr);
+		if (surf != value.surface.get()) {
+			// TODO: add ctr with uptr
+			SDLVideoFrame new_value{value.timestampUS, nullptr};
+			new_value.surface = {
+				surf,
+				&SDL_DestroySurface
+			};
+
+			return RealStream::push(std::move(new_value));
+		} else {
+			return RealStream::push(value);
+		}
+	}
+};
+

src/frame_streams/stream_manager.hpp

@@ -194,7 +194,7 @@ bool StreamManager::connect(Object src, Object sink, bool threaded) {
 		std::move(our_data),
 		[](Connection& con) -> void {
 			// there might be more stored
-			for (size_t i = 0; i < 10; i++) {
+			for (size_t i = 0; i < 64; i++) {
 				auto new_frame_opt = static_cast<inlineData*>(con.data.get())->reader->pop();
 				// TODO: frame interval estimates
 				if (new_frame_opt.has_value()) {

src/main_screen.cpp

@@ -6,6 +6,7 @@
 #include <solanaceae/contact/components.hpp>
 
 #include "./frame_streams/sdl/sdl_audio2_frame_stream2.hpp"
+#include "./frame_streams/sdl/sdl_video_frame_stream2.hpp"
 
 #include <imgui/imgui.h>
 
@@ -178,6 +179,27 @@ MainScreen::MainScreen(SimpleConfigModel&& conf_, SDL_Renderer* renderer_, Theme
 	} else {
 		std::cerr << "MS warning: no sdl audio: " << SDL_GetError() << "\n";
 	}
+
+	if (SDL_InitSubSystem(SDL_INIT_CAMERA)) {
+		{ // video in
+			ObjectHandle vsrc {os.registry(), os.registry().create()};
+			try {
+				vsrc.emplace<Components::FrameStream2Source<SDLVideoFrame>>(
+					std::make_unique<SDLVideo2InputDevice>()
+				);
+
+				vsrc.emplace<Components::StreamSource>(Components::StreamSource::create<SDLVideoFrame>("SDL Video Default Recording Device"));
+				vsrc.emplace<Components::TagDefaultTarget>();
+
+				os.throwEventConstruct(vsrc);
+			} catch (...) {
+				std::cerr << "MS error: failed constructing default video input source\n";
+				os.registry().destroy(vsrc);
+			}
+		}
+	} else {
+		std::cerr << "MS warning: no sdl camera: " << SDL_GetError() << "\n";
+	}
 }
 
 MainScreen::~MainScreen(void) {
@@ -324,6 +346,11 @@ Screen* MainScreen::render(float time_delta, bool&) {
 						ImGui::SetItemTooltip("Limiting compute can slow down things like filetransfers!");
 					}
 
+					ImGui::Separator();
+
+					ImGui::Text("render interval: %.0fms (%.2ffps)", _render_interval*1000.f, 1.f/_render_interval);
+					ImGui::Text("tick   interval: %.0fms (%.2ftps)", _min_tick_interval*1000.f, 1.f/_min_tick_interval);
+
 					ImGui::EndMenu();
 				}
 				if (ImGui::BeginMenu("Settings")) {
@@ -482,12 +509,12 @@ Screen* MainScreen::render(float time_delta, bool&) {
 	// min over non animations in all cases
 	_render_interval = std::min<float>(pm_interval, cg_interval);
 	_render_interval = std::min<float>(_render_interval, tc_unfinished_queue_interval);
+	_render_interval = std::min<float>(_render_interval, dvt_interval);
 
 	// low delay time window
 	if (!_window_hidden && _time_since_event < curr_profile.low_delay_window) {
 		_render_interval = std::min<float>(_render_interval, ctc_interval);
 		_render_interval = std::min<float>(_render_interval, msgtc_interval);
-		_render_interval = std::min<float>(_render_interval, dvt_interval);
 
 		_render_interval = std::clamp(
 			_render_interval,
@@ -498,7 +525,6 @@ Screen* MainScreen::render(float time_delta, bool&) {
 	} else if (!_window_hidden && _time_since_event < curr_profile.mid_delay_window) {
 		_render_interval = std::min<float>(_render_interval, ctc_interval);
 		_render_interval = std::min<float>(_render_interval, msgtc_interval);
-		_render_interval = std::min<float>(_render_interval, dvt_interval);
 
 		_render_interval = std::clamp(
 			_render_interval,

src/tox_av_voip_model.cpp

@@ -9,8 +9,18 @@
 #include "./frame_streams/multi_source.hpp"
 #include "./frame_streams/audio_stream_pop_reframer.hpp"
 
+#include "./frame_streams/sdl/video.hpp"
+#include "./frame_streams/sdl/video_push_converter.hpp"
+
+#include <cstring>
+
 #include <iostream>
 
+// fwd
+namespace Message {
+uint64_t getTimeMS(void);
+} // Message
+
 namespace Components {
 	struct ToxAVIncomingAV {
 		bool incoming_audio {false};
@@ -21,12 +31,15 @@ namespace Components {
 		ObjectHandle o;
 		// ptr?
 	};
-	// vid
+	struct ToxAVVideoSink {
+		ObjectHandle o;
+	};
 	struct ToxAVAudioSource {
 		ObjectHandle o;
-		// ptr?
 	};
-	// vid
+	struct ToxAVVideoSource {
+		ObjectHandle o;
+	};
 } // Components
 
 struct ToxAVCallAudioSink : public FrameStream2SinkI<AudioFrame2> {
@@ -84,6 +97,63 @@ struct ToxAVCallAudioSink : public FrameStream2SinkI<AudioFrame2> {
 	}
 };
 
+// exlusive
+struct ToxAVCallVideoSink : public FrameStream2SinkI<SDLVideoFrame> {
+	using stream_type = PushConversionVideoStream<LockedFrameStream2<SDLVideoFrame>>;
+	ToxAVI& _toxav;
+
+	// bitrate for enabled state
+	uint32_t _video_bitrate {2}; // HACK: hardcode to 2mbits (toxap wrongly multiplies internally by 1000)
+
+	uint32_t _fid;
+	std::shared_ptr<stream_type> _writer;
+
+	ToxAVCallVideoSink(ToxAVI& toxav, uint32_t fid) : _toxav(toxav), _fid(fid) {}
+	~ToxAVCallVideoSink(void) {
+		if (_writer) {
+			_writer = nullptr;
+			_toxav.toxavVideoSetBitRate(_fid, 0);
+		}
+	}
+
+	// sink
+	std::shared_ptr<FrameStream2I<SDLVideoFrame>> subscribe(void) override {
+		if (_writer) {
+			// max 1 (exclusive, composite video somewhere else)
+			return nullptr;
+		}
+
+		auto err = _toxav.toxavVideoSetBitRate(_fid, _video_bitrate);
+		if (err != TOXAV_ERR_BIT_RATE_SET_OK) {
+			return nullptr;
+		}
+
+		// toxav needs I420
+		_writer = std::make_shared<stream_type>(SDL_PIXELFORMAT_IYUV);
+
+		return _writer;
+	}
+
+	bool unsubscribe(const std::shared_ptr<FrameStream2I<SDLVideoFrame>>& sub) override {
+		if (!sub || !_writer) {
+			// nah
+			return false;
+		}
+
+		if (sub == _writer) {
+			_writer = nullptr;
+
+			/*auto err = */_toxav.toxavVideoSetBitRate(_fid, 0);
+			// print warning? on error?
+
+			return true;
+		}
+
+		// what
+		return false;
+	}
+};
+
 void ToxAVVoIPModel::addAudioSource(ObjectHandle session, uint32_t friend_number) {
 	auto& stream_source = session.get_or_emplace<Components::VoIP::StreamSources>().streams;
 
@@ -94,13 +164,11 @@ void ToxAVVoIPModel::addAudioSource(ObjectHandle session, uint32_t friend_number
 	incoming_audio.emplace<Components::FrameStream2Source<AudioFrame2>>(std::move(new_asrc));
 	incoming_audio.emplace<Components::StreamSource>(Components::StreamSource::create<AudioFrame2>("ToxAV Friend Call Incoming Audio"));
 
-	std::cout << "new incoming audio\n";
 	if (
 		const auto* defaults = session.try_get<Components::VoIP::DefaultConfig>();
 		defaults != nullptr && defaults->incoming_audio
 	) {
 		incoming_audio.emplace<Components::TagConnectToDefault>(); // depends on what was specified in enter()
-		std::cout << "with default\n";
 	}
 
 	stream_source.push_back(incoming_audio);
@@ -135,6 +203,55 @@ void ToxAVVoIPModel::addAudioSink(ObjectHandle session, uint32_t friend_number)
 	_os.throwEventConstruct(outgoing_audio);
 }
 
+void ToxAVVoIPModel::addVideoSource(ObjectHandle session, uint32_t friend_number) {
+	auto& stream_source = session.get_or_emplace<Components::VoIP::StreamSources>().streams;
+
+	ObjectHandle incoming_video {_os.registry(), _os.registry().create()};
+
+	auto new_vsrc = std::make_unique<FrameStream2MultiSource<SDLVideoFrame>>();
+	incoming_video.emplace<FrameStream2MultiSource<SDLVideoFrame>*>(new_vsrc.get());
+	incoming_video.emplace<Components::FrameStream2Source<SDLVideoFrame>>(std::move(new_vsrc));
+	incoming_video.emplace<Components::StreamSource>(Components::StreamSource::create<SDLVideoFrame>("ToxAV Friend Call Incoming Video"));
+
+	if (
+		const auto* defaults = session.try_get<Components::VoIP::DefaultConfig>();
+		defaults != nullptr && defaults->incoming_video
+	) {
+		incoming_video.emplace<Components::TagConnectToDefault>(); // depends on what was specified in enter()
+	}
+
+	stream_source.push_back(incoming_video);
+	session.emplace<Components::ToxAVVideoSource>(incoming_video);
+	// TODO: tie session to stream
+
+	_video_sources[friend_number] = incoming_video;
+
+	_os.throwEventConstruct(incoming_video);
+}
+
+void ToxAVVoIPModel::addVideoSink(ObjectHandle session, uint32_t friend_number) {
+	auto& stream_sinks = session.get_or_emplace<Components::VoIP::StreamSinks>().streams;
+	ObjectHandle outgoing_video {_os.registry(), _os.registry().create()};
+
+	auto new_vsink = std::make_unique<ToxAVCallVideoSink>(_av, friend_number);
+	outgoing_video.emplace<ToxAVCallVideoSink*>(new_vsink.get());
+	outgoing_video.emplace<Components::FrameStream2Sink<SDLVideoFrame>>(std::move(new_vsink));
+	outgoing_video.emplace<Components::StreamSink>(Components::StreamSink::create<SDLVideoFrame>("ToxAV Friend Call Outgoing Video"));
+
+	if (
+		const auto* defaults = session.try_get<Components::VoIP::DefaultConfig>();
+		defaults != nullptr && defaults->outgoing_video
+	) {
+		outgoing_video.emplace<Components::TagConnectToDefault>(); // depends on what was specified in enter()
+	}
+
+	stream_sinks.push_back(outgoing_video);
+	session.emplace<Components::ToxAVVideoSink>(outgoing_video);
+	// TODO: tie session to stream
+
+	_os.throwEventConstruct(outgoing_video);
+}
+
 void ToxAVVoIPModel::destroySession(ObjectHandle session) {
 	if (!static_cast<bool>(session)) {
 		return;
@@ -153,6 +270,18 @@ void ToxAVVoIPModel::destroySession(ObjectHandle session) {
 			_audio_sources.erase(it_asrc);
 		}
 	}
+	if (session.all_of<Components::ToxAVVideoSource>()) {
+		auto it_vsrc = std::find_if(
+			_video_sources.cbegin(), _video_sources.cend(),
+			[o = session.get<Components::ToxAVVideoSource>().o](const auto& it) {
+				return it.second == o;
+			}
+		);
+
+		if (it_vsrc != _video_sources.cend()) {
+			_video_sources.erase(it_vsrc);
+		}
+	}
 
 	// destory sources
 	if (auto* ss = session.try_get<Components::VoIP::StreamSources>(); ss != nullptr) {
@@ -237,6 +366,39 @@ void ToxAVVoIPModel::tick(void) {
 			}
 		}
 	}
+
+	for (const auto& [oc, vsink] : _os.registry().view<ToxAVCallVideoSink*>().each()) {
+		if (!vsink->_writer) {
+			continue;
+		}
+
+		for (size_t i = 0; i < 10; i++) {
+			auto new_frame_opt = vsink->_writer->pop();
+			if (!new_frame_opt.has_value()) {
+				break;
+			}
+			const auto& new_frame = new_frame_opt.value();
+
+			if (!new_frame.surface) {
+				// wtf?
+				continue;
+			}
+
+			// conversion is done in the sink's stream
+			SDL_Surface* surf = new_frame.surface.get();
+			assert(surf != nullptr);
+
+			SDL_LockSurface(surf);
+			_av.toxavVideoSendFrame(
+				vsink->_fid,
+				surf->w, surf->h,
+				static_cast<const uint8_t*>(surf->pixels),
+				static_cast<const uint8_t*>(surf->pixels) + surf->w * surf->h,
+				static_cast<const uint8_t*>(surf->pixels) + surf->w * surf->h + (surf->w/2) * (surf->h/2)
+			);
+			SDL_UnlockSurface(surf);
+		}
+	}
 }
 
 ObjectHandle ToxAVVoIPModel::enter(const Contact3 c, const Components::VoIP::DefaultConfig& defaults) {
@@ -311,6 +473,8 @@ bool ToxAVVoIPModel::accept(ObjectHandle session, const Components::VoIP::Defaul
 	// bitrate cb or what?
 	assert(!session.all_of<Components::ToxAVAudioSink>());
 	addAudioSink(session, friend_number);
+	assert(!session.all_of<Components::ToxAVVideoSink>());
+	addVideoSink(session, friend_number);
 
 	if (const auto* i_av = session.try_get<Components::ToxAVIncomingAV>(); i_av != nullptr) {
 		// create audio src
@@ -321,6 +485,8 @@ bool ToxAVVoIPModel::accept(ObjectHandle session, const Components::VoIP::Defaul
 
 		// create video src
 		if (i_av->incoming_video) {
+			assert(!session.all_of<Components::ToxAVVideoSource>());
+			addVideoSource(session, friend_number);
 		}
 	}
 
@@ -419,6 +585,11 @@ bool ToxAVVoIPModel::onEvent(const Events::FriendCallState& e) {
 				}
 
 				// video
+				if (s.is_accepting_v() && !o.all_of<Components::ToxAVVideoSink>()) {
+					addVideoSink(o, e.friend_number);
+				} else if (!s.is_accepting_v() && o.all_of<Components::ToxAVVideoSink>()) {
+					// remove vsink?
+				}
 
 				// add/update sources
 				// audio
@@ -429,6 +600,11 @@ bool ToxAVVoIPModel::onEvent(const Events::FriendCallState& e) {
 				}
 
 				// video
+				if (s.is_sending_v() && !o.all_of<Components::ToxAVVideoSource>()) {
+					addVideoSource(o, e.friend_number);
+				} else if (!s.is_sending_v() && o.all_of<Components::ToxAVVideoSource>()) {
+					// remove vsrc?
+				}
 			}
 		}
 	}
@@ -470,7 +646,68 @@ bool ToxAVVoIPModel::onEvent(const Events::FriendAudioFrame& e) {
 	return true;
 }
 
-bool ToxAVVoIPModel::onEvent(const Events::FriendVideoFrame&) {
+bool ToxAVVoIPModel::onEvent(const Events::FriendVideoFrame& e) {
+	auto vsrc_it = _video_sources.find(e.friend_number);
+	if (vsrc_it == _video_sources.cend()) {
+		// missing src from lookup table
+		return false;
+	}
+
+	auto vsrc = vsrc_it->second;
+
+	if (!static_cast<bool>(vsrc)) {
+		// missing src to put frame into ??
+		return false;
+	}
+
+	assert(vsrc.all_of<FrameStream2MultiSource<SDLVideoFrame>*>());
+	assert(vsrc.all_of<Components::FrameStream2Source<SDLVideoFrame>>());
+
+	auto* new_surf = SDL_CreateSurface(e.width, e.height, SDL_PIXELFORMAT_IYUV);
+	assert(new_surf);
+	if (SDL_LockSurface(new_surf)) {
+		// copy the data
+		// we know how the implementation works, its y u v consecutivlely
+		// y
+		for (size_t y = 0; y < e.height; y++) {
+			std::memcpy(
+				//static_cast<uint8_t*>(new_surf->pixels) + new_surf->pitch*y,
+				static_cast<uint8_t*>(new_surf->pixels) + e.width*y,
+				e.y.ptr + e.ystride*y,
+				e.width
+			);
+		}
+
+		// u
+		for (size_t y = 0; y < e.height/2; y++) {
+			std::memcpy(
+				static_cast<uint8_t*>(new_surf->pixels) + (e.width*e.height) + (e.width/2)*y,
+				e.u.ptr + e.ustride*y,
+				e.width/2
+			);
+		}
+
+		// v
+		for (size_t y = 0; y < e.height/2; y++) {
+			std::memcpy(
+				static_cast<uint8_t*>(new_surf->pixels) + (e.width*e.height) + ((e.width/2)*(e.height/2)) + (e.width/2)*y,
+				e.v.ptr + e.vstride*y,
+				e.width/2
+			);
+		}
+
+		SDL_UnlockSurface(new_surf);
+	}
+
+	vsrc.get<FrameStream2MultiSource<SDLVideoFrame>*>()->push({
+		// ms -> us
+		// would be nice if we had been giving this from toxcore
+		// TODO: make more precise
+		Message::getTimeMS() * 1000,
+		new_surf
+	});
+
+	SDL_DestroySurface(new_surf);
 	return false;
 }
 

src/tox_av_voip_model.hpp

@@ -16,11 +16,13 @@ class ToxAVVoIPModel : protected ToxAVEventI, public VoIPModelI {
 
 	// for faster lookup
 	std::unordered_map<uint32_t, ObjectHandle> _audio_sources;
+	std::unordered_map<uint32_t, ObjectHandle> _video_sources;
 
 	// TODO: virtual? strategy? protected?
 	virtual void addAudioSource(ObjectHandle session, uint32_t friend_number);
 	virtual void addAudioSink(ObjectHandle session, uint32_t friend_number);
-	// TODO: video
+	virtual void addVideoSource(ObjectHandle session, uint32_t friend_number);
+	virtual void addVideoSink(ObjectHandle session, uint32_t friend_number);
 
 	void destroySession(ObjectHandle session);