src/CMakeLists.txt

@@ -115,8 +115,6 @@ target_sources(tomato PUBLIC
 	./frame_streams/sdl/sdl_audio2_frame_stream2.hpp
 	./frame_streams/sdl/sdl_audio2_frame_stream2.cpp
 	./frame_streams/sdl/video.hpp
-	./frame_streams/sdl/video_push_converter.hpp
-	./frame_streams/sdl/video_push_converter.cpp
 
 	./stream_manager_ui.hpp
 	./stream_manager_ui.cpp

src/frame_streams/sdl/video_push_converter.cpp

@@ -1,58 +0,0 @@
-#include "./video_push_converter.hpp"
-
-SDL_Surface* convertYUY2_IYUV(SDL_Surface* surf) {
-	if (surf->format != SDL_PIXELFORMAT_YUY2) {
-		return nullptr;
-	}
-	if ((surf->w % 2) != 0) {
-		SDL_SetError("YUY2->IYUV does not support odd widths");
-		// hmmm, we dont handle odd widths
-		return nullptr;
-	}
-
-	SDL_LockSurface(surf);
-
-	SDL_Surface* conv_surf = SDL_CreateSurface(surf->w, surf->h, SDL_PIXELFORMAT_IYUV);
-	SDL_LockSurface(conv_surf);
-
-	// YUY2 is 4:2:2 packed
-	// Y is simple, we just copy it over
-	// U V are double the resolution (vertically), so we avg both
-	// Packed mode: Y0+U0+Y1+V0 (1 plane)
-
-	uint8_t* y_plane = static_cast<uint8_t*>(conv_surf->pixels);
-	uint8_t* u_plane = static_cast<uint8_t*>(conv_surf->pixels) + conv_surf->w*conv_surf->h;
-	uint8_t* v_plane = static_cast<uint8_t*>(conv_surf->pixels) + conv_surf->w*conv_surf->h + (conv_surf->w/2)*(conv_surf->h/2);
-
-	const uint8_t* yuy2_data = static_cast<const uint8_t*>(surf->pixels);
-
-	for (int y = 0; y < surf->h; y++) {
-		for (int x = 0; x < surf->w; x += 2) {
-			// every pixel uses 2 bytes
-			const uint8_t* yuy2_curser = yuy2_data + y*surf->w*2 + x*2;
-			uint8_t src_y0 = yuy2_curser[0];
-			uint8_t src_u = yuy2_curser[1];
-			uint8_t src_y1 = yuy2_curser[2];
-			uint8_t src_v = yuy2_curser[3];
-
-			y_plane[y*conv_surf->w + x] = src_y0;
-			y_plane[y*conv_surf->w + x+1] = src_y1;
-
-			size_t uv_index = (y/2) * (conv_surf->w/2) + x/2;
-			if (y % 2 == 0) {
-				// first write
-				u_plane[uv_index] = src_u;
-				v_plane[uv_index] = src_v;
-			} else {
-				// second write, mix with existing value
-				u_plane[uv_index] = (int(u_plane[uv_index]) + int(src_u)) / 2;
-				v_plane[uv_index] = (int(v_plane[uv_index]) + int(src_v)) / 2;
-			}
-		}
-	}
-
-	SDL_UnlockSurface(conv_surf);
-	SDL_UnlockSurface(surf);
-	return conv_surf;
-}
-

src/frame_streams/sdl/video_push_converter.hpp

@@ -1,89 +0,0 @@
-#pragma once
-
-#include "./video.hpp"
-#include "../frame_stream2.hpp"
-
-#include <cassert>
-
-#include <iostream> // meh
-
-static bool isFormatYUV(SDL_PixelFormat f) {
-	return
-		f == SDL_PIXELFORMAT_YV12 ||
-		f == SDL_PIXELFORMAT_IYUV ||
-		f == SDL_PIXELFORMAT_YUY2 ||
-		f == SDL_PIXELFORMAT_UYVY ||
-		f == SDL_PIXELFORMAT_YVYU ||
-		f == SDL_PIXELFORMAT_NV12 ||
-		f == SDL_PIXELFORMAT_NV21 ||
-		f == SDL_PIXELFORMAT_P010
-	;
-}
-
-SDL_Surface* convertYUY2_IYUV(SDL_Surface* surf);
-
-template<typename RealStream>
-struct PushConversionVideoStream : public RealStream {
-	SDL_PixelFormat _forced_format {SDL_PIXELFORMAT_IYUV};
-
-	template<typename... Args>
-	PushConversionVideoStream(SDL_PixelFormat forced_format, Args&&... args) : RealStream(std::forward<Args>(args)...), _forced_format(forced_format) {}
-	~PushConversionVideoStream(void) {}
-
-	bool push(const SDLVideoFrame& value) override {
-		SDL_Surface* surf = value.surface.get();
-		if (surf->format != _forced_format) {
-			//std::cerr << "DTC: need to convert from " << SDL_GetPixelFormatName(converted_surf->format) << " to SDL_PIXELFORMAT_IYUV\n";
-			if (surf->format == SDL_PIXELFORMAT_YUY2 && _forced_format == SDL_PIXELFORMAT_IYUV) {
-				// optimized custom impl
-
-				//auto start = Message::getTimeMS();
-				surf = convertYUY2_IYUV(surf);
-				//auto end = Message::getTimeMS();
-				// 3ms
-				//std::cerr << "DTC: timing " << SDL_GetPixelFormatName(converted_surf->format) << "->SDL_PIXELFORMAT_IYUV: " << end-start << "ms\n";
-			} else if (isFormatYUV(surf->format)) {
-				// TODO: fix sdl rgb->yuv conversion resulting in too dark (colorspace) issues
-				// https://github.com/libsdl-org/SDL/issues/10877
-
-				// meh, need to convert to rgb as a stopgap
-
-				//auto start = Message::getTimeMS();
-				SDL_Surface* tmp_conv_surf = SDL_ConvertSurfaceAndColorspace(surf, SDL_PIXELFORMAT_RGB24, nullptr, SDL_COLORSPACE_RGB_DEFAULT, 0);
-				//auto end = Message::getTimeMS();
-				// 1ms
-				//std::cerr << "DTC: timing " << SDL_GetPixelFormatName(converted_surf->format) << "->SDL_PIXELFORMAT_RGB24: " << end-start << "ms\n";
-
-				//start = Message::getTimeMS();
-				surf = SDL_ConvertSurfaceAndColorspace(tmp_conv_surf, _forced_format, nullptr, SDL_COLORSPACE_YUV_DEFAULT, 0);
-				//end = Message::getTimeMS();
-				// 60ms
-				//std::cerr << "DTC: timing SDL_PIXELFORMAT_RGB24->" << SDL_GetPixelFormatName(_forced_format) << ": " << end-start << "ms\n";
-
-				SDL_DestroySurface(tmp_conv_surf);
-			} else {
-				surf = SDL_ConvertSurface(surf, _forced_format);
-			}
-
-			if (surf == nullptr) {
-				// oh god
-				std::cerr << "DTC error: failed to convert surface to IYUV: " << SDL_GetError() << "\n";
-				return false;
-			}
-		}
-		assert(surf != nullptr);
-		if (surf != value.surface.get()) {
-			// TODO: add ctr with uptr
-			SDLVideoFrame new_value{value.timestampUS, nullptr};
-			new_value.surface = {
-				surf,
-				&SDL_DestroySurface
-			};
-
-			return RealStream::push(std::move(new_value));
-		} else {
-			return RealStream::push(value);
-		}
-	}
-};
-

src/tox_av_voip_model.cpp

@@ -9,18 +9,8 @@
 #include "./frame_streams/multi_source.hpp"
 #include "./frame_streams/audio_stream_pop_reframer.hpp"
 
-#include "./frame_streams/sdl/video.hpp"
-#include "./frame_streams/sdl/video_push_converter.hpp"
-
-#include <cstring>
-
 #include <iostream>
 
-// fwd
-namespace Message {
-uint64_t getTimeMS(void);
-} // Message
-
 namespace Components {
 	struct ToxAVIncomingAV {
 		bool incoming_audio {false};
@@ -31,15 +21,12 @@ namespace Components {
 		ObjectHandle o;
 		// ptr?
 	};
-	struct ToxAVVideoSink {
+	// vid
-		ObjectHandle o;
-	};
 	struct ToxAVAudioSource {
 		ObjectHandle o;
+		// ptr?
 	};
-	struct ToxAVVideoSource {
+	// vid
-		ObjectHandle o;
-	};
 } // Components
 
 struct ToxAVCallAudioSink : public FrameStream2SinkI<AudioFrame2> {
@@ -97,63 +84,6 @@ struct ToxAVCallAudioSink : public FrameStream2SinkI<AudioFrame2> {
 	}
 };
 
-// exlusive
-struct ToxAVCallVideoSink : public FrameStream2SinkI<SDLVideoFrame> {
-	using stream_type = PushConversionVideoStream<LockedFrameStream2<SDLVideoFrame>>;
-	ToxAVI& _toxav;
-
-	// bitrate for enabled state
-	uint32_t _video_bitrate {2}; // HACK: hardcode to 2mbits (toxap wrongly multiplies internally by 1000)
-
-	uint32_t _fid;
-	std::shared_ptr<stream_type> _writer;
-
-	ToxAVCallVideoSink(ToxAVI& toxav, uint32_t fid) : _toxav(toxav), _fid(fid) {}
-	~ToxAVCallVideoSink(void) {
-		if (_writer) {
-			_writer = nullptr;
-			_toxav.toxavVideoSetBitRate(_fid, 0);
-		}
-	}
-
-	// sink
-	std::shared_ptr<FrameStream2I<SDLVideoFrame>> subscribe(void) override {
-		if (_writer) {
-			// max 1 (exclusive, composite video somewhere else)
-			return nullptr;
-		}
-
-		auto err = _toxav.toxavVideoSetBitRate(_fid, _video_bitrate);
-		if (err != TOXAV_ERR_BIT_RATE_SET_OK) {
-			return nullptr;
-		}
-
-		// toxav needs I420
-		_writer = std::make_shared<stream_type>(SDL_PIXELFORMAT_IYUV);
-
-		return _writer;
-	}
-
-	bool unsubscribe(const std::shared_ptr<FrameStream2I<SDLVideoFrame>>& sub) override {
-		if (!sub || !_writer) {
-			// nah
-			return false;
-		}
-
-		if (sub == _writer) {
-			_writer = nullptr;
-
-			/*auto err = */_toxav.toxavVideoSetBitRate(_fid, 0);
-			// print warning? on error?
-
-			return true;
-		}
-
-		// what
-		return false;
-	}
-};
-
 void ToxAVVoIPModel::addAudioSource(ObjectHandle session, uint32_t friend_number) {
 	auto& stream_source = session.get_or_emplace<Components::VoIP::StreamSources>().streams;
 
@@ -164,11 +94,13 @@ void ToxAVVoIPModel::addAudioSource(ObjectHandle session, uint32_t friend_number
 	incoming_audio.emplace<Components::FrameStream2Source<AudioFrame2>>(std::move(new_asrc));
 	incoming_audio.emplace<Components::StreamSource>(Components::StreamSource::create<AudioFrame2>("ToxAV Friend Call Incoming Audio"));
 
+	std::cout << "new incoming audio\n";
 	if (
 		const auto* defaults = session.try_get<Components::VoIP::DefaultConfig>();
 		defaults != nullptr && defaults->incoming_audio
 	) {
 		incoming_audio.emplace<Components::TagConnectToDefault>(); // depends on what was specified in enter()
+		std::cout << "with default\n";
 	}
 
 	stream_source.push_back(incoming_audio);
@@ -203,55 +135,6 @@ void ToxAVVoIPModel::addAudioSink(ObjectHandle session, uint32_t friend_number)
 	_os.throwEventConstruct(outgoing_audio);
 }
 
-void ToxAVVoIPModel::addVideoSource(ObjectHandle session, uint32_t friend_number) {
-	auto& stream_source = session.get_or_emplace<Components::VoIP::StreamSources>().streams;
-
-	ObjectHandle incoming_video {_os.registry(), _os.registry().create()};
-
-	auto new_vsrc = std::make_unique<FrameStream2MultiSource<SDLVideoFrame>>();
-	incoming_video.emplace<FrameStream2MultiSource<SDLVideoFrame>*>(new_vsrc.get());
-	incoming_video.emplace<Components::FrameStream2Source<SDLVideoFrame>>(std::move(new_vsrc));
-	incoming_video.emplace<Components::StreamSource>(Components::StreamSource::create<SDLVideoFrame>("ToxAV Friend Call Incoming Video"));
-
-	if (
-		const auto* defaults = session.try_get<Components::VoIP::DefaultConfig>();
-		defaults != nullptr && defaults->incoming_video
-	) {
-		incoming_video.emplace<Components::TagConnectToDefault>(); // depends on what was specified in enter()
-	}
-
-	stream_source.push_back(incoming_video);
-	session.emplace<Components::ToxAVVideoSource>(incoming_video);
-	// TODO: tie session to stream
-
-	_video_sources[friend_number] = incoming_video;
-
-	_os.throwEventConstruct(incoming_video);
-}
-
-void ToxAVVoIPModel::addVideoSink(ObjectHandle session, uint32_t friend_number) {
-	auto& stream_sinks = session.get_or_emplace<Components::VoIP::StreamSinks>().streams;
-	ObjectHandle outgoing_video {_os.registry(), _os.registry().create()};
-
-	auto new_vsink = std::make_unique<ToxAVCallVideoSink>(_av, friend_number);
-	outgoing_video.emplace<ToxAVCallVideoSink*>(new_vsink.get());
-	outgoing_video.emplace<Components::FrameStream2Sink<SDLVideoFrame>>(std::move(new_vsink));
-	outgoing_video.emplace<Components::StreamSink>(Components::StreamSink::create<SDLVideoFrame>("ToxAV Friend Call Outgoing Video"));
-
-	if (
-		const auto* defaults = session.try_get<Components::VoIP::DefaultConfig>();
-		defaults != nullptr && defaults->outgoing_video
-	) {
-		outgoing_video.emplace<Components::TagConnectToDefault>(); // depends on what was specified in enter()
-	}
-
-	stream_sinks.push_back(outgoing_video);
-	session.emplace<Components::ToxAVVideoSink>(outgoing_video);
-	// TODO: tie session to stream
-
-	_os.throwEventConstruct(outgoing_video);
-}
-
 void ToxAVVoIPModel::destroySession(ObjectHandle session) {
 	if (!static_cast<bool>(session)) {
 		return;
@@ -270,18 +153,6 @@ void ToxAVVoIPModel::destroySession(ObjectHandle session) {
 			_audio_sources.erase(it_asrc);
 		}
 	}
-	if (session.all_of<Components::ToxAVVideoSource>()) {
-		auto it_vsrc = std::find_if(
-			_video_sources.cbegin(), _video_sources.cend(),
-			[o = session.get<Components::ToxAVVideoSource>().o](const auto& it) {
-				return it.second == o;
-			}
-		);
-
-		if (it_vsrc != _video_sources.cend()) {
-			_video_sources.erase(it_vsrc);
-		}
-	}
 
 	// destory sources
 	if (auto* ss = session.try_get<Components::VoIP::StreamSources>(); ss != nullptr) {
@@ -366,39 +237,6 @@ void ToxAVVoIPModel::tick(void) {
 			}
 		}
 	}
-
-	for (const auto& [oc, vsink] : _os.registry().view<ToxAVCallVideoSink*>().each()) {
-		if (!vsink->_writer) {
-			continue;
-		}
-
-		for (size_t i = 0; i < 10; i++) {
-			auto new_frame_opt = vsink->_writer->pop();
-			if (!new_frame_opt.has_value()) {
-				break;
-			}
-			const auto& new_frame = new_frame_opt.value();
-
-			if (!new_frame.surface) {
-				// wtf?
-				continue;
-			}
-
-			// conversion is done in the sink's stream
-			SDL_Surface* surf = new_frame.surface.get();
-			assert(surf != nullptr);
-
-			SDL_LockSurface(surf);
-			_av.toxavVideoSendFrame(
-				vsink->_fid,
-				surf->w, surf->h,
-				static_cast<const uint8_t*>(surf->pixels),
-				static_cast<const uint8_t*>(surf->pixels) + surf->w * surf->h,
-				static_cast<const uint8_t*>(surf->pixels) + surf->w * surf->h + (surf->w/2) * (surf->h/2)
-			);
-			SDL_UnlockSurface(surf);
-		}
-	}
 }
 
 ObjectHandle ToxAVVoIPModel::enter(const Contact3 c, const Components::VoIP::DefaultConfig& defaults) {
@@ -473,8 +311,6 @@ bool ToxAVVoIPModel::accept(ObjectHandle session, const Components::VoIP::Defaul
 	// bitrate cb or what?
 	assert(!session.all_of<Components::ToxAVAudioSink>());
 	addAudioSink(session, friend_number);
-	assert(!session.all_of<Components::ToxAVVideoSink>());
-	addVideoSink(session, friend_number);
 
 	if (const auto* i_av = session.try_get<Components::ToxAVIncomingAV>(); i_av != nullptr) {
 		// create audio src
@@ -485,8 +321,6 @@ bool ToxAVVoIPModel::accept(ObjectHandle session, const Components::VoIP::Defaul
 
 		// create video src
 		if (i_av->incoming_video) {
-			assert(!session.all_of<Components::ToxAVVideoSource>());
-			addVideoSource(session, friend_number);
 		}
 	}
 
@@ -585,11 +419,6 @@ bool ToxAVVoIPModel::onEvent(const Events::FriendCallState& e) {
 				}
 
 				// video
-				if (s.is_accepting_v() && !o.all_of<Components::ToxAVVideoSink>()) {
-					addVideoSink(o, e.friend_number);
-				} else if (!s.is_accepting_v() && o.all_of<Components::ToxAVVideoSink>()) {
-					// remove vsink?
-				}
 
 				// add/update sources
 				// audio
@@ -600,11 +429,6 @@ bool ToxAVVoIPModel::onEvent(const Events::FriendCallState& e) {
 				}
 
 				// video
-				if (s.is_sending_v() && !o.all_of<Components::ToxAVVideoSource>()) {
-					addVideoSource(o, e.friend_number);
-				} else if (!s.is_sending_v() && o.all_of<Components::ToxAVVideoSource>()) {
-					// remove vsrc?
-				}
 			}
 		}
 	}
@@ -646,66 +470,7 @@ bool ToxAVVoIPModel::onEvent(const Events::FriendAudioFrame& e) {
 	return true;
 }
 
-bool ToxAVVoIPModel::onEvent(const Events::FriendVideoFrame& e) {
+bool ToxAVVoIPModel::onEvent(const Events::FriendVideoFrame&) {
-	auto vsrc_it = _video_sources.find(e.friend_number);
-	if (vsrc_it == _video_sources.cend()) {
-		// missing src from lookup table
-		return false;
-	}
-
-	auto vsrc = vsrc_it->second;
-
-	if (!static_cast<bool>(vsrc)) {
-		// missing src to put frame into ??
-		return false;
-	}
-
-	assert(vsrc.all_of<FrameStream2MultiSource<SDLVideoFrame>*>());
-	assert(vsrc.all_of<Components::FrameStream2Source<SDLVideoFrame>>());
-
-	auto* new_surf = SDL_CreateSurface(e.width, e.height, SDL_PIXELFORMAT_IYUV);
-	assert(new_surf);
-	if (SDL_LockSurface(new_surf)) {
-		// copy the data
-		// we know how the implementation works, its y u v consecutivlely
-		// y
-		for (size_t y = 0; y < e.height; y++) {
-			std::memcpy(
-				//static_cast<uint8_t*>(new_surf->pixels) + new_surf->pitch*y,
-				static_cast<uint8_t*>(new_surf->pixels) + e.width*y,
-				e.y.ptr + e.ystride*y,
-				e.width
-			);
-		}
-
-		// u
-		for (size_t y = 0; y < e.height/2; y++) {
-			std::memcpy(
-				static_cast<uint8_t*>(new_surf->pixels) + (e.width*e.height) + (e.width/2)*y,
-				e.u.ptr + e.ustride*y,
-				e.width/2
-			);
-		}
-
-		// v
-		for (size_t y = 0; y < e.height/2; y++) {
-			std::memcpy(
-				static_cast<uint8_t*>(new_surf->pixels) + (e.width*e.height) + ((e.width/2)*(e.height/2)) + (e.width/2)*y,
-				e.v.ptr + e.vstride*y,
-				e.width/2
-			);
-		}
-
-		SDL_UnlockSurface(new_surf);
-	}
-
-	vsrc.get<FrameStream2MultiSource<SDLVideoFrame>*>()->push({
-		// ms -> us
-		Message::getTimeMS() * 1000, // TODO: make more precise
-		new_surf
-	});
-
-	SDL_DestroySurface(new_surf);
 	return false;
 }
 

src/tox_av_voip_model.hpp

@@ -16,13 +16,11 @@ class ToxAVVoIPModel : protected ToxAVEventI, public VoIPModelI {
 
 	// for faster lookup
 	std::unordered_map<uint32_t, ObjectHandle> _audio_sources;
-	std::unordered_map<uint32_t, ObjectHandle> _video_sources;
 
 	// TODO: virtual? strategy? protected?
 	virtual void addAudioSource(ObjectHandle session, uint32_t friend_number);
 	virtual void addAudioSink(ObjectHandle session, uint32_t friend_number);
-	virtual void addVideoSource(ObjectHandle session, uint32_t friend_number);
+	// TODO: video
-	virtual void addVideoSink(ObjectHandle session, uint32_t friend_number);
 
 	void destroySession(ObjectHandle session);