enable toxav in cd

input with reframer (magic fix, someone pls tell my why)

.github/workflows/cd.yml

@@ -22,10 +22,10 @@ jobs:
         submodules: recursive
 
     - name: Install Dependencies
-      run: sudo apt update && sudo apt -y install libsodium-dev cmake
+      run: sudo apt update && sudo apt -y install libsodium-dev cmake libvpx-dev libopus-dev
 
     - name: Configure CMake
-      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
+      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DTOMATO_TOX_AV=ON
 
     - name: Build
       run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} -j 4 -t tomato
@@ -101,7 +101,7 @@ jobs:
     - name: Configure CMake
       env:
         ANDROID_NDK_HOME: ${{steps.setup_ndk.outputs.ndk-path}}
-      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_TOOLCHAIN_FILE=/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=${{matrix.platform.vcpkg_toolkit}} -DANDROID=1 -DANDROID_PLATFORM=23 -DANDROID_ABI=${{matrix.platform.ndk_abi}} -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=${{steps.setup_ndk.outputs.ndk-path}}/build/cmake/android.toolchain.cmake -DSDLIMAGE_JPG_SHARED=OFF -DSDLIMAGE_PNG_SHARED=OFF -DTOMATO_MAIN_SO=ON
+      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_TOOLCHAIN_FILE=/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=${{matrix.platform.vcpkg_toolkit}} -DANDROID=1 -DANDROID_PLATFORM=23 -DANDROID_ABI=${{matrix.platform.ndk_abi}} -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=${{steps.setup_ndk.outputs.ndk-path}}/build/cmake/android.toolchain.cmake -DSDLIMAGE_JPG_SHARED=OFF -DSDLIMAGE_PNG_SHARED=OFF -DTOMATO_MAIN_SO=ON -DTOMATO_TOX_AV=ON
 
     - name: Build (tomato)
       run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} -j 4 -t tomato
@@ -164,7 +164,7 @@ jobs:
     #- uses: ilammy/setup-nasm@v1
 
     - name: Configure CMake
-      run: cmake -G Ninja -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=x64-windows-static -DSDLIMAGE_VENDORED=ON -DSDLIMAGE_DEPS_SHARED=ON -DSDLIMAGE_JXL=OFF -DSDLIMAGE_AVIF=OFF -DPKG_CONFIG_EXECUTABLE=C:/vcpkg/installed/x64-windows/tools/pkgconf/pkgconf.exe
+      run: cmake -G Ninja -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=x64-windows-static -DSDLIMAGE_VENDORED=ON -DSDLIMAGE_DEPS_SHARED=ON -DSDLIMAGE_JXL=OFF -DSDLIMAGE_AVIF=OFF -DPKG_CONFIG_EXECUTABLE=C:/vcpkg/installed/x64-windows/tools/pkgconf/pkgconf.exe -DTOMATO_TOX_AV=ON
 
     - name: Build
       run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} -t tomato
@@ -229,7 +229,7 @@ jobs:
     #- uses: ilammy/setup-nasm@v1
 
     - name: Configure CMake
-      run: cmake -G Ninja -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=x64-windows-static -DTOMATO_ASAN=ON -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded -DSDLIMAGE_VENDORED=ON -DSDLIMAGE_DEPS_SHARED=ON -DSDLIMAGE_JXL=OFF -DSDLIMAGE_AVIF=OFF -DPKG_CONFIG_EXECUTABLE=C:/vcpkg/installed/x64-windows/tools/pkgconf/pkgconf.exe
+      run: cmake -G Ninja -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=x64-windows-static -DTOMATO_ASAN=ON -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded -DSDLIMAGE_VENDORED=ON -DSDLIMAGE_DEPS_SHARED=ON -DSDLIMAGE_JXL=OFF -DSDLIMAGE_AVIF=OFF -DPKG_CONFIG_EXECUTABLE=C:/vcpkg/installed/x64-windows/tools/pkgconf/pkgconf.exe -DTOMATO_TOX_AV=ON
 
     - name: Build
       run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} -j 4 -t tomato

flake.nix

@@ -80,6 +80,7 @@
         ] ++ self.packages.${system}.default.dlopenBuildInputs;
 
         cmakeFlags = [
+          "-DTOMATO_TOX_AV=ON"
           "-DTOMATO_ASAN=OFF"
           "-DCMAKE_BUILD_TYPE=RelWithDebInfo"
           #"-DCMAKE_BUILD_TYPE=Debug"

src/CMakeLists.txt

@@ -107,6 +107,10 @@ target_sources(tomato PUBLIC
 	./frame_streams/audio_stream2.hpp
 	./frame_streams/stream_manager.hpp
 	./frame_streams/stream_manager.cpp
+	./frame_streams/locked_frame_stream.hpp
+	./frame_streams/multi_source.hpp
+
+	./frame_streams/voip_model.hpp
 
 	./frame_streams/sdl/sdl_audio2_frame_stream2.hpp
 	./frame_streams/sdl/sdl_audio2_frame_stream2.cpp
@@ -123,6 +127,9 @@ if (TOMATO_TOX_AV)
 	target_sources(tomato PUBLIC
 		./tox_av.hpp
 		./tox_av.cpp
+
+		./tox_av_voip_model.hpp
+		./tox_av_voip_model.cpp
 	)
 
 	target_compile_definitions(tomato PUBLIC TOMATO_TOX_AV)
@@ -162,3 +169,18 @@ target_link_libraries(tomato PUBLIC
 
 set_target_properties(tomato PROPERTIES POSITION_INDEPENDENT_CODE ON)
 
+########################################
+
+add_executable(test_frame_stream2_pop_reframer EXCLUDE_FROM_ALL
+	./frame_streams/frame_stream2.hpp
+	./frame_streams/audio_stream2.hpp
+	./frame_streams/locked_frame_stream.hpp
+	./frame_streams/multi_source.hpp
+
+	./frame_streams/test_pop_reframer.cpp
+)
+
+target_link_libraries(test_frame_stream2_pop_reframer
+	solanaceae_util
+)
+

src/chat_gui4.cpp

@@ -9,9 +9,13 @@
 #include <solanaceae/contact/components.hpp>
 #include <solanaceae/util/utils.hpp>
 
+#include "./frame_streams/voip_model.hpp"
+
 // HACK: remove them
 #include <solanaceae/tox_contacts/components.hpp>
 
+#include <entt/entity/entity.hpp>
+
 #include <imgui/imgui.h>
 #include <imgui/misc/cpp/imgui_stdlib.h>
 
@@ -30,6 +34,7 @@
 #include <fstream>
 #include <iomanip>
 #include <sstream>
+#include <string>
 #include <variant>
 
 namespace Components {
@@ -257,6 +262,97 @@ float ChatGui4::render(float time_delta) {
 
 			if (ImGui::BeginChild(chat_label.c_str(), {0, 0}, ImGuiChildFlags_Border, ImGuiWindowFlags_MenuBar)) {
 				if (ImGui::BeginMenuBar()) {
+					// check if contact has voip model
+					// use activesessioncomp instead?
+					if (_cr.all_of<VoIPModelI*>(*_selected_contact)) {
+						if (ImGui::BeginMenu("VoIP")) {
+							auto* voip_model = _cr.get<VoIPModelI*>(*_selected_contact);
+
+							std::vector<ObjectHandle> contact_sessions;
+							std::vector<ObjectHandle> acceptable_sessions;
+							for (const auto& [ov, o_vm, sc] : _os.registry().view<VoIPModelI*, Components::VoIP::SessionContact>().each()) {
+								if (o_vm != voip_model) {
+									continue;
+								}
+								if (sc.c != *_selected_contact) {
+									continue;
+								}
+
+								auto o = _os.objectHandle(ov);
+								contact_sessions.push_back(o);
+
+								if (!o.all_of<Components::VoIP::Incoming>()) {
+									continue; // not incoming
+								}
+
+								// state is ringing/not yet accepted
+								const auto* session_state = o.try_get<Components::VoIP::SessionState>();
+								if (session_state == nullptr) {
+									continue;
+								}
+
+								if (session_state->state != Components::VoIP::SessionState::State::RINGING) {
+									continue;
+								}
+								acceptable_sessions.push_back(o);
+							}
+
+							static Components::VoIP::DefaultConfig g_default_connections{};
+
+							if (ImGui::BeginMenu("default connections")) {
+								ImGui::MenuItem("incoming audio", nullptr, &g_default_connections.incoming_audio);
+								ImGui::MenuItem("incoming video", nullptr, &g_default_connections.incoming_video);
+								ImGui::Separator();
+								ImGui::MenuItem("outgoing audio", nullptr, &g_default_connections.outgoing_audio);
+								ImGui::MenuItem("outgoing video", nullptr, &g_default_connections.outgoing_video);
+								ImGui::EndMenu();
+							}
+
+							if (acceptable_sessions.size() < 2) {
+								if (ImGui::MenuItem("accept call", nullptr, false, !acceptable_sessions.empty())) {
+									voip_model->accept(acceptable_sessions.front(), g_default_connections);
+								}
+							} else {
+								if (ImGui::BeginMenu("accept call", !acceptable_sessions.empty())) {
+									for (const auto o : acceptable_sessions) {
+										std::string label = "accept #";
+										label += std::to_string(entt::to_integral(entt::to_entity(o.entity())));
+
+										if (ImGui::MenuItem(label.c_str())) {
+											voip_model->accept(o, g_default_connections);
+										}
+									}
+									ImGui::EndMenu();
+								}
+							}
+
+							// TODO: disable if already in call?
+							if (ImGui::Button(" call ")) {
+								voip_model->enter(*_selected_contact, g_default_connections);
+							}
+
+							if (contact_sessions.size() < 2) {
+								if (ImGui::MenuItem("leave/reject call", nullptr, false, !contact_sessions.empty())) {
+									voip_model->leave(contact_sessions.front());
+								}
+							} else {
+								if (ImGui::BeginMenu("leave/reject call")) {
+									// list
+									for (const auto o : contact_sessions) {
+										std::string label = "end #";
+										label += std::to_string(entt::to_integral(entt::to_entity(o.entity())));
+
+										if (ImGui::MenuItem(label.c_str())) {
+											voip_model->leave(o);
+										}
+									}
+									ImGui::EndMenu();
+								}
+							}
+
+							ImGui::EndMenu();
+						}
+					}
 					if (ImGui::BeginMenu("debug")) {
 						ImGui::Checkbox("show extra info", &_show_chat_extra_info);
 						ImGui::Checkbox("show avatar transfers", &_show_chat_avatar_tf);

src/frame_streams/audio_stream_pop_reframer.hpp

@@ -0,0 +1,103 @@
+#pragma once
+
+#include "./audio_stream2.hpp"
+
+// reframes audio frames to a specified size in ms
+// TODO: use absolute sample count instead??
+template<typename RealAudioStream>
+struct AudioStreamPopReFramer : public FrameStream2I<AudioFrame2> {
+	uint32_t _frame_length_ms {20};
+
+	// gotta be careful of the multithreaded nature
+	// and false(true) sharing
+	uint64_t _pad0{};
+	RealAudioStream _stream;
+	uint64_t _pad1{};
+
+	// dequeue?
+	std::vector<int16_t> _buffer;
+
+	uint32_t _sample_rate {48'000};
+	size_t _channels {0};
+
+	AudioStreamPopReFramer(uint32_t frame_length_ms = 20)
+	: _frame_length_ms(frame_length_ms) {
+	}
+
+	AudioStreamPopReFramer(uint32_t frame_length_ms, FrameStream2I<AudioFrame2>&& stream)
+	: _frame_length_ms(frame_length_ms), _stream(std::move(stream)) {
+	}
+
+	~AudioStreamPopReFramer(void) {}
+
+	size_t getDesiredSize(void) const {
+		return _frame_length_ms * _sample_rate * _channels / 1000;
+	}
+
+	int32_t size(void) override { return -1; }
+
+	std::optional<AudioFrame2> pop(void) override {
+		do {
+			auto new_in = _stream.pop();
+			if (new_in.has_value()) {
+				auto& new_value = new_in.value();
+
+				// changed
+				if (_sample_rate != new_value.sample_rate || _channels != new_value.channels) {
+					//if (_channels != 0) {
+					//    std::cerr << "ReFramer warning: reconfiguring, dropping buffer\n";
+					//}
+					_sample_rate = new_value.sample_rate;
+					_channels = new_value.channels;
+
+					// buffer does not exist or config changed and we discard
+					_buffer = {};
+				}
+
+				//std::cout << "new incoming frame is " << new_value.getSpan().size/new_value.channels*1000/new_value.sample_rate << "ms\n";
+
+				auto new_span = new_value.getSpan();
+
+				if (_buffer.empty()) {
+					_buffer = {new_span.cbegin(), new_span.cend()};
+				} else {
+					_buffer.insert(_buffer.cend(), new_span.cbegin(), new_span.cend());
+				}
+			} else if (_buffer.empty()) {
+				// first pop might result in invalid state
+				return std::nullopt;
+			} else {
+				// inner stream pop did not give a new value
+				break; // out of loop
+			}
+		} while (_buffer.size() < getDesiredSize());
+
+		const auto desired_size = getDesiredSize();
+
+		// > threshold?
+		if (_buffer.size() < desired_size) {
+			return std::nullopt;
+		}
+
+		// copy data
+		std::vector<int16_t> return_buffer(_buffer.cbegin(), _buffer.cbegin()+desired_size);
+
+		// now crop buffer (meh)
+		// move data from back to front
+		_buffer.erase(_buffer.cbegin(), _buffer.cbegin() + desired_size);
+
+		return AudioFrame2{
+			_sample_rate,
+			_channels,
+			std::move(return_buffer),
+		};
+	}
+
+	bool push(const AudioFrame2& value) override {
+		// might be worth it to instead do the work on push
+		//assert(false && "push reframing not implemented");
+		// passthrough
+		return _stream.push(value);
+	}
+};
+

src/frame_streams/locked_frame_stream.hpp

@@ -0,0 +1,46 @@
+#pragma once
+
+#include "./frame_stream2.hpp"
+
+#include <mutex>
+#include <deque>
+
+// threadsafe queue frame stream
+// protected by a simple mutex lock
+// prefer lockless queue implementations, when available
+template<typename FrameType>
+struct LockedFrameStream2 : public FrameStream2I<FrameType> {
+	std::mutex _lock;
+
+	std::deque<FrameType> _frames;
+
+	~LockedFrameStream2(void) {}
+
+	int32_t size(void) { return -1; }
+
+	std::optional<FrameType> pop(void) {
+		std::lock_guard lg{_lock};
+
+		if (_frames.empty()) {
+			return std::nullopt;
+		}
+
+		FrameType new_frame = std::move(_frames.front());
+		_frames.pop_front();
+
+		return std::move(new_frame);
+	}
+
+	bool push(const FrameType& value) {
+		std::lock_guard lg{_lock};
+
+		if (_frames.size() > 1024) {
+			return false; // hard limit
+		}
+
+		_frames.push_back(value);
+
+		return true;
+	}
+};
+

src/frame_streams/multi_source.hpp

@@ -0,0 +1,62 @@
+#pragma once
+
+#include "./locked_frame_stream.hpp"
+
+#include <cassert>
+
+// implements a stream that pushes to all sub streams
+template<typename FrameType, typename SubStreamType = LockedFrameStream2<FrameType>>
+struct FrameStream2MultiSource : public FrameStream2SourceI<FrameType>, public FrameStream2I<FrameType> {
+	using sub_stream_type_t = SubStreamType;
+
+	// pointer stability
+	std::vector<std::shared_ptr<SubStreamType>> _sub_streams;
+	std::mutex _sub_stream_lock; // accessing the _sub_streams array needs to be exclusive
+	// a simple lock here is ok, since this tends to be a rare operation,
+	// except for the push, which is always on the same thread
+
+	virtual ~FrameStream2MultiSource(void) {}
+
+	// TODO: forward args instead
+	std::shared_ptr<FrameStream2I<FrameType>> subscribe(void) override {
+		std::lock_guard lg{_sub_stream_lock};
+		return _sub_streams.emplace_back(std::make_unique<SubStreamType>());
+	}
+
+	bool unsubscribe(const std::shared_ptr<FrameStream2I<FrameType>>& sub) override {
+		std::lock_guard lg{_sub_stream_lock};
+		for (auto it = _sub_streams.begin(); it != _sub_streams.end(); it++) {
+			if (*it == sub) {
+				_sub_streams.erase(it);
+				return true;
+			}
+		}
+		return false; // ?
+	}
+
+	// stream interface
+
+	int32_t size(void) override {
+		// TODO: return something sensible?
+		return -1;
+	}
+
+	std::optional<FrameType> pop(void) override {
+		// nope
+		assert(false && "this logic is very frame type specific, provide an impl");
+		return std::nullopt;
+	}
+
+	// returns true if there are readers
+	bool push(const FrameType& value) override {
+		std::lock_guard lg{_sub_stream_lock};
+		bool have_readers{false};
+		for (auto& it : _sub_streams) {
+			[[maybe_unused]] auto _ = it->push(value);
+			have_readers = true; // even if queue full, we still continue believing in them
+			// maybe consider push return value?
+		}
+		return have_readers;
+	}
+};
+

src/frame_streams/sdl/sdl_audio2_frame_stream2.cpp

@@ -4,6 +4,8 @@
 #include <iostream>
 #include <optional>
 
+#include "../audio_stream_pop_reframer.hpp"
+
 // "thin" wrapper around sdl audio streams
 // we dont needs to get fance, as they already provide everything we need
 struct SDLAudio2StreamReader : public AudioFrame2Stream2I {
@@ -58,7 +60,7 @@ struct SDLAudio2StreamReader : public AudioFrame2Stream2I {
 			return std::nullopt;
 		}
 
-		return AudioFrame2 {
+		return AudioFrame2{
 			_sample_rate, _channels,
 			Span<int16_t>(_buffer.data(), read_bytes/sizeof(int16_t)),
 		};
@@ -127,11 +129,17 @@ std::shared_ptr<FrameStream2I<AudioFrame2>> SDLAudio2InputDevice::subscribe(void
 	// error check
 	SDL_BindAudioStream(_virtual_device_id, sdl_stream);
 
-	auto new_stream = std::make_shared<SDLAudio2StreamReader>();
+	//auto new_stream = std::make_shared<SDLAudio2StreamReader>();
-	// TODO: move to ctr
+	//// TODO: move to ctr
-	new_stream->_stream = {sdl_stream, &SDL_DestroyAudioStream};
+	//new_stream->_stream = {sdl_stream, &SDL_DestroyAudioStream};
-	new_stream->_sample_rate = spec.freq;
+	//new_stream->_sample_rate = spec.freq;
-	new_stream->_channels = spec.channels;
+	//new_stream->_channels = spec.channels;
+
+	auto new_stream = std::make_shared<AudioStreamPopReFramer<SDLAudio2StreamReader>>();
+	new_stream->_stream._stream = {sdl_stream, &SDL_DestroyAudioStream};
+	new_stream->_stream._sample_rate = spec.freq;
+	new_stream->_stream._channels = spec.channels;
+	new_stream->_frame_length_ms = 5; // WHY DOES THIS FIX MY ISSUE !!!
 
 	_streams.emplace_back(new_stream);
 

src/frame_streams/test_pop_reframer.cpp

@@ -0,0 +1,155 @@
+#include <iostream>
+
+#include "./audio_stream_pop_reframer.hpp"
+#include "./locked_frame_stream.hpp"
+
+#include <cassert>
+
+int main(void) {
+	{ // pump perfect
+		AudioStreamPopReFramer<LockedFrameStream2<AudioFrame2>> stream;
+		stream._frame_length_ms = 10;
+
+		AudioFrame2 f1 {
+			48'000,
+			1,
+			{},
+		};
+		f1.buffer = std::vector<int16_t>(
+			// perfect size
+			stream._frame_length_ms * f1.sample_rate * f1.channels / 1000,
+			0
+		);
+
+		{ // fill with sequential value
+			int16_t seq = 0;
+			for (auto& v : std::get<std::vector<int16_t>>(f1.buffer)) {
+				v = seq++;
+			}
+		}
+
+		stream.push(f1);
+
+		auto ret_opt = stream.pop();
+		assert(ret_opt);
+
+		auto& ret = ret_opt.value();
+		assert(ret.sample_rate == f1.sample_rate);
+		assert(ret.channels == f1.channels);
+		assert(ret.getSpan().size == f1.getSpan().size);
+		{
+			int16_t seq = 0;
+			for (const auto v : ret.getSpan()) {
+				assert(v == seq++);
+			}
+		}
+	}
+
+	{ // pump half
+		AudioStreamPopReFramer<LockedFrameStream2<AudioFrame2>> stream;
+		stream._frame_length_ms = 10;
+
+		AudioFrame2 f1 {
+			48'000,
+			1,
+			{},
+		};
+		f1.buffer = std::vector<int16_t>(
+			// perfect size
+			(stream._frame_length_ms * f1.sample_rate * f1.channels / 1000) / 2,
+			0
+		);
+		AudioFrame2 f2 {
+			48'000,
+			1,
+			{},
+		};
+		f2.buffer = std::vector<int16_t>(
+			// perfect size
+			(stream._frame_length_ms * f1.sample_rate * f1.channels / 1000) / 2,
+			0
+		);
+
+		{ // fill with sequential value
+			int16_t seq = 0;
+			for (auto& v : std::get<std::vector<int16_t>>(f1.buffer)) {
+				v = seq++;
+			}
+			for (auto& v : std::get<std::vector<int16_t>>(f2.buffer)) {
+				v = seq++;
+			}
+		}
+
+		stream.push(f1);
+		stream.push(f2);
+
+		// supposed to combine both
+		auto ret_opt = stream.pop();
+		assert(ret_opt);
+
+		auto& ret = ret_opt.value();
+		assert(ret.sample_rate == f1.sample_rate);
+		assert(ret.channels == f1.channels);
+		assert(ret.getSpan().size == stream._frame_length_ms * f1.sample_rate * f1.channels / 1000);
+		{
+			int16_t seq = 0;
+			for (const auto v : ret.getSpan()) {
+				assert(v == seq++);
+			}
+		}
+	}
+
+	{ // pump double
+		AudioStreamPopReFramer<LockedFrameStream2<AudioFrame2>> stream;
+		stream._frame_length_ms = 20;
+
+		AudioFrame2 f1 {
+			48'000,
+			2,
+			{},
+		};
+		f1.buffer = std::vector<int16_t>(
+			// perfect size
+			(stream._frame_length_ms * f1.sample_rate * f1.channels / 1000) * 2,
+			0
+		);
+		{ // fill with sequential value
+			int16_t seq = 0;
+			for (auto& v : std::get<std::vector<int16_t>>(f1.buffer)) {
+				v = seq++;
+			}
+		}
+
+		stream.push(f1);
+
+		// pop 2x
+		int16_t seq = 0;
+		{
+			auto ret_opt = stream.pop();
+			assert(ret_opt);
+
+			auto& ret = ret_opt.value();
+			assert(ret.sample_rate == f1.sample_rate);
+			assert(ret.channels == f1.channels);
+			assert(ret.getSpan().size == stream._frame_length_ms * f1.sample_rate * f1.channels / 1000);
+			for (const auto v : ret.getSpan()) {
+				assert(v == seq++);
+			}
+		}
+
+		{
+			auto ret_opt = stream.pop();
+			assert(ret_opt);
+
+			auto& ret = ret_opt.value();
+			assert(ret.sample_rate == f1.sample_rate);
+			assert(ret.channels == f1.channels);
+			assert(ret.getSpan().size == stream._frame_length_ms * f1.sample_rate * f1.channels / 1000);
+			for (const auto v : ret.getSpan()) {
+				assert(v == seq++);
+			}
+		}
+	}
+
+	return 0;
+}

src/frame_streams/voip_model.hpp

@@ -0,0 +1,77 @@
+#pragma once
+
+#include <solanaceae/contact/contact_model3.hpp>
+#include <solanaceae/object_store/fwd.hpp>
+
+struct VoIPModelI;
+
+namespace Components::VoIP {
+
+	struct TagVoIPSession {};
+
+	// getting called or invited by
+	struct Incoming {
+		Contact3 c{entt::null};
+	};
+
+	struct DefaultConfig {
+		bool incoming_audio {true};
+		bool incoming_video {true};
+		bool outgoing_audio {true};
+		bool outgoing_video {true};
+	};
+
+	// to talk to the model handling this session
+	//struct VoIPModel {
+		//VoIPModelI* ptr {nullptr};
+	//};
+
+	struct SessionState {
+		// ????
+		// incoming
+		// outgoing
+		enum class State {
+			RINGING,
+			CONNECTED,
+		} state;
+	};
+
+	struct SessionContact {
+		Contact3 c{entt::null};
+	};
+
+	struct StreamSources {
+		// list of all stream sources originating from this VoIP session
+		std::vector<Object> streams;
+	};
+
+	struct StreamSinks {
+		// list of all stream sinks going to this VoIP session
+		std::vector<Object> streams;
+	};
+
+} // Components::VoIP
+
+// TODO: events? piggyback on objects?
+
+// stream model instead?? -> more generic than "just" audio and video?
+// or specialized like this
+// streams abstract type in a nice way
+struct VoIPModelI {
+	virtual ~VoIPModelI(void) {}
+
+	// enters a call/voicechat/videocall ???
+	// - contact
+	// - default stream sources/sinks ?
+	// - enable a/v ? -> done on connecting to sources
+	// returns object tieing together the VoIP session
+	virtual ObjectHandle enter(const Contact3 c, const Components::VoIP::DefaultConfig& defaults = {true, true, true, true}) { (void)c,(void)defaults; return {}; }
+
+	// accept/join an invite to a session
+	virtual bool accept(ObjectHandle session, const Components::VoIP::DefaultConfig& defaults = {true, true, true, true}) { (void)session,(void)defaults; return false; }
+
+	// leaves a call
+	// - VoIP session object
+	virtual bool leave(ObjectHandle session) { (void)session; return false; }
+};
+

src/main_screen.cpp

@@ -25,13 +25,14 @@ MainScreen::MainScreen(SimpleConfigModel&& conf_, SDL_Renderer* renderer_, Theme
 	tc(save_path, save_password),
 	tpi(tc.getTox()),
 	ad(tc),
-#if TOMATO_TOX_AV
-	tav(tc.getTox()),
-#endif
 	tcm(cr, tc, tc),
 	tmm(rmm, cr, tcm, tc, tc),
 	ttm(rmm, cr, tcm, tc, tc, os),
 	tffom(cr, rmm, tcm, tc, tc),
+#if TOMATO_TOX_AV
+	tav(tc.getTox()),
+	tavvoip(os, tav, cr, tcm),
+#endif
 	theme(theme_),
 	mmil(rmm),
 	tam(/*rmm, */ os, cr, conf),
@@ -80,7 +81,7 @@ MainScreen::MainScreen(SimpleConfigModel&& conf_, SDL_Renderer* renderer_, Theme
 		g_provideInstance<ToxPrivateI>("ToxPrivateI", "host", &tpi);
 		g_provideInstance<ToxEventProviderI>("ToxEventProviderI", "host", &tc);
 #if TOMATO_TOX_AV
-		g_provideInstance<ToxAV>("ToxAV", "host", &tav);
+		g_provideInstance<ToxAVI>("ToxAVI", "host", &tav);
 #endif
 		g_provideInstance<ToxContactModel2>("ToxContactModel2", "host", &tcm);
 
@@ -526,8 +527,12 @@ Screen* MainScreen::tick(float time_delta, bool& quit) {
 
 #if TOMATO_TOX_AV
 	tav.toxavIterate();
+	// breaks it
 	// HACK: pow by 1.18 to increase 200 -> ~500
-	const float av_interval = std::pow(tav.toxavIterationInterval(), 1.18)/1000.f;
+	//const float av_interval = std::pow(tav.toxavIterationInterval(), 1.18)/1000.f;
+	const float av_interval = tav.toxavIterationInterval()/1000.f;
+
+	tavvoip.tick();
 #endif
 
 	tcm.iterate(time_delta); // compute

src/main_screen.hpp

@@ -39,6 +39,7 @@
 
 #if TOMATO_TOX_AV
 #include "./tox_av.hpp"
+#include "./tox_av_voip_model.hpp"
 #endif
 
 #include <string>
@@ -67,13 +68,14 @@ struct MainScreen final : public Screen {
 	ToxClient tc;
 	ToxPrivateImpl tpi;
 	AutoDirty ad;
-#if TOMATO_TOX_AV
-	ToxAV tav;
-#endif
 	ToxContactModel2 tcm;
 	ToxMessageManager tmm;
 	ToxTransferManager ttm;
 	ToxFriendFauxOfflineMessaging tffom;
+#if TOMATO_TOX_AV
+	ToxAVI tav;
+	ToxAVVoIPModel tavvoip;
+#endif
 
 	Theme& theme;
 

src/tox_av.cpp

@@ -7,18 +7,7 @@
 
 // https://almogfx.bandcamp.com/track/crushed-w-cassade
 
-struct ToxAVFriendCallState final {
+ToxAVI::ToxAVI(Tox* tox) : _tox(tox) {
-	const uint32_t state {TOXAV_FRIEND_CALL_STATE_NONE};
-
-	[[nodiscard]] bool is_error(void) const { return state & TOXAV_FRIEND_CALL_STATE_ERROR; }
-	[[nodiscard]] bool is_finished(void) const { return state & TOXAV_FRIEND_CALL_STATE_FINISHED; }
-	[[nodiscard]] bool is_sending_a(void) const { return state & TOXAV_FRIEND_CALL_STATE_SENDING_A; }
-	[[nodiscard]] bool is_sending_v(void) const { return state & TOXAV_FRIEND_CALL_STATE_SENDING_V; }
-	[[nodiscard]] bool is_accepting_a(void) const { return state & TOXAV_FRIEND_CALL_STATE_ACCEPTING_A; }
-	[[nodiscard]] bool is_accepting_v(void) const { return state & TOXAV_FRIEND_CALL_STATE_ACCEPTING_V; }
-};
-
-ToxAV::ToxAV(Tox* tox) : _tox(tox) {
 	Toxav_Err_New err_new {TOXAV_ERR_NEW_OK};
 	_tox_av = toxav_new(_tox, &err_new);
 	// TODO: throw
@@ -28,7 +17,7 @@ ToxAV::ToxAV(Tox* tox) : _tox(tox) {
 		_tox_av,
 		+[](ToxAV*, uint32_t friend_number, bool audio_enabled, bool video_enabled, void *user_data) {
 			assert(user_data != nullptr);
-			static_cast<ToxAV*>(user_data)->cb_call(friend_number, audio_enabled, video_enabled);
+			static_cast<ToxAVI*>(user_data)->cb_call(friend_number, audio_enabled, video_enabled);
 		},
 		this
 	);
@@ -36,7 +25,7 @@ ToxAV::ToxAV(Tox* tox) : _tox(tox) {
 		_tox_av,
 		+[](ToxAV*, uint32_t friend_number, uint32_t state, void *user_data) {
 			assert(user_data != nullptr);
-			static_cast<ToxAV*>(user_data)->cb_call_state(friend_number, state);
+			static_cast<ToxAVI*>(user_data)->cb_call_state(friend_number, state);
 		},
 		this
 	);
@@ -44,7 +33,7 @@ ToxAV::ToxAV(Tox* tox) : _tox(tox) {
 		_tox_av,
 		+[](ToxAV*, uint32_t friend_number, uint32_t audio_bit_rate, void *user_data) {
 			assert(user_data != nullptr);
-			static_cast<ToxAV*>(user_data)->cb_audio_bit_rate(friend_number, audio_bit_rate);
+			static_cast<ToxAVI*>(user_data)->cb_audio_bit_rate(friend_number, audio_bit_rate);
 		},
 		this
 	);
@@ -52,7 +41,7 @@ ToxAV::ToxAV(Tox* tox) : _tox(tox) {
 		_tox_av,
 		+[](ToxAV*, uint32_t friend_number, uint32_t video_bit_rate, void *user_data) {
 			assert(user_data != nullptr);
-			static_cast<ToxAV*>(user_data)->cb_video_bit_rate(friend_number, video_bit_rate);
+			static_cast<ToxAVI*>(user_data)->cb_video_bit_rate(friend_number, video_bit_rate);
 		},
 		this
 	);
@@ -60,7 +49,7 @@ ToxAV::ToxAV(Tox* tox) : _tox(tox) {
 		_tox_av,
 		+[](ToxAV*, uint32_t friend_number, const int16_t pcm[], size_t sample_count, uint8_t channels, uint32_t sampling_rate, void *user_data) {
 			assert(user_data != nullptr);
-			static_cast<ToxAV*>(user_data)->cb_audio_receive_frame(friend_number, pcm, sample_count, channels, sampling_rate);
+			static_cast<ToxAVI*>(user_data)->cb_audio_receive_frame(friend_number, pcm, sample_count, channels, sampling_rate);
 		},
 		this
 	);
@@ -75,83 +64,83 @@ ToxAV::ToxAV(Tox* tox) : _tox(tox) {
 			void *user_data
 		) {
 			assert(user_data != nullptr);
-			static_cast<ToxAV*>(user_data)->cb_video_receive_frame(friend_number, width, height, y, u, v, ystride, ustride, vstride);
+			static_cast<ToxAVI*>(user_data)->cb_video_receive_frame(friend_number, width, height, y, u, v, ystride, ustride, vstride);
 		},
 		this
 	);
 }
 
-ToxAV::~ToxAV(void) {
+ToxAVI::~ToxAVI(void) {
 	toxav_kill(_tox_av);
 }
 
-uint32_t ToxAV::toxavIterationInterval(void) const {
+uint32_t ToxAVI::toxavIterationInterval(void) const {
 	return toxav_iteration_interval(_tox_av);
 }
 
-void ToxAV::toxavIterate(void) {
+void ToxAVI::toxavIterate(void) {
 	toxav_iterate(_tox_av);
 }
 
-uint32_t ToxAV::toxavAudioIterationInterval(void) const {
+uint32_t ToxAVI::toxavAudioIterationInterval(void) const {
 	return toxav_audio_iteration_interval(_tox_av);
 }
 
-void ToxAV::toxavAudioIterate(void) {
+void ToxAVI::toxavAudioIterate(void) {
 	toxav_audio_iterate(_tox_av);
 }
 
-uint32_t ToxAV::toxavVideoIterationInterval(void) const {
+uint32_t ToxAVI::toxavVideoIterationInterval(void) const {
 	return toxav_video_iteration_interval(_tox_av);
 }
 
-void ToxAV::toxavVideoIterate(void) {
+void ToxAVI::toxavVideoIterate(void) {
 	toxav_video_iterate(_tox_av);
 }
 
-Toxav_Err_Call ToxAV::toxavCall(uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate) {
+Toxav_Err_Call ToxAVI::toxavCall(uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate) {
 	Toxav_Err_Call err {TOXAV_ERR_CALL_OK};
 	toxav_call(_tox_av, friend_number, audio_bit_rate, video_bit_rate, &err);
 	return err;
 }
 
-Toxav_Err_Answer ToxAV::toxavAnswer(uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate) {
+Toxav_Err_Answer ToxAVI::toxavAnswer(uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate) {
 	Toxav_Err_Answer err {TOXAV_ERR_ANSWER_OK};
 	toxav_answer(_tox_av, friend_number, audio_bit_rate, video_bit_rate, &err);
 	return err;
 }
 
-Toxav_Err_Call_Control ToxAV::toxavCallControl(uint32_t friend_number, Toxav_Call_Control control) {
+Toxav_Err_Call_Control ToxAVI::toxavCallControl(uint32_t friend_number, Toxav_Call_Control control) {
 	Toxav_Err_Call_Control err {TOXAV_ERR_CALL_CONTROL_OK};
 	toxav_call_control(_tox_av, friend_number, control, &err);
 	return err;
 }
 
-Toxav_Err_Send_Frame ToxAV::toxavAudioSendFrame(uint32_t friend_number, const int16_t pcm[], size_t sample_count, uint8_t channels, uint32_t sampling_rate) {
+Toxav_Err_Send_Frame ToxAVI::toxavAudioSendFrame(uint32_t friend_number, const int16_t pcm[], size_t sample_count, uint8_t channels, uint32_t sampling_rate) {
 	Toxav_Err_Send_Frame err {TOXAV_ERR_SEND_FRAME_OK};
 	toxav_audio_send_frame(_tox_av, friend_number, pcm, sample_count, channels, sampling_rate, &err);
 	return err;
 }
 
-Toxav_Err_Bit_Rate_Set ToxAV::toxavAudioSetBitRate(uint32_t friend_number, uint32_t bit_rate) {
+Toxav_Err_Bit_Rate_Set ToxAVI::toxavAudioSetBitRate(uint32_t friend_number, uint32_t bit_rate) {
 	Toxav_Err_Bit_Rate_Set err {TOXAV_ERR_BIT_RATE_SET_OK};
 	toxav_audio_set_bit_rate(_tox_av, friend_number, bit_rate, &err);
 	return err;
 }
 
-Toxav_Err_Send_Frame ToxAV::toxavVideoSendFrame(uint32_t friend_number, uint16_t width, uint16_t height, const uint8_t y[], const uint8_t u[], const uint8_t v[]) {
+Toxav_Err_Send_Frame ToxAVI::toxavVideoSendFrame(uint32_t friend_number, uint16_t width, uint16_t height, const uint8_t y[], const uint8_t u[], const uint8_t v[]) {
 	Toxav_Err_Send_Frame err {TOXAV_ERR_SEND_FRAME_OK};
 	toxav_video_send_frame(_tox_av, friend_number, width, height, y, u, v, &err);
 	return err;
 }
 
-Toxav_Err_Bit_Rate_Set ToxAV::toxavVideoSetBitRate(uint32_t friend_number, uint32_t bit_rate) {
+Toxav_Err_Bit_Rate_Set ToxAVI::toxavVideoSetBitRate(uint32_t friend_number, uint32_t bit_rate) {
 	Toxav_Err_Bit_Rate_Set err {TOXAV_ERR_BIT_RATE_SET_OK};
 	toxav_video_set_bit_rate(_tox_av, friend_number, bit_rate, &err);
 	return err;
 }
 
-void ToxAV::cb_call(uint32_t friend_number, bool audio_enabled, bool video_enabled) {
+void ToxAVI::cb_call(uint32_t friend_number, bool audio_enabled, bool video_enabled) {
 	std::cerr << "TOXAV: receiving call f:" << friend_number << " a:" << audio_enabled << " v:" << video_enabled << "\n";
 	//Toxav_Err_Answer err_answer { TOXAV_ERR_ANSWER_OK };
 	//toxav_answer(_tox_av, friend_number, 0, 0, &err_answer);
@@ -169,7 +158,7 @@ void ToxAV::cb_call(uint32_t friend_number, bool audio_enabled, bool video_enabl
 	);
 }
 
-void ToxAV::cb_call_state(uint32_t friend_number, uint32_t state) {
+void ToxAVI::cb_call_state(uint32_t friend_number, uint32_t state) {
 	//ToxAVFriendCallState w_state{state};
 
 	//w_state.is_error();
@@ -185,7 +174,7 @@ void ToxAV::cb_call_state(uint32_t friend_number, uint32_t state) {
 	);
 }
 
-void ToxAV::cb_audio_bit_rate(uint32_t friend_number, uint32_t audio_bit_rate) {
+void ToxAVI::cb_audio_bit_rate(uint32_t friend_number, uint32_t audio_bit_rate) {
 	std::cerr << "TOXAV: audio bitrate f:" << friend_number << " abr:" << audio_bit_rate << "\n";
 
 	dispatch(
@@ -197,7 +186,7 @@ void ToxAV::cb_audio_bit_rate(uint32_t friend_number, uint32_t audio_bit_rate) {
 	);
 }
 
-void ToxAV::cb_video_bit_rate(uint32_t friend_number, uint32_t video_bit_rate) {
+void ToxAVI::cb_video_bit_rate(uint32_t friend_number, uint32_t video_bit_rate) {
 	std::cerr << "TOXAV: video bitrate f:" << friend_number << " vbr:" << video_bit_rate << "\n";
 
 	dispatch(
@@ -209,7 +198,7 @@ void ToxAV::cb_video_bit_rate(uint32_t friend_number, uint32_t video_bit_rate) {
 	);
 }
 
-void ToxAV::cb_audio_receive_frame(uint32_t friend_number, const int16_t pcm[], size_t sample_count, uint8_t channels, uint32_t sampling_rate) {
+void ToxAVI::cb_audio_receive_frame(uint32_t friend_number, const int16_t pcm[], size_t sample_count, uint8_t channels, uint32_t sampling_rate) {
 	//std::cerr << "TOXAV: audio frame f:" <<  friend_number << " sc:" << sample_count << " ch:" << (int)channels << " sr:" << sampling_rate << "\n";
 
 	dispatch(
@@ -223,7 +212,7 @@ void ToxAV::cb_audio_receive_frame(uint32_t friend_number, const int16_t pcm[],
 	);
 }
 
-void ToxAV::cb_video_receive_frame(
+void ToxAVI::cb_video_receive_frame(
 	uint32_t friend_number,
 	uint16_t width, uint16_t height,
 	const uint8_t y[/*! max(width, abs(ystride)) * height */],

src/tox_av.hpp

@@ -82,14 +82,15 @@ struct ToxAVEventI {
 };
 using ToxAVEventProviderI = EventProviderI<ToxAVEventI>;
 
-struct ToxAV : public ToxAVEventProviderI{
+// TODO: seperate out implementation from interface
+struct ToxAVI : public ToxAVEventProviderI {
 	Tox* _tox = nullptr;
 	ToxAV* _tox_av = nullptr;
 
 	static constexpr const char* version {"0"};
 
-	ToxAV(Tox* tox);
+	ToxAVI(Tox* tox);
-	virtual ~ToxAV(void);
+	virtual ~ToxAVI(void);
 
 	// interface
 	// if iterate is called on a different thread, it will fire events there
@@ -134,3 +135,14 @@ struct ToxAV : public ToxAVEventProviderI{
 	);
 };
 
+struct ToxAVFriendCallState final {
+	const uint32_t state {TOXAV_FRIEND_CALL_STATE_NONE};
+
+	[[nodiscard]] bool is_error(void) const { return state & TOXAV_FRIEND_CALL_STATE_ERROR; }
+	[[nodiscard]] bool is_finished(void) const { return state & TOXAV_FRIEND_CALL_STATE_FINISHED; }
+	[[nodiscard]] bool is_sending_a(void) const { return state & TOXAV_FRIEND_CALL_STATE_SENDING_A; }
+	[[nodiscard]] bool is_sending_v(void) const { return state & TOXAV_FRIEND_CALL_STATE_SENDING_V; }
+	[[nodiscard]] bool is_accepting_a(void) const { return state & TOXAV_FRIEND_CALL_STATE_ACCEPTING_A; }
+	[[nodiscard]] bool is_accepting_v(void) const { return state & TOXAV_FRIEND_CALL_STATE_ACCEPTING_V; }
+};
+

src/tox_av_voip_model.cpp

@@ -0,0 +1,476 @@
+#include "./tox_av_voip_model.hpp"
+
+#include <solanaceae/object_store/object_store.hpp>
+#include <solanaceae/tox_contacts/components.hpp>
+
+#include "./frame_streams/stream_manager.hpp"
+#include "./frame_streams/audio_stream2.hpp"
+#include "./frame_streams/locked_frame_stream.hpp"
+#include "./frame_streams/multi_source.hpp"
+#include "./frame_streams/audio_stream_pop_reframer.hpp"
+
+#include <iostream>
+
+namespace Components {
+	struct ToxAVIncomingAV {
+		bool incoming_audio {false};
+		bool incoming_video {false};
+	};
+
+	struct ToxAVAudioSink {
+		ObjectHandle o;
+		// ptr?
+	};
+	// vid
+	struct ToxAVAudioSource {
+		ObjectHandle o;
+		// ptr?
+	};
+	// vid
+} // Components
+
+struct ToxAVCallAudioSink : public FrameStream2SinkI<AudioFrame2> {
+	ToxAVI& _toxav;
+
+	// bitrate for enabled state
+	uint32_t _audio_bitrate {32};
+
+	uint32_t _fid;
+	std::shared_ptr<AudioStreamPopReFramer<LockedFrameStream2<AudioFrame2>>> _writer;
+
+	ToxAVCallAudioSink(ToxAVI& toxav, uint32_t fid) : _toxav(toxav), _fid(fid) {}
+	~ToxAVCallAudioSink(void) {
+		if (_writer) {
+			_writer = nullptr;
+			_toxav.toxavAudioSetBitRate(_fid, 0);
+		}
+	}
+
+	// sink
+	std::shared_ptr<FrameStream2I<AudioFrame2>> subscribe(void) override {
+		if (_writer) {
+			// max 1 (exclusive for now)
+			return nullptr;
+		}
+
+		auto err = _toxav.toxavAudioSetBitRate(_fid, _audio_bitrate);
+		if (err != TOXAV_ERR_BIT_RATE_SET_OK) {
+			return nullptr;
+		}
+
+		// 20ms for now, 10ms would work too, further investigate stutters at 5ms (probably too slow interval rate)
+		_writer = std::make_shared<AudioStreamPopReFramer<LockedFrameStream2<AudioFrame2>>>(20);
+
+		return _writer;
+	}
+
+	bool unsubscribe(const std::shared_ptr<FrameStream2I<AudioFrame2>>& sub) override {
+		if (!sub || !_writer) {
+			// nah
+			return false;
+		}
+
+		if (sub == _writer) {
+			_writer = nullptr;
+
+			/*auto err = */_toxav.toxavAudioSetBitRate(_fid, 0);
+			// print warning? on error?
+
+			return true;
+		}
+
+		// what
+		return false;
+	}
+};
+
+void ToxAVVoIPModel::addAudioSource(ObjectHandle session, uint32_t friend_number) {
+	auto& stream_source = session.get_or_emplace<Components::VoIP::StreamSources>().streams;
+
+	ObjectHandle incoming_audio {_os.registry(), _os.registry().create()};
+
+	auto new_asrc = std::make_unique<FrameStream2MultiSource<AudioFrame2>>();
+	incoming_audio.emplace<FrameStream2MultiSource<AudioFrame2>*>(new_asrc.get());
+	incoming_audio.emplace<Components::FrameStream2Source<AudioFrame2>>(std::move(new_asrc));
+	incoming_audio.emplace<Components::StreamSource>(Components::StreamSource::create<AudioFrame2>("ToxAV Friend Call Incoming Audio"));
+
+	std::cout << "new incoming audio\n";
+	if (
+		const auto* defaults = session.try_get<Components::VoIP::DefaultConfig>();
+		defaults != nullptr && defaults->incoming_audio
+	) {
+		incoming_audio.emplace<Components::TagConnectToDefault>(); // depends on what was specified in enter()
+		std::cout << "with default\n";
+	}
+
+	stream_source.push_back(incoming_audio);
+	session.emplace<Components::ToxAVAudioSource>(incoming_audio);
+	// TODO: tie session to stream
+
+	_audio_sources[friend_number] = incoming_audio;
+
+	_os.throwEventConstruct(incoming_audio);
+}
+
+void ToxAVVoIPModel::addAudioSink(ObjectHandle session, uint32_t friend_number) {
+	auto& stream_sinks = session.get_or_emplace<Components::VoIP::StreamSinks>().streams;
+	ObjectHandle outgoing_audio {_os.registry(), _os.registry().create()};
+
+	auto new_asink = std::make_unique<ToxAVCallAudioSink>(_av, friend_number);
+	outgoing_audio.emplace<ToxAVCallAudioSink*>(new_asink.get());
+	outgoing_audio.emplace<Components::FrameStream2Sink<AudioFrame2>>(std::move(new_asink));
+	outgoing_audio.emplace<Components::StreamSink>(Components::StreamSink::create<AudioFrame2>("ToxAV Friend Call Outgoing Audio"));
+
+	if (
+		const auto* defaults = session.try_get<Components::VoIP::DefaultConfig>();
+		defaults != nullptr && defaults->outgoing_audio
+	) {
+		outgoing_audio.emplace<Components::TagConnectToDefault>(); // depends on what was specified in enter()
+	}
+
+	stream_sinks.push_back(outgoing_audio);
+	session.emplace<Components::ToxAVAudioSink>(outgoing_audio);
+	// TODO: tie session to stream
+
+	_os.throwEventConstruct(outgoing_audio);
+}
+
+void ToxAVVoIPModel::destroySession(ObjectHandle session) {
+	if (!static_cast<bool>(session)) {
+		return;
+	}
+
+	// remove lookup
+	if (session.all_of<Components::ToxAVAudioSource>()) {
+		auto it_asrc = std::find_if(
+			_audio_sources.cbegin(), _audio_sources.cend(),
+			[o = session.get<Components::ToxAVAudioSource>().o](const auto& it) {
+				return it.second == o;
+			}
+		);
+
+		if (it_asrc != _audio_sources.cend()) {
+			_audio_sources.erase(it_asrc);
+		}
+	}
+
+	// destory sources
+	if (auto* ss = session.try_get<Components::VoIP::StreamSources>(); ss != nullptr) {
+		for (const auto ssov : ss->streams) {
+
+			_os.throwEventDestroy(ssov);
+			_os.registry().destroy(ssov);
+		}
+	}
+
+	// destory sinks
+	if (auto* ss = session.try_get<Components::VoIP::StreamSinks>(); ss != nullptr) {
+		for (const auto ssov : ss->streams) {
+
+			_os.throwEventDestroy(ssov);
+			_os.registry().destroy(ssov);
+		}
+	}
+
+	// destory session
+	_os.throwEventDestroy(session);
+	_os.registry().destroy(session);
+}
+
+ToxAVVoIPModel::ToxAVVoIPModel(ObjectStore2& os, ToxAVI& av, Contact3Registry& cr, ToxContactModel2& tcm) :
+	_os(os), _av(av), _cr(cr), _tcm(tcm)
+{
+	_av.subscribe(this, ToxAV_Event::friend_call);
+	_av.subscribe(this, ToxAV_Event::friend_call_state);
+	_av.subscribe(this, ToxAV_Event::friend_audio_bitrate);
+	_av.subscribe(this, ToxAV_Event::friend_video_bitrate);
+	_av.subscribe(this, ToxAV_Event::friend_audio_frame);
+	_av.subscribe(this, ToxAV_Event::friend_video_frame);
+
+	// attach to all tox friend contacts
+
+	for (const auto& [cv, _] : _cr.view<Contact::Components::ToxFriendPersistent>().each()) {
+		_cr.emplace<VoIPModelI*>(cv, this);
+	}
+	// TODO: events
+}
+
+ToxAVVoIPModel::~ToxAVVoIPModel(void) {
+	for (const auto& [ov, voipmodel] : _os.registry().view<VoIPModelI*>().each()) {
+		if (voipmodel == this) {
+			destroySession(_os.objectHandle(ov));
+		}
+	}
+}
+
+void ToxAVVoIPModel::tick(void) {
+	for (const auto& [oc, asink] : _os.registry().view<ToxAVCallAudioSink*>().each()) {
+		if (!asink->_writer) {
+			continue;
+		}
+
+		for (size_t i = 0; i < 100; i++) {
+			auto new_frame_opt = asink->_writer->pop();
+			if (!new_frame_opt.has_value()) {
+				break;
+			}
+			const auto& new_frame = new_frame_opt.value();
+
+			//* @param sample_count Number of samples in this frame. Valid numbers here are
+			//*   `((sample rate) * (audio length) / 1000)`, where audio length can be
+			//*   2.5, 5, 10, 20, 40 or 60 milliseconds.
+
+			// we likely needs to subdivide/repackage
+			// frame size should be an option exposed to the user
+			// with 10ms as a default ?
+			// the larger the frame size, the less overhead but the more delay
+
+			auto err = _av.toxavAudioSendFrame(
+				asink->_fid,
+				new_frame.getSpan().ptr,
+				new_frame.getSpan().size / new_frame.channels,
+				new_frame.channels,
+				new_frame.sample_rate
+			);
+			if (err != TOXAV_ERR_SEND_FRAME_OK) {
+				std::cerr << "DTC: failed to send audio frame " << err << "\n";
+			}
+		}
+	}
+}
+
+ObjectHandle ToxAVVoIPModel::enter(const Contact3 c, const Components::VoIP::DefaultConfig& defaults) {
+	if (!_cr.all_of<Contact::Components::ToxFriendEphemeral>(c)) {
+		return {};
+	}
+
+	const auto friend_number = _cr.get<Contact::Components::ToxFriendEphemeral>(c).friend_number;
+
+	const auto err = _av.toxavCall(friend_number, 0, 0);
+	if (err != TOXAV_ERR_CALL_OK) {
+		std::cerr << "TAVVOIP error: failed to start call: " << err << "\n";
+		return {};
+	}
+
+	ObjectHandle new_session {_os.registry(), _os.registry().create()};
+
+	new_session.emplace<VoIPModelI*>(this);
+	new_session.emplace<Components::VoIP::TagVoIPSession>(); // ??
+	new_session.emplace<Components::VoIP::SessionContact>(c);
+	new_session.emplace<Components::VoIP::SessionState>().state = Components::VoIP::SessionState::State::RINGING;
+	new_session.emplace<Components::VoIP::DefaultConfig>(defaults);
+
+	_os.throwEventConstruct(new_session);
+	return new_session;
+}
+
+bool ToxAVVoIPModel::accept(ObjectHandle session, const Components::VoIP::DefaultConfig& defaults) {
+	if (!static_cast<bool>(session)) {
+		return false;
+	}
+
+	if (!session.all_of<
+		Components::VoIP::TagVoIPSession,
+		VoIPModelI*,
+		Components::VoIP::SessionContact,
+		Components::VoIP::Incoming
+	>()) {
+		return false;
+	}
+
+	// check if self
+	if (session.get<VoIPModelI*>() != this) {
+		return false;
+	}
+
+	const auto session_contact = session.get<Components::VoIP::SessionContact>().c;
+	if (!_cr.all_of<Contact::Components::ToxFriendEphemeral>(session_contact)) {
+		return false;
+	}
+
+	const auto friend_number = _cr.get<Contact::Components::ToxFriendEphemeral>(session_contact).friend_number;
+	auto err = _av.toxavAnswer(friend_number, 0, 0);
+	if (err != TOXAV_ERR_ANSWER_OK) {
+		std::cerr << "TOXAVVOIP error: ansering call failed: " << err << "\n";
+		// we simply let it be for now, it apears we can try ansering later again
+		// we also get an error here when the call is already in progress (:
+		return false;
+	}
+
+	session.emplace<Components::VoIP::DefaultConfig>(defaults);
+
+	// answer defaults to enabled receiving audio and video
+	// TODO: think about how we should handle this
+	// set to disabled? and enable on src connection?
+	// we already default disabled send and enabled on sink connection
+	//_av.toxavCallControl(friend_number, TOXAV_CALL_CONTROL_HIDE_VIDEO);
+	//_av.toxavCallControl(friend_number, TOXAV_CALL_CONTROL_MUTE_AUDIO);
+
+
+	// how do we know the other side is accepting audio
+	// bitrate cb or what?
+	assert(!session.all_of<Components::ToxAVAudioSink>());
+	addAudioSink(session, friend_number);
+
+	if (const auto* i_av = session.try_get<Components::ToxAVIncomingAV>(); i_av != nullptr) {
+		// create audio src
+		if (i_av->incoming_audio) {
+			assert(!session.all_of<Components::ToxAVAudioSource>());
+			addAudioSource(session, friend_number);
+		}
+
+		// create video src
+		if (i_av->incoming_video) {
+		}
+	}
+
+	session.get_or_emplace<Components::VoIP::SessionState>().state = Components::VoIP::SessionState::State::CONNECTED;
+	_os.throwEventUpdate(session);
+	return true;
+}
+
+bool ToxAVVoIPModel::leave(ObjectHandle session) {
+	// rename to end?
+
+	if (!static_cast<bool>(session)) {
+		return false;
+	}
+
+	if (!session.all_of<
+		Components::VoIP::TagVoIPSession,
+		VoIPModelI*,
+		Components::VoIP::SessionContact
+	>()) {
+		return false;
+	}
+
+	// check if self
+	if (session.get<VoIPModelI*>() != this) {
+		return false;
+	}
+
+	const auto session_contact = session.get<Components::VoIP::SessionContact>().c;
+	if (!_cr.all_of<Contact::Components::ToxFriendEphemeral>(session_contact)) {
+		return false;
+	}
+
+	const auto friend_number = _cr.get<Contact::Components::ToxFriendEphemeral>(session_contact).friend_number;
+	// check error? (we delete anyway)
+	_av.toxavCallControl(friend_number, Toxav_Call_Control::TOXAV_CALL_CONTROL_CANCEL);
+
+	destroySession(session);
+	return true;
+}
+
+bool ToxAVVoIPModel::onEvent(const Events::FriendCall& e) {
+	// new incoming call, create voip session, ready to be accepted
+	// (or rejected...)
+
+	const auto session_contact = _tcm.getContactFriend(e.friend_number);
+	if (!_cr.valid(session_contact)) {
+		return false;
+	}
+
+	ObjectHandle new_session {_os.registry(), _os.registry().create()};
+
+	new_session.emplace<VoIPModelI*>(this);
+	new_session.emplace<Components::VoIP::TagVoIPSession>(); // ??
+	new_session.emplace<Components::VoIP::Incoming>(session_contact); // in 1on1 its always the same contact, might leave blank
+	new_session.emplace<Components::VoIP::SessionContact>(session_contact);
+	new_session.emplace<Components::VoIP::SessionState>().state = Components::VoIP::SessionState::State::RINGING;
+	new_session.emplace<Components::ToxAVIncomingAV>(e.audio_enabled, e.video_enabled);
+
+	_os.throwEventConstruct(new_session);
+	return true;
+}
+
+bool ToxAVVoIPModel::onEvent(const Events::FriendCallState& e) {
+	const auto session_contact = _tcm.getContactFriend(e.friend_number);
+	if (!_cr.valid(session_contact)) {
+		return false;
+	}
+
+	ToxAVFriendCallState s{e.state};
+
+	// find session(s?)
+	// TODO: keep lookup table
+	for (const auto& [ov, voipmodel] : _os.registry().view<VoIPModelI*>().each()) {
+		if (voipmodel == this) {
+			auto o = _os.objectHandle(ov);
+
+			if (!o.all_of<Components::VoIP::SessionContact>()) {
+				continue;
+			}
+			if (session_contact != o.get<Components::VoIP::SessionContact>().c) {
+				continue;
+			}
+
+			if (s.is_error() || s.is_finished()) {
+				// destroy call
+				destroySession(o);
+			} else {
+				// remote accepted our call, or av send/recv conditions changed?
+				o.get<Components::VoIP::SessionState>().state = Components::VoIP::SessionState::State::CONNECTED; // set to in call ??
+
+				if (s.is_accepting_a() && !o.all_of<Components::ToxAVAudioSink>()) {
+					addAudioSink(o, e.friend_number);
+				} else if (!s.is_accepting_a() && o.all_of<Components::ToxAVAudioSink>()) {
+					// remove asink?
+				}
+
+				// video
+
+				// add/update sources
+				// audio
+				if (s.is_sending_a() && !o.all_of<Components::ToxAVAudioSource>()) {
+					addAudioSource(o, e.friend_number);
+				} else if (!s.is_sending_a() && o.all_of<Components::ToxAVAudioSource>()) {
+					// remove asrc?
+				}
+
+				// video
+			}
+		}
+	}
+
+	return true;
+}
+
+bool ToxAVVoIPModel::onEvent(const Events::FriendAudioBitrate&) {
+	return false;
+}
+
+bool ToxAVVoIPModel::onEvent(const Events::FriendVideoBitrate&) {
+	return false;
+}
+
+bool ToxAVVoIPModel::onEvent(const Events::FriendAudioFrame& e) {
+	auto asrc_it = _audio_sources.find(e.friend_number);
+	if (asrc_it == _audio_sources.cend()) {
+		// missing src from lookup table
+		return false;
+	}
+
+	auto asrc = asrc_it->second;
+
+	if (!static_cast<bool>(asrc)) {
+		// missing src to put frame into ??
+		return false;
+	}
+
+	assert(asrc.all_of<FrameStream2MultiSource<AudioFrame2>*>());
+	assert(asrc.all_of<Components::FrameStream2Source<AudioFrame2>>());
+
+	asrc.get<FrameStream2MultiSource<AudioFrame2>*>()->push(AudioFrame2{
+		e.sampling_rate,
+		e.channels,
+		std::vector<int16_t>(e.pcm.begin(), e.pcm.end()) // copy
+	});
+
+	return true;
+}
+
+bool ToxAVVoIPModel::onEvent(const Events::FriendVideoFrame&) {
+	return false;
+}
+

src/tox_av_voip_model.hpp

@@ -0,0 +1,46 @@
+#pragma once
+
+#include <solanaceae/object_store/fwd.hpp>
+#include <solanaceae/contact/contact_model3.hpp>
+#include <solanaceae/tox_contacts/tox_contact_model2.hpp>
+#include "./frame_streams/voip_model.hpp"
+#include "./tox_av.hpp"
+
+#include <unordered_map>
+
+class ToxAVVoIPModel : protected ToxAVEventI, public VoIPModelI {
+	ObjectStore2& _os;
+	ToxAVI& _av;
+	Contact3Registry& _cr;
+	ToxContactModel2& _tcm;
+
+	// for faster lookup
+	std::unordered_map<uint32_t, ObjectHandle> _audio_sources;
+
+	// TODO: virtual? strategy? protected?
+	virtual void addAudioSource(ObjectHandle session, uint32_t friend_number);
+	virtual void addAudioSink(ObjectHandle session, uint32_t friend_number);
+	// TODO: video
+
+	void destroySession(ObjectHandle session);
+
+	public:
+		ToxAVVoIPModel(ObjectStore2& os, ToxAVI& av, Contact3Registry& cr, ToxContactModel2& tcm);
+		~ToxAVVoIPModel(void);
+
+		void tick(void);
+
+	public: // voip model
+		ObjectHandle enter(const Contact3 c, const Components::VoIP::DefaultConfig& defaults) override;
+		bool accept(ObjectHandle session, const Components::VoIP::DefaultConfig& defaults) override;
+		bool leave(ObjectHandle session) override;
+
+	protected: // toxav events
+		bool onEvent(const Events::FriendCall&) override;
+		bool onEvent(const Events::FriendCallState&) override;
+		bool onEvent(const Events::FriendAudioBitrate&) override;
+		bool onEvent(const Events::FriendVideoBitrate&) override;
+		bool onEvent(const Events::FriendAudioFrame&) override;
+		bool onEvent(const Events::FriendVideoFrame&) override;
+};
+