enable toxav in cd

input with reframer (magic fix, someone pls tell my why)

.github/workflows/cd.yml

@@ -22,10 +22,10 @@ jobs:
         submodules: recursive
 
     - name: Install Dependencies
-      run: sudo apt update && sudo apt -y install libsodium-dev cmake
+      run: sudo apt update && sudo apt -y install libsodium-dev cmake libvpx-dev libopus-dev
 
     - name: Configure CMake
-      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
+      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DTOMATO_TOX_AV=ON
 
     - name: Build
       run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} -j 4 -t tomato
@@ -101,7 +101,7 @@ jobs:
     - name: Configure CMake
       env:
         ANDROID_NDK_HOME: ${{steps.setup_ndk.outputs.ndk-path}}
-      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_TOOLCHAIN_FILE=/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=${{matrix.platform.vcpkg_toolkit}} -DANDROID=1 -DANDROID_PLATFORM=23 -DANDROID_ABI=${{matrix.platform.ndk_abi}} -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=${{steps.setup_ndk.outputs.ndk-path}}/build/cmake/android.toolchain.cmake -DSDLIMAGE_JPG_SHARED=OFF -DSDLIMAGE_PNG_SHARED=OFF -DTOMATO_MAIN_SO=ON
+      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_TOOLCHAIN_FILE=/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=${{matrix.platform.vcpkg_toolkit}} -DANDROID=1 -DANDROID_PLATFORM=23 -DANDROID_ABI=${{matrix.platform.ndk_abi}} -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=${{steps.setup_ndk.outputs.ndk-path}}/build/cmake/android.toolchain.cmake -DSDLIMAGE_JPG_SHARED=OFF -DSDLIMAGE_PNG_SHARED=OFF -DTOMATO_MAIN_SO=ON -DTOMATO_TOX_AV=ON
 
     - name: Build (tomato)
       run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} -j 4 -t tomato
@@ -164,7 +164,7 @@ jobs:
     #- uses: ilammy/setup-nasm@v1
 
     - name: Configure CMake
-      run: cmake -G Ninja -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=x64-windows-static -DSDLIMAGE_VENDORED=ON -DSDLIMAGE_DEPS_SHARED=ON -DSDLIMAGE_JXL=OFF -DSDLIMAGE_AVIF=OFF -DPKG_CONFIG_EXECUTABLE=C:/vcpkg/installed/x64-windows/tools/pkgconf/pkgconf.exe
+      run: cmake -G Ninja -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=x64-windows-static -DSDLIMAGE_VENDORED=ON -DSDLIMAGE_DEPS_SHARED=ON -DSDLIMAGE_JXL=OFF -DSDLIMAGE_AVIF=OFF -DPKG_CONFIG_EXECUTABLE=C:/vcpkg/installed/x64-windows/tools/pkgconf/pkgconf.exe -DTOMATO_TOX_AV=ON
 
     - name: Build
       run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} -t tomato
@@ -229,7 +229,7 @@ jobs:
     #- uses: ilammy/setup-nasm@v1
 
     - name: Configure CMake
-      run: cmake -G Ninja -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=x64-windows-static -DTOMATO_ASAN=ON -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded -DSDLIMAGE_VENDORED=ON -DSDLIMAGE_DEPS_SHARED=ON -DSDLIMAGE_JXL=OFF -DSDLIMAGE_AVIF=OFF -DPKG_CONFIG_EXECUTABLE=C:/vcpkg/installed/x64-windows/tools/pkgconf/pkgconf.exe
+      run: cmake -G Ninja -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=x64-windows-static -DTOMATO_ASAN=ON -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded -DSDLIMAGE_VENDORED=ON -DSDLIMAGE_DEPS_SHARED=ON -DSDLIMAGE_JXL=OFF -DSDLIMAGE_AVIF=OFF -DPKG_CONFIG_EXECUTABLE=C:/vcpkg/installed/x64-windows/tools/pkgconf/pkgconf.exe -DTOMATO_TOX_AV=ON
 
     - name: Build
       run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} -j 4 -t tomato

CMakeLists.txt

@@ -27,19 +27,11 @@ message("II TOMATO_TOX_AV: ${TOMATO_TOX_AV}")
 if (TOMATO_ASAN)
 	if (${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU" OR ${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang")
 		if (NOT WIN32) # exclude mingw
-			add_compile_options(-fsanitize=address,undefined)
+			#link_libraries(-fsanitize=address)
 			link_libraries(-fsanitize=address,undefined)
 			#link_libraries(-fsanitize=undefined)
 			link_libraries(-static-libasan) # make it "work" on nix
-
-			#add_compile_options(-fsanitize=thread)
-			#link_libraries(-fsanitize=thread)
-
 			message("II enabled ASAN")
-			if (OFF) # TODO: switch for minimal runtime in deployed scenarios
-				add_compile_options(-fsanitize-minimal-runtime)
-				link_libraries(-fsanitize-minimal-runtime)
-			endif()
 		else()
 			message("!! can not enable ASAN on this platform (gcc/clang + win)")
 		endif()

external/CMakeLists.txt

@@ -24,4 +24,3 @@ add_subdirectory(./libwebp)
 add_subdirectory(./qoi)
 add_subdirectory(./sdl_image)
 
-add_subdirectory(./spscqueue)

external/spscqueue/CMakeLists.txt

@@ -1,9 +0,0 @@
-cmake_minimum_required(VERSION 3.9 FATAL_ERROR)
-
-add_library(SPSCQueue INTERFACE
-	./SPSCQueue.h
-)
-
-target_compile_features(SPSCQueue INTERFACE cxx_std_17)
-target_include_directories(SPSCQueue INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}")
-

external/spscqueue/SPSCQueue.h

@@ -1,237 +0,0 @@
-/*
-Copyright (c) 2020 Erik Rigtorp <erik@rigtorp.se>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
- */
-
-#pragma once
-
-#include <atomic>
-#include <cassert>
-#include <cstddef>
-#include <memory> // std::allocator
-#include <new>    // std::hardware_destructive_interference_size
-#include <stdexcept>
-#include <type_traits> // std::enable_if, std::is_*_constructible
-
-#ifdef __has_cpp_attribute
-#if __has_cpp_attribute(nodiscard)
-#define RIGTORP_NODISCARD [[nodiscard]]
-#endif
-#endif
-#ifndef RIGTORP_NODISCARD
-#define RIGTORP_NODISCARD
-#endif
-
-namespace rigtorp {
-
-template <typename T, typename Allocator = std::allocator<T>> class SPSCQueue {
-
-#if defined(__cpp_if_constexpr) && defined(__cpp_lib_void_t)
-  template <typename Alloc2, typename = void>
-  struct has_allocate_at_least : std::false_type {};
-
-  template <typename Alloc2>
-  struct has_allocate_at_least<
-      Alloc2, std::void_t<typename Alloc2::value_type,
-                          decltype(std::declval<Alloc2 &>().allocate_at_least(
-                              size_t{}))>> : std::true_type {};
-#endif
-
-public:
-  explicit SPSCQueue(const size_t capacity,
-                     const Allocator &allocator = Allocator())
-      : capacity_(capacity), allocator_(allocator) {
-    // The queue needs at least one element
-    if (capacity_ < 1) {
-      capacity_ = 1;
-    }
-    capacity_++; // Needs one slack element
-    // Prevent overflowing size_t
-    if (capacity_ > SIZE_MAX - 2 * kPadding) {
-      capacity_ = SIZE_MAX - 2 * kPadding;
-    }
-
-#if defined(__cpp_if_constexpr) && defined(__cpp_lib_void_t)
-    if constexpr (has_allocate_at_least<Allocator>::value) {
-      auto res = allocator_.allocate_at_least(capacity_ + 2 * kPadding);
-      slots_ = res.ptr;
-      capacity_ = res.count - 2 * kPadding;
-    } else {
-      slots_ = std::allocator_traits<Allocator>::allocate(
-          allocator_, capacity_ + 2 * kPadding);
-    }
-#else
-    slots_ = std::allocator_traits<Allocator>::allocate(
-        allocator_, capacity_ + 2 * kPadding);
-#endif
-
-    static_assert(alignof(SPSCQueue<T>) == kCacheLineSize, "");
-    static_assert(sizeof(SPSCQueue<T>) >= 3 * kCacheLineSize, "");
-    assert(reinterpret_cast<char *>(&readIdx_) -
-               reinterpret_cast<char *>(&writeIdx_) >=
-           static_cast<std::ptrdiff_t>(kCacheLineSize));
-  }
-
-  ~SPSCQueue() {
-    while (front()) {
-      pop();
-    }
-    std::allocator_traits<Allocator>::deallocate(allocator_, slots_,
-                                                 capacity_ + 2 * kPadding);
-  }
-
-  // non-copyable and non-movable
-  SPSCQueue(const SPSCQueue &) = delete;
-  SPSCQueue &operator=(const SPSCQueue &) = delete;
-
-  template <typename... Args>
-  void emplace(Args &&...args) noexcept(
-      std::is_nothrow_constructible<T, Args &&...>::value) {
-    static_assert(std::is_constructible<T, Args &&...>::value,
-                  "T must be constructible with Args&&...");
-    auto const writeIdx = writeIdx_.load(std::memory_order_relaxed);
-    auto nextWriteIdx = writeIdx + 1;
-    if (nextWriteIdx == capacity_) {
-      nextWriteIdx = 0;
-    }
-    while (nextWriteIdx == readIdxCache_) {
-      readIdxCache_ = readIdx_.load(std::memory_order_acquire);
-    }
-    new (&slots_[writeIdx + kPadding]) T(std::forward<Args>(args)...);
-    writeIdx_.store(nextWriteIdx, std::memory_order_release);
-  }
-
-  template <typename... Args>
-  RIGTORP_NODISCARD bool try_emplace(Args &&...args) noexcept(
-      std::is_nothrow_constructible<T, Args &&...>::value) {
-    static_assert(std::is_constructible<T, Args &&...>::value,
-                  "T must be constructible with Args&&...");
-    auto const writeIdx = writeIdx_.load(std::memory_order_relaxed);
-    auto nextWriteIdx = writeIdx + 1;
-    if (nextWriteIdx == capacity_) {
-      nextWriteIdx = 0;
-    }
-    if (nextWriteIdx == readIdxCache_) {
-      readIdxCache_ = readIdx_.load(std::memory_order_acquire);
-      if (nextWriteIdx == readIdxCache_) {
-        return false;
-      }
-    }
-    new (&slots_[writeIdx + kPadding]) T(std::forward<Args>(args)...);
-    writeIdx_.store(nextWriteIdx, std::memory_order_release);
-    return true;
-  }
-
-  void push(const T &v) noexcept(std::is_nothrow_copy_constructible<T>::value) {
-    static_assert(std::is_copy_constructible<T>::value,
-                  "T must be copy constructible");
-    emplace(v);
-  }
-
-  template <typename P, typename = typename std::enable_if<
-                            std::is_constructible<T, P &&>::value>::type>
-  void push(P &&v) noexcept(std::is_nothrow_constructible<T, P &&>::value) {
-    emplace(std::forward<P>(v));
-  }
-
-  RIGTORP_NODISCARD bool
-  try_push(const T &v) noexcept(std::is_nothrow_copy_constructible<T>::value) {
-    static_assert(std::is_copy_constructible<T>::value,
-                  "T must be copy constructible");
-    return try_emplace(v);
-  }
-
-  template <typename P, typename = typename std::enable_if<
-                            std::is_constructible<T, P &&>::value>::type>
-  RIGTORP_NODISCARD bool
-  try_push(P &&v) noexcept(std::is_nothrow_constructible<T, P &&>::value) {
-    return try_emplace(std::forward<P>(v));
-  }
-
-  RIGTORP_NODISCARD T *front() noexcept {
-    auto const readIdx = readIdx_.load(std::memory_order_relaxed);
-    if (readIdx == writeIdxCache_) {
-      writeIdxCache_ = writeIdx_.load(std::memory_order_acquire);
-      if (writeIdxCache_ == readIdx) {
-        return nullptr;
-      }
-    }
-    return &slots_[readIdx + kPadding];
-  }
-
-  void pop() noexcept {
-    static_assert(std::is_nothrow_destructible<T>::value,
-                  "T must be nothrow destructible");
-    auto const readIdx = readIdx_.load(std::memory_order_relaxed);
-    assert(writeIdx_.load(std::memory_order_acquire) != readIdx &&
-           "Can only call pop() after front() has returned a non-nullptr");
-    slots_[readIdx + kPadding].~T();
-    auto nextReadIdx = readIdx + 1;
-    if (nextReadIdx == capacity_) {
-      nextReadIdx = 0;
-    }
-    readIdx_.store(nextReadIdx, std::memory_order_release);
-  }
-
-  RIGTORP_NODISCARD size_t size() const noexcept {
-    std::ptrdiff_t diff = writeIdx_.load(std::memory_order_acquire) -
-                          readIdx_.load(std::memory_order_acquire);
-    if (diff < 0) {
-      diff += capacity_;
-    }
-    return static_cast<size_t>(diff);
-  }
-
-  RIGTORP_NODISCARD bool empty() const noexcept {
-    return writeIdx_.load(std::memory_order_acquire) ==
-           readIdx_.load(std::memory_order_acquire);
-  }
-
-  RIGTORP_NODISCARD size_t capacity() const noexcept { return capacity_ - 1; }
-
-private:
-#ifdef __cpp_lib_hardware_interference_size
-  static constexpr size_t kCacheLineSize =
-      std::hardware_destructive_interference_size;
-#else
-  static constexpr size_t kCacheLineSize = 64;
-#endif
-
-  // Padding to avoid false sharing between slots_ and adjacent allocations
-  static constexpr size_t kPadding = (kCacheLineSize - 1) / sizeof(T) + 1;
-
-private:
-  size_t capacity_;
-  T *slots_;
-#if defined(__has_cpp_attribute) && __has_cpp_attribute(no_unique_address)
-  Allocator allocator_ [[no_unique_address]];
-#else
-  Allocator allocator_;
-#endif
-
-  // Align to cache line size in order to avoid false sharing
-  // readIdxCache_ and writeIdxCache_ is used to reduce the amount of cache
-  // coherency traffic
-  alignas(kCacheLineSize) std::atomic<size_t> writeIdx_ = {0};
-  alignas(kCacheLineSize) size_t readIdxCache_ = 0;
-  alignas(kCacheLineSize) std::atomic<size_t> readIdx_ = {0};
-  alignas(kCacheLineSize) size_t writeIdxCache_ = 0;
-};
-} // namespace rigtorp

flake.nix

@@ -81,7 +81,6 @@
 
         cmakeFlags = [
           "-DTOMATO_TOX_AV=ON"
-
           "-DTOMATO_ASAN=OFF"
           "-DCMAKE_BUILD_TYPE=RelWithDebInfo"
           #"-DCMAKE_BUILD_TYPE=Debug"

src/CMakeLists.txt

@@ -103,20 +103,24 @@ target_sources(tomato PUBLIC
 	./chat_gui4.hpp
 	./chat_gui4.cpp
 
-	./stream_manager.hpp
+	./frame_streams/frame_stream2.hpp
+	./frame_streams/audio_stream2.hpp
+	./frame_streams/stream_manager.hpp
+	./frame_streams/stream_manager.cpp
+	./frame_streams/locked_frame_stream.hpp
+	./frame_streams/multi_source.hpp
+
+	./frame_streams/voip_model.hpp
+
+	./frame_streams/sdl/sdl_audio2_frame_stream2.hpp
+	./frame_streams/sdl/sdl_audio2_frame_stream2.cpp
+	./frame_streams/sdl/video.hpp
+
 	./stream_manager_ui.hpp
 	./stream_manager_ui.cpp
 
 	./debug_video_tap.hpp
 	./debug_video_tap.cpp
-
-	./content/content.hpp
-	./content/frame_stream2.hpp
-	./content/sdl_video_frame_stream2.hpp
-	./content/sdl_video_frame_stream2.cpp
-	./content/audio_stream.hpp
-	./content/sdl_audio_frame_stream2.hpp
-	./content/sdl_audio_frame_stream2.cpp
 )
 
 if (TOMATO_TOX_AV)
@@ -124,8 +128,8 @@ if (TOMATO_TOX_AV)
 		./tox_av.hpp
 		./tox_av.cpp
 
-		./debug_tox_call.hpp
+		./tox_av_voip_model.hpp
-		./debug_tox_call.cpp
+		./tox_av_voip_model.cpp
 	)
 
 	target_compile_definitions(tomato PUBLIC TOMATO_TOX_AV)
@@ -158,8 +162,6 @@ target_link_libraries(tomato PUBLIC
 	libwebpmux # the f why (needed for anim encode)
 	qoi
 	SDL3_image::SDL3_image
-
-	SPSCQueue
 )
 
 # probably not enough
@@ -167,3 +169,18 @@ target_link_libraries(tomato PUBLIC
 
 set_target_properties(tomato PROPERTIES POSITION_INDEPENDENT_CODE ON)
 
+########################################
+
+add_executable(test_frame_stream2_pop_reframer EXCLUDE_FROM_ALL
+	./frame_streams/frame_stream2.hpp
+	./frame_streams/audio_stream2.hpp
+	./frame_streams/locked_frame_stream.hpp
+	./frame_streams/multi_source.hpp
+
+	./frame_streams/test_pop_reframer.cpp
+)
+
+target_link_libraries(test_frame_stream2_pop_reframer
+	solanaceae_util
+)
+

src/chat_gui/file_selector.cpp

@@ -111,6 +111,7 @@ void FileSelector::render(void) {
 				}
 			}
 
+			try {
 				// do sorting here
 				// TODO: cache the result (lol)
 				if (ImGuiTableSortSpecs* sorts_specs = ImGui::TableGetSortSpecs(); sorts_specs != nullptr && sorts_specs->SpecsCount >= 1) {
@@ -162,6 +163,9 @@ void FileSelector::render(void) {
 						break; default: ;
 					}
 				}
+			} catch (...) {
+				// we likely saw a file disapear
+			}
 
 			for (auto const& dir_entry : dirs) {
 				if (ImGui::TableNextColumn()) {

src/chat_gui4.cpp

@@ -9,9 +9,13 @@
 #include <solanaceae/contact/components.hpp>
 #include <solanaceae/util/utils.hpp>
 
+#include "./frame_streams/voip_model.hpp"
+
 // HACK: remove them
 #include <solanaceae/tox_contacts/components.hpp>
 
+#include <entt/entity/entity.hpp>
+
 #include <imgui/imgui.h>
 #include <imgui/misc/cpp/imgui_stdlib.h>
 
@@ -30,6 +34,7 @@
 #include <fstream>
 #include <iomanip>
 #include <sstream>
+#include <string>
 #include <variant>
 
 namespace Components {
@@ -257,6 +262,97 @@ float ChatGui4::render(float time_delta) {
 
 			if (ImGui::BeginChild(chat_label.c_str(), {0, 0}, ImGuiChildFlags_Border, ImGuiWindowFlags_MenuBar)) {
 				if (ImGui::BeginMenuBar()) {
+					// check if contact has voip model
+					// use activesessioncomp instead?
+					if (_cr.all_of<VoIPModelI*>(*_selected_contact)) {
+						if (ImGui::BeginMenu("VoIP")) {
+							auto* voip_model = _cr.get<VoIPModelI*>(*_selected_contact);
+
+							std::vector<ObjectHandle> contact_sessions;
+							std::vector<ObjectHandle> acceptable_sessions;
+							for (const auto& [ov, o_vm, sc] : _os.registry().view<VoIPModelI*, Components::VoIP::SessionContact>().each()) {
+								if (o_vm != voip_model) {
+									continue;
+								}
+								if (sc.c != *_selected_contact) {
+									continue;
+								}
+
+								auto o = _os.objectHandle(ov);
+								contact_sessions.push_back(o);
+
+								if (!o.all_of<Components::VoIP::Incoming>()) {
+									continue; // not incoming
+								}
+
+								// state is ringing/not yet accepted
+								const auto* session_state = o.try_get<Components::VoIP::SessionState>();
+								if (session_state == nullptr) {
+									continue;
+								}
+
+								if (session_state->state != Components::VoIP::SessionState::State::RINGING) {
+									continue;
+								}
+								acceptable_sessions.push_back(o);
+							}
+
+							static Components::VoIP::DefaultConfig g_default_connections{};
+
+							if (ImGui::BeginMenu("default connections")) {
+								ImGui::MenuItem("incoming audio", nullptr, &g_default_connections.incoming_audio);
+								ImGui::MenuItem("incoming video", nullptr, &g_default_connections.incoming_video);
+								ImGui::Separator();
+								ImGui::MenuItem("outgoing audio", nullptr, &g_default_connections.outgoing_audio);
+								ImGui::MenuItem("outgoing video", nullptr, &g_default_connections.outgoing_video);
+								ImGui::EndMenu();
+							}
+
+							if (acceptable_sessions.size() < 2) {
+								if (ImGui::MenuItem("accept call", nullptr, false, !acceptable_sessions.empty())) {
+									voip_model->accept(acceptable_sessions.front(), g_default_connections);
+								}
+							} else {
+								if (ImGui::BeginMenu("accept call", !acceptable_sessions.empty())) {
+									for (const auto o : acceptable_sessions) {
+										std::string label = "accept #";
+										label += std::to_string(entt::to_integral(entt::to_entity(o.entity())));
+
+										if (ImGui::MenuItem(label.c_str())) {
+											voip_model->accept(o, g_default_connections);
+										}
+									}
+									ImGui::EndMenu();
+								}
+							}
+
+							// TODO: disable if already in call?
+							if (ImGui::Button(" call ")) {
+								voip_model->enter(*_selected_contact, g_default_connections);
+							}
+
+							if (contact_sessions.size() < 2) {
+								if (ImGui::MenuItem("leave/reject call", nullptr, false, !contact_sessions.empty())) {
+									voip_model->leave(contact_sessions.front());
+								}
+							} else {
+								if (ImGui::BeginMenu("leave/reject call")) {
+									// list
+									for (const auto o : contact_sessions) {
+										std::string label = "end #";
+										label += std::to_string(entt::to_integral(entt::to_entity(o.entity())));
+
+										if (ImGui::MenuItem(label.c_str())) {
+											voip_model->leave(o);
+										}
+									}
+									ImGui::EndMenu();
+								}
+							}
+
+							ImGui::EndMenu();
+						}
+					}
 					if (ImGui::BeginMenu("debug")) {
 						ImGui::Checkbox("show extra info", &_show_chat_extra_info);
 						ImGui::Checkbox("show avatar transfers", &_show_chat_avatar_tf);

src/content/audio_stream.hpp

@@ -1,72 +0,0 @@
-#pragma once
-
-#include "./frame_stream2.hpp"
-
-#include <solanaceae/util/span.hpp>
-
-#include <cstdint>
-#include <variant>
-#include <vector>
-
-// raw audio
-// channels make samples interleaved,
-// planar channels are not supported
-struct AudioFrame {
-	// sequence number, to detect gaps
-	uint32_t seq {0};
-	// TODO: maybe use ts instead to discard old?
-	// since buffer size is variable, some timestamp would be needed to estimate the lost time
-
-	// samples per second
-	uint32_t sample_rate {48'000};
-
-	size_t channels {0};
-	std::variant<
-		std::vector<int16_t>, // S16, platform endianess
-		Span<int16_t>, // non owning variant, for direct consumption
-
-		std::vector<float>, // f32
-		Span<float> // non owning variant, for direct consumption
-	> buffer;
-
-	// helpers
-
-	bool isS16(void) const {
-		return
-			std::holds_alternative<std::vector<int16_t>>(buffer) ||
-			std::holds_alternative<Span<int16_t>>(buffer)
-		;
-	}
-	bool isF32(void) const {
-		return
-			std::holds_alternative<std::vector<float>>(buffer) ||
-			std::holds_alternative<Span<float>>(buffer)
-		;
-	}
-	template<typename T>
-	Span<T> getSpan(void) const {
-		static_assert(std::is_same_v<int16_t, T> || std::is_same_v<float, T>);
-		if constexpr (std::is_same_v<int16_t, T>) {
-			assert(isS16());
-			if (std::holds_alternative<std::vector<int16_t>>(buffer)) {
-				return Span<int16_t>{std::get<std::vector<int16_t>>(buffer)};
-			} else {
-				return std::get<Span<int16_t>>(buffer);
-			}
-		} else if constexpr (std::is_same_v<float, T>) {
-			assert(isF32());
-			if (std::holds_alternative<std::vector<float>>(buffer)) {
-				return Span<float>{std::get<std::vector<float>>(buffer)};
-			} else {
-				return std::get<Span<float>>(buffer);
-			}
-		}
-		return {};
-	}
-};
-
-using AudioFrameStream2I = FrameStream2I<AudioFrame>;
-
-using AudioFrameStream2MultiSource = FrameStream2MultiSource<AudioFrame>;
-using AudioFrameStream2 = AudioFrameStream2MultiSource::sub_stream_type_t; // just use the default for now
-

src/content/content.hpp

@@ -1,36 +0,0 @@
-#pragma once
-
-#include <entt/container/dense_set.hpp>
-
-#include <solanaceae/object_store/object_store.hpp>
-#include <solanaceae/contact/contact_model3.hpp>
-#include <solanaceae/message3/registry_message_model.hpp>
-
-#include <solanaceae/file/file2.hpp>
-
-namespace Content1::Components {
-
-	// TODO: design it as a tree?
-
-	// or something
-	struct TagFile {};
-	struct TagAudioStream {};
-	struct TagVideoStream {};
-
-	struct TimingTiedTo {
-		entt::dense_set<ObjectHandle> ties;
-	};
-
-	// the associated messages, if any
-	// useful if you want to update progress on the message
-	struct Messages {
-		std::vector<Message3Handle> messages;
-	};
-
-	// ?
-	struct SuspectedParticipants {
-		entt::dense_set<Contact3> participants;
-	};
-
-} // Content::Components
-

src/content/frame_stream2.hpp

@@ -1,153 +0,0 @@
-#pragma once
-
-#include <cstdint>
-#include <memory>
-#include <optional>
-#include <vector>
-#include <mutex>
-
-#include <SPSCQueue.h>
-
-// Frames ofen consist of:
-// - seq id // incremental sequential id, gaps in ids can be used to detect loss
-// - data // the frame data
-// eg:
-//struct ExampleFrame {
-	//int64_t seq_id {0};
-	//std::vector<uint8_t> data;
-//};
-
-template<typename FrameType>
-struct FrameStream2I {
-	virtual ~FrameStream2I(void) {}
-
-	// get number of available frames
-	[[nodiscard]] virtual int32_t size(void) = 0;
-
-	// get next frame
-	// TODO: optional instead?
-	// data sharing? -> no, data is copied for each fsr, if concurency supported
-	[[nodiscard]] virtual std::optional<FrameType> pop(void) = 0;
-
-	// returns true if there are readers (or we dont know)
-	virtual bool push(const FrameType& value) = 0;
-};
-
-template<typename FrameType>
-struct FrameStream2SourceI {
-	virtual ~FrameStream2SourceI(void) {}
-	[[nodiscard]] virtual std::shared_ptr<FrameStream2I<FrameType>> subscribe(void) = 0;
-	virtual bool unsubscribe(const std::shared_ptr<FrameStream2I<FrameType>>& sub) = 0;
-};
-
-template<typename FrameType>
-struct FrameStream2SinkI {
-	virtual ~FrameStream2SinkI(void) {}
-	[[nodiscard]] virtual std::shared_ptr<FrameStream2I<FrameType>> subscribe(void) = 0;
-	virtual bool unsubscribe(const std::shared_ptr<FrameStream2I<FrameType>>& sub) = 0;
-};
-
-// needs count frames queue size
-// having ~1-2sec buffer size is often sufficent
-template<typename FrameType>
-struct QueuedFrameStream2 : public FrameStream2I<FrameType> {
-	using frame_type = FrameType;
-
-	rigtorp::SPSCQueue<FrameType> _queue;
-
-	// discard values if queue full
-	// will block if not lossy and full on push
-	const bool _lossy {true};
-
-	explicit QueuedFrameStream2(size_t queue_size, bool lossy = true) : _queue(queue_size), _lossy(lossy) {}
-
-	int32_t size(void) override {
-		return _queue.size();
-	}
-
-	std::optional<FrameType> pop(void) override {
-		auto* ret_ptr = _queue.front();
-		if (ret_ptr == nullptr) {
-			return std::nullopt;
-		}
-
-		// move away
-		FrameType ret = std::move(*ret_ptr);
-
-		_queue.pop();
-
-		return ret;
-	}
-
-	bool push(const FrameType& value) override {
-		if (_lossy) {
-			[[maybe_unused]] auto _ = _queue.try_emplace(value);
-			// TODO: maybe return ?
-		} else {
-			_queue.push(value);
-		}
-		return true;
-	}
-};
-
-// implements a stream that pushes to all sub streams
-// release all streams before destructing! // TODO: improve lifetime here, maybe some shared semaphore?
-template<typename FrameType, typename SubStreamType = QueuedFrameStream2<FrameType>>
-struct FrameStream2MultiSource : public FrameStream2SourceI<FrameType>, public FrameStream2I<FrameType> {
-	using sub_stream_type_t = SubStreamType;
-
-	// pointer stability
-	std::vector<std::shared_ptr<SubStreamType>> _sub_streams;
-	std::mutex _sub_stream_lock; // accessing the _sub_streams array needs to be exclusive
-	// a simple lock here is ok, since this tends to be a rare operation,
-	// except for the push, which is always on the same thread
-
-	virtual ~FrameStream2MultiSource(void) {}
-
-	// TODO: forward args instead
-	std::shared_ptr<FrameStream2I<FrameType>> subscribe(void) override {
-		// TODO: args???
-		size_t queue_size = 8;
-		bool lossy = true;
-
-		std::lock_guard lg{_sub_stream_lock};
-		return _sub_streams.emplace_back(std::make_unique<SubStreamType>(queue_size, lossy));
-	}
-
-	bool unsubscribe(const std::shared_ptr<FrameStream2I<FrameType>>& sub) override {
-		std::lock_guard lg{_sub_stream_lock};
-		for (auto it = _sub_streams.begin(); it != _sub_streams.end(); it++) {
-			if (*it == sub) {
-				_sub_streams.erase(it);
-				return true;
-			}
-		}
-		return false; // ?
-	}
-
-	// stream interface
-
-	int32_t size(void) override {
-		// TODO: return something sensible?
-		return -1;
-	}
-
-	std::optional<FrameType> pop(void) override {
-		// nope
-		assert(false && "this logic is very frame type specific, provide an impl");
-		return std::nullopt;
-	}
-
-	// returns true if there are readers
-	bool push(const FrameType& value) override {
-		std::lock_guard lg{_sub_stream_lock};
-		bool have_readers{false};
-		for (auto& it : _sub_streams) {
-			[[maybe_unused]] auto _ = it->push(value);
-			have_readers = true; // even if queue full, we still continue believing in them
-			// maybe consider push return value?
-		}
-		return have_readers;
-	}
-};
-

src/content/sdl_video_frame_stream2.cpp

@@ -1,167 +0,0 @@
-#include "./sdl_video_frame_stream2.hpp"
-#include "SDL3/SDL_camera.h"
-#include "SDL3/SDL_pixels.h"
-
-#include <chrono>
-#include <cstdint>
-#include <iostream>
-#include <memory>
-#include <thread>
-
-// TODO: move out and create lazy cam for each device
-SDLVideoCameraContent::SDLVideoCameraContent(void) {
-	int devcount {0};
-	//SDL_CameraDeviceID *devices = SDL_GetCameraDevices(&devcount);
-	SDL_CameraID *devices = SDL_GetCameras(&devcount);
-	std::cout << "SDL Camera Driver: " << SDL_GetCurrentCameraDriver() << "\n";
-
-	if (devices == nullptr || devcount < 1) {
-		throw int(1); // TODO: proper exception?
-	}
-
-	std::cout << "### found cameras:\n";
-	for (int i = 0; i < devcount; i++) {
-		const SDL_CameraID device = devices[i];
-
-		const char *name = SDL_GetCameraName(device);
-		std::cout << "  - Camera #" << i << ": " << name << "\n";
-
-		int speccount {0};
-		SDL_CameraSpec** specs = SDL_GetCameraSupportedFormats(device, &speccount);
-		if (specs == nullptr) {
-			std::cout << "    - no supported spec\n";
-		} else {
-			for (int spec_i = 0; spec_i < speccount; spec_i++) {
-				std::cout << "    - " << specs[spec_i]->width << "x" << specs[spec_i]->height << "@" << float(specs[spec_i]->framerate_numerator)/specs[spec_i]->framerate_denominator << "fps " << SDL_GetPixelFormatName(specs[spec_i]->format) << "\n";
-
-			}
-			SDL_free(specs);
-		}
-	}
-
-	{
-		SDL_CameraSpec spec {
-			// FORCE a diffrent pixel format
-			//SDL_PIXELFORMAT_RGBA8888,
-			//SDL_PIXELFORMAT_UNKNOWN,
-			//SDL_PIXELFORMAT_IYUV,
-			SDL_PIXELFORMAT_YUY2,
-
-			//SDL_COLORSPACE_UNKNOWN,
-			//SDL_COLORSPACE_SRGB,
-			//SDL_COLORSPACE_SRGB_LINEAR,
-			SDL_COLORSPACE_YUV_DEFAULT,
-
-			//1280, 720,
-			//640, 360,
-			//640, 480,
-			696, 392,
-
-			//1, 30
-			30, 1
-		};
-		_camera = {
-			//SDL_OpenCamera(devices[devcount-1], &spec),
-			SDL_OpenCamera(devices[0], nullptr),
-			//SDL_OpenCamera(devices[0], &spec),
-			&SDL_CloseCamera
-		};
-		SDL_GetCameraFormat(_camera.get(), &spec);
-	}
-	SDL_free(devices);
-	if (!static_cast<bool>(_camera)) {
-		throw int(2);
-	}
-
-	while (SDL_GetCameraPermissionState(_camera.get()) == 0) {
-		std::cerr << "permission for camera not granted\n";
-		std::this_thread::sleep_for(std::chrono::milliseconds(10));
-	}
-
-	if (SDL_GetCameraPermissionState(_camera.get()) < 0) {
-		std::cerr << "user denied camera permission\n";
-		throw int(3);
-	}
-
-	SDL_CameraSpec spec;
-	float fps {1.f};
-	if (!SDL_GetCameraFormat(_camera.get(), &spec)) {
-		// meh
-		throw int(5);
-	} else {
-		fps = float(spec.framerate_numerator)/float(spec.framerate_denominator);
-		std::cout << "camera fps: " << fps << "fps (" << spec.framerate_numerator << "/" << spec.framerate_denominator << ")\n";
-		auto* format_name = SDL_GetPixelFormatName(spec.format);
-		std::cout << "camera format: " << format_name << "\n";
-	}
-
-	_thread = std::thread([this, fps](void) {
-		while (!_thread_should_quit) {
-			Uint64 timestampNS = 0;
-			SDL_Surface* sdl_frame_next = SDL_AcquireCameraFrame(_camera.get(), &timestampNS);
-
-			// no new frame yet, or error
-			if (sdl_frame_next == nullptr) {
-				// only sleep 1/10, we expected a frame
-				std::this_thread::sleep_for(std::chrono::milliseconds(int64_t((1000/fps) / 10)));
-				continue;
-			}
-
-#if 0
-			{ // test copy to trigger bug
-				SDL_Surface* test_surf = SDL_CreateSurface(
-					sdl_frame_next->w,
-					sdl_frame_next->h,
-					SDL_PIXELFORMAT_RGBA8888
-				);
-
-				assert(test_surf != nullptr);
-
-				SDL_BlitSurface(sdl_frame_next, nullptr, test_surf, nullptr);
-
-				SDL_DestroySurface(test_surf);
-			}
-#endif
-
-			bool someone_listening {false};
-			{
-				SDLVideoFrame new_frame_non_owning {
-					timestampNS/1000,
-					sdl_frame_next
-				};
-
-				// creates surface copies
-				someone_listening = push(new_frame_non_owning);
-			}
-			SDL_ReleaseCameraFrame(_camera.get(), sdl_frame_next);
-
-			if (someone_listening) {
-				// double the interval on acquire
-				std::this_thread::sleep_for(std::chrono::milliseconds(int64_t((1000/fps)*0.5)));
-			} else {
-				std::cerr << "i guess no one is listening\n";
-				// we just sleep 4x as long, bc no one is listening
-				std::this_thread::sleep_for(std::chrono::milliseconds(int64_t((1000/fps)*4)));
-			}
-		}
-	});
-}
-
-SDLVideoCameraContent::~SDLVideoCameraContent(void) {
-	_thread_should_quit = true;
-	_thread.join();
-	// TODO: what to do if readers are still present?
-
-	// HACK: sdl is buggy and freezes otherwise. it is likely still possible, but rare to freeze here
-	// flush unused frames
-#if 1
-	while (true) {
-		SDL_Surface* sdl_frame_next = SDL_AcquireCameraFrame(_camera.get(), nullptr);
-		if (sdl_frame_next != nullptr) {
-			SDL_ReleaseCameraFrame(_camera.get(), sdl_frame_next);
-		} else {
-			break;
-		}
-	}
-#endif
-}

src/content/sdl_video_frame_stream2.hpp

@@ -1,71 +0,0 @@
-#pragma once
-
-#include "./frame_stream2.hpp"
-
-#include <SDL3/SDL.h>
-
-#include <cstdint>
-#include <thread>
-
-inline void nopSurfaceDestructor(SDL_Surface*) {}
-
-// this is very sdl specific
-struct SDLVideoFrame {
-	// TODO: sequence numbering?
-	// micro seconds (nano is way too much)
-	uint64_t timestampUS {0};
-
-	std::unique_ptr<SDL_Surface, decltype(&SDL_DestroySurface)> surface {nullptr, &SDL_DestroySurface};
-
-	// special non-owning constructor?
-	SDLVideoFrame(
-		uint64_t ts,
-		SDL_Surface* surf
-	) {
-		timestampUS = ts;
-		surface = {surf, &nopSurfaceDestructor};
-	}
-	SDLVideoFrame(SDLVideoFrame&& other) = default;
-	// copy
-	SDLVideoFrame(const SDLVideoFrame& other) {
-		timestampUS = other.timestampUS;
-		if (static_cast<bool>(other.surface)) {
-			//surface = {
-			//    SDL_CreateSurface(
-			//        other.surface->w,
-			//        other.surface->h,
-			//        other.surface->format
-			//    ),
-			//    &SDL_DestroySurface
-			//};
-			//SDL_BlitSurface(other.surface.get(), nullptr, surface.get(), nullptr);
-			surface = {
-				SDL_DuplicateSurface(other.surface.get()),
-				&SDL_DestroySurface
-			};
-		}
-	}
-	SDLVideoFrame& operator=(const SDLVideoFrame& other) = delete;
-};
-
-using SDLVideoFrameStream2MultiSource = FrameStream2MultiSource<SDLVideoFrame>;
-using SDLVideoFrameStream2 = SDLVideoFrameStream2MultiSource::sub_stream_type_t; // just use the default for now
-
-struct SDLVideoCameraContent : public SDLVideoFrameStream2MultiSource {
-	// meh, empty default
-	std::unique_ptr<SDL_Camera, decltype(&SDL_CloseCamera)> _camera {nullptr, &SDL_CloseCamera};
-	std::atomic<bool> _thread_should_quit {false};
-	std::thread _thread;
-
-	// construct source and start thread
-	// TODO: optimize so the thread is not always running
-	SDLVideoCameraContent(void);
-
-	// stops the thread and closes the camera
-	~SDLVideoCameraContent(void);
-
-	// make only some of writer public
-	using SDLVideoFrameStream2MultiSource::subscribe;
-	using SDLVideoFrameStream2MultiSource::unsubscribe;
-};
-

src/content/stream_reader.hpp

@@ -1,15 +0,0 @@
-#pragma once
-
-#include <solanaceae/util/span.hpp>
-
-// most media that can be counted as "stream" comes in packets/frames/messages
-// so this class provides an interface for ideal async fetching of frames
-struct RawFrameStreamReaderI {
-	// return the number of ready frames in cache
-	// returns -1 if unknown
-	virtual int64_t have(void) = 0;
-
-	// get next frame, empty if none
-	virtual ByteSpan getNext(void) = 0;
-};
-

src/content/stream_reader_sdl_audio.cpp

@@ -1,39 +0,0 @@
-#include "./stream_reader_sdl_audio.hpp"
-
-SDLAudioFrameStreamReader::SDLAudioFrameStreamReader(int32_t buffer_size) : _buffer_size(buffer_size),  _stream{nullptr, &SDL_DestroyAudioStream} {
-	_buffer.resize(_buffer_size);
-
-	const SDL_AudioSpec spec = { SDL_AUDIO_S16, 1, 48000 };
-
-	_stream = {
-		SDL_OpenAudioDeviceStream(SDL_AUDIO_DEVICE_DEFAULT_CAPTURE, &spec, nullptr, nullptr),
-		&SDL_DestroyAudioStream
-	};
-}
-
-Span<int16_t> SDLAudioFrameStreamReader::getNextAudioFrame(void) {
-	const int32_t needed_bytes = (_buffer.size() - _remaining_size) * sizeof(int16_t);
-	const auto read_bytes = SDL_GetAudioStreamData(_stream.get(), _buffer.data()+_remaining_size, needed_bytes);
-	if (read_bytes < 0) {
-		// error
-		return {};
-	}
-	if (read_bytes < needed_bytes) {
-		// HACK: we are just assuming here that sdl never gives us half a sample!
-		_remaining_size += read_bytes / sizeof(int16_t);
-		return {};
-	}
-
-	_remaining_size = 0;
-	return Span<int16_t>{_buffer};
-}
-
-int64_t SDLAudioFrameStreamReader::have(void) {
-	return -1;
-}
-
-ByteSpan SDLAudioFrameStreamReader::getNext(void) {
-	auto next_frame_span = getNextAudioFrame();
-	return ByteSpan{reinterpret_cast<const uint8_t*>(next_frame_span.ptr), next_frame_span.size};
-}
-

src/content/stream_reader_sdl_audio.hpp

@@ -1,31 +0,0 @@
-#pragma once
-
-#include "./stream_reader.hpp"
-
-#include <SDL3/SDL.h>
-
-#include <cstdint>
-#include <cstdio>
-#include <memory>
-
-struct SDLAudioFrameStreamReader : public RawFrameStreamReaderI {
-	// count samples per buffer
-	const int32_t _buffer_size {1024};
-	std::vector<int16_t> _buffer;
-	size_t _remaining_size {0}; // data still in buffer, that was remaining from last call and not enough to fill a full frame
-
-	// meh, empty default
-	std::unique_ptr<SDL_AudioStream, decltype(&SDL_DestroyAudioStream)> _stream;
-
-	// buffer_size in number of samples
-	SDLAudioFrameStreamReader(int32_t buffer_size = 1024);
-
-	// data owned by StreamReader, overwritten by next call to getNext*()
-	Span<int16_t> getNextAudioFrame(void);
-
-	public: // interface
-		int64_t have(void) override;
-
-		ByteSpan getNext(void) override;
-};
-

src/content/stream_reader_sdl_video.cpp

@@ -1,84 +0,0 @@
-#include "./stream_reader_sdl_video.hpp"
-
-#include <iostream>
-
-SDLVideoFrameStreamReader::SDLVideoFrameStreamReader() : _camera{nullptr, &SDL_CloseCamera}, _surface{nullptr, &SDL_DestroySurface} {
-	// enumerate
-	int devcount = 0;
-	SDL_CameraDeviceID *devices = SDL_GetCameraDevices(&devcount);
-
-	if (devices == nullptr || devcount < 1) {
-		throw int(1); // TODO: proper exception?
-	}
-
-	std::cout << "### found cameras:\n";
-	for (int i = 0; i < devcount; i++) {
-		const SDL_CameraDeviceID device = devices[i];
-		char *name = SDL_GetCameraDeviceName(device);
-
-		std::cout << "  - Camera #" << i << ": " << name << "\n";
-
-		SDL_free(name);
-	}
-
-	_camera = {
-		SDL_OpenCameraDevice(devices[0], nullptr),
-		&SDL_CloseCamera
-	};
-	if (!static_cast<bool>(_camera)) {
-		throw int(2);
-	}
-
-	SDL_CameraSpec spec;
-	if (SDL_GetCameraFormat(_camera.get(), &spec) < 0) {
-		// meh
-	} else {
-		// interval
-		float interval = float(spec.interval_numerator)/float(spec.interval_denominator);
-		std::cout << "camera interval: " << interval*1000 << "ms\n";
-	}
-}
-
-SDLVideoFrameStreamReader::VideoFrame SDLVideoFrameStreamReader::getNextVideoFrameRGBA(void) {
-	if (!static_cast<bool>(_camera)) {
-		return {};
-	}
-
-	Uint64 timestampNS = 0;
-	SDL_Surface* frame_next = SDL_AcquireCameraFrame(_camera.get(), &timestampNS);
-
-	// no new frame yet, or error
-	if (frame_next == nullptr) {
-		//std::cout << "failed acquiring frame\n";
-		return {};
-	}
-
-	// TODO: investigate zero copy
-	_surface = {
-		SDL_ConvertSurfaceFormat(frame_next, SDL_PIXELFORMAT_RGBA8888),
-		&SDL_DestroySurface
-	};
-
-	SDL_ReleaseCameraFrame(_camera.get(), frame_next);
-
-	SDL_LockSurface(_surface.get());
-
-	return {
-		_surface->w,
-		_surface->h,
-		timestampNS,
-		{
-			reinterpret_cast<const uint8_t*>(_surface->pixels),
-			uint64_t(_surface->w*_surface->h*4) // rgba
-		}
-	};
-}
-
-int64_t SDLVideoFrameStreamReader::have(void) {
-	return -1;
-}
-
-ByteSpan SDLVideoFrameStreamReader::getNext(void) {
-	return {};
-}
-

src/content/stream_reader_sdl_video.hpp

@@ -1,34 +0,0 @@
-#pragma once
-
-#include "./stream_reader.hpp"
-
-#include <SDL3/SDL.h>
-
-#include <cstdint>
-#include <cstdio>
-#include <memory>
-
-struct SDLVideoFrameStreamReader : public RawFrameStreamReaderI {
-	// meh, empty default
-	std::unique_ptr<SDL_Camera, decltype(&SDL_CloseCamera)> _camera;
-	std::unique_ptr<SDL_Surface, decltype(&SDL_DestroySurface)> _surface;
-
-	SDLVideoFrameStreamReader(void);
-
-	struct VideoFrame {
-		int32_t width {0};
-		int32_t height {0};
-
-		uint64_t timestampNS {0};
-
-		ByteSpan data;
-	};
-
-	// data owned by StreamReader, overwritten by next call to getNext*()
-	VideoFrame getNextVideoFrameRGBA(void);
-
-	public: // interface
-		int64_t have(void) override;
-		ByteSpan getNext(void) override;
-};
-

src/debug_tox_call.cpp

@@ -1,718 +0,0 @@
-#include "./debug_tox_call.hpp"
-
-#include "./stream_manager.hpp"
-#include "./content/audio_stream.hpp"
-#include "./content/sdl_video_frame_stream2.hpp"
-
-#include <SDL3/SDL.h>
-
-#include <imgui/imgui.h>
-
-#include <cstring>
-#include <cstdint>
-
-#include <iostream>
-#include <memory>
-#include <optional>
-
-// fwd
-namespace Message {
-	uint64_t getTimeMS();
-}
-
-static bool isFormatPlanar(SDL_PixelFormat f) {
-	return
-		f == SDL_PIXELFORMAT_YV12 ||
-		f == SDL_PIXELFORMAT_IYUV ||
-		f == SDL_PIXELFORMAT_YUY2 ||
-		f == SDL_PIXELFORMAT_UYVY ||
-		f == SDL_PIXELFORMAT_YVYU ||
-		f == SDL_PIXELFORMAT_NV12 ||
-		f == SDL_PIXELFORMAT_NV21 ||
-		f == SDL_PIXELFORMAT_P010
-	;
-}
-
-static SDL_Surface* convertYUY2_IYUV(SDL_Surface* surf) {
-	if (surf->format != SDL_PIXELFORMAT_YUY2) {
-		return nullptr;
-	}
-	if ((surf->w % 2) != 0) {
-		SDL_SetError("YUY2->IYUV does not support odd widths");
-		// hmmm, we dont handle odd widths
-		return nullptr;
-	}
-
-	SDL_LockSurface(surf);
-
-	SDL_Surface* conv_surf = SDL_CreateSurface(surf->w, surf->h, SDL_PIXELFORMAT_IYUV);
-	SDL_LockSurface(conv_surf);
-
-	// YUY2 is 4:2:2 packed
-	// Y is simple, we just copy it over
-	// U V are double the resolution (vertically), so we avg both
-	// Packed mode: Y0+U0+Y1+V0 (1 plane)
-
-	uint8_t* y_plane = static_cast<uint8_t*>(conv_surf->pixels);
-	uint8_t* u_plane = static_cast<uint8_t*>(conv_surf->pixels) + conv_surf->w*conv_surf->h;
-	uint8_t* v_plane = static_cast<uint8_t*>(conv_surf->pixels) + conv_surf->w*conv_surf->h + (conv_surf->w/2)*(conv_surf->h/2);
-
-	const uint8_t* yuy2_data = static_cast<const uint8_t*>(surf->pixels);
-
-	for (int y = 0; y < surf->h; y++) {
-		for (int x = 0; x < surf->w; x += 2) {
-			// every pixel uses 2 bytes
-			const uint8_t* yuy2_curser = yuy2_data + y*surf->w*2 + x*2;
-			uint8_t src_y0 = yuy2_curser[0];
-			uint8_t src_u = yuy2_curser[1];
-			uint8_t src_y1 = yuy2_curser[2];
-			uint8_t src_v = yuy2_curser[3];
-
-			y_plane[y*conv_surf->w + x] = src_y0;
-			y_plane[y*conv_surf->w + x+1] = src_y1;
-
-			size_t uv_index = (y/2) * (conv_surf->w/2) + x/2;
-			if (y % 2 == 0) {
-				// first write
-				u_plane[uv_index] = src_u;
-				v_plane[uv_index] = src_v;
-			} else {
-				// second write, mix with existing value
-				u_plane[uv_index] = (int(u_plane[uv_index]) + int(src_u)) / 2;
-				v_plane[uv_index] = (int(v_plane[uv_index]) + int(src_v)) / 2;
-			}
-		}
-	}
-
-	SDL_UnlockSurface(conv_surf);
-	SDL_UnlockSurface(surf);
-	return conv_surf;
-}
-
-struct PushConversionQueuedVideoStream : public QueuedFrameStream2<SDLVideoFrame> {
-	SDL_PixelFormat _forced_format {SDL_PIXELFORMAT_IYUV};
-
-	PushConversionQueuedVideoStream(size_t queue_size, bool lossy = true) : QueuedFrameStream2<SDLVideoFrame>(queue_size, lossy) {}
-	~PushConversionQueuedVideoStream(void) {}
-
-	bool push(const SDLVideoFrame& value) override {
-		SDL_Surface* converted_surf = value.surface.get();
-		if (converted_surf->format != _forced_format) {
-			//std::cerr << "DTC: need to convert from " << SDL_GetPixelFormatName(converted_surf->format) << " to SDL_PIXELFORMAT_IYUV\n";
-			if (converted_surf->format == SDL_PIXELFORMAT_YUY2 && _forced_format == SDL_PIXELFORMAT_IYUV) {
-				// optimized custom impl
-
-				//auto start = Message::getTimeMS();
-				converted_surf = convertYUY2_IYUV(converted_surf);
-				//auto end = Message::getTimeMS();
-				// 3ms
-				//std::cerr << "DTC: timing " << SDL_GetPixelFormatName(converted_surf->format) << "->SDL_PIXELFORMAT_IYUV: " << end-start << "ms\n";
-			} else if (isFormatPlanar(converted_surf->format)) {
-				// meh, need to convert to rgb as a stopgap
-
-				//auto start = Message::getTimeMS();
-				//SDL_Surface* tmp_conv_surf = SDL_ConvertSurfaceAndColorspace(converted_surf, SDL_PIXELFORMAT_RGBA32, nullptr, SDL_COLORSPACE_RGB_DEFAULT, 0);
-				SDL_Surface* tmp_conv_surf = SDL_ConvertSurfaceAndColorspace(converted_surf, SDL_PIXELFORMAT_RGB24, nullptr, SDL_COLORSPACE_RGB_DEFAULT, 0);
-				//auto end = Message::getTimeMS();
-				// 1ms
-				//std::cerr << "DTC: timing " << SDL_GetPixelFormatName(converted_surf->format) << "->SDL_PIXELFORMAT_RGB24: " << end-start << "ms\n";
-
-				// TODO: fix sdl rgb->yuv conversion resulting in too dark (colorspace) issues
-				//start = Message::getTimeMS();
-				converted_surf = SDL_ConvertSurfaceAndColorspace(tmp_conv_surf, SDL_PIXELFORMAT_IYUV, nullptr, SDL_COLORSPACE_YUV_DEFAULT, 0);
-				//end = Message::getTimeMS();
-				// 60ms
-				//std::cerr << "DTC: timing SDL_PIXELFORMAT_RGB24->SDL_PIXELFORMAT_IYUV: " << end-start << "ms\n";
-
-				SDL_DestroySurface(tmp_conv_surf);
-			} else {
-				converted_surf = SDL_ConvertSurface(converted_surf, SDL_PIXELFORMAT_IYUV);
-			}
-
-			if (converted_surf == nullptr) {
-				// oh god
-				std::cerr << "DTC error: failed to convert surface to IYUV: " << SDL_GetError() << "\n";
-				return false;
-			}
-		}
-		assert(converted_surf != nullptr);
-		if (converted_surf != value.surface.get()) {
-			// TODO: add ctr with uptr
-			SDLVideoFrame new_value{value.timestampUS, nullptr};
-			new_value.surface = {
-				converted_surf,
-				&SDL_DestroySurface
-			};
-
-			return QueuedFrameStream2<SDLVideoFrame>::push(std::move(new_value));
-		} else {
-			return QueuedFrameStream2<SDLVideoFrame>::push(value);
-		}
-	}
-};
-
-// exlusive
-// TODO: replace with something better than a queue
-struct ToxAVCallVideoSink : public FrameStream2SinkI<SDLVideoFrame> {
-	ToxAV& _toxav;
-
-	// bitrate for enabled state
-	uint32_t _video_bitrate {2};
-
-	uint32_t _fid;
-	std::shared_ptr<PushConversionQueuedVideoStream> _writer;
-
-	ToxAVCallVideoSink(ToxAV& toxav, uint32_t fid) : _toxav(toxav), _fid(fid) {}
-	~ToxAVCallVideoSink(void) {
-		if (_writer) {
-			_writer = nullptr;
-			_toxav.toxavVideoSetBitRate(_fid, 0);
-		}
-	}
-
-	// sink
-	std::shared_ptr<FrameStream2I<SDLVideoFrame>> subscribe(void) override {
-		if (_writer) {
-			// max 1 (exclusive, composite video somewhere else)
-			return nullptr;
-		}
-
-		auto err = _toxav.toxavVideoSetBitRate(_fid, _video_bitrate);
-		if (err != TOXAV_ERR_BIT_RATE_SET_OK) {
-			return nullptr;
-		}
-
-		_writer = std::make_shared<PushConversionQueuedVideoStream>(10, true);
-
-		return _writer;
-	}
-
-	bool unsubscribe(const std::shared_ptr<FrameStream2I<SDLVideoFrame>>& sub) override {
-		if (!sub || !_writer) {
-			// nah
-			return false;
-		}
-
-		if (sub == _writer) {
-			_writer = nullptr;
-
-			/*auto err = */_toxav.toxavVideoSetBitRate(_fid, 0);
-			// print warning? on error?
-
-			return true;
-		}
-
-		// what
-		return false;
-	}
-};
-
-// TODO: make proper adapter
-struct AudioStreamReFramer {
-	FrameStream2I<AudioFrame>* _stream {nullptr};
-	uint32_t frame_length_ms {10};
-
-	uint32_t own_seq_counter {0};
-
-	std::vector<int16_t> buffer;
-	size_t samples_in_buffer {0}; // absolute, so divide by ch for actual length
-
-	uint32_t seq {0};
-	uint32_t sample_rate {48'000};
-	size_t channels {0};
-
-
-	std::optional<AudioFrame> pop(void) {
-		assert(_stream != nullptr);
-
-		auto new_in = _stream->pop();
-		if (new_in.has_value()) {
-			auto& new_value = new_in.value();
-			assert(new_value.isS16());
-			if (!new_value.isS16()) {
-				return std::nullopt;
-			}
-
-			if (
-				(buffer.empty()) || // buffer not yet inited
-				(sample_rate != new_value.sample_rate || channels != new_value.channels) // not the same
-			) {
-				seq = 0;
-				sample_rate = new_value.sample_rate;
-				channels = new_value.channels;
-
-				// buffer does not exist or config changed and we discard
-				// preallocate to 2x desired buffer size
-				buffer = std::vector<int16_t>(2 * (channels*sample_rate*frame_length_ms)/1000);
-				samples_in_buffer = 0;
-			}
-
-			// TODO: this will be very important in the future
-			// replace seq with timestapsUS like video??
-#if 0
-			// some time / seq comparison shit
-			if (seq != 0 && new_value.seq != 0) {
-				if (seq+1 != new_value.seq) {
-					// we skipped shit
-					// TODO: insert silence to pad?
-
-					// drop existing
-					samples_in_buffer = 0;
-				}
-			}
-#endif
-
-			// update to latest
-			seq = new_value.seq;
-
-
-			auto new_span = new_value.getSpan<int16_t>();
-
-			//std::cout << "new incoming frame is " << new_value.getSpan<int16_t>().size/new_value.channels*1000/new_value.sample_rate << "ms\n";
-
-			// now append
-			// buffer too small
-			if (buffer.size() - samples_in_buffer < new_value.getSpan<int16_t>().size) {
-				buffer.resize(buffer.size() + new_value.getSpan<int16_t>().size - (buffer.size() - samples_in_buffer));
-			}
-
-			// TODO: memcpy
-			for (size_t i = 0; i < new_span.size; i++) {
-				buffer.at(samples_in_buffer+i) = new_span[i];
-			}
-			samples_in_buffer += new_span.size;
-		} else if (buffer.empty() || samples_in_buffer == 0) {
-			// first pop might result in invalid state
-			return std::nullopt;
-		}
-
-		const size_t desired_size {frame_length_ms * sample_rate * channels / 1000};
-
-		// > threshold?
-		if (samples_in_buffer < desired_size) {
-			return std::nullopt;
-		}
-
-		std::vector<int16_t> return_buffer(desired_size);
-		// copy data
-		for (size_t i = 0; i < return_buffer.size(); i++) {
-			return_buffer.at(i) = buffer.at(i);
-		}
-
-		// now crop buffer (meh)
-		// move data from back to front
-		for (size_t i = 0; i < samples_in_buffer-return_buffer.size(); i++) {
-			buffer.at(i) = buffer.at(desired_size + i);
-		}
-		samples_in_buffer -= return_buffer.size();
-
-		return AudioFrame{
-			own_seq_counter++,
-			sample_rate,
-			channels,
-			std::move(return_buffer),
-		};
-	}
-};
-
-struct ToxAVCallAudioSink : public FrameStream2SinkI<AudioFrame> {
-	ToxAV& _toxav;
-
-	// bitrate for enabled state
-	uint32_t _audio_bitrate {32};
-
-	uint32_t _fid;
-	std::shared_ptr<QueuedFrameStream2<AudioFrame>> _writer;
-
-	ToxAVCallAudioSink(ToxAV& toxav, uint32_t fid) : _toxav(toxav), _fid(fid) {}
-	~ToxAVCallAudioSink(void) {
-		if (_writer) {
-			_writer = nullptr;
-			_toxav.toxavAudioSetBitRate(_fid, 0);
-		}
-	}
-
-	// sink
-	std::shared_ptr<FrameStream2I<AudioFrame>> subscribe(void) override {
-		if (_writer) {
-			// max 1 (exclusive for now)
-			return nullptr;
-		}
-
-		auto err = _toxav.toxavAudioSetBitRate(_fid, _audio_bitrate);
-		if (err != TOXAV_ERR_BIT_RATE_SET_OK) {
-			return nullptr;
-		}
-
-		_writer = std::make_shared<QueuedFrameStream2<AudioFrame>>(16, false);
-
-		return _writer;
-	}
-
-	bool unsubscribe(const std::shared_ptr<FrameStream2I<AudioFrame>>& sub) override {
-		if (!sub || !_writer) {
-			// nah
-			return false;
-		}
-
-		if (sub == _writer) {
-			_writer = nullptr;
-
-			/*auto err = */_toxav.toxavAudioSetBitRate(_fid, 0);
-			// print warning? on error?
-
-			return true;
-		}
-
-		// what
-		return false;
-	}
-};
-
-DebugToxCall::DebugToxCall(ObjectStore2& os, ToxAV& toxav, TextureUploaderI& tu) : _os(os), _toxav(toxav), _tu(tu) {
-	_toxav.subscribe(this, ToxAV_Event::friend_call);
-	_toxav.subscribe(this, ToxAV_Event::friend_call_state);
-	_toxav.subscribe(this, ToxAV_Event::friend_audio_bitrate);
-	_toxav.subscribe(this, ToxAV_Event::friend_video_bitrate);
-	_toxav.subscribe(this, ToxAV_Event::friend_audio_frame);
-	_toxav.subscribe(this, ToxAV_Event::friend_video_frame);
-}
-
-DebugToxCall::~DebugToxCall(void) {
-	// destroy all calls/connections/sources/sinks here
-
-	for (auto& [fid, call] : _calls) {
-		if (static_cast<bool>(call.incoming_vsrc)) {
-			_os.throwEventDestroy(call.incoming_vsrc);
-			call.incoming_vsrc.destroy();
-		}
-		if (static_cast<bool>(call.incoming_asrc)) {
-			_os.throwEventDestroy(call.incoming_asrc);
-			call.incoming_asrc.destroy();
-		}
-		if (static_cast<bool>(call.outgoing_vsink)) {
-			_os.throwEventDestroy(call.outgoing_vsink);
-			call.outgoing_vsink.destroy();
-		}
-		if (static_cast<bool>(call.outgoing_asink)) {
-			_os.throwEventDestroy(call.outgoing_asink);
-			call.outgoing_asink.destroy();
-		}
-	}
-}
-
-void DebugToxCall::tick(float) {
-	// pump sinks to tox
-	// TODO: own thread or direct on push (requires thread save toxcore)
-	// TODO: pump at double the frame rate
-	for (const auto& [oc, vsink] : _os.registry().view<ToxAVCallVideoSink*>().each()) {
-		if (!vsink->_writer) {
-			continue;
-		}
-
-		for (size_t i = 0; i < 10; i++) {
-			auto new_frame_opt = vsink->_writer->pop();
-			if (!new_frame_opt.has_value()) {
-				break;
-			}
-
-			if (!new_frame_opt.value().surface) {
-				// wtf?
-				continue;
-			}
-
-			// conversion is done in the sinks stream
-			SDL_Surface* surf = new_frame_opt.value().surface.get();
-			assert(surf != nullptr);
-
-			SDL_LockSurface(surf);
-			_toxav.toxavVideoSendFrame(
-				vsink->_fid,
-				surf->w, surf->h,
-				static_cast<const uint8_t*>(surf->pixels),
-				static_cast<const uint8_t*>(surf->pixels) + surf->w * surf->h,
-				static_cast<const uint8_t*>(surf->pixels) + surf->w * surf->h + (surf->w/2) * (surf->h/2)
-			);
-			SDL_UnlockSurface(surf);
-		}
-	}
-
-	for (const auto& [oc, asink, asrf] : _os.registry().view<ToxAVCallAudioSink*, AudioStreamReFramer>().each()) {
-		if (!asink->_writer) {
-			continue;
-		}
-
-		asrf._stream = asink->_writer.get();
-
-		for (size_t i = 0; i < 10; i++) {
-			//auto new_frame_opt = asink->_writer->pop();
-			auto new_frame_opt = asrf.pop();
-			if (!new_frame_opt.has_value()) {
-				break;
-			}
-			const auto& new_frame = new_frame_opt.value();
-			assert(new_frame.isS16());
-
-//* @param sample_count Number of samples in this frame. Valid numbers here are
-//*   `((sample rate) * (audio length) / 1000)`, where audio length can be
-//*   2.5, 5, 10, 20, 40 or 60 milliseconds.
-
-			// we likely needs to subdivide/repackage
-			// frame size should be an option exposed to the user
-			// with 10ms as a default ?
-			// the larger the frame size, the less overhead but the more delay
-
-			auto err = _toxav.toxavAudioSendFrame(
-				asink->_fid,
-				new_frame.getSpan<int16_t>().ptr,
-				new_frame.getSpan<int16_t>().size / new_frame.channels,
-				new_frame.channels,
-				new_frame.sample_rate
-			);
-			if (err != TOXAV_ERR_SEND_FRAME_OK) {
-				std::cerr << "DTC: failed to send audio frame " << err << "\n";
-			}
-		}
-	}
-}
-
-float DebugToxCall::render(void) {
-	float next_frame {2.f};
-	if (ImGui::Begin("toxav debug")) {
-		ImGui::Text("Calls:");
-		ImGui::Indent();
-		for (auto& [fid, call] : _calls) {
-			ImGui::PushID(fid);
-
-			ImGui::Text("fid:%d state:%d", fid, call.state);
-			if (call.incoming) {
-				ImGui::SameLine();
-				if (ImGui::SmallButton("answer")) {
-					const auto ret = _toxav.toxavAnswer(fid, 0, 0);
-					//const auto ret = _toxav.toxavAnswer(fid, 0, 1); // 1mbit/s
-					//const auto ret = _toxav.toxavAnswer(fid, 0, 2); // 2mbit/s
-					//const auto ret = _toxav.toxavAnswer(fid, 0, 100); // 100mbit/s
-					//const auto ret = _toxav.toxavAnswer(fid, 0, 2500); // 2500mbit/s
-					if (ret == TOXAV_ERR_ANSWER_OK) {
-						call.incoming = false;
-
-						// create sinks
-						call.outgoing_vsink = {_os.registry(), _os.registry().create()};
-						{
-							auto new_vsink = std::make_unique<ToxAVCallVideoSink>(_toxav, fid);
-							call.outgoing_vsink.emplace<ToxAVCallVideoSink*>(new_vsink.get());
-							call.outgoing_vsink.emplace<Components::FrameStream2Sink<SDLVideoFrame>>(std::move(new_vsink));
-							call.outgoing_vsink.emplace<Components::StreamSink>(Components::StreamSink::create<SDLVideoFrame>("ToxAV Friend Call Outgoing Video"));
-							_os.throwEventConstruct(call.outgoing_vsink);
-						}
-						call.outgoing_asink = {_os.registry(), _os.registry().create()};
-						{
-							auto new_asink = std::make_unique<ToxAVCallAudioSink>(_toxav, fid);
-							call.outgoing_asink.emplace<ToxAVCallAudioSink*>(new_asink.get());
-							call.outgoing_asink.emplace<AudioStreamReFramer>().frame_length_ms = 10;
-							call.outgoing_asink.emplace<Components::FrameStream2Sink<AudioFrame>>(std::move(new_asink));
-							call.outgoing_asink.emplace<Components::StreamSink>(Components::StreamSink::create<AudioFrame>("ToxAV Friend Call Outgoing Audio"));
-							_os.throwEventConstruct(call.outgoing_asink);
-						}
-
-						// create sources
-						if (call.incoming_v) {
-							call.incoming_vsrc = {_os.registry(), _os.registry().create()};
-							{
-								auto new_vsrc = std::make_unique<SDLVideoFrameStream2MultiSource>();
-								call.incoming_vsrc.emplace<SDLVideoFrameStream2MultiSource*>(new_vsrc.get());
-								call.incoming_vsrc.emplace<Components::FrameStream2Source<SDLVideoFrame>>(std::move(new_vsrc));
-								call.incoming_vsrc.emplace<Components::StreamSource>(Components::StreamSource::create<SDLVideoFrame>("ToxAV Friend Call Incoming Video"));
-								_os.throwEventConstruct(call.incoming_vsrc);
-							}
-						}
-						if (call.incoming_a) {
-							call.incoming_asrc = {_os.registry(), _os.registry().create()};
-							{
-								auto new_asrc = std::make_unique<AudioFrameStream2MultiSource>();
-								call.incoming_asrc.emplace<AudioFrameStream2MultiSource*>(new_asrc.get());
-								call.incoming_asrc.emplace<Components::FrameStream2Source<AudioFrame>>(std::move(new_asrc));
-								call.incoming_asrc.emplace<Components::StreamSource>(Components::StreamSource::create<AudioFrame>("ToxAV Friend Call Incoming Audio"));
-								_os.throwEventConstruct(call.incoming_asrc);
-							}
-						}
-					}
-				}
-			} else if (call.state != TOXAV_FRIEND_CALL_STATE_FINISHED) {
-				next_frame = std::min(next_frame, 0.1f);
-				ImGui::SameLine();
-				if (ImGui::SmallButton("hang up")) {
-					const auto ret = _toxav.toxavCallControl(fid, TOXAV_CALL_CONTROL_CANCEL);
-					if (ret == TOXAV_ERR_CALL_CONTROL_OK) {
-						// we hung up
-						// not sure if its possible for toxcore to tell this us too when the other side does this at the same time?
-						call.state = TOXAV_FRIEND_CALL_STATE_FINISHED;
-
-						// TODO: stream manager disconnectAll()
-						if (static_cast<bool>(call.incoming_vsrc)) {
-							_os.throwEventDestroy(call.incoming_vsrc);
-							call.incoming_vsrc.destroy();
-						}
-						if (static_cast<bool>(call.incoming_asrc)) {
-							_os.throwEventDestroy(call.incoming_asrc);
-							call.incoming_asrc.destroy();
-						}
-						if (static_cast<bool>(call.outgoing_vsink)) {
-							_os.throwEventDestroy(call.outgoing_vsink);
-							call.outgoing_vsink.destroy();
-						}
-						if (static_cast<bool>(call.outgoing_asink)) {
-							_os.throwEventDestroy(call.outgoing_asink);
-							call.outgoing_asink.destroy();
-						}
-					}
-				}
-
-				//if (ImGui::BeginCombo("audio src", "---")) {
-				//    ImGui::EndCombo();
-				//}
-				//if (ImGui::BeginCombo("video src", "---")) {
-				//    ImGui::EndCombo();
-				//}
-			}
-
-			ImGui::PopID();
-		}
-		ImGui::Unindent();
-	}
-	ImGui::End();
-
-	return next_frame;
-}
-
-bool DebugToxCall::onEvent(const Events::FriendCall& e)  {
-	auto& call = _calls[e.friend_number];
-	call.incoming = true;
-	call.incoming_a = e.audio_enabled;
-	call.incoming_v = e.video_enabled;
-	call.state = TOXAV_FRIEND_CALL_STATE_NONE;
-
-	return true;
-}
-
-bool DebugToxCall::onEvent(const Events::FriendCallState& e)  {
-	auto& call = _calls[e.friend_number];
-	call.state = e.state;
-
-	if (
-		(call.state & TOXAV_FRIEND_CALL_STATE_FINISHED) != 0 ||
-		(call.state & TOXAV_FRIEND_CALL_STATE_ERROR) != 0
-	) {
-		if (static_cast<bool>(call.incoming_vsrc)) {
-			_os.throwEventDestroy(call.incoming_vsrc);
-			call.incoming_vsrc.destroy();
-		}
-		if (static_cast<bool>(call.incoming_asrc)) {
-			_os.throwEventDestroy(call.incoming_asrc);
-			call.incoming_asrc.destroy();
-		}
-		if (static_cast<bool>(call.outgoing_vsink)) {
-			_os.throwEventDestroy(call.outgoing_vsink);
-			call.outgoing_vsink.destroy();
-		}
-		if (static_cast<bool>(call.outgoing_asink)) {
-			_os.throwEventDestroy(call.outgoing_asink);
-			call.outgoing_asink.destroy();
-		}
-	}
-
-	return true;
-}
-
-bool DebugToxCall::onEvent(const Events::FriendAudioBitrate&)  {
-	return false;
-}
-
-bool DebugToxCall::onEvent(const Events::FriendVideoBitrate&)  {
-	return false;
-}
-
-bool DebugToxCall::onEvent(const Events::FriendAudioFrame& e)  {
-	auto& call = _calls[e.friend_number];
-
-	if (!static_cast<bool>(call.incoming_asrc)) {
-		// missing src to put frame into ??
-		return false;
-	}
-
-	assert(call.incoming_asrc.all_of<AudioFrameStream2MultiSource*>());
-	assert(call.incoming_asrc.all_of<Components::FrameStream2Source<AudioFrame>>());
-
-	call.num_a_frames++;
-
-	call.incoming_asrc.get<AudioFrameStream2MultiSource*>()->push(AudioFrame{
-		0, //seq
-		e.sampling_rate,
-		e.channels,
-		std::vector<int16_t>(e.pcm.begin(), e.pcm.end()) // copy
-	});
-
-	return true;
-}
-
-bool DebugToxCall::onEvent(const Events::FriendVideoFrame& e)  {
-	// TODO: skip if we dont know about this call
-	auto& call = _calls[e.friend_number];
-
-	if (!static_cast<bool>(call.incoming_vsrc)) {
-		// missing src to put frame into ??
-		return false;
-	}
-
-	assert(call.incoming_vsrc.all_of<SDLVideoFrameStream2MultiSource*>());
-	assert(call.incoming_vsrc.all_of<Components::FrameStream2Source<SDLVideoFrame>>());
-
-	call.num_v_frames++;
-
-	auto* new_surf = SDL_CreateSurface(e.width, e.height, SDL_PIXELFORMAT_IYUV);
-	assert(new_surf);
-	if (SDL_LockSurface(new_surf)) {
-		// copy the data
-		// we know how the implementation works, its y u v consecutivlely
-		// y
-		for (size_t y = 0; y < e.height; y++) {
-			std::memcpy(
-				//static_cast<uint8_t*>(new_surf->pixels) + new_surf->pitch*y,
-				static_cast<uint8_t*>(new_surf->pixels) + e.width*y,
-				e.y.ptr + e.ystride*y,
-				e.width
-			);
-		}
-
-		// u
-		for (size_t y = 0; y < e.height/2; y++) {
-			std::memcpy(
-				static_cast<uint8_t*>(new_surf->pixels) + (e.width*e.height) + (e.width/2)*y,
-				e.u.ptr + e.ustride*y,
-				e.width/2
-			);
-		}
-
-		// v
-		for (size_t y = 0; y < e.height/2; y++) {
-			std::memcpy(
-				static_cast<uint8_t*>(new_surf->pixels) + (e.width*e.height) + ((e.width/2)*(e.height/2)) + (e.width/2)*y,
-				e.v.ptr + e.vstride*y,
-				e.width/2
-			);
-		}
-
-		SDL_UnlockSurface(new_surf);
-	}
-
-	call.incoming_vsrc.get<SDLVideoFrameStream2MultiSource*>()->push({
-		// ms -> us
-		Message::getTimeMS() * 1000, // TODO: make more precise
-		new_surf
-	});
-
-	SDL_DestroySurface(new_surf);
-
-	return true;
-}
-

src/debug_tox_call.hpp

@@ -1,52 +0,0 @@
-#pragma once
-
-//#include <solanaceae/object_store/fwd.hpp>
-#include <solanaceae/object_store/object_store.hpp>
-#include "./tox_av.hpp"
-#include "./texture_uploader.hpp"
-
-#include <map>
-#include <cstdint>
-
-class DebugToxCall : public ToxAVEventI {
-	ObjectStore2& _os;
-	ToxAV& _toxav;
-	TextureUploaderI& _tu;
-
-	struct Call {
-		bool incoming {false};
-		bool incoming_a {false};
-		bool incoming_v {false};
-
-		uint32_t state {0}; // ? just last state ?
-
-		uint32_t incomming_abr {0};
-		uint32_t incomming_vbr {0};
-
-		size_t num_a_frames {0};
-		size_t num_v_frames {0};
-
-		ObjectHandle incoming_vsrc;
-		ObjectHandle incoming_asrc;
-
-		ObjectHandle outgoing_vsink;
-		ObjectHandle outgoing_asink;
-	};
-	// tox friend id -> call
-	std::map<uint32_t, Call> _calls;
-
-	public:
-		DebugToxCall(ObjectStore2& os, ToxAV& toxav, TextureUploaderI& tu);
-		~DebugToxCall(void);
-
-		void tick(float time_delta);
-		float render(void);
-
-	protected: // toxav events
-		bool onEvent(const Events::FriendCall&) override;
-		bool onEvent(const Events::FriendCallState&) override;
-		bool onEvent(const Events::FriendAudioBitrate&) override;
-		bool onEvent(const Events::FriendVideoBitrate&) override;
-		bool onEvent(const Events::FriendAudioFrame&) override;
-		bool onEvent(const Events::FriendVideoFrame&) override;
-};

src/debug_video_tap.cpp

@@ -8,92 +8,233 @@
 
 #include <imgui/imgui.h>
 
-#include "./content/sdl_video_frame_stream2.hpp"
+#include "./frame_streams/sdl/video.hpp"
+#include "./frame_streams/frame_stream2.hpp"
 
 #include <string>
 #include <memory>
+#include <mutex>
+#include <deque>
+
+#include <thread>
+#include <chrono>
+#include <atomic>
+
 #include <iostream>
 
-struct DebugVideoTapSink : public FrameStream2SinkI<SDLVideoFrame> {
+// fwd
-	std::shared_ptr<QueuedFrameStream2<SDLVideoFrame>> _writer;
+namespace Message {
+uint64_t getTimeMS(void);
+}
 
-	DebugVideoTapSink(void) {}
+// threadsafe queue frame stream
+// protected by a simple mutex lock
+template<typename FrameType>
+struct LockedFrameStream2 : public FrameStream2I<FrameType> {
+	std::mutex _lock;
+
+	std::deque<FrameType> _frames;
+
+	~LockedFrameStream2(void) {}
+
+	int32_t size(void) { return -1; }
+
+	std::optional<FrameType> pop(void) {
+		std::lock_guard lg{_lock};
+
+		if (_frames.empty()) {
+			return std::nullopt;
+		}
+
+		FrameType new_frame = std::move(_frames.front());
+		_frames.pop_front();
+
+		return std::move(new_frame);
+	}
+
+	bool push(const FrameType& value) {
+		std::lock_guard lg{_lock};
+
+		_frames.push_back(value);
+
+		return true;
+	}
+};
+
+struct DebugVideoTapSink : public FrameStream2SinkI<SDLVideoFrame> {
+	TextureUploaderI& _tu;
+
+	uint32_t _id_counter {0};
+
+	struct Writer {
+		struct View {
+			uint32_t _id {0}; // for stable imgui ids
+
+			uint64_t _tex {0};
+			uint32_t _tex_w {0};
+			uint32_t _tex_h {0};
+
+			bool _mirror {false}; // flip horizontally
+
+			uint64_t _v_last_ts {0}; // us
+			float _v_interval_avg {0.f}; // s
+		} view;
+
+		std::shared_ptr<LockedFrameStream2<SDLVideoFrame>> stream;
+	};
+	std::vector<Writer> _writers;
+
+	DebugVideoTapSink(TextureUploaderI& tu) : _tu(tu) {}
 	~DebugVideoTapSink(void) {}
 
 	// sink
 	std::shared_ptr<FrameStream2I<SDLVideoFrame>> subscribe(void) override {
-		if (_writer) {
+		_writers.emplace_back(Writer{
-			// max 1 (exclusive)
+			Writer::View{_id_counter++},
-			return nullptr;
+			std::make_shared<LockedFrameStream2<SDLVideoFrame>>()
-		}
+		});
 
-		_writer = std::make_shared<QueuedFrameStream2<SDLVideoFrame>>(1, true);
+		return _writers.back().stream;
-
-		return _writer;
 	}
 
 	bool unsubscribe(const std::shared_ptr<FrameStream2I<SDLVideoFrame>>& sub) override {
-		if (!sub || !_writer) {
+		if (!sub || _writers.empty()) {
 			// nah
 			return false;
 		}
 
-		if (sub == _writer) {
+		for (auto it = _writers.cbegin(); it != _writers.cend(); it++) {
-			_writer =  nullptr;
+			if (it->stream == sub) {
+				_tu.destroy(it->view._tex);
+				_writers.erase(it);
 				return true;
 			}
+		}
 
 		// what
 		return false;
 	}
 };
 
+struct DebugVideoTestSource : public FrameStream2SourceI<SDLVideoFrame> {
+	std::vector<std::shared_ptr<LockedFrameStream2<SDLVideoFrame>>> _readers;
+
+	std::atomic_bool _stop {false};
+	std::thread _thread;
+
+	DebugVideoTestSource(void) {
+		std::cout << "DVTS: starting new test video source\n";
+		_thread = std::thread([this](void) {
+			while (!_stop) {
+				if (!_readers.empty()) {
+					auto* surf = SDL_CreateSurface(960, 720, SDL_PIXELFORMAT_ARGB32);
+
+					// color
+					static auto start_time = Message::getTimeMS();
+					const float time = (Message::getTimeMS() - start_time)/1000.f;
+					SDL_ClearSurface(surf, std::sin(time), std::cos(time), 0.5f, 1.f);
+
+					SDLVideoFrame frame{ // non-owning
+						Message::getTimeMS()*1000,
+						surf,
+					};
+
+					for (auto& stream : _readers) {
+						stream->push(frame); // copy
+					}
+
+					SDL_DestroySurface(surf);
+				}
+
+				std::this_thread::sleep_for(std::chrono::milliseconds(50));
+			}
+		});
+	}
+	~DebugVideoTestSource(void) {
+		_stop = true;
+		_thread.join();
+	}
+
+	std::shared_ptr<FrameStream2I<SDLVideoFrame>> subscribe(void) override {
+		return _readers.emplace_back(std::make_shared<LockedFrameStream2<SDLVideoFrame>>());
+	}
+
+	bool unsubscribe(const std::shared_ptr<FrameStream2I<SDLVideoFrame>>& sub) override {
+		for (auto it = _readers.cbegin(); it != _readers.cend(); it++) {
+			if (it->get() == sub.get()) {
+				_readers.erase(it);
+				return true;
+			}
+		}
+
+		return false;
+	}
+};
+
 DebugVideoTap::DebugVideoTap(ObjectStore2& os, StreamManager& sm, TextureUploaderI& tu) : _os(os), _sm(sm), _tu(tu) {
 	// post self as video sink
 	_tap = {_os.registry(), _os.registry().create()};
 	try {
-		auto dvts = std::make_unique<DebugVideoTapSink>();
+		auto dvts = std::make_unique<DebugVideoTapSink>(_tu);
 		_tap.emplace<DebugVideoTapSink*>(dvts.get()); // to get our data back
 		_tap.emplace<Components::FrameStream2Sink<SDLVideoFrame>>(
 			std::move(dvts)
 		);
 
 		_tap.emplace<Components::StreamSink>(Components::StreamSink::create<SDLVideoFrame>("DebugVideoTap"));
+		_tap.emplace<Components::TagDefaultTarget>();
+
+		_os.throwEventConstruct(_tap);
 	} catch (...) {
 		_os.registry().destroy(_tap);
 	}
+
+	_src = {_os.registry(), _os.registry().create()};
+	try {
+		auto dvts = std::make_unique<DebugVideoTestSource>();
+		_src.emplace<DebugVideoTestSource*>(dvts.get());
+		_src.emplace<Components::FrameStream2Source<SDLVideoFrame>>(
+			std::move(dvts)
+		);
+
+		_src.emplace<Components::StreamSource>(Components::StreamSource::create<SDLVideoFrame>("DebugVideoTest"));
+
+		_os.throwEventConstruct(_src);
+	} catch (...) {
+		_os.registry().destroy(_src);
+	}
 }
 
 DebugVideoTap::~DebugVideoTap(void) {
 	if (static_cast<bool>(_tap)) {
 		_os.registry().destroy(_tap);
 	}
+	if (static_cast<bool>(_src)) {
+		_os.registry().destroy(_src);
+	}
 }
 
 float DebugVideoTap::render(void) {
-	if (ImGui::Begin("DebugVideoTap")) {
+	float min_interval {2.f};
-		{ // first pull the latest img from sink and update the texture
+	auto& dvtsw = _tap.get<DebugVideoTapSink*>()->_writers;
-			assert(static_cast<bool>(_tap));
+	for (auto& [view, stream] : dvtsw) {
-
+		std::string window_title {"DebugVideoTap #"};
-			auto& dvtsw = _tap.get<DebugVideoTapSink*>()->_writer;
+		window_title += std::to_string(view._id);
-			if (dvtsw) {
+		ImGui::SetNextWindowSize({250, 250}, ImGuiCond_Appearing);
-				while (true) {
+		if (ImGui::Begin(window_title.c_str())) {
-					auto new_frame_opt = dvtsw->pop();
+			while (auto new_frame_opt = stream->pop()) {
-					if (new_frame_opt.has_value()) {
 				// timing
-						if (_v_last_ts == 0) {
+				if (view._v_last_ts == 0) {
-							_v_last_ts = new_frame_opt.value().timestampUS;
+					view._v_last_ts = new_frame_opt.value().timestampUS;
 				} else {
-							auto delta = int64_t(new_frame_opt.value().timestampUS) - int64_t(_v_last_ts);
+					auto delta = int64_t(new_frame_opt.value().timestampUS) - int64_t(view._v_last_ts);
-							_v_last_ts = new_frame_opt.value().timestampUS;
+					view._v_last_ts = new_frame_opt.value().timestampUS;
 
-							//delta = std::min<int64_t>(delta, 10*1000*1000);
+					if (view._v_interval_avg == 0) {
-
+						view._v_interval_avg = delta/1'000'000.f;
-							if (_v_interval_avg == 0) {
-								_v_interval_avg = delta/1'000'000.f;
 					} else {
 						const float r = 0.2f;
-								_v_interval_avg = _v_interval_avg * (1.f-r) + (delta/1'000'000.f) * r;
+						view._v_interval_avg = view._v_interval_avg * (1.f-r) + (delta/1'000'000.f) * r;
 					}
 				}
 
@@ -108,9 +249,9 @@ float DebugVideoTap::render(void) {
 				}
 
 				SDL_LockSurface(converted_surf);
-						if (_tex == 0 || (int)_tex_w != converted_surf->w || (int)_tex_h != converted_surf->h) {
+				if (view._tex == 0 || (int)view._tex_w != converted_surf->w || (int)view._tex_h != converted_surf->h) {
-							_tu.destroy(_tex);
+					_tu.destroy(view._tex);
-							_tex = _tu.uploadRGBA(
+					view._tex = _tu.uploadRGBA(
 						static_cast<const uint8_t*>(converted_surf->pixels),
 						converted_surf->w,
 						converted_surf->h,
@@ -118,10 +259,10 @@ float DebugVideoTap::render(void) {
 						TextureUploaderI::STREAMING
 					);
 
-							_tex_w = converted_surf->w;
+					view._tex_w = converted_surf->w;
-							_tex_h = converted_surf->h;
+					view._tex_h = converted_surf->h;
 				} else {
-							_tu.updateRGBA(_tex, static_cast<const uint8_t*>(converted_surf->pixels), converted_surf->w * converted_surf->h * 4);
+					_tu.updateRGBA(view._tex, static_cast<const uint8_t*>(converted_surf->pixels), converted_surf->w * converted_surf->h * 4);
 				}
 				SDL_UnlockSurface(converted_surf);
 
@@ -129,82 +270,27 @@ float DebugVideoTap::render(void) {
 					// clean up temp
 					SDL_DestroySurface(converted_surf);
 				}
-					} else {
-						break;
-					}
-				}
-			}
 			}
 
-		// list sources dropdown to connect too
+			ImGui::Checkbox("mirror", &view._mirror);
-		std::string preview_label {"none"};
-		if (static_cast<bool>(_selected_src)) {
-			preview_label = std::to_string(entt::to_integral(entt::to_entity(_selected_src.entity()))) + " (" + _selected_src.get<Components::StreamSource>().name + ")";
-		}
-
-		if (ImGui::BeginCombo("selected source", preview_label.c_str())) {
-			if (ImGui::Selectable("none")) {
-				switchTo({});
-			}
-
-			for (const auto& [oc, ss] : _os.registry().view<Components::StreamSource>().each()) {
-				if (ss.frame_type_name != entt::type_name<SDLVideoFrame>::value()) {
-					continue;
-				}
-				std::string label = std::to_string(entt::to_integral(entt::to_entity(oc))) + " (" + ss.name + ")";
-				if (ImGui::Selectable(label.c_str())) {
-					switchTo({_os.registry(), oc});
-				}
-			}
-
-			ImGui::EndCombo();
-		}
-
-		//ImGui::SetNextItemWidth(0);
-		ImGui::Checkbox("mirror", &_mirror);
 
 			// img here
-		if (_tex != 0) {
+			if (view._tex != 0) {
 				ImGui::SameLine();
-			ImGui::Text("moving avg interval: %f", _v_interval_avg);
+				ImGui::Text("moving avg interval: %f", view._v_interval_avg);
 				const float img_w = ImGui::GetContentRegionAvail().x;
 				ImGui::Image(
-				reinterpret_cast<ImTextureID>(_tex),
+					reinterpret_cast<ImTextureID>(view._tex),
-				ImVec2{img_w, img_w * float(_tex_h)/_tex_w},
+					ImVec2{img_w, img_w * float(view._tex_h)/view._tex_w},
-				ImVec2{_mirror?1.f:0.f, 0},
+					ImVec2{view._mirror?1.f:0.f, 0},
-				ImVec2{_mirror?0.f:1.f, 1}
+					ImVec2{view._mirror?0.f:1.f, 1}
 				);
 			}
+
 		}
 		ImGui::End();
-
-	if (_v_interval_avg != 0) {
-		return _v_interval_avg;
-	} else {
-		return 2.f;
-	}
-}
-
-void DebugVideoTap::switchTo(ObjectHandle o) {
-	if (o == _selected_src) {
-		std::cerr << "DVT: switch to same ...\n";
-		return;
-	}
-
-	_tu.destroy(_tex);
-	_tex = 0;
-	_v_last_ts = 0;
-	_v_interval_avg = 0;
-
-	if (static_cast<bool>(_selected_src)) {
-		_sm.disconnect<SDLVideoFrame>(_selected_src, _tap);
-	}
-
-	if (static_cast<bool>(o) && _sm.connect<SDLVideoFrame>(o, _tap)) {
-		_selected_src = o;
-	} else {
-		std::cerr << "DVT: cleared video source\n";
-		_selected_src = {};
 	}
+
+	return min_interval;
 }
 

src/debug_video_tap.hpp

@@ -1,34 +1,23 @@
 #pragma once
 
-#include <cstdint>
 #include <solanaceae/object_store/fwd.hpp>
-#include "./stream_manager.hpp"
+#include "./frame_streams/stream_manager.hpp"
 #include "./texture_uploader.hpp"
 
 // provides a sink and a small window displaying a SDLVideoFrame
+// HACK: provides a test video source
 class DebugVideoTap {
 	ObjectStore2& _os;
 	StreamManager& _sm;
 	TextureUploaderI& _tu;
 
-	ObjectHandle _selected_src;
 	ObjectHandle _tap;
-
+	ObjectHandle _src;
-	uint64_t _tex {0};
-	uint32_t _tex_w {0};
-	uint32_t _tex_h {0};
-
-	bool _mirror {false}; // flip horizontally
-
-	uint64_t _v_last_ts {0}; // us
-	float _v_interval_avg {0.f}; // s
 
 	public:
 		DebugVideoTap(ObjectStore2& os, StreamManager& sm, TextureUploaderI& tu);
 		~DebugVideoTap(void);
 
 		float render(void);
-
-		void switchTo(ObjectHandle o);
 };
 

src/frame_streams/audio_stream2.hpp

@@ -0,0 +1,39 @@
+#pragma once
+
+#include "./frame_stream2.hpp"
+
+#include <solanaceae/util/span.hpp>
+
+#include <cstdint>
+#include <variant>
+#include <vector>
+
+// raw audio
+// channels make samples interleaved,
+// planar channels are not supported
+// s16 only stopgap audio frame (simplified)
+struct AudioFrame2 {
+	// samples per second
+	uint32_t sample_rate {48'000};
+
+	// only >0 is valid
+	size_t channels {0};
+
+	std::variant<
+		std::vector<int16_t>, // S16, platform endianess
+		Span<int16_t> // non owning variant, for direct consumption
+	> buffer;
+
+	// helpers
+	Span<int16_t> getSpan(void) const {
+		if (std::holds_alternative<std::vector<int16_t>>(buffer)) {
+			return Span<int16_t>{std::get<std::vector<int16_t>>(buffer)};
+		} else {
+			return std::get<Span<int16_t>>(buffer);
+		}
+		return {};
+	}
+};
+
+using AudioFrame2Stream2I = FrameStream2I<AudioFrame2>;
+

src/frame_streams/audio_stream_pop_reframer.hpp

@@ -0,0 +1,103 @@
+#pragma once
+
+#include "./audio_stream2.hpp"
+
+// reframes audio frames to a specified size in ms
+// TODO: use absolute sample count instead??
+template<typename RealAudioStream>
+struct AudioStreamPopReFramer : public FrameStream2I<AudioFrame2> {
+	uint32_t _frame_length_ms {20};
+
+	// gotta be careful of the multithreaded nature
+	// and false(true) sharing
+	uint64_t _pad0{};
+	RealAudioStream _stream;
+	uint64_t _pad1{};
+
+	// dequeue?
+	std::vector<int16_t> _buffer;
+
+	uint32_t _sample_rate {48'000};
+	size_t _channels {0};
+
+	AudioStreamPopReFramer(uint32_t frame_length_ms = 20)
+	: _frame_length_ms(frame_length_ms) {
+	}
+
+	AudioStreamPopReFramer(uint32_t frame_length_ms, FrameStream2I<AudioFrame2>&& stream)
+	: _frame_length_ms(frame_length_ms), _stream(std::move(stream)) {
+	}
+
+	~AudioStreamPopReFramer(void) {}
+
+	size_t getDesiredSize(void) const {
+		return _frame_length_ms * _sample_rate * _channels / 1000;
+	}
+
+	int32_t size(void) override { return -1; }
+
+	std::optional<AudioFrame2> pop(void) override {
+		do {
+			auto new_in = _stream.pop();
+			if (new_in.has_value()) {
+				auto& new_value = new_in.value();
+
+				// changed
+				if (_sample_rate != new_value.sample_rate || _channels != new_value.channels) {
+					//if (_channels != 0) {
+					//    std::cerr << "ReFramer warning: reconfiguring, dropping buffer\n";
+					//}
+					_sample_rate = new_value.sample_rate;
+					_channels = new_value.channels;
+
+					// buffer does not exist or config changed and we discard
+					_buffer = {};
+				}
+
+				//std::cout << "new incoming frame is " << new_value.getSpan().size/new_value.channels*1000/new_value.sample_rate << "ms\n";
+
+				auto new_span = new_value.getSpan();
+
+				if (_buffer.empty()) {
+					_buffer = {new_span.cbegin(), new_span.cend()};
+				} else {
+					_buffer.insert(_buffer.cend(), new_span.cbegin(), new_span.cend());
+				}
+			} else if (_buffer.empty()) {
+				// first pop might result in invalid state
+				return std::nullopt;
+			} else {
+				// inner stream pop did not give a new value
+				break; // out of loop
+			}
+		} while (_buffer.size() < getDesiredSize());
+
+		const auto desired_size = getDesiredSize();
+
+		// > threshold?
+		if (_buffer.size() < desired_size) {
+			return std::nullopt;
+		}
+
+		// copy data
+		std::vector<int16_t> return_buffer(_buffer.cbegin(), _buffer.cbegin()+desired_size);
+
+		// now crop buffer (meh)
+		// move data from back to front
+		_buffer.erase(_buffer.cbegin(), _buffer.cbegin() + desired_size);
+
+		return AudioFrame2{
+			_sample_rate,
+			_channels,
+			std::move(return_buffer),
+		};
+	}
+
+	bool push(const AudioFrame2& value) override {
+		// might be worth it to instead do the work on push
+		//assert(false && "push reframing not implemented");
+		// passthrough
+		return _stream.push(value);
+	}
+};
+

src/frame_streams/frame_stream2.hpp

@@ -0,0 +1,47 @@
+#pragma once
+
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <vector>
+
+// Frames often consist of:
+// - seq id // incremental sequential id, gaps in ids can be used to detect loss
+// - or timestamp
+// - data // the frame data
+// eg:
+//struct ExampleFrame {
+	//int64_t seq_id {0};
+	//std::vector<uint8_t> data;
+//};
+
+template<typename FrameType>
+struct FrameStream2I {
+	virtual ~FrameStream2I(void) {}
+
+	// get number of available frames
+	// returns -1 if unknown
+	[[nodiscard]] virtual int32_t size(void) = 0;
+
+	// get next frame
+	// data sharing? -> no, data is copied for each fsr, if concurency supported
+	[[nodiscard]] virtual std::optional<FrameType> pop(void) = 0;
+
+	// returns true if there are readers (or we dont know)
+	virtual bool push(const FrameType& value) = 0;
+};
+
+template<typename FrameType>
+struct FrameStream2SourceI {
+	virtual ~FrameStream2SourceI(void) {}
+	[[nodiscard]] virtual std::shared_ptr<FrameStream2I<FrameType>> subscribe(void) = 0;
+	virtual bool unsubscribe(const std::shared_ptr<FrameStream2I<FrameType>>& sub) = 0;
+};
+
+template<typename FrameType>
+struct FrameStream2SinkI {
+	virtual ~FrameStream2SinkI(void) {}
+	[[nodiscard]] virtual std::shared_ptr<FrameStream2I<FrameType>> subscribe(void) = 0;
+	virtual bool unsubscribe(const std::shared_ptr<FrameStream2I<FrameType>>& sub) = 0;
+};
+

src/frame_streams/locked_frame_stream.hpp

@@ -0,0 +1,46 @@
+#pragma once
+
+#include "./frame_stream2.hpp"
+
+#include <mutex>
+#include <deque>
+
+// threadsafe queue frame stream
+// protected by a simple mutex lock
+// prefer lockless queue implementations, when available
+template<typename FrameType>
+struct LockedFrameStream2 : public FrameStream2I<FrameType> {
+	std::mutex _lock;
+
+	std::deque<FrameType> _frames;
+
+	~LockedFrameStream2(void) {}
+
+	int32_t size(void) { return -1; }
+
+	std::optional<FrameType> pop(void) {
+		std::lock_guard lg{_lock};
+
+		if (_frames.empty()) {
+			return std::nullopt;
+		}
+
+		FrameType new_frame = std::move(_frames.front());
+		_frames.pop_front();
+
+		return std::move(new_frame);
+	}
+
+	bool push(const FrameType& value) {
+		std::lock_guard lg{_lock};
+
+		if (_frames.size() > 1024) {
+			return false; // hard limit
+		}
+
+		_frames.push_back(value);
+
+		return true;
+	}
+};
+

src/frame_streams/multi_source.hpp

@@ -0,0 +1,62 @@
+#pragma once
+
+#include "./locked_frame_stream.hpp"
+
+#include <cassert>
+
+// implements a stream that pushes to all sub streams
+template<typename FrameType, typename SubStreamType = LockedFrameStream2<FrameType>>
+struct FrameStream2MultiSource : public FrameStream2SourceI<FrameType>, public FrameStream2I<FrameType> {
+	using sub_stream_type_t = SubStreamType;
+
+	// pointer stability
+	std::vector<std::shared_ptr<SubStreamType>> _sub_streams;
+	std::mutex _sub_stream_lock; // accessing the _sub_streams array needs to be exclusive
+	// a simple lock here is ok, since this tends to be a rare operation,
+	// except for the push, which is always on the same thread
+
+	virtual ~FrameStream2MultiSource(void) {}
+
+	// TODO: forward args instead
+	std::shared_ptr<FrameStream2I<FrameType>> subscribe(void) override {
+		std::lock_guard lg{_sub_stream_lock};
+		return _sub_streams.emplace_back(std::make_unique<SubStreamType>());
+	}
+
+	bool unsubscribe(const std::shared_ptr<FrameStream2I<FrameType>>& sub) override {
+		std::lock_guard lg{_sub_stream_lock};
+		for (auto it = _sub_streams.begin(); it != _sub_streams.end(); it++) {
+			if (*it == sub) {
+				_sub_streams.erase(it);
+				return true;
+			}
+		}
+		return false; // ?
+	}
+
+	// stream interface
+
+	int32_t size(void) override {
+		// TODO: return something sensible?
+		return -1;
+	}
+
+	std::optional<FrameType> pop(void) override {
+		// nope
+		assert(false && "this logic is very frame type specific, provide an impl");
+		return std::nullopt;
+	}
+
+	// returns true if there are readers
+	bool push(const FrameType& value) override {
+		std::lock_guard lg{_sub_stream_lock};
+		bool have_readers{false};
+		for (auto& it : _sub_streams) {
+			[[maybe_unused]] auto _ = it->push(value);
+			have_readers = true; // even if queue full, we still continue believing in them
+			// maybe consider push return value?
+		}
+		return have_readers;
+	}
+};
+

src/frame_streams/sdl/sdl_audio2_frame_stream2.cpp

@@ -1,32 +1,33 @@
-#include "./sdl_audio_frame_stream2.hpp"
+#include "./sdl_audio2_frame_stream2.hpp"
 
+#include <cassert>
 #include <iostream>
+#include <optional>
+
+#include "../audio_stream_pop_reframer.hpp"
 
 // "thin" wrapper around sdl audio streams
 // we dont needs to get fance, as they already provide everything we need
-struct SDLAudioStreamReader : public AudioFrameStream2I {
+struct SDLAudio2StreamReader : public AudioFrame2Stream2I {
 	std::unique_ptr<SDL_AudioStream, decltype(&SDL_DestroyAudioStream)> _stream;
 
-	uint32_t _seq_counter {0};
-
 	uint32_t _sample_rate {48'000};
 	size_t _channels {0};
-	SDL_AudioFormat _format {SDL_AUDIO_S16};
 
+	// buffer gets reused!
 	std::vector<int16_t> _buffer;
 
-	SDLAudioStreamReader(void) : _stream(nullptr, nullptr) {}
+	SDLAudio2StreamReader(void) : _stream(nullptr, nullptr) {}
-	SDLAudioStreamReader(SDLAudioStreamReader&& other) :
+	SDLAudio2StreamReader(SDLAudio2StreamReader&& other) :
 		_stream(std::move(other._stream)),
 		_sample_rate(other._sample_rate),
-		_channels(other._channels),
+		_channels(other._channels)
-		_format(other._format)
 	{
-		static const size_t buffer_size {960*_channels};
+		const size_t buffer_size {960*_channels};
 		_buffer.resize(buffer_size);
 	}
 
-	~SDLAudioStreamReader(void) {
+	~SDLAudio2StreamReader(void) {
 		if (_stream) {
 			SDL_UnbindAudioStream(_stream.get());
 		}
@@ -39,11 +40,13 @@ struct SDLAudioStreamReader : public AudioFrameStream2I {
 		return -1;
 	}
 
-	std::optional<AudioFrame> pop(void) override {
+	std::optional<AudioFrame2> pop(void) override {
 		assert(_stream);
-		assert(_format == SDL_AUDIO_S16);
+		if (!_stream) {
+			return std::nullopt;
+		}
 
-		static const size_t buffer_size {960*_channels};
+		const size_t buffer_size {960*_channels};
 		_buffer.resize(buffer_size); // noop?
 
 		const auto read_bytes = SDL_GetAudioStreamData(
@@ -57,31 +60,30 @@ struct SDLAudioStreamReader : public AudioFrameStream2I {
 			return std::nullopt;
 		}
 
-		return AudioFrame {
+		return AudioFrame2{
-			_seq_counter++,
 			_sample_rate, _channels,
 			Span<int16_t>(_buffer.data(), read_bytes/sizeof(int16_t)),
 		};
 	}
 
-	bool push(const AudioFrame&) override {
+	bool push(const AudioFrame2&) override {
 		// TODO: make universal sdl stream wrapper (combine with SDLAudioOutputDeviceDefaultInstance)
 		assert(false && "read only");
 		return false;
 	}
 };
 
-SDLAudioInputDevice::SDLAudioInputDevice(void) : SDLAudioInputDevice(SDL_AUDIO_DEVICE_DEFAULT_RECORDING) {
+SDLAudio2InputDevice::SDLAudio2InputDevice(void) : SDLAudio2InputDevice(SDL_AUDIO_DEVICE_DEFAULT_RECORDING) {
 }
 
-SDLAudioInputDevice::SDLAudioInputDevice(SDL_AudioDeviceID conf_device_id) : _configured_device_id(conf_device_id) {
+SDLAudio2InputDevice::SDLAudio2InputDevice(SDL_AudioDeviceID conf_device_id) : _configured_device_id(conf_device_id) {
 	if (_configured_device_id == 0) {
 		// TODO: proper error handling
 		throw int(1);
 	}
 }
 
-SDLAudioInputDevice::~SDLAudioInputDevice(void) {
+SDLAudio2InputDevice::~SDLAudio2InputDevice(void) {
 	_streams.clear();
 
 	if (_virtual_device_id != 0) {
@@ -90,7 +92,7 @@ SDLAudioInputDevice::~SDLAudioInputDevice(void) {
 	}
 }
 
-std::shared_ptr<FrameStream2I<AudioFrame>> SDLAudioInputDevice::subscribe(void) {
+std::shared_ptr<FrameStream2I<AudioFrame2>> SDLAudio2InputDevice::subscribe(void) {
 	if (_virtual_device_id == 0) {
 		// first stream, open device
 		// this spec is more like a hint to the hardware
@@ -108,7 +110,7 @@ std::shared_ptr<FrameStream2I<AudioFrame>> SDLAudioInputDevice::subscribe(void)
 	}
 
 	SDL_AudioSpec spec {
-		SDL_AUDIO_S16,
+		SDL_AUDIO_S16, // required, as AudioFrame2 only supports s16
 		1, // TODO: conf
 		48'000,
 	};
@@ -127,19 +129,24 @@ std::shared_ptr<FrameStream2I<AudioFrame>> SDLAudioInputDevice::subscribe(void)
 	// error check
 	SDL_BindAudioStream(_virtual_device_id, sdl_stream);
 
-	auto new_stream = std::make_shared<SDLAudioStreamReader>();
+	//auto new_stream = std::make_shared<SDLAudio2StreamReader>();
-	// TODO: move to ctr
+	//// TODO: move to ctr
-	new_stream->_stream = {sdl_stream, &SDL_DestroyAudioStream};
+	//new_stream->_stream = {sdl_stream, &SDL_DestroyAudioStream};
-	new_stream->_sample_rate = spec.freq;
+	//new_stream->_sample_rate = spec.freq;
-	new_stream->_channels = spec.channels;
+	//new_stream->_channels = spec.channels;
-	new_stream->_format = spec.format;
+
+	auto new_stream = std::make_shared<AudioStreamPopReFramer<SDLAudio2StreamReader>>();
+	new_stream->_stream._stream = {sdl_stream, &SDL_DestroyAudioStream};
+	new_stream->_stream._sample_rate = spec.freq;
+	new_stream->_stream._channels = spec.channels;
+	new_stream->_frame_length_ms = 5; // WHY DOES THIS FIX MY ISSUE !!!
 
 	_streams.emplace_back(new_stream);
 
 	return new_stream;
 }
 
-bool SDLAudioInputDevice::unsubscribe(const std::shared_ptr<FrameStream2I<AudioFrame>>& sub) {
+bool SDLAudio2InputDevice::unsubscribe(const std::shared_ptr<FrameStream2I<AudioFrame2>>& sub) {
 	for (auto it = _streams.cbegin(); it != _streams.cend(); it++) {
 		if (*it == sub) {
 			_streams.erase(it);
@@ -158,58 +165,55 @@ bool SDLAudioInputDevice::unsubscribe(const std::shared_ptr<FrameStream2I<AudioF
 }
 
 // does not need to be visible in the header
-struct SDLAudioOutputDeviceDefaultInstance : public AudioFrameStream2I {
+struct SDLAudio2OutputDeviceDefaultInstance : public AudioFrame2Stream2I {
 	std::unique_ptr<SDL_AudioStream, decltype(&SDL_DestroyAudioStream)> _stream;
 
 	uint32_t _last_sample_rate {48'000};
 	size_t _last_channels {0};
-	SDL_AudioFormat _last_format {SDL_AUDIO_S16};
 
 	// TODO: audio device
-	SDLAudioOutputDeviceDefaultInstance(void);
+	SDLAudio2OutputDeviceDefaultInstance(void);
-	SDLAudioOutputDeviceDefaultInstance(SDLAudioOutputDeviceDefaultInstance&& other);
+	SDLAudio2OutputDeviceDefaultInstance(SDLAudio2OutputDeviceDefaultInstance&& other);
 
-	~SDLAudioOutputDeviceDefaultInstance(void);
+	~SDLAudio2OutputDeviceDefaultInstance(void);
 
 	int32_t size(void) override;
-	std::optional<AudioFrame> pop(void) override;
+	std::optional<AudioFrame2> pop(void) override;
-	bool push(const AudioFrame& value) override;
+	bool push(const AudioFrame2& value) override;
 };
 
-SDLAudioOutputDeviceDefaultInstance::SDLAudioOutputDeviceDefaultInstance(void) : _stream(nullptr, nullptr) {
+SDLAudio2OutputDeviceDefaultInstance::SDLAudio2OutputDeviceDefaultInstance(void) : _stream(nullptr, nullptr) {
 }
 
-SDLAudioOutputDeviceDefaultInstance::SDLAudioOutputDeviceDefaultInstance(SDLAudioOutputDeviceDefaultInstance&& other) : _stream(std::move(other._stream)) {
+SDLAudio2OutputDeviceDefaultInstance::SDLAudio2OutputDeviceDefaultInstance(SDLAudio2OutputDeviceDefaultInstance&& other) : _stream(std::move(other._stream)) {
 }
 
-SDLAudioOutputDeviceDefaultInstance::~SDLAudioOutputDeviceDefaultInstance(void) {
+SDLAudio2OutputDeviceDefaultInstance::~SDLAudio2OutputDeviceDefaultInstance(void) {
 }
-int32_t SDLAudioOutputDeviceDefaultInstance::size(void) {
+int32_t SDLAudio2OutputDeviceDefaultInstance::size(void) {
 	return -1;
 }
 
-std::optional<AudioFrame> SDLAudioOutputDeviceDefaultInstance::pop(void) {
+std::optional<AudioFrame2> SDLAudio2OutputDeviceDefaultInstance::pop(void) {
 	assert(false);
 	// this is an output device, there is no data to pop
 	return std::nullopt;
 }
 
-bool SDLAudioOutputDeviceDefaultInstance::push(const AudioFrame& value) {
+bool SDLAudio2OutputDeviceDefaultInstance::push(const AudioFrame2& value) {
 	if (!static_cast<bool>(_stream)) {
 		return false;
 	}
 
-	// verify here the fame has the same sample type, channel count and sample freq
+	// verify here the fame has the same channel count and sample freq
 	// if something changed, we need to either use a temporary stream, just for conversion, or reopen _stream with the new params
 	// because of data temporality, the second options looks like a better candidate
 	if (
 		value.sample_rate != _last_sample_rate ||
-		value.channels != _last_channels ||
+		value.channels != _last_channels
-		(value.isF32() && _last_format != SDL_AUDIO_F32) ||
-		(value.isS16() && _last_format != SDL_AUDIO_S16)
 	) {
 		const SDL_AudioSpec spec = {
-			static_cast<SDL_AudioFormat>((value.isF32() ? SDL_AUDIO_F32 :  SDL_AUDIO_S16)),
+			static_cast<SDL_AudioFormat>(SDL_AUDIO_S16),
 			static_cast<int>(value.channels),
 			static_cast<int>(value.sample_rate)
 		};
@@ -219,8 +223,7 @@ bool SDLAudioOutputDeviceDefaultInstance::push(const AudioFrame& value) {
 		std::cerr << "SDLAOD: audio format changed\n";
 	}
 
-	if (value.isS16()) {
+	auto data = value.getSpan();
-		auto data = value.getSpan<int16_t>();
 
 	if (data.size == 0) {
 		std::cerr << "empty audio frame??\n";
@@ -230,35 +233,22 @@ bool SDLAudioOutputDeviceDefaultInstance::push(const AudioFrame& value) {
 		std::cerr << "put data error\n";
 		return false; // return true?
 	}
-	} else if (value.isF32()) {
-		auto data = value.getSpan<float>();
-
-		if (data.size == 0) {
-			std::cerr << "empty audio frame??\n";
-		}
-
-		if (!SDL_PutAudioStreamData(_stream.get(), data.ptr, data.size * sizeof(float))) {
-			std::cerr << "put data error\n";
-			return false; // return true?
-		}
-	}
 
 	_last_sample_rate = value.sample_rate;
 	_last_channels = value.channels;
-	_last_format = value.isF32() ? SDL_AUDIO_F32 :  SDL_AUDIO_S16;
 
 	return true;
 }
 
 
-SDLAudioOutputDeviceDefaultSink::~SDLAudioOutputDeviceDefaultSink(void) {
+SDLAudio2OutputDeviceDefaultSink::~SDLAudio2OutputDeviceDefaultSink(void) {
 	// TODO: pause and close device?
 }
 
-std::shared_ptr<FrameStream2I<AudioFrame>> SDLAudioOutputDeviceDefaultSink::subscribe(void) {
+std::shared_ptr<FrameStream2I<AudioFrame2>> SDLAudio2OutputDeviceDefaultSink::subscribe(void) {
-	auto new_instance = std::make_shared<SDLAudioOutputDeviceDefaultInstance>();
+	auto new_instance = std::make_shared<SDLAudio2OutputDeviceDefaultInstance>();
 
-	constexpr SDL_AudioSpec spec = { SDL_AUDIO_S16, 1, 48000 };
+	constexpr SDL_AudioSpec spec = { SDL_AUDIO_S16, 2, 48000 };
 
 	new_instance->_stream = {
 		SDL_OpenAudioDeviceStream(SDL_AUDIO_DEVICE_DEFAULT_PLAYBACK, &spec, nullptr, nullptr),
@@ -266,7 +256,6 @@ std::shared_ptr<FrameStream2I<AudioFrame>> SDLAudioOutputDeviceDefaultSink::subs
 	};
 	new_instance->_last_sample_rate = spec.freq;
 	new_instance->_last_channels = spec.channels;
-	new_instance->_last_format = spec.format;
 
 	if (!static_cast<bool>(new_instance->_stream)) {
 		std::cerr << "SDL open audio device failed!\n";
@@ -279,7 +268,8 @@ std::shared_ptr<FrameStream2I<AudioFrame>> SDLAudioOutputDeviceDefaultSink::subs
 	return new_instance;
 }
 
-bool SDLAudioOutputDeviceDefaultSink::unsubscribe(const std::shared_ptr<FrameStream2I<AudioFrame>>& sub) {
+bool SDLAudio2OutputDeviceDefaultSink::unsubscribe(const std::shared_ptr<FrameStream2I<AudioFrame2>>& sub) {
+	// TODO: i think we should keep track of them
 	if (!sub) {
 		return false;
 	}

src/frame_streams/sdl/sdl_audio2_frame_stream2.hpp

@@ -1,7 +1,7 @@
 #pragma once
 
-#include "./frame_stream2.hpp"
+#include "../frame_stream2.hpp"
-#include "./audio_stream.hpp"
+#include "../audio_stream2.hpp"
 
 #include <SDL3/SDL.h>
 
@@ -13,31 +13,31 @@
 // source
 // opens device
 // creates a sdl audio stream for each subscribed reader stream
-struct SDLAudioInputDevice : public FrameStream2SourceI<AudioFrame> {
+struct SDLAudio2InputDevice : public FrameStream2SourceI<AudioFrame2> {
 	// held by instances
 	using sdl_stream_type = std::unique_ptr<SDL_AudioStream, decltype(&SDL_DestroyAudioStream)>;
 
 	SDL_AudioDeviceID _configured_device_id {0};
 	SDL_AudioDeviceID _virtual_device_id {0};
 
-	std::vector<std::shared_ptr<FrameStream2I<AudioFrame>>> _streams;
+	std::vector<std::shared_ptr<FrameStream2I<AudioFrame2>>> _streams;
 
-	SDLAudioInputDevice(void);
+	SDLAudio2InputDevice(void);
-	SDLAudioInputDevice(SDL_AudioDeviceID conf_device_id);
+	SDLAudio2InputDevice(SDL_AudioDeviceID conf_device_id);
-	~SDLAudioInputDevice(void);
+	~SDLAudio2InputDevice(void);
 
-	std::shared_ptr<FrameStream2I<AudioFrame>> subscribe(void) override;
+	std::shared_ptr<FrameStream2I<AudioFrame2>> subscribe(void) override;
-	bool unsubscribe(const std::shared_ptr<FrameStream2I<AudioFrame>>& sub) override;
+	bool unsubscribe(const std::shared_ptr<FrameStream2I<AudioFrame2>>& sub) override;
 };
 
 // sink
 // constructs entirely new streams, since sdl handles sync and mixing for us (or should)
-struct SDLAudioOutputDeviceDefaultSink : public FrameStream2SinkI<AudioFrame> {
+struct SDLAudio2OutputDeviceDefaultSink : public FrameStream2SinkI<AudioFrame2> {
 	// TODO: pause device?
 
-	~SDLAudioOutputDeviceDefaultSink(void);
+	~SDLAudio2OutputDeviceDefaultSink(void);
 
-	std::shared_ptr<FrameStream2I<AudioFrame>> subscribe(void) override;
+	std::shared_ptr<FrameStream2I<AudioFrame2>> subscribe(void) override;
-	bool unsubscribe(const std::shared_ptr<FrameStream2I<AudioFrame>>& sub) override;
+	bool unsubscribe(const std::shared_ptr<FrameStream2I<AudioFrame2>>& sub) override;
 };
 

src/frame_streams/sdl/video.hpp

@@ -0,0 +1,41 @@
+#pragma once
+
+#include <SDL3/SDL.h>
+
+#include <cstdint>
+#include <memory>
+
+// https://youtu.be/71Iw4Q74OaE
+
+inline void nopSurfaceDestructor(SDL_Surface*) {}
+
+// this is very sdl specific
+// but allows us to autoconvert between formats (to a degree)
+struct SDLVideoFrame {
+	// micro seconds (nano is way too much)
+	uint64_t timestampUS {0};
+
+	std::unique_ptr<SDL_Surface, decltype(&SDL_DestroySurface)> surface {nullptr, &SDL_DestroySurface};
+
+	// special non-owning constructor
+	SDLVideoFrame(
+		uint64_t ts,
+		SDL_Surface* surf
+	) {
+		timestampUS = ts;
+		surface = {surf, &nopSurfaceDestructor};
+	}
+	SDLVideoFrame(SDLVideoFrame&& other) = default;
+	// copy
+	SDLVideoFrame(const SDLVideoFrame& other) {
+		timestampUS = other.timestampUS;
+		if (static_cast<bool>(other.surface)) {
+			surface = {
+				SDL_DuplicateSurface(other.surface.get()),
+				&SDL_DestroySurface
+			};
+		}
+	}
+	SDLVideoFrame& operator=(const SDLVideoFrame& other) = delete;
+};
+

src/frame_streams/stream_manager.cpp

@@ -0,0 +1,206 @@
+#include "./stream_manager.hpp"
+
+StreamManager::Connection::Connection(
+	ObjectHandle src_,
+	ObjectHandle sink_,
+	std::unique_ptr<Data>&& data_,
+	std::function<void(Connection&)>&& pump_fn_,
+	std::function<void(Connection&)>&& unsubscribe_fn_,
+	bool on_main_thread_
+) :
+	src(src_),
+	sink(sink_),
+	data(std::move(data_)),
+	pump_fn(std::move(pump_fn_)),
+	unsubscribe_fn(std::move(unsubscribe_fn_)),
+	on_main_thread(on_main_thread_)
+{
+	if (!on_main_thread) {
+		// start thread
+		pump_thread = std::thread([this](void) {
+			while (!stop) {
+				pump_fn(*this);
+				std::this_thread::sleep_for(std::chrono::milliseconds(5));
+			}
+			finished = true;
+		});
+	}
+}
+
+StreamManager::StreamManager(ObjectStore2& os) : _os(os) {
+	_os.subscribe(this, ObjectStore_Event::object_construct);
+	//_os.subscribe(this, ObjectStore_Event::object_update);
+	_os.subscribe(this, ObjectStore_Event::object_destroy);
+}
+
+StreamManager::~StreamManager(void) {
+	// stop all connetions
+	for (const auto& con : _connections) {
+		con->stop = true;
+		if (!con->on_main_thread) {
+			con->pump_thread.join(); // we skip the finished check and wait
+		}
+		con->unsubscribe_fn(*con);
+	}
+}
+
+bool StreamManager::connect(Object src, Object sink, bool threaded) {
+	auto h_src = _os.objectHandle(src);
+	auto h_sink = _os.objectHandle(sink);
+	if (!static_cast<bool>(h_src) || !static_cast<bool>(h_sink)) {
+		// an object does not exist
+		return false;
+	}
+
+	// get src and sink comps
+	if (!h_src.all_of<Components::StreamSource>()) {
+		// src not stream source
+		return false;
+	}
+
+	if (!h_sink.all_of<Components::StreamSink>()) {
+		// sink not stream sink
+		return false;
+	}
+
+	const auto& ssrc = h_src.get<Components::StreamSource>();
+	const auto& ssink = h_sink.get<Components::StreamSink>();
+
+	// compare type
+	if (ssrc.frame_type_name != ssink.frame_type_name) {
+		return false;
+	}
+
+	// always fail in debug mode
+	assert(static_cast<bool>(ssrc.connect_fn));
+	if (!static_cast<bool>(ssrc.connect_fn)) {
+		return false;
+	}
+
+	// use connect fn from src
+	return ssrc.connect_fn(*this, src, sink, threaded);
+}
+
+bool StreamManager::disconnect(Object src, Object sink) {
+	auto res = std::find_if(
+		_connections.cbegin(), _connections.cend(),
+		[&](const auto& a) { return a->src == src && a->sink == sink; }
+	);
+	if (res == _connections.cend()) {
+		// not found
+		return false;
+	}
+
+	// do disconnect
+	(*res)->stop = true;
+
+	return true;
+}
+
+bool StreamManager::disconnectAll(Object o) {
+	bool succ {false};
+	for (const auto& con : _connections) {
+		if (con->src == o || con->sink == o) {
+			con->stop = true;
+			succ = true;
+		}
+	}
+
+	return succ;
+}
+
+// do we need the time delta?
+float StreamManager::tick(float) {
+	// pump all mainthread connections
+	for (auto it = _connections.begin(); it != _connections.end();) {
+		auto& con = **it;
+
+		if (!static_cast<bool>(con.src) || !static_cast<bool>(con.sink)) {
+			// either side disappeard without disconnectAll
+			// TODO: warn/error log
+			con.stop = true;
+		}
+
+		if (con.on_main_thread) {
+			con.pump_fn(con);
+		}
+
+		if (con.stop && (con.finished || con.on_main_thread)) {
+			if (!con.on_main_thread) {
+				assert(con.pump_thread.joinable());
+				con.pump_thread.join();
+			}
+			con.unsubscribe_fn(con);
+			it = _connections.erase(it);
+		} else {
+			it++;
+		}
+	}
+
+	// return min over intervals instead
+	return 2.f; // TODO: 2sec makes mainthread connections unusable
+}
+
+bool StreamManager::onEvent(const ObjectStore::Events::ObjectConstruct& e) {
+	if (!e.e.any_of<Components::StreamSink, Components::StreamSource>()) {
+		return false;
+	}
+
+	// update default targets
+	if (e.e.all_of<Components::TagDefaultTarget>()) {
+		if (e.e.all_of<Components::StreamSource>()) {
+			_default_sources[e.e.get<Components::StreamSource>().frame_type_name] = e.e;
+		} else { // sink
+			_default_sinks[e.e.get<Components::StreamSink>().frame_type_name] = e.e;
+		}
+	}
+
+	// connect to default
+	// only ever do this on new objects
+	if (e.e.all_of<Components::TagConnectToDefault>()) {
+		if (e.e.all_of<Components::StreamSource>()) {
+			auto it_d_sink = _default_sinks.find(e.e.get<Components::StreamSource>().frame_type_name);
+			if (it_d_sink != _default_sinks.cend()) {
+				// TODO: threaded
+				connect(e.e, it_d_sink->second);
+			}
+		} else { // sink
+			auto it_d_src = _default_sources.find(e.e.get<Components::StreamSink>().frame_type_name);
+			if (it_d_src != _default_sources.cend()) {
+				// TODO: threaded
+				connect(it_d_src->second, e.e);
+			}
+		}
+	}
+
+	return false;
+}
+
+bool StreamManager::onEvent(const ObjectStore::Events::ObjectUpdate&) {
+	// what do we do here?
+	return false;
+}
+
+bool StreamManager::onEvent(const ObjectStore::Events::ObjectDestory& e) {
+	// typeless
+	for (auto it = _default_sources.cbegin(); it != _default_sources.cend();) {
+		if (it->second == e.e) {
+			it = _default_sources.erase(it);
+		} else {
+			it++;
+		}
+	}
+	for (auto it = _default_sinks.cbegin(); it != _default_sinks.cend();) {
+		if (it->second == e.e) {
+			it = _default_sinks.erase(it);
+		} else {
+			it++;
+		}
+	}
+
+	// TODO: destroy connections
+	// TODO: auto reconnect default following devices if another default exists
+
+	return false;
+}
+

src/frame_streams/stream_manager.hpp

@@ -0,0 +1,222 @@
+#pragma once
+
+#include <solanaceae/object_store/fwd.hpp>
+#include <solanaceae/object_store/object_store.hpp>
+
+#include <entt/core/type_info.hpp>
+
+#include "./frame_stream2.hpp"
+
+#include <unordered_map>
+#include <vector>
+#include <memory>
+#include <algorithm>
+#include <thread>
+#include <chrono>
+#include <atomic>
+
+// fwd
+class StreamManager;
+
+namespace Components {
+
+	// mark a source or sink as the(a) default
+	struct TagDefaultTarget {};
+
+	// mark a source/sink as to be connected to a default sink/source
+	// of the same type
+	struct TagConnectToDefault {};
+
+	struct StreamSource {
+		std::string name;
+		std::string frame_type_name;
+
+		std::function<bool(StreamManager&, Object, Object, bool)> connect_fn;
+
+		template<typename FrameType>
+		static StreamSource create(const std::string& name);
+	};
+
+	struct StreamSink {
+		std::string name;
+		std::string frame_type_name;
+
+		template<typename FrameType>
+		static StreamSink create(const std::string& name);
+	};
+
+	template<typename FrameType>
+	using FrameStream2Source = std::unique_ptr<FrameStream2SourceI<FrameType>>;
+
+	template<typename FrameType>
+	using FrameStream2Sink = std::unique_ptr<FrameStream2SinkI<FrameType>>;
+
+} // Components
+
+
+class StreamManager : protected ObjectStoreEventI {
+	friend class StreamManagerUI; // TODO: make this go away
+	ObjectStore2& _os;
+
+	struct Connection {
+		ObjectHandle src;
+		ObjectHandle sink;
+
+		struct Data {
+			virtual ~Data(void) {}
+		};
+		std::unique_ptr<Data> data; // stores reader writer type erased
+		std::function<void(Connection&)> pump_fn; // TODO: make it return next interval?
+		std::function<void(Connection&)> unsubscribe_fn;
+
+		bool on_main_thread {true};
+		std::atomic_bool stop {false}; // disconnect
+		std::atomic_bool finished {false}; // disconnect
+
+		// pump thread
+		std::thread pump_thread;
+
+		// frame interval counters and estimates
+		// TODO
+
+		Connection(void) = default;
+		Connection(
+			ObjectHandle src_,
+			ObjectHandle sink_,
+			std::unique_ptr<Data>&& data_,
+			std::function<void(Connection&)>&& pump_fn_,
+			std::function<void(Connection&)>&& unsubscribe_fn_,
+			bool on_main_thread_ = true
+		);
+	};
+	std::vector<std::unique_ptr<Connection>> _connections;
+
+	std::unordered_map<std::string, Object> _default_sources;
+	std::unordered_map<std::string, Object> _default_sinks;
+
+	public:
+		StreamManager(ObjectStore2& os);
+		virtual ~StreamManager(void);
+
+		template<typename FrameType>
+		bool connect(Object src, Object sink, bool threaded = true);
+
+		bool connect(Object src, Object sink, bool threaded = true);
+		bool disconnect(Object src, Object sink);
+		bool disconnectAll(Object o);
+
+		// do we need the time delta?
+		float tick(float);
+
+	protected:
+		bool onEvent(const ObjectStore::Events::ObjectConstruct&) override;
+		bool onEvent(const ObjectStore::Events::ObjectUpdate&) override;
+		bool onEvent(const ObjectStore::Events::ObjectDestory&) override;
+};
+
+// template impls
+
+namespace Components {
+
+	// we require the complete sm type here
+	template<typename FrameType>
+	StreamSource StreamSource::create(const std::string& name) {
+		return StreamSource{
+			name,
+			std::string{entt::type_name<FrameType>::value()},
+			+[](StreamManager& sm, Object src, Object sink, bool threaded) {
+				return sm.connect<FrameType>(src, sink, threaded);
+			},
+		};
+	}
+
+	template<typename FrameType>
+	StreamSink StreamSink::create(const std::string& name) {
+		return StreamSink{
+			name,
+			std::string{entt::type_name<FrameType>::value()},
+		};
+	}
+
+} // Components
+
+template<typename FrameType>
+bool StreamManager::connect(Object src, Object sink, bool threaded) {
+	auto res = std::find_if(
+		_connections.cbegin(), _connections.cend(),
+		[&](const auto& a) { return a->src == src && a->sink == sink; }
+	);
+	if (res != _connections.cend()) {
+		// already exists
+		return false;
+	}
+
+	auto h_src = _os.objectHandle(src);
+	auto h_sink = _os.objectHandle(sink);
+	if (!static_cast<bool>(h_src) || !static_cast<bool>(h_sink)) {
+		// an object does not exist
+		return false;
+	}
+
+	if (!h_src.all_of<Components::FrameStream2Source<FrameType>>()) {
+		// src not stream source
+		return false;
+	}
+
+	if (!h_sink.all_of<Components::FrameStream2Sink<FrameType>>()) {
+		// sink not stream sink
+		return false;
+	}
+
+	auto& src_stream = h_src.get<Components::FrameStream2Source<FrameType>>();
+	auto& sink_stream = h_sink.get<Components::FrameStream2Sink<FrameType>>();
+
+	struct inlineData : public Connection::Data {
+		virtual ~inlineData(void) {}
+		std::shared_ptr<FrameStream2I<FrameType>> reader;
+		std::shared_ptr<FrameStream2I<FrameType>> writer;
+	};
+
+	auto our_data = std::make_unique<inlineData>();
+
+	our_data->reader = src_stream->subscribe();
+	if (!our_data->reader) {
+		return false;
+	}
+	our_data->writer = sink_stream->subscribe();
+	if (!our_data->writer) {
+		return false;
+	}
+
+	_connections.push_back(std::make_unique<Connection>(
+		h_src,
+		h_sink,
+		std::move(our_data),
+		[](Connection& con) -> void {
+			// there might be more stored
+			for (size_t i = 0; i < 10; i++) {
+				auto new_frame_opt = static_cast<inlineData*>(con.data.get())->reader->pop();
+				// TODO: frame interval estimates
+				if (new_frame_opt.has_value()) {
+					static_cast<inlineData*>(con.data.get())->writer->push(new_frame_opt.value());
+				} else {
+					break;
+				}
+			}
+		},
+		[](Connection& con) -> void {
+			auto* src_stream_ptr = con.src.try_get<Components::FrameStream2Source<FrameType>>();
+			if (src_stream_ptr != nullptr) {
+				(*src_stream_ptr)->unsubscribe(static_cast<inlineData*>(con.data.get())->reader);
+			}
+			auto* sink_stream_ptr = con.sink.try_get<Components::FrameStream2Sink<FrameType>>();
+			if (sink_stream_ptr != nullptr) {
+				(*sink_stream_ptr)->unsubscribe(static_cast<inlineData*>(con.data.get())->writer);
+			}
+		},
+		!threaded
+	));
+
+	return true;
+}
+

src/frame_streams/test_pop_reframer.cpp

@@ -0,0 +1,155 @@
+#include <iostream>
+
+#include "./audio_stream_pop_reframer.hpp"
+#include "./locked_frame_stream.hpp"
+
+#include <cassert>
+
+int main(void) {
+	{ // pump perfect
+		AudioStreamPopReFramer<LockedFrameStream2<AudioFrame2>> stream;
+		stream._frame_length_ms = 10;
+
+		AudioFrame2 f1 {
+			48'000,
+			1,
+			{},
+		};
+		f1.buffer = std::vector<int16_t>(
+			// perfect size
+			stream._frame_length_ms * f1.sample_rate * f1.channels / 1000,
+			0
+		);
+
+		{ // fill with sequential value
+			int16_t seq = 0;
+			for (auto& v : std::get<std::vector<int16_t>>(f1.buffer)) {
+				v = seq++;
+			}
+		}
+
+		stream.push(f1);
+
+		auto ret_opt = stream.pop();
+		assert(ret_opt);
+
+		auto& ret = ret_opt.value();
+		assert(ret.sample_rate == f1.sample_rate);
+		assert(ret.channels == f1.channels);
+		assert(ret.getSpan().size == f1.getSpan().size);
+		{
+			int16_t seq = 0;
+			for (const auto v : ret.getSpan()) {
+				assert(v == seq++);
+			}
+		}
+	}
+
+	{ // pump half
+		AudioStreamPopReFramer<LockedFrameStream2<AudioFrame2>> stream;
+		stream._frame_length_ms = 10;
+
+		AudioFrame2 f1 {
+			48'000,
+			1,
+			{},
+		};
+		f1.buffer = std::vector<int16_t>(
+			// perfect size
+			(stream._frame_length_ms * f1.sample_rate * f1.channels / 1000) / 2,
+			0
+		);
+		AudioFrame2 f2 {
+			48'000,
+			1,
+			{},
+		};
+		f2.buffer = std::vector<int16_t>(
+			// perfect size
+			(stream._frame_length_ms * f1.sample_rate * f1.channels / 1000) / 2,
+			0
+		);
+
+		{ // fill with sequential value
+			int16_t seq = 0;
+			for (auto& v : std::get<std::vector<int16_t>>(f1.buffer)) {
+				v = seq++;
+			}
+			for (auto& v : std::get<std::vector<int16_t>>(f2.buffer)) {
+				v = seq++;
+			}
+		}
+
+		stream.push(f1);
+		stream.push(f2);
+
+		// supposed to combine both
+		auto ret_opt = stream.pop();
+		assert(ret_opt);
+
+		auto& ret = ret_opt.value();
+		assert(ret.sample_rate == f1.sample_rate);
+		assert(ret.channels == f1.channels);
+		assert(ret.getSpan().size == stream._frame_length_ms * f1.sample_rate * f1.channels / 1000);
+		{
+			int16_t seq = 0;
+			for (const auto v : ret.getSpan()) {
+				assert(v == seq++);
+			}
+		}
+	}
+
+	{ // pump double
+		AudioStreamPopReFramer<LockedFrameStream2<AudioFrame2>> stream;
+		stream._frame_length_ms = 20;
+
+		AudioFrame2 f1 {
+			48'000,
+			2,
+			{},
+		};
+		f1.buffer = std::vector<int16_t>(
+			// perfect size
+			(stream._frame_length_ms * f1.sample_rate * f1.channels / 1000) * 2,
+			0
+		);
+		{ // fill with sequential value
+			int16_t seq = 0;
+			for (auto& v : std::get<std::vector<int16_t>>(f1.buffer)) {
+				v = seq++;
+			}
+		}
+
+		stream.push(f1);
+
+		// pop 2x
+		int16_t seq = 0;
+		{
+			auto ret_opt = stream.pop();
+			assert(ret_opt);
+
+			auto& ret = ret_opt.value();
+			assert(ret.sample_rate == f1.sample_rate);
+			assert(ret.channels == f1.channels);
+			assert(ret.getSpan().size == stream._frame_length_ms * f1.sample_rate * f1.channels / 1000);
+			for (const auto v : ret.getSpan()) {
+				assert(v == seq++);
+			}
+		}
+
+		{
+			auto ret_opt = stream.pop();
+			assert(ret_opt);
+
+			auto& ret = ret_opt.value();
+			assert(ret.sample_rate == f1.sample_rate);
+			assert(ret.channels == f1.channels);
+			assert(ret.getSpan().size == stream._frame_length_ms * f1.sample_rate * f1.channels / 1000);
+			for (const auto v : ret.getSpan()) {
+				assert(v == seq++);
+			}
+		}
+	}
+
+	return 0;
+}

src/frame_streams/voip_model.hpp

@@ -0,0 +1,77 @@
+#pragma once
+
+#include <solanaceae/contact/contact_model3.hpp>
+#include <solanaceae/object_store/fwd.hpp>
+
+struct VoIPModelI;
+
+namespace Components::VoIP {
+
+	struct TagVoIPSession {};
+
+	// getting called or invited by
+	struct Incoming {
+		Contact3 c{entt::null};
+	};
+
+	struct DefaultConfig {
+		bool incoming_audio {true};
+		bool incoming_video {true};
+		bool outgoing_audio {true};
+		bool outgoing_video {true};
+	};
+
+	// to talk to the model handling this session
+	//struct VoIPModel {
+		//VoIPModelI* ptr {nullptr};
+	//};
+
+	struct SessionState {
+		// ????
+		// incoming
+		// outgoing
+		enum class State {
+			RINGING,
+			CONNECTED,
+		} state;
+	};
+
+	struct SessionContact {
+		Contact3 c{entt::null};
+	};
+
+	struct StreamSources {
+		// list of all stream sources originating from this VoIP session
+		std::vector<Object> streams;
+	};
+
+	struct StreamSinks {
+		// list of all stream sinks going to this VoIP session
+		std::vector<Object> streams;
+	};
+
+} // Components::VoIP
+
+// TODO: events? piggyback on objects?
+
+// stream model instead?? -> more generic than "just" audio and video?
+// or specialized like this
+// streams abstract type in a nice way
+struct VoIPModelI {
+	virtual ~VoIPModelI(void) {}
+
+	// enters a call/voicechat/videocall ???
+	// - contact
+	// - default stream sources/sinks ?
+	// - enable a/v ? -> done on connecting to sources
+	// returns object tieing together the VoIP session
+	virtual ObjectHandle enter(const Contact3 c, const Components::VoIP::DefaultConfig& defaults = {true, true, true, true}) { (void)c,(void)defaults; return {}; }
+
+	// accept/join an invite to a session
+	virtual bool accept(ObjectHandle session, const Components::VoIP::DefaultConfig& defaults = {true, true, true, true}) { (void)session,(void)defaults; return false; }
+
+	// leaves a call
+	// - VoIP session object
+	virtual bool leave(ObjectHandle session) { (void)session; return false; }
+};
+

src/main.cpp

@@ -12,9 +12,6 @@
 
 #include "./start_screen.hpp"
 
-#include "./content/sdl_video_frame_stream2.hpp"
-#include "./content/sdl_audio_frame_stream2.hpp"
-
 #include <filesystem>
 #include <memory>
 #include <iostream>
@@ -31,6 +28,8 @@ int main(int argc, char** argv) {
 
 	runSysCheck();
 
+	SDL_SetAppMetadata("tomato", "0.0.0-wip", nullptr);
+
 #ifdef __ANDROID__
 	// change current working dir to internal storage
 	std::filesystem::current_path(SDL_GetAndroidInternalStoragePath());
@@ -75,51 +74,10 @@ int main(int argc, char** argv) {
 
 	std::cout << "SDL Renderer: " << SDL_GetRendererName(renderer.get()) << "\n";
 
-	// optionally init audio and camera
-	if (!SDL_Init(SDL_INIT_AUDIO)) {
-		std::cerr << "SDL_Init AUDIO failed (" << SDL_GetError() << ")\n";
-	} else if (false) {
-		SDLAudioInputDevice aid;
-		auto reader = aid.subscribe();
-
-		auto writer = SDLAudioOutputDeviceDefaultSink{}.subscribe();
-
-		for (size_t i = 0; i < 200; i++) {
-			std::this_thread::sleep_for(std::chrono::milliseconds(10));
-			auto new_frame_opt = reader->pop();
-			if (new_frame_opt.has_value()) {
-				std::cout << "audio frame was seq:" << new_frame_opt.value().seq << " sr:" << new_frame_opt.value().sample_rate << " " << (new_frame_opt.value().isS16()?"S16":"F32") << " l:" << (new_frame_opt.value().isS16()?new_frame_opt.value().getSpan<int16_t>().size:new_frame_opt.value().getSpan<float>().size) << "\n";
-				writer->push(new_frame_opt.value());
-			} else {
-				std::cout << "no audio frame\n";
-			}
-		}
-
-		aid.unsubscribe(reader);
-	}
-
-	if (!SDL_Init(SDL_INIT_CAMERA)) {
-		std::cerr << "SDL_Init CAMERA failed (" << SDL_GetError() << ")\n";
-	} else if (false) { // HACK
-		std::cerr << "CAMERA initialized\n";
-		SDLVideoCameraContent vcc;
-		auto reader = vcc.subscribe();
-		for (size_t i = 0; i < 20; i++) {
-			std::this_thread::sleep_for(std::chrono::milliseconds(50));
-			auto new_frame_opt = reader->pop();
-			if (new_frame_opt.has_value()) {
-				std::cout << "video frame was " << new_frame_opt.value().surface->w << "x" << new_frame_opt.value().surface->h << " " << new_frame_opt.value().timestampUS << "us " << new_frame_opt.value().surface->format << "sf\n";
-			}
-		}
-		vcc.unsubscribe(reader);
-	}
-	std::cout << "after sdl video stuffery\n";
-
 	IMGUI_CHECKVERSION();
 	ImGui::CreateContext();
 
 	// TODO: test android behaviour
-	// -> its too big, dpi does not take eye-screen-distance into account
 	float display_scale = SDL_GetWindowDisplayScale(window.get());
 	if (display_scale < 0.001f) {
 		// error?

src/main_screen.cpp

@@ -5,15 +5,12 @@
 
 #include <solanaceae/contact/components.hpp>
 
+#include "./frame_streams/sdl/sdl_audio2_frame_stream2.hpp"
+
 #include <imgui/imgui.h>
 
 #include <SDL3/SDL.h>
 
-#include "./content/sdl_video_frame_stream2.hpp"
-#include "content/audio_stream.hpp"
-#include "content/sdl_audio_frame_stream2.hpp"
-#include "stream_manager.hpp"
-
 #include <memory>
 #include <cmath>
 #include <string_view>
@@ -28,14 +25,14 @@ MainScreen::MainScreen(SimpleConfigModel&& conf_, SDL_Renderer* renderer_, Theme
 	tc(save_path, save_password),
 	tpi(tc.getTox()),
 	ad(tc),
-#if TOMATO_TOX_AV
-	tav(tc.getTox()),
-	dtc(os, tav, sdlrtu),
-#endif
 	tcm(cr, tc, tc),
 	tmm(rmm, cr, tcm, tc, tc),
 	ttm(rmm, cr, tcm, tc, tc, os),
 	tffom(cr, rmm, tcm, tc, tc),
+#if TOMATO_TOX_AV
+	tav(tc.getTox()),
+	tavvoip(os, tav, cr, tcm),
+#endif
 	theme(theme_),
 	mmil(rmm),
 	tam(/*rmm, */ os, cr, conf),
@@ -47,10 +44,10 @@ MainScreen::MainScreen(SimpleConfigModel&& conf_, SDL_Renderer* renderer_, Theme
 	cg(conf, os, rmm, cr, sdlrtu, contact_tc, msg_tc, theme),
 	sw(conf),
 	osui(os),
-	smui(os, sm),
-	dvt(os, sm, sdlrtu),
 	tuiu(tc, conf),
-	tdch(tpi)
+	tdch(tpi),
+	smui(os, sm),
+	dvt(os, sm, sdlrtu)
 {
 	tel.subscribeAll(tc);
 
@@ -84,7 +81,7 @@ MainScreen::MainScreen(SimpleConfigModel&& conf_, SDL_Renderer* renderer_, Theme
 		g_provideInstance<ToxPrivateI>("ToxPrivateI", "host", &tpi);
 		g_provideInstance<ToxEventProviderI>("ToxEventProviderI", "host", &tc);
 #if TOMATO_TOX_AV
-		g_provideInstance<ToxAV>("ToxAV", "host", &tav);
+		g_provideInstance<ToxAVI>("ToxAVI", "host", &tav);
 #endif
 		g_provideInstance<ToxContactModel2>("ToxContactModel2", "host", &tcm);
 
@@ -146,55 +143,45 @@ MainScreen::MainScreen(SimpleConfigModel&& conf_, SDL_Renderer* renderer_, Theme
 
 	conf.dump();
 
-	{ // add system av devices
+	if (SDL_InitSubSystem(SDL_INIT_AUDIO)) {
-		if (false) {
+		// add system audio devices
-			ObjectHandle vsrc {os.registry(), os.registry().create()};
+		{ // audio in
-			try {
-				vsrc.emplace<Components::FrameStream2Source<SDLVideoFrame>>(
-					std::make_unique<SDLVideoCameraContent>()
-				);
-
-				vsrc.emplace<Components::StreamSource>(Components::StreamSource::create<SDLVideoFrame>("WebCam"));
-
-				os.throwEventConstruct(vsrc);
-			} catch (...) {
-				os.registry().destroy(vsrc);
-			}
-		}
-
-		if (true) { // audio in
 			ObjectHandle asrc {os.registry(), os.registry().create()};
 			try {
-				asrc.emplace<Components::FrameStream2Source<AudioFrame>>(
+				asrc.emplace<Components::FrameStream2Source<AudioFrame2>>(
-					std::make_unique<SDLAudioInputDevice>()
+					std::make_unique<SDLAudio2InputDevice>()
 				);
 
-				asrc.emplace<Components::StreamSource>(Components::StreamSource::create<AudioFrame>("SDL Audio Default Recording Device"));
+				asrc.emplace<Components::StreamSource>(Components::StreamSource::create<AudioFrame2>("SDL Audio Default Recording Device"));
+				asrc.emplace<Components::TagDefaultTarget>();
 
 				os.throwEventConstruct(asrc);
 			} catch (...) {
 				os.registry().destroy(asrc);
 			}
 		}
-
 		{ // audio out
 			ObjectHandle asink {os.registry(), os.registry().create()};
 			try {
-				asink.emplace<Components::FrameStream2Sink<AudioFrame>>(
+				asink.emplace<Components::FrameStream2Sink<AudioFrame2>>(
-					std::make_unique<SDLAudioOutputDeviceDefaultSink>()
+					std::make_unique<SDLAudio2OutputDeviceDefaultSink>()
 				);
 
-				asink.emplace<Components::StreamSink>(Components::StreamSink::create<AudioFrame>("SDL Audio Default Playback Device"));
+				asink.emplace<Components::StreamSink>(Components::StreamSink::create<AudioFrame2>("SDL Audio Default Playback Device"));
+				asink.emplace<Components::TagDefaultTarget>();
 
 				os.throwEventConstruct(asink);
 			} catch (...) {
 				os.registry().destroy(asink);
 			}
 		}
+	} else {
+		std::cerr << "MS warning: no sdl audio: " << SDL_GetError() << "\n";
 	}
 }
 
 MainScreen::~MainScreen(void) {
+	// TODO: quit sdl audio
 }
 
 bool MainScreen::handleEvent(SDL_Event& e) {
@@ -308,19 +295,16 @@ Screen* MainScreen::render(float time_delta, bool&) {
 	}
 	// ACTUALLY NOT IF RENDERED, MOVED LOGIC TO ABOVE
 	// it might unload textures, so it needs to be done before rendering
-	float animation_interval = contact_tc.update();
+	const float ctc_interval = contact_tc.update();
-	animation_interval = std::min<float>(animation_interval, msg_tc.update());
+	const float msgtc_interval = msg_tc.update();
 
 	const float cg_interval = cg.render(time_delta); // render
 	sw.render(); // render
 	osui.render();
-	smui.render();
-	animation_interval = std::min<float>(animation_interval, dvt.render());
 	tuiu.render(); // render
 	tdch.render(); // render
-#if TOMATO_TOX_AV
+	smui.render();
-	animation_interval = std::min<float>(animation_interval, dtc.render());
+	const float dvt_interval = dvt.render();
-#endif
 
 	{ // main window menubar injection
 		if (ImGui::Begin("tomato")) {
@@ -501,7 +485,9 @@ Screen* MainScreen::render(float time_delta, bool&) {
 
 	// low delay time window
 	if (!_window_hidden && _time_since_event < curr_profile.low_delay_window) {
-		_render_interval = std::min<float>(_render_interval, animation_interval);
+		_render_interval = std::min<float>(_render_interval, ctc_interval);
+		_render_interval = std::min<float>(_render_interval, msgtc_interval);
+		_render_interval = std::min<float>(_render_interval, dvt_interval);
 
 		_render_interval = std::clamp(
 			_render_interval,
@@ -510,7 +496,9 @@ Screen* MainScreen::render(float time_delta, bool&) {
 		);
 	// mid delay time window
 	} else if (!_window_hidden && _time_since_event < curr_profile.mid_delay_window) {
-		_render_interval = std::min<float>(_render_interval, animation_interval);
+		_render_interval = std::min<float>(_render_interval, ctc_interval);
+		_render_interval = std::min<float>(_render_interval, msgtc_interval);
+		_render_interval = std::min<float>(_render_interval, dvt_interval);
 
 		_render_interval = std::clamp(
 			_render_interval,
@@ -533,18 +521,22 @@ Screen* MainScreen::render(float time_delta, bool&) {
 }
 
 Screen* MainScreen::tick(float time_delta, bool& quit) {
+	const float sm_interval = sm.tick(time_delta);
+
 	quit = !tc.iterate(time_delta); // compute
 
 #if TOMATO_TOX_AV
 	tav.toxavIterate();
+	// breaks it
+	// HACK: pow by 1.18 to increase 200 -> ~500
+	//const float av_interval = std::pow(tav.toxavIterationInterval(), 1.18)/1000.f;
 	const float av_interval = tav.toxavIterationInterval()/1000.f;
-	dtc.tick(time_delta);
+
+	tavvoip.tick();
 #endif
 
 	tcm.iterate(time_delta); // compute
 
-	const float sm_interval = sm.tick(time_delta);
-
 	const float fo_interval = tffom.tick(time_delta);
 
 	tam.iterate(); // compute
@@ -572,6 +564,10 @@ Screen* MainScreen::tick(float time_delta, bool& quit) {
 		std::pow(tc.toxIterationInterval(), 1.6f)/1000.f,
 		pm_interval
 	);
+	_min_tick_interval = std::min<float>(
+		_min_tick_interval,
+		sm_interval
+	);
 	_min_tick_interval = std::min<float>(
 		_min_tick_interval,
 		fo_interval
@@ -584,11 +580,6 @@ Screen* MainScreen::tick(float time_delta, bool& quit) {
 	);
 #endif
 
-	_min_tick_interval = std::min<float>(
-		_min_tick_interval,
-		sm_interval
-	);
-
 	//std::cout << "MS: min tick interval: " << _min_tick_interval << "\n";
 
 	switch (_compute_perf_mode) {

src/main_screen.hpp

@@ -11,13 +11,12 @@
 #include <solanaceae/plugin/plugin_manager.hpp>
 #include <solanaceae/toxcore/tox_event_logger.hpp>
 #include "./tox_private_impl.hpp"
+#include "./frame_streams/stream_manager.hpp"
 
 #include <solanaceae/tox_contacts/tox_contact_model2.hpp>
 #include <solanaceae/tox_messages/tox_message_manager.hpp>
 #include <solanaceae/tox_messages/tox_transfer_manager.hpp>
 
-#include "./stream_manager.hpp"
-
 #include "./tox_client.hpp"
 #include "./auto_dirty.hpp"
 
@@ -32,15 +31,15 @@
 #include "./chat_gui4.hpp"
 #include "./chat_gui/settings_window.hpp"
 #include "./object_store_ui.hpp"
-#include "./stream_manager_ui.hpp"
-#include "./debug_video_tap.hpp"
 #include "./tox_ui_utils.hpp"
 #include "./tox_dht_cap_histo.hpp"
 #include "./tox_friend_faux_offline_messaging.hpp"
+#include "./stream_manager_ui.hpp"
+#include "./debug_video_tap.hpp"
 
 #if TOMATO_TOX_AV
 #include "./tox_av.hpp"
-#include "./debug_tox_call.hpp"
+#include "./tox_av_voip_model.hpp"
 #endif
 
 #include <string>
@@ -69,14 +68,14 @@ struct MainScreen final : public Screen {
 	ToxClient tc;
 	ToxPrivateImpl tpi;
 	AutoDirty ad;
-#if TOMATO_TOX_AV
-	ToxAV tav;
-	DebugToxCall dtc;
-#endif
 	ToxContactModel2 tcm;
 	ToxMessageManager tmm;
 	ToxTransferManager ttm;
 	ToxFriendFauxOfflineMessaging tffom;
+#if TOMATO_TOX_AV
+	ToxAVI tav;
+	ToxAVVoIPModel tavvoip;
+#endif
 
 	Theme& theme;
 
@@ -94,10 +93,10 @@ struct MainScreen final : public Screen {
 	ChatGui4 cg;
 	SettingsWindow sw;
 	ObjectStoreUI osui;
-	StreamManagerUI smui;
-	DebugVideoTap dvt;
 	ToxUIUtils tuiu;
 	ToxDHTCapHisto tdch;
+	StreamManagerUI smui;
+	DebugVideoTap dvt;
 
 	PluginManager pm; // last, so it gets destroyed first
 

src/stream_manager.hpp

@@ -1,320 +0,0 @@
-#pragma once
-
-#include <solanaceae/object_store/fwd.hpp>
-#include <solanaceae/object_store/object_store.hpp>
-
-#include <entt/core/type_info.hpp>
-
-#include "./content/frame_stream2.hpp"
-
-#include <vector>
-#include <memory>
-#include <algorithm>
-#include <thread>
-#include <chrono>
-
-// fwd
-class StreamManager;
-
-namespace Components {
-	struct StreamSource {
-		std::string name;
-		std::string frame_type_name;
-
-		std::function<bool(StreamManager&, Object, Object, bool)> connect_fn;
-
-		template<typename FrameType>
-		static StreamSource create(const std::string& name);
-	};
-
-	struct StreamSink {
-		std::string name;
-		std::string frame_type_name;
-
-		template<typename FrameType>
-		static StreamSink create(const std::string& name);
-	};
-
-	template<typename FrameType>
-	using FrameStream2Source = std::unique_ptr<FrameStream2SourceI<FrameType>>;
-
-	template<typename FrameType>
-	using FrameStream2Sink = std::unique_ptr<FrameStream2SinkI<FrameType>>;
-
-} // Components
-
-
-class StreamManager {
-	friend class StreamManagerUI; // TODO: make this go away
-	ObjectStore2& _os;
-
-	struct Connection {
-		ObjectHandle src;
-		ObjectHandle sink;
-
-		struct Data {
-			virtual ~Data(void) {}
-		};
-		std::unique_ptr<Data> data; // stores reader writer type erased
-		std::function<void(Connection&)> pump_fn;
-		std::function<void(Connection&)> unsubscribe_fn;
-
-		bool on_main_thread {true};
-		std::atomic_bool stop {false}; // disconnect
-		std::atomic_bool finished {false}; // disconnect
-
-		// pump thread
-		std::thread pump_thread;
-
-		// frame interval counters and estimates
-
-		Connection(void) = default;
-		Connection(
-			ObjectHandle src_,
-			ObjectHandle sink_,
-			std::unique_ptr<Data>&& data_,
-			std::function<void(Connection&)>&& pump_fn_,
-			std::function<void(Connection&)>&& unsubscribe_fn_,
-			bool on_main_thread_ = true
-		) :
-			src(src_),
-			sink(sink_),
-			data(std::move(data_)),
-			pump_fn(std::move(pump_fn_)),
-			unsubscribe_fn(std::move(unsubscribe_fn_)),
-			on_main_thread(on_main_thread_)
-		{
-			if (!on_main_thread) {
-				// start thread
-				pump_thread = std::thread([this](void) {
-					while (!stop) {
-						pump_fn(*this);
-						std::this_thread::sleep_for(std::chrono::milliseconds(5));
-					}
-					finished = true;
-				});
-			}
-		}
-	};
-	std::vector<std::unique_ptr<Connection>> _connections;
-
-	public:
-		StreamManager(ObjectStore2& os) : _os(os) {}
-		virtual ~StreamManager(void) {
-			// stop all connetions
-			for (const auto& con : _connections) {
-				con->stop = true;
-				if (!con->on_main_thread) {
-					con->pump_thread.join(); // we skip the finished check and wait
-				}
-				con->unsubscribe_fn(*con);
-			}
-		}
-
-		// stream type is FrameStream2I<FrameType>
-		// TODO: improve this design
-		// src and sink need to be a FrameStream2MultiStream<FrameType>
-		template<typename FrameType>
-		bool connect(Object src, Object sink, bool threaded = true) {
-			auto res = std::find_if(
-				_connections.cbegin(), _connections.cend(),
-				[&](const auto& a) { return a->src == src && a->sink == sink; }
-			);
-			if (res != _connections.cend()) {
-				// already exists
-				return false;
-			}
-
-			auto h_src = _os.objectHandle(src);
-			auto h_sink = _os.objectHandle(sink);
-			if (!static_cast<bool>(h_src) || !static_cast<bool>(h_sink)) {
-				// an object does not exist
-				return false;
-			}
-
-			if (!h_src.all_of<Components::FrameStream2Source<FrameType>>()) {
-				// src not stream source
-				return false;
-			}
-
-			if (!h_sink.all_of<Components::FrameStream2Sink<FrameType>>()) {
-				// sink not stream sink
-				return false;
-			}
-
-			auto& src_stream = h_src.get<Components::FrameStream2Source<FrameType>>();
-			auto& sink_stream = h_sink.get<Components::FrameStream2Sink<FrameType>>();
-
-			struct inlineData : public Connection::Data {
-				virtual ~inlineData(void) {}
-				std::shared_ptr<FrameStream2I<FrameType>> reader;
-				std::shared_ptr<FrameStream2I<FrameType>> writer;
-			};
-
-			auto our_data = std::make_unique<inlineData>();
-
-			our_data->reader = src_stream->subscribe();
-			if (!our_data->reader) {
-				return false;
-			}
-			our_data->writer = sink_stream->subscribe();
-			if (!our_data->writer) {
-				return false;
-			}
-
-			_connections.push_back(std::make_unique<Connection>(
-				h_src,
-				h_sink,
-				std::move(our_data),
-				[](Connection& con) -> void {
-					// there might be more stored
-					for (size_t i = 0; i < 10; i++) {
-						auto new_frame_opt = static_cast<inlineData*>(con.data.get())->reader->pop();
-						// TODO: frame interval estimates
-						if (new_frame_opt.has_value()) {
-							static_cast<inlineData*>(con.data.get())->writer->push(new_frame_opt.value());
-						} else {
-							break;
-						}
-					}
-				},
-				[](Connection& con) -> void {
-					auto* src_stream_ptr = con.src.try_get<Components::FrameStream2Source<FrameType>>();
-					if (src_stream_ptr != nullptr) {
-						(*src_stream_ptr)->unsubscribe(static_cast<inlineData*>(con.data.get())->reader);
-					}
-					auto* sink_stream_ptr = con.sink.try_get<Components::FrameStream2Sink<FrameType>>();
-					if (sink_stream_ptr != nullptr) {
-						(*sink_stream_ptr)->unsubscribe(static_cast<inlineData*>(con.data.get())->writer);
-					}
-				},
-				!threaded
-			));
-
-			return true;
-		}
-
-		bool connect(Object src, Object sink, bool threaded = true) {
-			auto h_src = _os.objectHandle(src);
-			auto h_sink = _os.objectHandle(sink);
-			if (!static_cast<bool>(h_src) || !static_cast<bool>(h_sink)) {
-				// an object does not exist
-				return false;
-			}
-
-			// get src and sink comps
-			if (!h_src.all_of<Components::StreamSource>()) {
-				// src not stream source
-				return false;
-			}
-
-			if (!h_sink.all_of<Components::StreamSink>()) {
-				// sink not stream sink
-				return false;
-			}
-
-			const auto& ssrc = h_src.get<Components::StreamSource>();
-			const auto& ssink = h_sink.get<Components::StreamSink>();
-
-			// compare type
-			if (ssrc.frame_type_name != ssink.frame_type_name) {
-				return false;
-			}
-
-			// always fail in debug mode
-			assert(static_cast<bool>(ssrc.connect_fn));
-			if (!static_cast<bool>(ssrc.connect_fn)) {
-				return false;
-			}
-
-			// use connect fn from src
-			return ssrc.connect_fn(*this, src, sink, threaded);
-		}
-
-		template<typename StreamType>
-		bool disconnect(Object src, Object sink) {
-			auto res = std::find_if(
-				_connections.cbegin(), _connections.cend(),
-				[&](const auto& a) { return a->src == src && a->sink == sink; }
-			);
-			if (res == _connections.cend()) {
-				// not found
-				return false;
-			}
-
-			// do disconnect
-			(*res)->stop = true;
-
-			return true;
-		}
-
-		template<typename StreamType>
-		bool disconnectAll(Object o) {
-			bool succ {false};
-			for (const auto& con : _connections) {
-				if (con->src == o || con->sink == o) {
-					con->stop = true;
-					succ = true;
-				}
-			}
-
-			return succ;
-		}
-
-		// do we need the time delta?
-		float tick(float) {
-			// pump all mainthread connections
-			for (auto it = _connections.begin(); it != _connections.end();) {
-				auto& con = **it;
-
-				if (!static_cast<bool>(con.src) || !static_cast<bool>(con.sink)) {
-					// either side disappeard without disconnectAll
-					// TODO: warn/error log
-					con.stop = true;
-				}
-
-				if (con.on_main_thread) {
-					con.pump_fn(con);
-				}
-
-				if (con.stop && (con.finished || con.on_main_thread)) {
-					if (!con.on_main_thread) {
-						assert(con.pump_thread.joinable());
-						con.pump_thread.join();
-					}
-					con.unsubscribe_fn(con);
-					it = _connections.erase(it);
-				} else {
-					it++;
-				}
-			}
-
-			// return min over intervals instead
-			return 0.01f;
-		}
-};
-
-namespace Components {
-
-	// we require the complete sm type here
-	template<typename FrameType>
-	StreamSource StreamSource::create(const std::string& name) {
-		return StreamSource{
-			name,
-			std::string{entt::type_name<FrameType>::value()},
-			+[](StreamManager& sm, Object src, Object sink, bool threaded) {
-				return sm.connect<FrameType>(src, sink, threaded);
-			},
-		};
-	}
-
-	template<typename FrameType>
-	StreamSink StreamSink::create(const std::string& name) {
-		return StreamSink{
-			name,
-			std::string{entt::type_name<FrameType>::value()},
-		};
-	}
-
-} // Components
-

src/stream_manager_ui.cpp

@@ -56,6 +56,12 @@ void StreamManagerUI::render(void) {
 					ImGui::TableNextColumn();
 					ImGui::Text("%d", entt::to_integral(entt::to_entity(oc)));
 
+					if (_os.registry().all_of<Components::TagDefaultTarget>(oc)) {
+						ImGui::TableSetBgColor(ImGuiTableBgTarget_RowBg1, ImGui::GetColorU32(ImVec4{0.6f, 0.f, 0.6f, 0.25f}));
+					} else if (_os.registry().all_of<Components::TagConnectToDefault>(oc)) {
+						ImGui::TableSetBgColor(ImGuiTableBgTarget_RowBg1, ImGui::GetColorU32(ImVec4{0.6f, 0.6f, 0.f, 0.25f}));
+					}
+
 					const auto *ssrc = _os.registry().try_get<Components::StreamSource>(oc);
 					ImGui::TableNextColumn();
 					ImGui::TextUnformatted(ssrc!=nullptr?ssrc->name.c_str():"none");
@@ -118,6 +124,12 @@ void StreamManagerUI::render(void) {
 					ImGui::TableNextColumn();
 					ImGui::Text("%d", entt::to_integral(entt::to_entity(oc)));
 
+					if (_os.registry().all_of<Components::TagDefaultTarget>(oc)) {
+						ImGui::TableSetBgColor(ImGuiTableBgTarget_RowBg1, ImGui::GetColorU32(ImVec4{0.6f, 0.f, 0.6f, 0.25f}));
+					} else if (_os.registry().all_of<Components::TagConnectToDefault>(oc)) {
+						ImGui::TableSetBgColor(ImGuiTableBgTarget_RowBg1, ImGui::GetColorU32(ImVec4{0.6f, 0.6f, 0.f, 0.25f}));
+					}
+
 					const auto *ssink = _os.registry().try_get<Components::StreamSink>(oc);
 					ImGui::TableNextColumn();
 					ImGui::TextUnformatted(ssink!=nullptr?ssink->name.c_str():"none");

src/stream_manager_ui.hpp

@@ -1,13 +1,13 @@
 #pragma once
 
 #include <solanaceae/object_store/fwd.hpp>
-#include "./stream_manager.hpp"
+#include "./frame_streams/stream_manager.hpp"
 
 class StreamManagerUI {
 	ObjectStore2& _os;
 	StreamManager& _sm;
 
-	bool _show_window {true};
+	bool _show_window {false};
 
 	public:
 		StreamManagerUI(ObjectStore2& os, StreamManager& sm);

src/tox_av.cpp

@@ -7,18 +7,7 @@
 
 // https://almogfx.bandcamp.com/track/crushed-w-cassade
 
-struct ToxAVFriendCallState final {
+ToxAVI::ToxAVI(Tox* tox) : _tox(tox) {
-	const uint32_t state {TOXAV_FRIEND_CALL_STATE_NONE};
-
-	[[nodiscard]] bool is_error(void) const { return state & TOXAV_FRIEND_CALL_STATE_ERROR; }
-	[[nodiscard]] bool is_finished(void) const { return state & TOXAV_FRIEND_CALL_STATE_FINISHED; }
-	[[nodiscard]] bool is_sending_a(void) const { return state & TOXAV_FRIEND_CALL_STATE_SENDING_A; }
-	[[nodiscard]] bool is_sending_v(void) const { return state & TOXAV_FRIEND_CALL_STATE_SENDING_V; }
-	[[nodiscard]] bool is_accepting_a(void) const { return state & TOXAV_FRIEND_CALL_STATE_ACCEPTING_A; }
-	[[nodiscard]] bool is_accepting_v(void) const { return state & TOXAV_FRIEND_CALL_STATE_ACCEPTING_V; }
-};
-
-ToxAV::ToxAV(Tox* tox) : _tox(tox) {
 	Toxav_Err_New err_new {TOXAV_ERR_NEW_OK};
 	_tox_av = toxav_new(_tox, &err_new);
 	// TODO: throw
@@ -28,7 +17,7 @@ ToxAV::ToxAV(Tox* tox) : _tox(tox) {
 		_tox_av,
 		+[](ToxAV*, uint32_t friend_number, bool audio_enabled, bool video_enabled, void *user_data) {
 			assert(user_data != nullptr);
-			static_cast<ToxAV*>(user_data)->cb_call(friend_number, audio_enabled, video_enabled);
+			static_cast<ToxAVI*>(user_data)->cb_call(friend_number, audio_enabled, video_enabled);
 		},
 		this
 	);
@@ -36,7 +25,7 @@ ToxAV::ToxAV(Tox* tox) : _tox(tox) {
 		_tox_av,
 		+[](ToxAV*, uint32_t friend_number, uint32_t state, void *user_data) {
 			assert(user_data != nullptr);
-			static_cast<ToxAV*>(user_data)->cb_call_state(friend_number, state);
+			static_cast<ToxAVI*>(user_data)->cb_call_state(friend_number, state);
 		},
 		this
 	);
@@ -44,7 +33,7 @@ ToxAV::ToxAV(Tox* tox) : _tox(tox) {
 		_tox_av,
 		+[](ToxAV*, uint32_t friend_number, uint32_t audio_bit_rate, void *user_data) {
 			assert(user_data != nullptr);
-			static_cast<ToxAV*>(user_data)->cb_audio_bit_rate(friend_number, audio_bit_rate);
+			static_cast<ToxAVI*>(user_data)->cb_audio_bit_rate(friend_number, audio_bit_rate);
 		},
 		this
 	);
@@ -52,7 +41,7 @@ ToxAV::ToxAV(Tox* tox) : _tox(tox) {
 		_tox_av,
 		+[](ToxAV*, uint32_t friend_number, uint32_t video_bit_rate, void *user_data) {
 			assert(user_data != nullptr);
-			static_cast<ToxAV*>(user_data)->cb_video_bit_rate(friend_number, video_bit_rate);
+			static_cast<ToxAVI*>(user_data)->cb_video_bit_rate(friend_number, video_bit_rate);
 		},
 		this
 	);
@@ -60,7 +49,7 @@ ToxAV::ToxAV(Tox* tox) : _tox(tox) {
 		_tox_av,
 		+[](ToxAV*, uint32_t friend_number, const int16_t pcm[], size_t sample_count, uint8_t channels, uint32_t sampling_rate, void *user_data) {
 			assert(user_data != nullptr);
-			static_cast<ToxAV*>(user_data)->cb_audio_receive_frame(friend_number, pcm, sample_count, channels, sampling_rate);
+			static_cast<ToxAVI*>(user_data)->cb_audio_receive_frame(friend_number, pcm, sample_count, channels, sampling_rate);
 		},
 		this
 	);
@@ -75,83 +64,83 @@ ToxAV::ToxAV(Tox* tox) : _tox(tox) {
 			void *user_data
 		) {
 			assert(user_data != nullptr);
-			static_cast<ToxAV*>(user_data)->cb_video_receive_frame(friend_number, width, height, y, u, v, ystride, ustride, vstride);
+			static_cast<ToxAVI*>(user_data)->cb_video_receive_frame(friend_number, width, height, y, u, v, ystride, ustride, vstride);
 		},
 		this
 	);
 }
 
-ToxAV::~ToxAV(void) {
+ToxAVI::~ToxAVI(void) {
 	toxav_kill(_tox_av);
 }
 
-uint32_t ToxAV::toxavIterationInterval(void) const {
+uint32_t ToxAVI::toxavIterationInterval(void) const {
 	return toxav_iteration_interval(_tox_av);
 }
 
-void ToxAV::toxavIterate(void) {
+void ToxAVI::toxavIterate(void) {
 	toxav_iterate(_tox_av);
 }
 
-uint32_t ToxAV::toxavAudioIterationInterval(void) const {
+uint32_t ToxAVI::toxavAudioIterationInterval(void) const {
 	return toxav_audio_iteration_interval(_tox_av);
 }
 
-void ToxAV::toxavAudioIterate(void) {
+void ToxAVI::toxavAudioIterate(void) {
 	toxav_audio_iterate(_tox_av);
 }
 
-uint32_t ToxAV::toxavVideoIterationInterval(void) const {
+uint32_t ToxAVI::toxavVideoIterationInterval(void) const {
 	return toxav_video_iteration_interval(_tox_av);
 }
 
-void ToxAV::toxavVideoIterate(void) {
+void ToxAVI::toxavVideoIterate(void) {
 	toxav_video_iterate(_tox_av);
 }
 
-Toxav_Err_Call ToxAV::toxavCall(uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate) {
+Toxav_Err_Call ToxAVI::toxavCall(uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate) {
 	Toxav_Err_Call err {TOXAV_ERR_CALL_OK};
 	toxav_call(_tox_av, friend_number, audio_bit_rate, video_bit_rate, &err);
 	return err;
 }
 
-Toxav_Err_Answer ToxAV::toxavAnswer(uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate) {
+Toxav_Err_Answer ToxAVI::toxavAnswer(uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate) {
 	Toxav_Err_Answer err {TOXAV_ERR_ANSWER_OK};
 	toxav_answer(_tox_av, friend_number, audio_bit_rate, video_bit_rate, &err);
 	return err;
 }
 
-Toxav_Err_Call_Control ToxAV::toxavCallControl(uint32_t friend_number, Toxav_Call_Control control) {
+Toxav_Err_Call_Control ToxAVI::toxavCallControl(uint32_t friend_number, Toxav_Call_Control control) {
 	Toxav_Err_Call_Control err {TOXAV_ERR_CALL_CONTROL_OK};
 	toxav_call_control(_tox_av, friend_number, control, &err);
 	return err;
 }
 
-Toxav_Err_Send_Frame ToxAV::toxavAudioSendFrame(uint32_t friend_number, const int16_t pcm[], size_t sample_count, uint8_t channels, uint32_t sampling_rate) {
+Toxav_Err_Send_Frame ToxAVI::toxavAudioSendFrame(uint32_t friend_number, const int16_t pcm[], size_t sample_count, uint8_t channels, uint32_t sampling_rate) {
 	Toxav_Err_Send_Frame err {TOXAV_ERR_SEND_FRAME_OK};
 	toxav_audio_send_frame(_tox_av, friend_number, pcm, sample_count, channels, sampling_rate, &err);
 	return err;
 }
 
-Toxav_Err_Bit_Rate_Set ToxAV::toxavAudioSetBitRate(uint32_t friend_number, uint32_t bit_rate) {
+Toxav_Err_Bit_Rate_Set ToxAVI::toxavAudioSetBitRate(uint32_t friend_number, uint32_t bit_rate) {
 	Toxav_Err_Bit_Rate_Set err {TOXAV_ERR_BIT_RATE_SET_OK};
 	toxav_audio_set_bit_rate(_tox_av, friend_number, bit_rate, &err);
 	return err;
 }
 
-Toxav_Err_Send_Frame ToxAV::toxavVideoSendFrame(uint32_t friend_number, uint16_t width, uint16_t height, const uint8_t y[], const uint8_t u[], const uint8_t v[]) {
+Toxav_Err_Send_Frame ToxAVI::toxavVideoSendFrame(uint32_t friend_number, uint16_t width, uint16_t height, const uint8_t y[], const uint8_t u[], const uint8_t v[]) {
 	Toxav_Err_Send_Frame err {TOXAV_ERR_SEND_FRAME_OK};
 	toxav_video_send_frame(_tox_av, friend_number, width, height, y, u, v, &err);
 	return err;
 }
 
-Toxav_Err_Bit_Rate_Set ToxAV::toxavVideoSetBitRate(uint32_t friend_number, uint32_t bit_rate) {
+Toxav_Err_Bit_Rate_Set ToxAVI::toxavVideoSetBitRate(uint32_t friend_number, uint32_t bit_rate) {
 	Toxav_Err_Bit_Rate_Set err {TOXAV_ERR_BIT_RATE_SET_OK};
 	toxav_video_set_bit_rate(_tox_av, friend_number, bit_rate, &err);
 	return err;
 }
 
-void ToxAV::cb_call(uint32_t friend_number, bool audio_enabled, bool video_enabled) {
+void ToxAVI::cb_call(uint32_t friend_number, bool audio_enabled, bool video_enabled) {
 	std::cerr << "TOXAV: receiving call f:" << friend_number << " a:" << audio_enabled << " v:" << video_enabled << "\n";
 	//Toxav_Err_Answer err_answer { TOXAV_ERR_ANSWER_OK };
 	//toxav_answer(_tox_av, friend_number, 0, 0, &err_answer);
@@ -169,7 +158,7 @@ void ToxAV::cb_call(uint32_t friend_number, bool audio_enabled, bool video_enabl
 	);
 }
 
-void ToxAV::cb_call_state(uint32_t friend_number, uint32_t state) {
+void ToxAVI::cb_call_state(uint32_t friend_number, uint32_t state) {
 	//ToxAVFriendCallState w_state{state};
 
 	//w_state.is_error();
@@ -185,7 +174,7 @@ void ToxAV::cb_call_state(uint32_t friend_number, uint32_t state) {
 	);
 }
 
-void ToxAV::cb_audio_bit_rate(uint32_t friend_number, uint32_t audio_bit_rate) {
+void ToxAVI::cb_audio_bit_rate(uint32_t friend_number, uint32_t audio_bit_rate) {
 	std::cerr << "TOXAV: audio bitrate f:" << friend_number << " abr:" << audio_bit_rate << "\n";
 
 	dispatch(
@@ -197,7 +186,7 @@ void ToxAV::cb_audio_bit_rate(uint32_t friend_number, uint32_t audio_bit_rate) {
 	);
 }
 
-void ToxAV::cb_video_bit_rate(uint32_t friend_number, uint32_t video_bit_rate) {
+void ToxAVI::cb_video_bit_rate(uint32_t friend_number, uint32_t video_bit_rate) {
 	std::cerr << "TOXAV: video bitrate f:" << friend_number << " vbr:" << video_bit_rate << "\n";
 
 	dispatch(
@@ -209,7 +198,7 @@ void ToxAV::cb_video_bit_rate(uint32_t friend_number, uint32_t video_bit_rate) {
 	);
 }
 
-void ToxAV::cb_audio_receive_frame(uint32_t friend_number, const int16_t pcm[], size_t sample_count, uint8_t channels, uint32_t sampling_rate) {
+void ToxAVI::cb_audio_receive_frame(uint32_t friend_number, const int16_t pcm[], size_t sample_count, uint8_t channels, uint32_t sampling_rate) {
 	//std::cerr << "TOXAV: audio frame f:" <<  friend_number << " sc:" << sample_count << " ch:" << (int)channels << " sr:" << sampling_rate << "\n";
 
 	dispatch(
@@ -223,7 +212,7 @@ void ToxAV::cb_audio_receive_frame(uint32_t friend_number, const int16_t pcm[],
 	);
 }
 
-void ToxAV::cb_video_receive_frame(
+void ToxAVI::cb_video_receive_frame(
 	uint32_t friend_number,
 	uint16_t width, uint16_t height,
 	const uint8_t y[/*! max(width, abs(ystride)) * height */],

src/tox_av.hpp

@@ -82,14 +82,15 @@ struct ToxAVEventI {
 };
 using ToxAVEventProviderI = EventProviderI<ToxAVEventI>;
 
-struct ToxAV : public ToxAVEventProviderI{
+// TODO: seperate out implementation from interface
+struct ToxAVI : public ToxAVEventProviderI {
 	Tox* _tox = nullptr;
 	ToxAV* _tox_av = nullptr;
 
 	static constexpr const char* version {"0"};
 
-	ToxAV(Tox* tox);
+	ToxAVI(Tox* tox);
-	virtual ~ToxAV(void);
+	virtual ~ToxAVI(void);
 
 	// interface
 	// if iterate is called on a different thread, it will fire events there
@@ -134,3 +135,14 @@ struct ToxAV : public ToxAVEventProviderI{
 	);
 };
 
+struct ToxAVFriendCallState final {
+	const uint32_t state {TOXAV_FRIEND_CALL_STATE_NONE};
+
+	[[nodiscard]] bool is_error(void) const { return state & TOXAV_FRIEND_CALL_STATE_ERROR; }
+	[[nodiscard]] bool is_finished(void) const { return state & TOXAV_FRIEND_CALL_STATE_FINISHED; }
+	[[nodiscard]] bool is_sending_a(void) const { return state & TOXAV_FRIEND_CALL_STATE_SENDING_A; }
+	[[nodiscard]] bool is_sending_v(void) const { return state & TOXAV_FRIEND_CALL_STATE_SENDING_V; }
+	[[nodiscard]] bool is_accepting_a(void) const { return state & TOXAV_FRIEND_CALL_STATE_ACCEPTING_A; }
+	[[nodiscard]] bool is_accepting_v(void) const { return state & TOXAV_FRIEND_CALL_STATE_ACCEPTING_V; }
+};
+

src/tox_av_voip_model.cpp

@@ -0,0 +1,476 @@
+#include "./tox_av_voip_model.hpp"
+
+#include <solanaceae/object_store/object_store.hpp>
+#include <solanaceae/tox_contacts/components.hpp>
+
+#include "./frame_streams/stream_manager.hpp"
+#include "./frame_streams/audio_stream2.hpp"
+#include "./frame_streams/locked_frame_stream.hpp"
+#include "./frame_streams/multi_source.hpp"
+#include "./frame_streams/audio_stream_pop_reframer.hpp"
+
+#include <iostream>
+
+namespace Components {
+	struct ToxAVIncomingAV {
+		bool incoming_audio {false};
+		bool incoming_video {false};
+	};
+
+	struct ToxAVAudioSink {
+		ObjectHandle o;
+		// ptr?
+	};
+	// vid
+	struct ToxAVAudioSource {
+		ObjectHandle o;
+		// ptr?
+	};
+	// vid
+} // Components
+
+struct ToxAVCallAudioSink : public FrameStream2SinkI<AudioFrame2> {
+	ToxAVI& _toxav;
+
+	// bitrate for enabled state
+	uint32_t _audio_bitrate {32};
+
+	uint32_t _fid;
+	std::shared_ptr<AudioStreamPopReFramer<LockedFrameStream2<AudioFrame2>>> _writer;
+
+	ToxAVCallAudioSink(ToxAVI& toxav, uint32_t fid) : _toxav(toxav), _fid(fid) {}
+	~ToxAVCallAudioSink(void) {
+		if (_writer) {
+			_writer = nullptr;
+			_toxav.toxavAudioSetBitRate(_fid, 0);
+		}
+	}
+
+	// sink
+	std::shared_ptr<FrameStream2I<AudioFrame2>> subscribe(void) override {
+		if (_writer) {
+			// max 1 (exclusive for now)
+			return nullptr;
+		}
+
+		auto err = _toxav.toxavAudioSetBitRate(_fid, _audio_bitrate);
+		if (err != TOXAV_ERR_BIT_RATE_SET_OK) {
+			return nullptr;
+		}
+
+		// 20ms for now, 10ms would work too, further investigate stutters at 5ms (probably too slow interval rate)
+		_writer = std::make_shared<AudioStreamPopReFramer<LockedFrameStream2<AudioFrame2>>>(20);
+
+		return _writer;
+	}
+
+	bool unsubscribe(const std::shared_ptr<FrameStream2I<AudioFrame2>>& sub) override {
+		if (!sub || !_writer) {
+			// nah
+			return false;
+		}
+
+		if (sub == _writer) {
+			_writer = nullptr;
+
+			/*auto err = */_toxav.toxavAudioSetBitRate(_fid, 0);
+			// print warning? on error?
+
+			return true;
+		}
+
+		// what
+		return false;
+	}
+};
+
+void ToxAVVoIPModel::addAudioSource(ObjectHandle session, uint32_t friend_number) {
+	auto& stream_source = session.get_or_emplace<Components::VoIP::StreamSources>().streams;
+
+	ObjectHandle incoming_audio {_os.registry(), _os.registry().create()};
+
+	auto new_asrc = std::make_unique<FrameStream2MultiSource<AudioFrame2>>();
+	incoming_audio.emplace<FrameStream2MultiSource<AudioFrame2>*>(new_asrc.get());
+	incoming_audio.emplace<Components::FrameStream2Source<AudioFrame2>>(std::move(new_asrc));
+	incoming_audio.emplace<Components::StreamSource>(Components::StreamSource::create<AudioFrame2>("ToxAV Friend Call Incoming Audio"));
+
+	std::cout << "new incoming audio\n";
+	if (
+		const auto* defaults = session.try_get<Components::VoIP::DefaultConfig>();
+		defaults != nullptr && defaults->incoming_audio
+	) {
+		incoming_audio.emplace<Components::TagConnectToDefault>(); // depends on what was specified in enter()
+		std::cout << "with default\n";
+	}
+
+	stream_source.push_back(incoming_audio);
+	session.emplace<Components::ToxAVAudioSource>(incoming_audio);
+	// TODO: tie session to stream
+
+	_audio_sources[friend_number] = incoming_audio;
+
+	_os.throwEventConstruct(incoming_audio);
+}
+
+void ToxAVVoIPModel::addAudioSink(ObjectHandle session, uint32_t friend_number) {
+	auto& stream_sinks = session.get_or_emplace<Components::VoIP::StreamSinks>().streams;
+	ObjectHandle outgoing_audio {_os.registry(), _os.registry().create()};
+
+	auto new_asink = std::make_unique<ToxAVCallAudioSink>(_av, friend_number);
+	outgoing_audio.emplace<ToxAVCallAudioSink*>(new_asink.get());
+	outgoing_audio.emplace<Components::FrameStream2Sink<AudioFrame2>>(std::move(new_asink));
+	outgoing_audio.emplace<Components::StreamSink>(Components::StreamSink::create<AudioFrame2>("ToxAV Friend Call Outgoing Audio"));
+
+	if (
+		const auto* defaults = session.try_get<Components::VoIP::DefaultConfig>();
+		defaults != nullptr && defaults->outgoing_audio
+	) {
+		outgoing_audio.emplace<Components::TagConnectToDefault>(); // depends on what was specified in enter()
+	}
+
+	stream_sinks.push_back(outgoing_audio);
+	session.emplace<Components::ToxAVAudioSink>(outgoing_audio);
+	// TODO: tie session to stream
+
+	_os.throwEventConstruct(outgoing_audio);
+}
+
+void ToxAVVoIPModel::destroySession(ObjectHandle session) {
+	if (!static_cast<bool>(session)) {
+		return;
+	}
+
+	// remove lookup
+	if (session.all_of<Components::ToxAVAudioSource>()) {
+		auto it_asrc = std::find_if(
+			_audio_sources.cbegin(), _audio_sources.cend(),
+			[o = session.get<Components::ToxAVAudioSource>().o](const auto& it) {
+				return it.second == o;
+			}
+		);
+
+		if (it_asrc != _audio_sources.cend()) {
+			_audio_sources.erase(it_asrc);
+		}
+	}
+
+	// destory sources
+	if (auto* ss = session.try_get<Components::VoIP::StreamSources>(); ss != nullptr) {
+		for (const auto ssov : ss->streams) {
+
+			_os.throwEventDestroy(ssov);
+			_os.registry().destroy(ssov);
+		}
+	}
+
+	// destory sinks
+	if (auto* ss = session.try_get<Components::VoIP::StreamSinks>(); ss != nullptr) {
+		for (const auto ssov : ss->streams) {
+
+			_os.throwEventDestroy(ssov);
+			_os.registry().destroy(ssov);
+		}
+	}
+
+	// destory session
+	_os.throwEventDestroy(session);
+	_os.registry().destroy(session);
+}
+
+ToxAVVoIPModel::ToxAVVoIPModel(ObjectStore2& os, ToxAVI& av, Contact3Registry& cr, ToxContactModel2& tcm) :
+	_os(os), _av(av), _cr(cr), _tcm(tcm)
+{
+	_av.subscribe(this, ToxAV_Event::friend_call);
+	_av.subscribe(this, ToxAV_Event::friend_call_state);
+	_av.subscribe(this, ToxAV_Event::friend_audio_bitrate);
+	_av.subscribe(this, ToxAV_Event::friend_video_bitrate);
+	_av.subscribe(this, ToxAV_Event::friend_audio_frame);
+	_av.subscribe(this, ToxAV_Event::friend_video_frame);
+
+	// attach to all tox friend contacts
+
+	for (const auto& [cv, _] : _cr.view<Contact::Components::ToxFriendPersistent>().each()) {
+		_cr.emplace<VoIPModelI*>(cv, this);
+	}
+	// TODO: events
+}
+
+ToxAVVoIPModel::~ToxAVVoIPModel(void) {
+	for (const auto& [ov, voipmodel] : _os.registry().view<VoIPModelI*>().each()) {
+		if (voipmodel == this) {
+			destroySession(_os.objectHandle(ov));
+		}
+	}
+}
+
+void ToxAVVoIPModel::tick(void) {
+	for (const auto& [oc, asink] : _os.registry().view<ToxAVCallAudioSink*>().each()) {
+		if (!asink->_writer) {
+			continue;
+		}
+
+		for (size_t i = 0; i < 100; i++) {
+			auto new_frame_opt = asink->_writer->pop();
+			if (!new_frame_opt.has_value()) {
+				break;
+			}
+			const auto& new_frame = new_frame_opt.value();
+
+			//* @param sample_count Number of samples in this frame. Valid numbers here are
+			//*   `((sample rate) * (audio length) / 1000)`, where audio length can be
+			//*   2.5, 5, 10, 20, 40 or 60 milliseconds.
+
+			// we likely needs to subdivide/repackage
+			// frame size should be an option exposed to the user
+			// with 10ms as a default ?
+			// the larger the frame size, the less overhead but the more delay
+
+			auto err = _av.toxavAudioSendFrame(
+				asink->_fid,
+				new_frame.getSpan().ptr,
+				new_frame.getSpan().size / new_frame.channels,
+				new_frame.channels,
+				new_frame.sample_rate
+			);
+			if (err != TOXAV_ERR_SEND_FRAME_OK) {
+				std::cerr << "DTC: failed to send audio frame " << err << "\n";
+			}
+		}
+	}
+}
+
+ObjectHandle ToxAVVoIPModel::enter(const Contact3 c, const Components::VoIP::DefaultConfig& defaults) {
+	if (!_cr.all_of<Contact::Components::ToxFriendEphemeral>(c)) {
+		return {};
+	}
+
+	const auto friend_number = _cr.get<Contact::Components::ToxFriendEphemeral>(c).friend_number;
+
+	const auto err = _av.toxavCall(friend_number, 0, 0);
+	if (err != TOXAV_ERR_CALL_OK) {
+		std::cerr << "TAVVOIP error: failed to start call: " << err << "\n";
+		return {};
+	}
+
+	ObjectHandle new_session {_os.registry(), _os.registry().create()};
+
+	new_session.emplace<VoIPModelI*>(this);
+	new_session.emplace<Components::VoIP::TagVoIPSession>(); // ??
+	new_session.emplace<Components::VoIP::SessionContact>(c);
+	new_session.emplace<Components::VoIP::SessionState>().state = Components::VoIP::SessionState::State::RINGING;
+	new_session.emplace<Components::VoIP::DefaultConfig>(defaults);
+
+	_os.throwEventConstruct(new_session);
+	return new_session;
+}
+
+bool ToxAVVoIPModel::accept(ObjectHandle session, const Components::VoIP::DefaultConfig& defaults) {
+	if (!static_cast<bool>(session)) {
+		return false;
+	}
+
+	if (!session.all_of<
+		Components::VoIP::TagVoIPSession,
+		VoIPModelI*,
+		Components::VoIP::SessionContact,
+		Components::VoIP::Incoming
+	>()) {
+		return false;
+	}
+
+	// check if self
+	if (session.get<VoIPModelI*>() != this) {
+		return false;
+	}
+
+	const auto session_contact = session.get<Components::VoIP::SessionContact>().c;
+	if (!_cr.all_of<Contact::Components::ToxFriendEphemeral>(session_contact)) {
+		return false;
+	}
+
+	const auto friend_number = _cr.get<Contact::Components::ToxFriendEphemeral>(session_contact).friend_number;
+	auto err = _av.toxavAnswer(friend_number, 0, 0);
+	if (err != TOXAV_ERR_ANSWER_OK) {
+		std::cerr << "TOXAVVOIP error: ansering call failed: " << err << "\n";
+		// we simply let it be for now, it apears we can try ansering later again
+		// we also get an error here when the call is already in progress (:
+		return false;
+	}
+
+	session.emplace<Components::VoIP::DefaultConfig>(defaults);
+
+	// answer defaults to enabled receiving audio and video
+	// TODO: think about how we should handle this
+	// set to disabled? and enable on src connection?
+	// we already default disabled send and enabled on sink connection
+	//_av.toxavCallControl(friend_number, TOXAV_CALL_CONTROL_HIDE_VIDEO);
+	//_av.toxavCallControl(friend_number, TOXAV_CALL_CONTROL_MUTE_AUDIO);
+
+
+	// how do we know the other side is accepting audio
+	// bitrate cb or what?
+	assert(!session.all_of<Components::ToxAVAudioSink>());
+	addAudioSink(session, friend_number);
+
+	if (const auto* i_av = session.try_get<Components::ToxAVIncomingAV>(); i_av != nullptr) {
+		// create audio src
+		if (i_av->incoming_audio) {
+			assert(!session.all_of<Components::ToxAVAudioSource>());
+			addAudioSource(session, friend_number);
+		}
+
+		// create video src
+		if (i_av->incoming_video) {
+		}
+	}
+
+	session.get_or_emplace<Components::VoIP::SessionState>().state = Components::VoIP::SessionState::State::CONNECTED;
+	_os.throwEventUpdate(session);
+	return true;
+}
+
+bool ToxAVVoIPModel::leave(ObjectHandle session) {
+	// rename to end?
+
+	if (!static_cast<bool>(session)) {
+		return false;
+	}
+
+	if (!session.all_of<
+		Components::VoIP::TagVoIPSession,
+		VoIPModelI*,
+		Components::VoIP::SessionContact
+	>()) {
+		return false;
+	}
+
+	// check if self
+	if (session.get<VoIPModelI*>() != this) {
+		return false;
+	}
+
+	const auto session_contact = session.get<Components::VoIP::SessionContact>().c;
+	if (!_cr.all_of<Contact::Components::ToxFriendEphemeral>(session_contact)) {
+		return false;
+	}
+
+	const auto friend_number = _cr.get<Contact::Components::ToxFriendEphemeral>(session_contact).friend_number;
+	// check error? (we delete anyway)
+	_av.toxavCallControl(friend_number, Toxav_Call_Control::TOXAV_CALL_CONTROL_CANCEL);
+
+	destroySession(session);
+	return true;
+}
+
+bool ToxAVVoIPModel::onEvent(const Events::FriendCall& e) {
+	// new incoming call, create voip session, ready to be accepted
+	// (or rejected...)
+
+	const auto session_contact = _tcm.getContactFriend(e.friend_number);
+	if (!_cr.valid(session_contact)) {
+		return false;
+	}
+
+	ObjectHandle new_session {_os.registry(), _os.registry().create()};
+
+	new_session.emplace<VoIPModelI*>(this);
+	new_session.emplace<Components::VoIP::TagVoIPSession>(); // ??
+	new_session.emplace<Components::VoIP::Incoming>(session_contact); // in 1on1 its always the same contact, might leave blank
+	new_session.emplace<Components::VoIP::SessionContact>(session_contact);
+	new_session.emplace<Components::VoIP::SessionState>().state = Components::VoIP::SessionState::State::RINGING;
+	new_session.emplace<Components::ToxAVIncomingAV>(e.audio_enabled, e.video_enabled);
+
+	_os.throwEventConstruct(new_session);
+	return true;
+}
+
+bool ToxAVVoIPModel::onEvent(const Events::FriendCallState& e) {
+	const auto session_contact = _tcm.getContactFriend(e.friend_number);
+	if (!_cr.valid(session_contact)) {
+		return false;
+	}
+
+	ToxAVFriendCallState s{e.state};
+
+	// find session(s?)
+	// TODO: keep lookup table
+	for (const auto& [ov, voipmodel] : _os.registry().view<VoIPModelI*>().each()) {
+		if (voipmodel == this) {
+			auto o = _os.objectHandle(ov);
+
+			if (!o.all_of<Components::VoIP::SessionContact>()) {
+				continue;
+			}
+			if (session_contact != o.get<Components::VoIP::SessionContact>().c) {
+				continue;
+			}
+
+			if (s.is_error() || s.is_finished()) {
+				// destroy call
+				destroySession(o);
+			} else {
+				// remote accepted our call, or av send/recv conditions changed?
+				o.get<Components::VoIP::SessionState>().state = Components::VoIP::SessionState::State::CONNECTED; // set to in call ??
+
+				if (s.is_accepting_a() && !o.all_of<Components::ToxAVAudioSink>()) {
+					addAudioSink(o, e.friend_number);
+				} else if (!s.is_accepting_a() && o.all_of<Components::ToxAVAudioSink>()) {
+					// remove asink?
+				}
+
+				// video
+
+				// add/update sources
+				// audio
+				if (s.is_sending_a() && !o.all_of<Components::ToxAVAudioSource>()) {
+					addAudioSource(o, e.friend_number);
+				} else if (!s.is_sending_a() && o.all_of<Components::ToxAVAudioSource>()) {
+					// remove asrc?
+				}
+
+				// video
+			}
+		}
+	}
+
+	return true;
+}
+
+bool ToxAVVoIPModel::onEvent(const Events::FriendAudioBitrate&) {
+	return false;
+}
+
+bool ToxAVVoIPModel::onEvent(const Events::FriendVideoBitrate&) {
+	return false;
+}
+
+bool ToxAVVoIPModel::onEvent(const Events::FriendAudioFrame& e) {
+	auto asrc_it = _audio_sources.find(e.friend_number);
+	if (asrc_it == _audio_sources.cend()) {
+		// missing src from lookup table
+		return false;
+	}
+
+	auto asrc = asrc_it->second;
+
+	if (!static_cast<bool>(asrc)) {
+		// missing src to put frame into ??
+		return false;
+	}
+
+	assert(asrc.all_of<FrameStream2MultiSource<AudioFrame2>*>());
+	assert(asrc.all_of<Components::FrameStream2Source<AudioFrame2>>());
+
+	asrc.get<FrameStream2MultiSource<AudioFrame2>*>()->push(AudioFrame2{
+		e.sampling_rate,
+		e.channels,
+		std::vector<int16_t>(e.pcm.begin(), e.pcm.end()) // copy
+	});
+
+	return true;
+}
+
+bool ToxAVVoIPModel::onEvent(const Events::FriendVideoFrame&) {
+	return false;
+}
+

src/tox_av_voip_model.hpp

@@ -0,0 +1,46 @@
+#pragma once
+
+#include <solanaceae/object_store/fwd.hpp>
+#include <solanaceae/contact/contact_model3.hpp>
+#include <solanaceae/tox_contacts/tox_contact_model2.hpp>
+#include "./frame_streams/voip_model.hpp"
+#include "./tox_av.hpp"
+
+#include <unordered_map>
+
+class ToxAVVoIPModel : protected ToxAVEventI, public VoIPModelI {
+	ObjectStore2& _os;
+	ToxAVI& _av;
+	Contact3Registry& _cr;
+	ToxContactModel2& _tcm;
+
+	// for faster lookup
+	std::unordered_map<uint32_t, ObjectHandle> _audio_sources;
+
+	// TODO: virtual? strategy? protected?
+	virtual void addAudioSource(ObjectHandle session, uint32_t friend_number);
+	virtual void addAudioSink(ObjectHandle session, uint32_t friend_number);
+	// TODO: video
+
+	void destroySession(ObjectHandle session);
+
+	public:
+		ToxAVVoIPModel(ObjectStore2& os, ToxAVI& av, Contact3Registry& cr, ToxContactModel2& tcm);
+		~ToxAVVoIPModel(void);
+
+		void tick(void);
+
+	public: // voip model
+		ObjectHandle enter(const Contact3 c, const Components::VoIP::DefaultConfig& defaults) override;
+		bool accept(ObjectHandle session, const Components::VoIP::DefaultConfig& defaults) override;
+		bool leave(ObjectHandle session) override;
+
+	protected: // toxav events
+		bool onEvent(const Events::FriendCall&) override;
+		bool onEvent(const Events::FriendCallState&) override;
+		bool onEvent(const Events::FriendAudioBitrate&) override;
+		bool onEvent(const Events::FriendVideoBitrate&) override;
+		bool onEvent(const Events::FriendAudioFrame&) override;
+		bool onEvent(const Events::FriendVideoFrame&) override;
+};
+