register file transfer types to os inspector (does not show anything...)

.github/workflows/cd.yml

@@ -74,7 +74,7 @@ jobs:
         git pull
 
     - name: Install Dependencies
-      run: vcpkg install libsodium:x64-windows-static pthreads:x64-windows-static pkgconf:x64-windows
+      run: vcpkg install pkgconf:x64-windows libsodium:x64-windows-static pthreads:x64-windows-static opus:x64-windows-static libvpx:x64-windows-static
 
     # setup vs env
     - uses: ilammy/msvc-dev-cmd@v1
@@ -134,7 +134,7 @@ jobs:
         git pull
 
     - name: Install Dependencies
-      run: vcpkg install libsodium:x64-windows-static pthreads:x64-windows-static pkgconf:x64-windows
+      run: vcpkg install pkgconf:x64-windows libsodium:x64-windows-static pthreads:x64-windows-static opus:x64-windows-static libvpx:x64-windows-static
 
     # setup vs env
     - uses: ilammy/msvc-dev-cmd@v1

.github/workflows/ci.yml

@@ -65,7 +65,7 @@ jobs:
         git pull
 
     - name: Install Dependencies
-      run: vcpkg install libsodium:x64-windows-static pthreads:x64-windows-static pkgconf:x64-windows
+      run: vcpkg install pkgconf:x64-windows libsodium:x64-windows-static pthreads:x64-windows-static opus:x64-windows-static libvpx:x64-windows-static
 
     # setup vs env
     - uses: ilammy/msvc-dev-cmd@v1

CMakeLists.txt

@@ -23,10 +23,17 @@ option(TOMATO_ASAN "Build tomato with asan (gcc/clang/msvc)" OFF)
 if (TOMATO_ASAN)
 	if (${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU" OR ${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang")
 		if (NOT WIN32) # exclude mingw
-			#link_libraries(-fsanitize=address)
+			add_compile_options(-fsanitize=address,undefined)
 			link_libraries(-fsanitize=address,undefined)
-			#link_libraries(-fsanitize=undefined)
+
+			#add_compile_options(-fsanitize=thread)
+			#link_libraries(-fsanitize=thread)
+
 			message("II enabled ASAN")
+			if (OFF) # TODO: switch for minimal runtime in deployed scenarios
+				add_compile_options(-fsanitize-minimal-runtime)
+				link_libraries(-fsanitize-minimal-runtime)
+			endif()
 		else()
 			message("!! can not enable ASAN on this platform (gcc/clang + win)")
 		endif()

flake.nix

@@ -74,6 +74,8 @@
           #(libsodium.override { stdenv = pkgs.pkgsStatic.stdenv; })
           #pkgsStatic.libsodium
           libsodium
+          libopus
+          libvpx
         ] ++ self.packages.${system}.default.dlopenBuildInputs;
 
         cmakeFlags = [

src/CMakeLists.txt

@@ -16,6 +16,9 @@ add_executable(tomato
 	./tox_client.cpp
 	./auto_dirty.hpp
 	./auto_dirty.cpp
+	./tox_private_impl.hpp
+	./tox_av.hpp
+	./tox_av.cpp
 
 	./theme.hpp
 
@@ -63,6 +66,10 @@ add_executable(tomato
 	./chat_gui/settings_window.hpp
 	./chat_gui/settings_window.cpp
 
+	./imgui_entt_entity_editor.hpp
+	./object_store_ui.hpp
+	./object_store_ui.cpp
+
 	./tox_ui_utils.hpp
 	./tox_ui_utils.cpp
 
@@ -74,6 +81,14 @@ add_executable(tomato
 
 	./chat_gui4.hpp
 	./chat_gui4.cpp
+
+	./content/content.hpp
+	./content/frame_stream2.hpp
+	./content/sdl_video_frame_stream2.hpp
+	./content/sdl_video_frame_stream2.cpp
+	./content/audio_stream.hpp
+	./content/sdl_audio_frame_stream2.hpp
+	./content/sdl_audio_frame_stream2.cpp
 )
 
 target_compile_features(tomato PUBLIC cxx_std_17)

src/content/SPSCQueue.h

@@ -0,0 +1,237 @@
+/*
+Copyright (c) 2020 Erik Rigtorp <erik@rigtorp.se>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+ */
+
+#pragma once
+
+#include <atomic>
+#include <cassert>
+#include <cstddef>
+#include <memory> // std::allocator
+#include <new>    // std::hardware_destructive_interference_size
+#include <stdexcept>
+#include <type_traits> // std::enable_if, std::is_*_constructible
+
+#ifdef __has_cpp_attribute
+#if __has_cpp_attribute(nodiscard)
+#define RIGTORP_NODISCARD [[nodiscard]]
+#endif
+#endif
+#ifndef RIGTORP_NODISCARD
+#define RIGTORP_NODISCARD
+#endif
+
+namespace rigtorp {
+
+template <typename T, typename Allocator = std::allocator<T>> class SPSCQueue {
+
+#if defined(__cpp_if_constexpr) && defined(__cpp_lib_void_t)
+  template <typename Alloc2, typename = void>
+  struct has_allocate_at_least : std::false_type {};
+
+  template <typename Alloc2>
+  struct has_allocate_at_least<
+      Alloc2, std::void_t<typename Alloc2::value_type,
+                          decltype(std::declval<Alloc2 &>().allocate_at_least(
+                              size_t{}))>> : std::true_type {};
+#endif
+
+public:
+  explicit SPSCQueue(const size_t capacity,
+                     const Allocator &allocator = Allocator())
+      : capacity_(capacity), allocator_(allocator) {
+    // The queue needs at least one element
+    if (capacity_ < 1) {
+      capacity_ = 1;
+    }
+    capacity_++; // Needs one slack element
+    // Prevent overflowing size_t
+    if (capacity_ > SIZE_MAX - 2 * kPadding) {
+      capacity_ = SIZE_MAX - 2 * kPadding;
+    }
+
+#if defined(__cpp_if_constexpr) && defined(__cpp_lib_void_t)
+    if constexpr (has_allocate_at_least<Allocator>::value) {
+      auto res = allocator_.allocate_at_least(capacity_ + 2 * kPadding);
+      slots_ = res.ptr;
+      capacity_ = res.count - 2 * kPadding;
+    } else {
+      slots_ = std::allocator_traits<Allocator>::allocate(
+          allocator_, capacity_ + 2 * kPadding);
+    }
+#else
+    slots_ = std::allocator_traits<Allocator>::allocate(
+        allocator_, capacity_ + 2 * kPadding);
+#endif
+
+    static_assert(alignof(SPSCQueue<T>) == kCacheLineSize, "");
+    static_assert(sizeof(SPSCQueue<T>) >= 3 * kCacheLineSize, "");
+    assert(reinterpret_cast<char *>(&readIdx_) -
+               reinterpret_cast<char *>(&writeIdx_) >=
+           static_cast<std::ptrdiff_t>(kCacheLineSize));
+  }
+
+  ~SPSCQueue() {
+    while (front()) {
+      pop();
+    }
+    std::allocator_traits<Allocator>::deallocate(allocator_, slots_,
+                                                 capacity_ + 2 * kPadding);
+  }
+
+  // non-copyable and non-movable
+  SPSCQueue(const SPSCQueue &) = delete;
+  SPSCQueue &operator=(const SPSCQueue &) = delete;
+
+  template <typename... Args>
+  void emplace(Args &&...args) noexcept(
+      std::is_nothrow_constructible<T, Args &&...>::value) {
+    static_assert(std::is_constructible<T, Args &&...>::value,
+                  "T must be constructible with Args&&...");
+    auto const writeIdx = writeIdx_.load(std::memory_order_relaxed);
+    auto nextWriteIdx = writeIdx + 1;
+    if (nextWriteIdx == capacity_) {
+      nextWriteIdx = 0;
+    }
+    while (nextWriteIdx == readIdxCache_) {
+      readIdxCache_ = readIdx_.load(std::memory_order_acquire);
+    }
+    new (&slots_[writeIdx + kPadding]) T(std::forward<Args>(args)...);
+    writeIdx_.store(nextWriteIdx, std::memory_order_release);
+  }
+
+  template <typename... Args>
+  RIGTORP_NODISCARD bool try_emplace(Args &&...args) noexcept(
+      std::is_nothrow_constructible<T, Args &&...>::value) {
+    static_assert(std::is_constructible<T, Args &&...>::value,
+                  "T must be constructible with Args&&...");
+    auto const writeIdx = writeIdx_.load(std::memory_order_relaxed);
+    auto nextWriteIdx = writeIdx + 1;
+    if (nextWriteIdx == capacity_) {
+      nextWriteIdx = 0;
+    }
+    if (nextWriteIdx == readIdxCache_) {
+      readIdxCache_ = readIdx_.load(std::memory_order_acquire);
+      if (nextWriteIdx == readIdxCache_) {
+        return false;
+      }
+    }
+    new (&slots_[writeIdx + kPadding]) T(std::forward<Args>(args)...);
+    writeIdx_.store(nextWriteIdx, std::memory_order_release);
+    return true;
+  }
+
+  void push(const T &v) noexcept(std::is_nothrow_copy_constructible<T>::value) {
+    static_assert(std::is_copy_constructible<T>::value,
+                  "T must be copy constructible");
+    emplace(v);
+  }
+
+  template <typename P, typename = typename std::enable_if<
+                            std::is_constructible<T, P &&>::value>::type>
+  void push(P &&v) noexcept(std::is_nothrow_constructible<T, P &&>::value) {
+    emplace(std::forward<P>(v));
+  }
+
+  RIGTORP_NODISCARD bool
+  try_push(const T &v) noexcept(std::is_nothrow_copy_constructible<T>::value) {
+    static_assert(std::is_copy_constructible<T>::value,
+                  "T must be copy constructible");
+    return try_emplace(v);
+  }
+
+  template <typename P, typename = typename std::enable_if<
+                            std::is_constructible<T, P &&>::value>::type>
+  RIGTORP_NODISCARD bool
+  try_push(P &&v) noexcept(std::is_nothrow_constructible<T, P &&>::value) {
+    return try_emplace(std::forward<P>(v));
+  }
+
+  RIGTORP_NODISCARD T *front() noexcept {
+    auto const readIdx = readIdx_.load(std::memory_order_relaxed);
+    if (readIdx == writeIdxCache_) {
+      writeIdxCache_ = writeIdx_.load(std::memory_order_acquire);
+      if (writeIdxCache_ == readIdx) {
+        return nullptr;
+      }
+    }
+    return &slots_[readIdx + kPadding];
+  }
+
+  void pop() noexcept {
+    static_assert(std::is_nothrow_destructible<T>::value,
+                  "T must be nothrow destructible");
+    auto const readIdx = readIdx_.load(std::memory_order_relaxed);
+    assert(writeIdx_.load(std::memory_order_acquire) != readIdx &&
+           "Can only call pop() after front() has returned a non-nullptr");
+    slots_[readIdx + kPadding].~T();
+    auto nextReadIdx = readIdx + 1;
+    if (nextReadIdx == capacity_) {
+      nextReadIdx = 0;
+    }
+    readIdx_.store(nextReadIdx, std::memory_order_release);
+  }
+
+  RIGTORP_NODISCARD size_t size() const noexcept {
+    std::ptrdiff_t diff = writeIdx_.load(std::memory_order_acquire) -
+                          readIdx_.load(std::memory_order_acquire);
+    if (diff < 0) {
+      diff += capacity_;
+    }
+    return static_cast<size_t>(diff);
+  }
+
+  RIGTORP_NODISCARD bool empty() const noexcept {
+    return writeIdx_.load(std::memory_order_acquire) ==
+           readIdx_.load(std::memory_order_acquire);
+  }
+
+  RIGTORP_NODISCARD size_t capacity() const noexcept { return capacity_ - 1; }
+
+private:
+#ifdef __cpp_lib_hardware_interference_size
+  static constexpr size_t kCacheLineSize =
+      std::hardware_destructive_interference_size;
+#else
+  static constexpr size_t kCacheLineSize = 64;
+#endif
+
+  // Padding to avoid false sharing between slots_ and adjacent allocations
+  static constexpr size_t kPadding = (kCacheLineSize - 1) / sizeof(T) + 1;
+
+private:
+  size_t capacity_;
+  T *slots_;
+#if defined(__has_cpp_attribute) && __has_cpp_attribute(no_unique_address)
+  Allocator allocator_ [[no_unique_address]];
+#else
+  Allocator allocator_;
+#endif
+
+  // Align to cache line size in order to avoid false sharing
+  // readIdxCache_ and writeIdxCache_ is used to reduce the amount of cache
+  // coherency traffic
+  alignas(kCacheLineSize) std::atomic<size_t> writeIdx_ = {0};
+  alignas(kCacheLineSize) size_t readIdxCache_ = 0;
+  alignas(kCacheLineSize) std::atomic<size_t> readIdx_ = {0};
+  alignas(kCacheLineSize) size_t writeIdxCache_ = 0;
+};
+} // namespace rigtorp

src/content/audio_stream.hpp

@@ -0,0 +1,69 @@
+#pragma once
+
+#include "./frame_stream2.hpp"
+
+#include <solanaceae/util/span.hpp>
+
+#include <cstdint>
+#include <variant>
+#include <vector>
+
+// raw audio
+// channels make samples interleaved,
+// planar channels are not supported
+struct AudioFrame {
+	// sequence number, to detect gaps
+	uint32_t seq {0};
+	// TODO: maybe use ts instead to discard old?
+	// since buffer size is variable, some timestamp would be needed to estimate the lost time
+
+	// samples per second
+	uint32_t sample_rate {48'000};
+
+	size_t channels {0};
+	std::variant<
+		std::vector<int16_t>, // S16, platform endianess
+		Span<int16_t>, // non owning variant, for direct consumption
+
+		std::vector<float>, // f32
+		Span<float> // non owning variant, for direct consumption
+	> buffer;
+
+	// helpers
+
+	bool isS16(void) const {
+		return
+			std::holds_alternative<std::vector<int16_t>>(buffer) ||
+			std::holds_alternative<Span<int16_t>>(buffer)
+		;
+	}
+	bool isF32(void) const {
+		return
+			std::holds_alternative<std::vector<float>>(buffer) ||
+			std::holds_alternative<Span<float>>(buffer)
+		;
+	}
+	template<typename T>
+	Span<T> getSpan(void) const {
+		static_assert(std::is_same_v<int16_t, T> || std::is_same_v<float, T>);
+		if constexpr (std::is_same_v<int16_t, T>) {
+			assert(isS16());
+			if (std::holds_alternative<std::vector<int16_t>>(buffer)) {
+				return Span<int16_t>{std::get<std::vector<int16_t>>(buffer)};
+			} else {
+				return std::get<Span<int16_t>>(buffer);
+			}
+		} else if constexpr (std::is_same_v<float, T>) {
+			assert(isF32());
+			if (std::holds_alternative<std::vector<float>>(buffer)) {
+				return Span<float>{std::get<std::vector<float>>(buffer)};
+			} else {
+				return std::get<Span<float>>(buffer);
+			}
+		}
+		return {};
+	}
+};
+
+using AudioFrameStream2I = FrameStream2I<AudioFrame>;
+

src/content/content.hpp

@@ -0,0 +1,49 @@
+#pragma once
+
+#include <entt/container/dense_set.hpp>
+
+#include <solanaceae/object_store/object_store.hpp>
+#include <solanaceae/contact/contact_model3.hpp>
+#include <solanaceae/message3/registry_message_model.hpp>
+
+#include <solanaceae/file/file2.hpp>
+
+namespace Content1::Components {
+
+	// TODO: design it as a tree?
+
+	// or something
+	struct TagFile {};
+	struct TagAudioStream {};
+	struct TagVideoStream {};
+
+	struct TimingTiedTo {
+		entt::dense_set<ObjectHandle> ties;
+	};
+
+	// the associated messages, if any
+	// useful if you want to update progress on the message
+	struct Messages {
+		std::vector<Message3Handle> messages;
+	};
+
+	// ?
+	struct SuspectedParticipants {
+		entt::dense_set<Contact3> participants;
+	};
+
+	struct ReadHeadHint {
+		// points to the first byte we want
+		// this is just a hint, that can be set from outside
+		// to guide the sequential "piece picker" strategy
+		// the strategy *should* set this to the first byte we dont yet have
+		uint64_t offset_into_file {0u};
+	};
+
+} // Content::Components
+
+// TODO: i have no idea
+struct RawFile2ReadFromContentFactoryI {
+	virtual std::shared_ptr<File2I> open(ObjectHandle h) = 0;
+};
+

src/content/frame_stream2.hpp

@@ -0,0 +1,132 @@
+#pragma once
+
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <vector>
+#include <mutex>
+
+#include "./SPSCQueue.h"
+
+// Frames ofen consist of:
+// - seq id // incremental sequential id, gaps in ids can be used to detect loss
+// - data // the frame data
+// eg:
+//struct ExampleFrame {
+	//int64_t seq_id {0};
+	//std::vector<uint8_t> data;
+//};
+
+template<typename FrameType>
+struct FrameStream2I {
+	virtual ~FrameStream2I(void) {}
+
+	// get number of available frames
+	[[nodiscard]] virtual int32_t size(void) = 0;
+
+	// get next frame
+	// TODO: optional instead?
+	// data sharing? -> no, data is copied for each fsr, if concurency supported
+	[[nodiscard]] virtual std::optional<FrameType> pop(void) = 0;
+
+	// returns true if there are readers (or we dont know)
+	virtual bool push(const FrameType& value) = 0;
+};
+
+// needs count frames queue size
+// having ~1-2sec buffer size is often sufficent
+template<typename FrameType>
+struct QueuedFrameStream2 : public FrameStream2I<FrameType> {
+	using frame_type = FrameType;
+
+	rigtorp::SPSCQueue<FrameType> _queue;
+
+	// discard values if queue full
+	// will block if not lossy and full on push
+	const bool _lossy {true};
+
+	explicit QueuedFrameStream2(size_t queue_size, bool lossy = true) : _queue(queue_size), _lossy(lossy) {}
+
+	int32_t size(void) override {
+		return _queue.size();
+	}
+
+	std::optional<FrameType> pop(void) override {
+		auto* ret_ptr = _queue.front();
+		if (ret_ptr == nullptr) {
+			return std::nullopt;
+		}
+
+		// move away
+		FrameType ret = std::move(*ret_ptr);
+
+		_queue.pop();
+
+		return ret;
+	}
+
+	bool push(const FrameType& value) override {
+		if (_lossy) {
+			[[maybe_unused]] auto _ = _queue.try_emplace(value);
+			// TODO: maybe return ?
+		} else {
+			_queue.push(value);
+		}
+		return true;
+	}
+};
+
+// implements a stream that pops or pushes to all sub streams
+// you need to mind the direction you intend it to use
+// release all streams before destructing! // TODO: improve lifetime here, maybe some shared semaphore?
+template<typename FrameType, typename SubStreamType = QueuedFrameStream2<FrameType>>
+struct FrameStream2MultiStream : public FrameStream2I<FrameType> {
+	using sub_stream_type_t = SubStreamType;
+
+	// pointer stability
+	std::vector<std::unique_ptr<SubStreamType>> _sub_streams;
+	std::mutex _sub_stream_lock; // accessing the _sub_streams array needs to be exclusive
+	// a simple lock here is ok, since this tends to be a rare operation,
+	// except for the push, which is always on the same thread
+
+	// TODO: forward args instead
+	SubStreamType* aquireSubStream(size_t queue_size = 10, bool lossy = true) {
+		std::lock_guard lg{_sub_stream_lock};
+		return _sub_streams.emplace_back(std::make_unique<SubStreamType>(queue_size, lossy)).get();
+	}
+
+	void releaseSubStream(SubStreamType* sub) {
+		std::lock_guard lg{_sub_stream_lock};
+		for (auto it = _sub_streams.begin(); it != _sub_streams.end(); it++) {
+			if (it->get() == sub) {
+				_sub_streams.erase(it);
+				break;
+			}
+		}
+	}
+
+	// stream interface
+
+	int32_t size(void) override {
+		// TODO: return something sensible?
+		return -1;
+	}
+
+	std::optional<FrameType> pop(void) override {
+		assert(false && "this logic is very frame type specific, provide an impl");
+		return std::nullopt;
+	}
+
+	// returns true if there are readers
+	bool push(const FrameType& value) override {
+		std::lock_guard lg{_sub_stream_lock};
+		bool have_readers{false};
+		for (auto& it : _sub_streams) {
+			[[maybe_unused]] auto _ = it->push(value);
+			have_readers = true; // even if queue full, we still continue believing in them
+			// maybe consider push return value?
+		}
+		return have_readers;
+	}
+};
+

src/content/sdl_audio_frame_stream2.cpp

@@ -0,0 +1,170 @@
+#include "./sdl_audio_frame_stream2.hpp"
+
+#include <iostream>
+#include <vector>
+
+SDLAudioInputDeviceDefault::SDLAudioInputDeviceDefault(void) : _stream{nullptr, &SDL_DestroyAudioStream} {
+	constexpr SDL_AudioSpec spec = { SDL_AUDIO_S16, 1, 48000 };
+
+	_stream = {
+		SDL_OpenAudioDeviceStream(SDL_AUDIO_DEVICE_DEFAULT_CAPTURE, &spec, nullptr, nullptr),
+		&SDL_DestroyAudioStream
+	};
+
+	if (!static_cast<bool>(_stream)) {
+		std::cerr << "SDL open audio device failed!\n";
+	}
+
+	const auto audio_device_id = SDL_GetAudioStreamDevice(_stream.get());
+	SDL_ResumeAudioDevice(audio_device_id);
+
+	static constexpr size_t buffer_size {512*2}; // in samples
+	const auto interval_ms {(buffer_size * 1000) / spec.freq};
+
+	_thread = std::thread([this, interval_ms, spec](void) {
+		while (!_thread_should_quit) {
+			//static std::vector<int16_t> buffer(buffer_size);
+			static AudioFrame tmp_frame {
+				0, // TODO: seq
+				spec.freq, spec.channels,
+				std::vector<int16_t>(buffer_size)
+			};
+
+			auto& buffer = std::get<std::vector<int16_t>>(tmp_frame.buffer);
+			buffer.resize(buffer_size);
+
+			const auto read_bytes = SDL_GetAudioStreamData(
+				_stream.get(),
+				buffer.data(),
+				buffer.size()*sizeof(int16_t)
+			);
+			//if (read_bytes != 0) {
+				//std::cerr << "read " << read_bytes << "/" << buffer.size()*sizeof(int16_t) << " audio bytes\n";
+			//}
+
+			// no new frame yet, or error
+			if (read_bytes <= 0) {
+				// only sleep 1/5, we expected a frame
+				std::this_thread::sleep_for(std::chrono::milliseconds(int64_t(interval_ms/5)));
+				continue;
+			}
+
+			buffer.resize(read_bytes/sizeof(int16_t)); // this might be costly?
+
+			bool someone_listening {false};
+			someone_listening = push(tmp_frame);
+
+			if (someone_listening) {
+				// double the interval on acquire
+				std::this_thread::sleep_for(std::chrono::milliseconds(int64_t(interval_ms/2)));
+			} else {
+				std::cerr << "i guess no one is listening\n";
+				// we just sleep 32x as long, bc no one is listening
+				// with the hardcoded settings, this is ~320ms
+				// TODO: just hardcode something like 500ms?
+				// TODO: suspend
+				std::this_thread::sleep_for(std::chrono::milliseconds(int64_t(interval_ms*32)));
+			}
+		}
+	});
+}
+
+SDLAudioInputDeviceDefault::~SDLAudioInputDeviceDefault(void) {
+	// TODO: pause audio device?
+	_thread_should_quit = true;
+	_thread.join();
+	// TODO: what to do if readers are still present?
+}
+
+int32_t SDLAudioOutputDeviceDefaultInstance::size(void) {
+	return -1;
+}
+
+std::optional<AudioFrame> SDLAudioOutputDeviceDefaultInstance::pop(void) {
+	assert(false);
+	// this is an output device, there is no data to pop
+	return std::nullopt;
+}
+
+bool SDLAudioOutputDeviceDefaultInstance::push(const AudioFrame& value) {
+	if (!static_cast<bool>(_stream)) {
+		return false;
+	}
+
+	// verify here the fame has the same sample type, channel count and sample freq
+	// if something changed, we need to either use a temporary stream, just for conversion, or reopen _stream with the new params
+	// because of data temporality, the second options looks like a better candidate
+	if (
+		value.sample_rate != _last_sample_rate ||
+		value.channels != _last_channels ||
+		(value.isF32() && _last_format != SDL_AUDIO_F32) ||
+		(value.isS16() && _last_format != SDL_AUDIO_S16)
+	) {
+		const auto device_id = SDL_GetAudioStreamDevice(_stream.get());
+		SDL_FlushAudioStream(_stream.get());
+
+		const SDL_AudioSpec spec = {
+			static_cast<SDL_AudioFormat>((value.isF32() ? SDL_AUDIO_F32 :  SDL_AUDIO_S16)),
+			static_cast<int>(value.channels),
+			static_cast<int>(value.sample_rate)
+		};
+
+		_stream = {
+			SDL_OpenAudioDeviceStream(device_id, &spec, nullptr, nullptr),
+			&SDL_DestroyAudioStream
+		};
+	}
+
+	// HACK
+	assert(value.isS16());
+
+	auto data = value.getSpan<int16_t>();
+
+	if (data.size == 0) {
+		std::cerr << "empty audio frame??\n";
+	}
+
+	if (SDL_PutAudioStreamData(_stream.get(), data.ptr, data.size * sizeof(int16_t)) < 0) {
+		std::cerr << "put data error\n";
+		return false; // return true?
+	}
+
+	_last_sample_rate = value.sample_rate;
+	_last_channels = value.channels;
+	_last_format = value.isF32() ? SDL_AUDIO_F32 :  SDL_AUDIO_S16;
+
+	return true;
+}
+
+SDLAudioOutputDeviceDefaultInstance::SDLAudioOutputDeviceDefaultInstance(void) : _stream(nullptr, nullptr) {
+}
+
+SDLAudioOutputDeviceDefaultInstance::SDLAudioOutputDeviceDefaultInstance(SDLAudioOutputDeviceDefaultInstance&& other) : _stream(std::move(other._stream)) {
+}
+
+SDLAudioOutputDeviceDefaultInstance::~SDLAudioOutputDeviceDefaultInstance(void) {
+}
+
+SDLAudioOutputDeviceDefaultInstance SDLAudioOutputDeviceDefaultFactory::create(void) {
+	SDLAudioOutputDeviceDefaultInstance new_instance;
+
+	constexpr SDL_AudioSpec spec = { SDL_AUDIO_S16, 1, 48000 };
+
+	new_instance._stream = {
+		SDL_OpenAudioDeviceStream(SDL_AUDIO_DEVICE_DEFAULT_OUTPUT, &spec, nullptr, nullptr),
+		&SDL_DestroyAudioStream
+	};
+	new_instance._last_sample_rate = spec.freq;
+	new_instance._last_channels = spec.channels;
+	new_instance._last_format = spec.format;
+
+	if (!static_cast<bool>(new_instance._stream)) {
+		std::cerr << "SDL open audio device failed!\n";
+	}
+
+	const auto audio_device_id = SDL_GetAudioStreamDevice(new_instance._stream.get());
+	SDL_ResumeAudioDevice(audio_device_id);
+
+	return new_instance;
+}
+

src/content/sdl_audio_frame_stream2.hpp

@@ -0,0 +1,61 @@
+#pragma once
+
+#include "./frame_stream2.hpp"
+#include "./audio_stream.hpp"
+
+#include <SDL3/SDL.h>
+
+#include <cstdint>
+#include <variant>
+#include <vector>
+#include <thread>
+
+// we dont have to multicast ourself, because sdl streams and virtual devices already do this, but we do it anyway
+using AudioFrameStream2MultiStream = FrameStream2MultiStream<AudioFrame>;
+using AudioFrameStream2 = AudioFrameStream2MultiStream::sub_stream_type_t; // just use the default for now
+
+// object components?
+
+// source
+struct SDLAudioInputDeviceDefault : protected AudioFrameStream2MultiStream {
+	std::unique_ptr<SDL_AudioStream, decltype(&SDL_DestroyAudioStream)> _stream;
+
+	std::atomic<bool> _thread_should_quit {false};
+	std::thread _thread;
+
+	// construct source and start thread
+	// TODO: optimize so the thread is not always running
+	SDLAudioInputDeviceDefault(void);
+
+	// stops the thread and closes the device?
+	~SDLAudioInputDeviceDefault(void);
+
+	using AudioFrameStream2MultiStream::aquireSubStream;
+	using AudioFrameStream2MultiStream::releaseSubStream;
+};
+
+// sink
+struct SDLAudioOutputDeviceDefaultInstance : protected AudioFrameStream2I {
+	std::unique_ptr<SDL_AudioStream, decltype(&SDL_DestroyAudioStream)> _stream;
+
+	uint32_t _last_sample_rate {48'000};
+	size_t _last_channels {0};
+	SDL_AudioFormat _last_format {0};
+
+	SDLAudioOutputDeviceDefaultInstance(void);
+	SDLAudioOutputDeviceDefaultInstance(SDLAudioOutputDeviceDefaultInstance&& other);
+
+	~SDLAudioOutputDeviceDefaultInstance(void);
+
+	int32_t size(void) override;
+	std::optional<AudioFrame> pop(void) override;
+	bool push(const AudioFrame& value) override;
+};
+
+// constructs entirely new streams, since sdl handles sync and mixing for us (or should)
+struct SDLAudioOutputDeviceDefaultFactory {
+	// TODO: pause device?
+
+	SDLAudioOutputDeviceDefaultInstance create(void);
+};
+

src/content/sdl_video_frame_stream2.cpp

@@ -0,0 +1,142 @@
+#include "./sdl_video_frame_stream2.hpp"
+
+#include <chrono>
+#include <cstdint>
+#include <iostream>
+#include <memory>
+#include <thread>
+
+SDLVideoCameraContent::SDLVideoCameraContent(void) {
+	int devcount {0};
+	SDL_CameraDeviceID *devices = SDL_GetCameraDevices(&devcount);
+	std::cout << "SDL Camera Driver: " << SDL_GetCurrentCameraDriver() << "\n";
+
+	if (devices == nullptr || devcount < 1) {
+		throw int(1); // TODO: proper exception?
+	}
+
+	std::cout << "### found cameras:\n";
+	for (int i = 0; i < devcount; i++) {
+		const SDL_CameraDeviceID device = devices[i];
+
+		char *name = SDL_GetCameraDeviceName(device);
+		std::cout << "  - Camera #" << i << ": " << name << "\n";
+		SDL_free(name);
+
+		int speccount {0};
+		SDL_CameraSpec* specs = SDL_GetCameraDeviceSupportedFormats(device, &speccount);
+		if (specs == nullptr) {
+			std::cout << "    - no supported spec\n";
+		} else {
+			for (int spec_i = 0; spec_i < speccount; spec_i++) {
+				std::cout << "    - " << specs[spec_i].width << "x" << specs[spec_i].height << "@" << float(specs[spec_i].interval_denominator)/specs[spec_i].interval_numerator << " " << SDL_GetPixelFormatName(specs[spec_i].format) << "\n";
+
+			}
+
+			SDL_free(specs);
+		}
+	}
+
+	{
+		SDL_CameraSpec spec {
+			// FORCE a diffrent pixel format
+			SDL_PIXELFORMAT_RGBA8888,
+
+			//1280, 720,
+			//640, 360,
+			640, 480,
+
+			1, 30
+		};
+		_camera = {
+			SDL_OpenCameraDevice(devices[0], &spec),
+			&SDL_CloseCamera
+		};
+	}
+	SDL_free(devices);
+	if (!static_cast<bool>(_camera)) {
+		throw int(2);
+	}
+
+	SDL_CameraSpec spec;
+	float interval {0.1f};
+	if (SDL_GetCameraFormat(_camera.get(), &spec) < 0) {
+		// meh
+	} else {
+		// interval
+		interval = float(spec.interval_numerator)/float(spec.interval_denominator);
+		std::cout << "camera interval: " << interval*1000 << "ms\n";
+		auto* format_name = SDL_GetPixelFormatName(spec.format);
+		std::cout << "camera format: " << format_name << "\n";
+	}
+
+	_thread = std::thread([this, interval](void) {
+		while (!_thread_should_quit) {
+			Uint64 timestampNS = 0;
+			SDL_Surface* sdl_frame_next = SDL_AcquireCameraFrame(_camera.get(), &timestampNS);
+
+			// no new frame yet, or error
+			if (sdl_frame_next == nullptr) {
+				// only sleep 1/10, we expected a frame
+				std::this_thread::sleep_for(std::chrono::milliseconds(int64_t(interval*1000 / 10)));
+				continue;
+			}
+
+#if 0
+			{ // test copy to trigger bug
+				SDL_Surface* test_surf = SDL_CreateSurface(
+					sdl_frame_next->w,
+					sdl_frame_next->h,
+					SDL_PIXELFORMAT_RGBA8888
+				);
+
+				assert(test_surf != nullptr);
+
+				SDL_BlitSurface(sdl_frame_next, nullptr, test_surf, nullptr);
+
+				SDL_DestroySurface(test_surf);
+			}
+#endif
+
+			bool someone_listening {false};
+			{
+				SDLVideoFrame new_frame_non_owning {
+					timestampNS,
+					sdl_frame_next
+				};
+
+				// creates surface copies
+				someone_listening = push(new_frame_non_owning);
+			}
+			SDL_ReleaseCameraFrame(_camera.get(), sdl_frame_next);
+
+			if (someone_listening) {
+				// double the interval on acquire
+				std::this_thread::sleep_for(std::chrono::milliseconds(int64_t(interval*1000*0.5)));
+			} else {
+				std::cerr << "i guess no one is listening\n";
+				// we just sleep 4x as long, bc no one is listening
+				std::this_thread::sleep_for(std::chrono::milliseconds(int64_t(interval*1000*4)));
+			}
+		}
+	});
+}
+
+SDLVideoCameraContent::~SDLVideoCameraContent(void) {
+	_thread_should_quit = true;
+	_thread.join();
+	// TODO: what to do if readers are still present?
+
+	// HACK: sdl is buggy and freezes otherwise. it is likely still possible, but rare to freeze here
+	// flush unused frames
+#if 1
+	while (true) {
+		SDL_Surface* sdl_frame_next = SDL_AcquireCameraFrame(_camera.get(), nullptr);
+		if (sdl_frame_next != nullptr) {
+			SDL_ReleaseCameraFrame(_camera.get(), sdl_frame_next);
+		} else {
+			break;
+		}
+	}
+#endif
+}

src/content/sdl_video_frame_stream2.hpp

@@ -0,0 +1,65 @@
+#pragma once
+
+#include "./frame_stream2.hpp"
+
+#include <SDL3/SDL.h>
+
+#include <cstdint>
+#include <thread>
+
+inline void nopSurfaceDestructor(SDL_Surface*) {}
+
+// this is very sdl specific
+struct SDLVideoFrame {
+	// TODO: sequence numbering?
+	uint64_t timestampNS {0};
+
+	std::unique_ptr<SDL_Surface, decltype(&SDL_DestroySurface)> surface {nullptr, &SDL_DestroySurface};
+
+	// special non-owning constructor?
+	SDLVideoFrame(
+		uint64_t ts,
+		SDL_Surface* surf
+	) {
+		timestampNS = ts;
+		surface = {surf, &nopSurfaceDestructor};
+	}
+	// copy
+	SDLVideoFrame(const SDLVideoFrame& other) {
+		timestampNS = other.timestampNS;
+		if (static_cast<bool>(other.surface)) {
+			surface = {
+				SDL_CreateSurface(
+					other.surface->w,
+					other.surface->h,
+					SDL_PIXELFORMAT_RGBA8888 // meh
+				),
+				&SDL_DestroySurface
+			};
+			SDL_BlitSurface(other.surface.get(), nullptr, surface.get(), nullptr);
+		}
+	}
+	SDLVideoFrame& operator=(const SDLVideoFrame& other) = delete;
+};
+
+using SDLVideoFrameStream2MultiStream = FrameStream2MultiStream<SDLVideoFrame>;
+using SDLVideoFrameStream2 = SDLVideoFrameStream2MultiStream::sub_stream_type_t; // just use the default for now
+
+struct SDLVideoCameraContent : protected SDLVideoFrameStream2MultiStream {
+	// meh, empty default
+	std::unique_ptr<SDL_Camera, decltype(&SDL_CloseCamera)> _camera {nullptr, &SDL_CloseCamera};
+	std::atomic<bool> _thread_should_quit {false};
+	std::thread _thread;
+
+	// construct source and start thread
+	// TODO: optimize so the thread is not always running
+	SDLVideoCameraContent(void);
+
+	// stops the thread and closes the camera
+	~SDLVideoCameraContent(void);
+
+	// make only some of writer public
+	using SDLVideoFrameStream2MultiStream::aquireSubStream;
+	using SDLVideoFrameStream2MultiStream::releaseSubStream;
+};
+

src/content/stream_reader.hpp

@@ -0,0 +1,15 @@
+#pragma once
+
+#include <solanaceae/util/span.hpp>
+
+// most media that can be counted as "stream" comes in packets/frames/messages
+// so this class provides an interface for ideal async fetching of frames
+struct RawFrameStreamReaderI {
+	// return the number of ready frames in cache
+	// returns -1 if unknown
+	virtual int64_t have(void) = 0;
+
+	// get next frame, empty if none
+	virtual ByteSpan getNext(void) = 0;
+};
+

src/content/stream_reader_sdl_audio.cpp

@@ -0,0 +1,39 @@
+#include "./stream_reader_sdl_audio.hpp"
+
+SDLAudioFrameStreamReader::SDLAudioFrameStreamReader(int32_t buffer_size) : _buffer_size(buffer_size),  _stream{nullptr, &SDL_DestroyAudioStream} {
+	_buffer.resize(_buffer_size);
+
+	const SDL_AudioSpec spec = { SDL_AUDIO_S16, 1, 48000 };
+
+	_stream = {
+		SDL_OpenAudioDeviceStream(SDL_AUDIO_DEVICE_DEFAULT_CAPTURE, &spec, nullptr, nullptr),
+		&SDL_DestroyAudioStream
+	};
+}
+
+Span<int16_t> SDLAudioFrameStreamReader::getNextAudioFrame(void) {
+	const int32_t needed_bytes = (_buffer.size() - _remaining_size) * sizeof(int16_t);
+	const auto read_bytes = SDL_GetAudioStreamData(_stream.get(), _buffer.data()+_remaining_size, needed_bytes);
+	if (read_bytes < 0) {
+		// error
+		return {};
+	}
+	if (read_bytes < needed_bytes) {
+		// HACK: we are just assuming here that sdl never gives us half a sample!
+		_remaining_size += read_bytes / sizeof(int16_t);
+		return {};
+	}
+
+	_remaining_size = 0;
+	return Span<int16_t>{_buffer};
+}
+
+int64_t SDLAudioFrameStreamReader::have(void) {
+	return -1;
+}
+
+ByteSpan SDLAudioFrameStreamReader::getNext(void) {
+	auto next_frame_span = getNextAudioFrame();
+	return ByteSpan{reinterpret_cast<const uint8_t*>(next_frame_span.ptr), next_frame_span.size};
+}
+

src/content/stream_reader_sdl_audio.hpp

@@ -0,0 +1,31 @@
+#pragma once
+
+#include "./stream_reader.hpp"
+
+#include <SDL3/SDL.h>
+
+#include <cstdint>
+#include <cstdio>
+#include <memory>
+
+struct SDLAudioFrameStreamReader : public RawFrameStreamReaderI {
+	// count samples per buffer
+	const int32_t _buffer_size {1024};
+	std::vector<int16_t> _buffer;
+	size_t _remaining_size {0}; // data still in buffer, that was remaining from last call and not enough to fill a full frame
+
+	// meh, empty default
+	std::unique_ptr<SDL_AudioStream, decltype(&SDL_DestroyAudioStream)> _stream;
+
+	// buffer_size in number of samples
+	SDLAudioFrameStreamReader(int32_t buffer_size = 1024);
+
+	// data owned by StreamReader, overwritten by next call to getNext*()
+	Span<int16_t> getNextAudioFrame(void);
+
+	public: // interface
+		int64_t have(void) override;
+
+		ByteSpan getNext(void) override;
+};
+

src/content/stream_reader_sdl_video.cpp

@@ -0,0 +1,84 @@
+#include "./stream_reader_sdl_video.hpp"
+
+#include <iostream>
+
+SDLVideoFrameStreamReader::SDLVideoFrameStreamReader() : _camera{nullptr, &SDL_CloseCamera}, _surface{nullptr, &SDL_DestroySurface} {
+	// enumerate
+	int devcount = 0;
+	SDL_CameraDeviceID *devices = SDL_GetCameraDevices(&devcount);
+
+	if (devices == nullptr || devcount < 1) {
+		throw int(1); // TODO: proper exception?
+	}
+
+	std::cout << "### found cameras:\n";
+	for (int i = 0; i < devcount; i++) {
+		const SDL_CameraDeviceID device = devices[i];
+		char *name = SDL_GetCameraDeviceName(device);
+
+		std::cout << "  - Camera #" << i << ": " << name << "\n";
+
+		SDL_free(name);
+	}
+
+	_camera = {
+		SDL_OpenCameraDevice(devices[0], nullptr),
+		&SDL_CloseCamera
+	};
+	if (!static_cast<bool>(_camera)) {
+		throw int(2);
+	}
+
+	SDL_CameraSpec spec;
+	if (SDL_GetCameraFormat(_camera.get(), &spec) < 0) {
+		// meh
+	} else {
+		// interval
+		float interval = float(spec.interval_numerator)/float(spec.interval_denominator);
+		std::cout << "camera interval: " << interval*1000 << "ms\n";
+	}
+}
+
+SDLVideoFrameStreamReader::VideoFrame SDLVideoFrameStreamReader::getNextVideoFrameRGBA(void) {
+	if (!static_cast<bool>(_camera)) {
+		return {};
+	}
+
+	Uint64 timestampNS = 0;
+	SDL_Surface* frame_next = SDL_AcquireCameraFrame(_camera.get(), &timestampNS);
+
+	// no new frame yet, or error
+	if (frame_next == nullptr) {
+		//std::cout << "failed acquiring frame\n";
+		return {};
+	}
+
+	// TODO: investigate zero copy
+	_surface = {
+		SDL_ConvertSurfaceFormat(frame_next, SDL_PIXELFORMAT_RGBA8888),
+		&SDL_DestroySurface
+	};
+
+	SDL_ReleaseCameraFrame(_camera.get(), frame_next);
+
+	SDL_LockSurface(_surface.get());
+
+	return {
+		_surface->w,
+		_surface->h,
+		timestampNS,
+		{
+			reinterpret_cast<const uint8_t*>(_surface->pixels),
+			uint64_t(_surface->w*_surface->h*4) // rgba
+		}
+	};
+}
+
+int64_t SDLVideoFrameStreamReader::have(void) {
+	return -1;
+}
+
+ByteSpan SDLVideoFrameStreamReader::getNext(void) {
+	return {};
+}
+

src/content/stream_reader_sdl_video.hpp

@@ -0,0 +1,34 @@
+#pragma once
+
+#include "./stream_reader.hpp"
+
+#include <SDL3/SDL.h>
+
+#include <cstdint>
+#include <cstdio>
+#include <memory>
+
+struct SDLVideoFrameStreamReader : public RawFrameStreamReaderI {
+	// meh, empty default
+	std::unique_ptr<SDL_Camera, decltype(&SDL_CloseCamera)> _camera;
+	std::unique_ptr<SDL_Surface, decltype(&SDL_DestroySurface)> _surface;
+
+	SDLVideoFrameStreamReader(void);
+
+	struct VideoFrame {
+		int32_t width {0};
+		int32_t height {0};
+
+		uint64_t timestampNS {0};
+
+		ByteSpan data;
+	};
+
+	// data owned by StreamReader, overwritten by next call to getNext*()
+	VideoFrame getNextVideoFrameRGBA(void);
+
+	public: // interface
+		int64_t have(void) override;
+		ByteSpan getNext(void) override;
+};
+

src/image_loader.hpp

@@ -27,6 +27,8 @@ struct ImageLoaderI {
 
 		// only positive values are valid
 		ImageResult crop(int32_t c_x, int32_t c_y, int32_t c_w, int32_t c_h) const;
+
+		// TODO: scale
 	};
 	virtual ImageResult loadFromMemoryRGBA(const uint8_t* data, uint64_t data_size) = 0;
 };

src/imgui_entt_entity_editor.hpp

@@ -0,0 +1,319 @@
+// for the license, see the end of the file
+#pragma once
+
+#include "entt/entity/fwd.hpp"
+#include <map>
+#include <set>
+#include <functional>
+#include <string>
+
+#include <entt/entt.hpp>
+#include <imgui.h>
+
+#ifndef MM_IEEE_ASSERT
+	#define MM_IEEE_ASSERT(x) assert(x)
+#endif
+
+#define MM_IEEE_IMGUI_PAYLOAD_TYPE_ENTITY "MM_IEEE_ENTITY"
+
+#ifndef MM_IEEE_ENTITY_WIDGET
+	#define MM_IEEE_ENTITY_WIDGET ::MM::EntityWidget
+#endif
+
+namespace MM {
+
+template <class EntityType>
+inline void EntityWidget(EntityType& e, entt::basic_registry<EntityType>& reg, bool dropTarget = false)
+{
+	ImGui::PushID(static_cast<int>(entt::to_integral(e)));
+
+	if (reg.valid(e)) {
+		ImGui::Text("ID: %d", entt::to_integral(e));
+	} else {
+		ImGui::Text("Invalid Entity");
+	}
+
+	if (reg.valid(e)) {
+		if (ImGui::BeginDragDropSource(ImGuiDragDropFlags_SourceAllowNullID)) {
+			ImGui::SetDragDropPayload(MM_IEEE_IMGUI_PAYLOAD_TYPE_ENTITY, &e, sizeof(e));
+			ImGui::Text("ID: %d", entt::to_integral(e));
+			ImGui::EndDragDropSource();
+		}
+	}
+
+	if (dropTarget && ImGui::BeginDragDropTarget()) {
+		if (const ImGuiPayload* payload = ImGui::AcceptDragDropPayload(MM_IEEE_IMGUI_PAYLOAD_TYPE_ENTITY)) {
+			e = *(EntityType*)payload->Data;
+		}
+
+		ImGui::EndDragDropTarget();
+	}
+
+	ImGui::PopID();
+}
+
+template <class Component, class EntityType>
+void ComponentEditorWidget([[maybe_unused]] entt::basic_registry<EntityType>& registry, [[maybe_unused]] EntityType entity) {}
+
+template <class Component, class EntityType>
+void ComponentAddAction(entt::basic_registry<EntityType>& registry, EntityType entity)
+{
+	registry.template emplace<Component>(entity);
+}
+
+template <class Component, class EntityType>
+void ComponentRemoveAction(entt::basic_registry<EntityType>& registry, EntityType entity)
+{
+	registry.template remove<Component>(entity);
+}
+
+template <class EntityType>
+class EntityEditor {
+public:
+	using Registry = entt::basic_registry<EntityType>;
+	using ComponentTypeID = entt::id_type;
+
+	struct ComponentInfo {
+		using Callback = std::function<void(Registry&, EntityType)>;
+		std::string name;
+		Callback widget, create, destroy;
+	};
+
+	bool show_window = true;
+
+private:
+	std::map<ComponentTypeID, ComponentInfo> component_infos;
+
+	bool entityHasComponent(Registry& registry, EntityType& entity, ComponentTypeID type_id)
+	{
+		const auto* storage_ptr = registry.storage(type_id);
+		return storage_ptr != nullptr && storage_ptr->contains(entity);
+	}
+
+public:
+	template <class Component>
+	ComponentInfo& registerComponent(const ComponentInfo& component_info)
+	{
+		auto index = entt::type_hash<Component>::value();
+		auto insert_info = component_infos.insert_or_assign(index, component_info);
+		MM_IEEE_ASSERT(insert_info.second);
+		return std::get<ComponentInfo>(*insert_info.first);
+	}
+
+	template <class Component>
+	ComponentInfo& registerComponent(const std::string& name, typename ComponentInfo::Callback widget)
+	{
+		return registerComponent<Component>(ComponentInfo{
+			name,
+			widget,
+			ComponentAddAction<Component, EntityType>,
+			ComponentRemoveAction<Component, EntityType>,
+		});
+	}
+
+	template <class Component>
+	ComponentInfo& registerComponent(const std::string& name)
+	{
+		return registerComponent<Component>(name, ComponentEditorWidget<Component, EntityType>);
+	}
+
+	void renderEditor(Registry& registry, EntityType& e)
+	{
+		ImGui::TextUnformatted("Editing:");
+		ImGui::SameLine();
+
+		MM_IEEE_ENTITY_WIDGET(e, registry, true);
+
+		if (ImGui::Button("New")) {
+			e = registry.create();
+		}
+		if (registry.valid(e)) {
+			ImGui::SameLine();
+
+			if (ImGui::Button("Clone")) {
+				auto old_e = e;
+				e = registry.create();
+
+				// create a copy of an entity component by component
+				for (auto &&curr: registry.storage()) {
+					if (auto &storage = curr.second; storage.contains(old_e)) {
+						// TODO: do something with the return value. returns false on failure.
+						storage.push(e, storage.value(old_e));
+					}
+				}
+			}
+			ImGui::SameLine();
+
+			ImGui::Dummy({10, 0}); // space destroy a bit, to not accidentally click it
+			ImGui::SameLine();
+
+			// red button
+			ImGui::PushStyleColor(ImGuiCol_Button, ImVec4(0.65f, 0.15f, 0.15f, 1.f));
+			ImGui::PushStyleColor(ImGuiCol_ButtonHovered, ImVec4(0.8f, 0.3f, 0.3f, 1.f));
+			ImGui::PushStyleColor(ImGuiCol_ButtonActive, ImVec4(1.f, 0.2f, 0.2f, 1.f));
+			if (ImGui::Button("Destroy")) {
+				registry.destroy(e);
+				e = entt::null;
+			}
+			ImGui::PopStyleColor(3);
+		}
+
+		ImGui::Separator();
+
+		if (registry.valid(e)) {
+			ImGui::PushID(static_cast<int>(entt::to_integral(e)));
+			std::map<ComponentTypeID, ComponentInfo> has_not;
+			for (auto& [component_type_id, ci] : component_infos) {
+				if (entityHasComponent(registry, e, component_type_id)) {
+					ImGui::PushID(component_type_id);
+					if (ImGui::Button("-")) {
+						ci.destroy(registry, e);
+						ImGui::PopID();
+						continue; // early out to prevent access to deleted data
+					} else {
+						ImGui::SameLine();
+					}
+
+					if (ImGui::CollapsingHeader(ci.name.c_str())) {
+						ImGui::Indent(30.f);
+						ImGui::PushID("Widget");
+						ci.widget(registry, e);
+						ImGui::PopID();
+						ImGui::Unindent(30.f);
+					}
+					ImGui::PopID();
+				} else {
+					has_not[component_type_id] = ci;
+				}
+			}
+
+			if (!has_not.empty()) {
+				if (ImGui::Button("+ Add Component")) {
+					ImGui::OpenPopup("Add Component");
+				}
+
+				if (ImGui::BeginPopup("Add Component")) {
+					ImGui::TextUnformatted("Available:");
+					ImGui::Separator();
+
+					for (auto& [component_type_id, ci] : has_not) {
+						ImGui::PushID(component_type_id);
+						if (ImGui::Selectable(ci.name.c_str())) {
+							ci.create(registry, e);
+						}
+						ImGui::PopID();
+					}
+					ImGui::EndPopup();
+				}
+			}
+			ImGui::PopID();
+		}
+	}
+
+	void renderEntityList(Registry& registry, std::set<ComponentTypeID>& comp_list)
+	{
+		ImGui::Text("Components Filter:");
+		ImGui::SameLine();
+		if (ImGui::SmallButton("clear")) {
+			comp_list.clear();
+		}
+
+		ImGui::Indent();
+
+		for (const auto& [component_type_id, ci] : component_infos) {
+			bool is_in_list = comp_list.count(component_type_id);
+			bool active = is_in_list;
+
+			ImGui::Checkbox(ci.name.c_str(), &active);
+
+			if (is_in_list && !active) { // remove
+				comp_list.erase(component_type_id);
+			} else if (!is_in_list && active) { // add
+				comp_list.emplace(component_type_id);
+			}
+		}
+
+		ImGui::Unindent();
+		ImGui::Separator();
+
+		if (comp_list.empty()) {
+			ImGui::Text("Orphans:");
+			for (EntityType e : registry.template storage<EntityType>()) {
+				if (registry.orphan(e)) {
+					MM_IEEE_ENTITY_WIDGET(e, registry, false);
+				}
+			}
+		} else {
+			entt::basic_runtime_view<entt::basic_sparse_set<EntityType>> view{};
+			for (const auto type : comp_list) {
+				auto* storage_ptr = registry.storage(type);
+				if (storage_ptr != nullptr) {
+					view.iterate(*storage_ptr);
+				}
+			}
+
+			// TODO: add support for exclude
+
+			ImGui::Text("%lu Entities Matching:", view.size_hint());
+
+			if (ImGui::BeginChild("entity list")) {
+				for (auto e : view) {
+					MM_IEEE_ENTITY_WIDGET(e, registry, false);
+				}
+			}
+			ImGui::EndChild();
+		}
+	}
+
+	// displays both, editor and list
+	// uses static internally, use only as a quick way to get going!
+	void renderSimpleCombo(Registry& registry, EntityType& e)
+	{
+		if (show_window) {
+			ImGui::SetNextWindowSize(ImVec2(550, 400), ImGuiCond_FirstUseEver);
+			if (ImGui::Begin("Entity Editor", &show_window)) {
+				if (ImGui::BeginChild("list", {200, 0}, true)) {
+					static std::set<ComponentTypeID> comp_list;
+					renderEntityList(registry, comp_list);
+				}
+				ImGui::EndChild();
+
+				ImGui::SameLine();
+
+				if (ImGui::BeginChild("editor")) {
+					renderEditor(registry, e);
+				}
+				ImGui::EndChild();
+
+			}
+			ImGui::End();
+		}
+	}
+
+};
+
+} // MM
+
+// MIT License
+
+// Copyright (c) 2019-2022 Erik Scholz
+// Copyright (c) 2020 Gnik Droy
+
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+

src/main.cpp

@@ -9,6 +9,8 @@
 #include "./chat_gui/theme.hpp"
 
 #include "./start_screen.hpp"
+#include "./content/sdl_audio_frame_stream2.hpp"
+#include "./content/sdl_video_frame_stream2.hpp"
 
 #include <memory>
 #include <iostream>
@@ -58,6 +60,46 @@ int main(int argc, char** argv) {
 		}
 	}
 
+	// optionally init audio and camera
+	if (SDL_Init(SDL_INIT_AUDIO) < 0) {
+		std::cerr << "SDL_Init AUDIO failed (" << SDL_GetError() << ")\n";
+	} else if (false) {
+		SDLAudioInputDeviceDefault aidd;
+		auto* reader = aidd.aquireSubStream();
+
+		auto writer = SDLAudioOutputDeviceDefaultFactory{}.create();
+
+		for (size_t i = 0; i < 20; i++) {
+			std::this_thread::sleep_for(std::chrono::milliseconds(10));
+			auto new_frame_opt = reader->pop();
+			if (new_frame_opt.has_value()) {
+				std::cout << "audio frame was seq:" << new_frame_opt.value().seq << " sr:" << new_frame_opt.value().sample_rate << " " << (new_frame_opt.value().isS16()?"S16":"F32") << " l:" << (new_frame_opt.value().isS16()?new_frame_opt.value().getSpan<int16_t>().size:new_frame_opt.value().getSpan<float>().size) << "\n";
+				writer.push(new_frame_opt.value());
+			} else {
+				std::cout << "no audio frame\n";
+			}
+		}
+
+		aidd.releaseSubStream(reader);
+	}
+
+	if (SDL_Init(SDL_INIT_CAMERA) < 0) {
+		std::cerr << "SDL_Init CAMERA failed (" << SDL_GetError() << ")\n";
+	} else if (false) { // HACK
+		std::cerr << "CAMERA initialized\n";
+		SDLVideoCameraContent vcc;
+		auto* reader = vcc.aquireSubStream();
+		for (size_t i = 0; i < 20; i++) {
+			std::this_thread::sleep_for(std::chrono::milliseconds(50));
+			auto new_frame_opt = reader->pop();
+			if (new_frame_opt.has_value()) {
+				std::cout << "video frame was " << new_frame_opt.value().surface->w << "x" << new_frame_opt.value().surface->h << " " << new_frame_opt.value().timestampNS << "ns\n";
+			}
+		}
+		vcc.releaseSubStream(reader);
+	}
+	std::cout << "after sdl video stuffery\n";
+
 	IMGUI_CHECKVERSION();
 	ImGui::CreateContext();
 

src/main_screen.cpp

@@ -20,6 +20,7 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, Theme& theme_, std::string save_
 	tc(save_path, save_password),
 	tpi(tc.getTox()),
 	ad(tc),
+	tav(tc.getTox()),
 	tcm(cr, tc, tc),
 	tmm(rmm, cr, tcm, tc, tc),
 	ttm(rmm, cr, tcm, tc, tc),
@@ -34,6 +35,7 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, Theme& theme_, std::string save_
 	msg_tc(mil, sdlrtu),
 	cg(conf, rmm, cr, sdlrtu, contact_tc, msg_tc, theme),
 	sw(conf),
+	osui(os),
 	tuiu(tc, conf),
 	tdch(tpi)
 {
@@ -68,6 +70,7 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, Theme& theme_, std::string save_
 		g_provideInstance<ToxI>("ToxI", "host", &tc);
 		g_provideInstance<ToxPrivateI>("ToxPrivateI", "host", &tpi);
 		g_provideInstance<ToxEventProviderI>("ToxEventProviderI", "host", &tc);
+		g_provideInstance<ToxAV>("ToxAV", "host", &tav);
 		g_provideInstance<ToxContactModel2>("ToxContactModel2", "host", &tcm);
 
 		// TODO: pm?
@@ -244,6 +247,7 @@ Screen* MainScreen::render(float time_delta, bool&) {
 
 	const float cg_interval = cg.render(time_delta); // render
 	sw.render(); // render
+	osui.render();
 	tuiu.render(); // render
 	tdch.render(); // render
 

src/main_screen.hpp

@@ -11,6 +11,7 @@
 #include <solanaceae/plugin/plugin_manager.hpp>
 #include <solanaceae/toxcore/tox_event_logger.hpp>
 #include "./tox_private_impl.hpp"
+#include "./tox_av.hpp"
 
 #include <solanaceae/tox_contacts/tox_contact_model2.hpp>
 #include <solanaceae/tox_messages/tox_message_manager.hpp>
@@ -29,6 +30,7 @@
 
 #include "./chat_gui4.hpp"
 #include "./chat_gui/settings_window.hpp"
+#include "./object_store_ui.hpp"
 #include "./tox_ui_utils.hpp"
 #include "./tox_dht_cap_histo.hpp"
 #include "./tox_friend_faux_offline_messaging.hpp"
@@ -57,6 +59,7 @@ struct MainScreen final : public Screen {
 	ToxClient tc;
 	ToxPrivateImpl tpi;
 	AutoDirty ad;
+	ToxAV tav;
 	ToxContactModel2 tcm;
 	ToxMessageManager tmm;
 	ToxTransferManager ttm;
@@ -77,6 +80,7 @@ struct MainScreen final : public Screen {
 
 	ChatGui4 cg;
 	SettingsWindow sw;
+	ObjectStoreUI osui;
 	ToxUIUtils tuiu;
 	ToxDHTCapHisto tdch;
 

src/object_store_ui.cpp

@@ -0,0 +1,42 @@
+#include "./object_store_ui.hpp"
+
+#include <solanaceae/object_store/meta_components.hpp>
+
+#include <imgui/imgui.h>
+
+#include <solanaceae/message3/components.hpp>
+
+ObjectStoreUI::ObjectStoreUI(
+	ObjectStore2& os
+) : _os(os) {
+	_ee.show_window = false;
+
+	_ee.registerComponent<ObjectStore::Components::ID>("ID");
+	_ee.registerComponent<ObjectStore::Components::DataCompressionType>("DataCompressionType");
+
+	_ee.registerComponent<Message::Components::Transfer::FileInfo>("Transfer::FileInfo");
+	_ee.registerComponent<Message::Components::Transfer::FileInfoLocal>("Transfer::FileInfoLocal");
+}
+
+void ObjectStoreUI::render(void) {
+	{ // main window menubar injection
+		// assumes the window "tomato" was rendered already by cg
+		if (ImGui::Begin("tomato")) {
+			if (ImGui::BeginMenuBar()) {
+				ImGui::Separator();
+				if (ImGui::BeginMenu("ObjectStore")) {
+					if (ImGui::MenuItem("Inspector")) {
+						_ee.show_window = true;
+					}
+					ImGui::EndMenu();
+				}
+				ImGui::EndMenuBar();
+			}
+		}
+		ImGui::End();
+	}
+
+	static Object selected_ent {entt::null};
+	_ee.renderSimpleCombo(_os.registry(), selected_ent);
+}
+

src/object_store_ui.hpp

@@ -0,0 +1,19 @@
+#pragma once
+
+#include <solanaceae/object_store/object_store.hpp>
+
+#include "./imgui_entt_entity_editor.hpp"
+
+class ObjectStoreUI {
+	ObjectStore2& _os;
+
+	MM::EntityEditor<Object> _ee;
+
+	public:
+		ObjectStoreUI(
+			ObjectStore2& os
+		);
+
+		void render(void);
+};
+

src/tox_av.cpp

@@ -0,0 +1,82 @@
+#include "./tox_av.hpp"
+
+#include <cassert>
+
+// https://almogfx.bandcamp.com/track/crushed-w-cassade
+
+ToxAV::ToxAV(Tox* tox) : _tox(tox) {
+	Toxav_Err_New err_new {TOXAV_ERR_NEW_OK};
+	_tox_av = toxav_new(_tox, &err_new);
+	// TODO: throw
+	assert(err_new == TOXAV_ERR_NEW_OK);
+}
+ToxAV::~ToxAV(void) {
+	toxav_kill(_tox_av);
+}
+
+uint32_t ToxAV::toxavIterationInterval(void) const {
+	return toxav_iteration_interval(_tox_av);
+}
+
+void ToxAV::toxavIterate(void) {
+	toxav_iterate(_tox_av);
+}
+
+uint32_t ToxAV::toxavAudioIterationInterval(void) const {
+	return toxav_audio_iteration_interval(_tox_av);
+}
+
+void ToxAV::toxavAudioIterate(void) {
+	toxav_audio_iterate(_tox_av);
+}
+
+uint32_t ToxAV::toxavVideoIterationInterval(void) const {
+	return toxav_video_iteration_interval(_tox_av);
+}
+
+void ToxAV::toxavVideoIterate(void) {
+	toxav_video_iterate(_tox_av);
+}
+
+Toxav_Err_Call ToxAV::toxavCall(uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate) {
+	Toxav_Err_Call err {TOXAV_ERR_CALL_OK};
+	toxav_call(_tox_av, friend_number, audio_bit_rate, video_bit_rate, &err);
+	return err;
+}
+
+Toxav_Err_Answer ToxAV::toxavAnswer(uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate) {
+	Toxav_Err_Answer err {TOXAV_ERR_ANSWER_OK};
+	toxav_answer(_tox_av, friend_number, audio_bit_rate, video_bit_rate, &err);
+	return err;
+}
+
+Toxav_Err_Call_Control ToxAV::toxavCallControl(uint32_t friend_number, Toxav_Call_Control control) {
+	Toxav_Err_Call_Control err {TOXAV_ERR_CALL_CONTROL_OK};
+	toxav_call_control(_tox_av, friend_number, control, &err);
+	return err;
+}
+
+Toxav_Err_Send_Frame ToxAV::toxavAudioSendFrame(uint32_t friend_number, const int16_t pcm[], size_t sample_count, uint8_t channels, uint32_t sampling_rate) {
+	Toxav_Err_Send_Frame err {TOXAV_ERR_SEND_FRAME_OK};
+	toxav_audio_send_frame(_tox_av, friend_number, pcm, sample_count, channels, sampling_rate, &err);
+	return err;
+}
+
+Toxav_Err_Bit_Rate_Set ToxAV::toxavAudioSetBitRate(uint32_t friend_number, uint32_t bit_rate) {
+	Toxav_Err_Bit_Rate_Set err {TOXAV_ERR_BIT_RATE_SET_OK};
+	toxav_audio_set_bit_rate(_tox_av, friend_number, bit_rate, &err);
+	return err;
+}
+
+Toxav_Err_Send_Frame ToxAV::toxavVideoSendFrame(uint32_t friend_number, uint16_t width, uint16_t height, const uint8_t y[], const uint8_t u[], const uint8_t v[]) {
+	Toxav_Err_Send_Frame err {TOXAV_ERR_SEND_FRAME_OK};
+	toxav_video_send_frame(_tox_av, friend_number, width, height, y, u, v, &err);
+	return err;
+}
+
+Toxav_Err_Bit_Rate_Set ToxAV::toxavVideoSetBitRate(uint32_t friend_number, uint32_t bit_rate) {
+	Toxav_Err_Bit_Rate_Set err {TOXAV_ERR_BIT_RATE_SET_OK};
+	toxav_video_set_bit_rate(_tox_av, friend_number, bit_rate, &err);
+	return err;
+}
+

src/tox_av.hpp

@@ -0,0 +1,37 @@
+#pragma once
+
+#include <tox/toxav.h>
+
+struct ToxAV {
+	Tox* _tox = nullptr;
+	ToxAV* _tox_av = nullptr;
+
+	ToxAV(Tox* tox);
+	virtual ~ToxAV(void);
+
+	// interface
+	uint32_t toxavIterationInterval(void) const;
+	void toxavIterate(void);
+
+	uint32_t toxavAudioIterationInterval(void) const;
+	void toxavAudioIterate(void);
+	uint32_t toxavVideoIterationInterval(void) const;
+	void toxavVideoIterate(void);
+
+	Toxav_Err_Call toxavCall(uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate);
+	Toxav_Err_Answer toxavAnswer(uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate);
+	Toxav_Err_Call_Control toxavCallControl(uint32_t friend_number, Toxav_Call_Control control);
+	Toxav_Err_Send_Frame toxavAudioSendFrame(uint32_t friend_number, const int16_t pcm[], size_t sample_count, uint8_t channels, uint32_t sampling_rate);
+	Toxav_Err_Bit_Rate_Set toxavAudioSetBitRate(uint32_t friend_number, uint32_t bit_rate);
+	Toxav_Err_Send_Frame toxavVideoSendFrame(uint32_t friend_number, uint16_t width, uint16_t height, const uint8_t y[/*! height * width */], const uint8_t u[/*! height/2 * width/2 */], const uint8_t v[/*! height/2 * width/2 */]);
+	Toxav_Err_Bit_Rate_Set toxavVideoSetBitRate(uint32_t friend_number, uint32_t bit_rate);
+
+//int32_t toxav_add_av_groupchat(Tox *tox, toxav_audio_data_cb *audio_callback, void *userdata);
+//int32_t toxav_join_av_groupchat(Tox *tox, uint32_t friendnumber, const uint8_t data[], uint16_t length, toxav_audio_data_cb *audio_callback, void *userdata);
+//int32_t toxav_group_send_audio(Tox *tox, uint32_t groupnumber, const int16_t pcm[], uint32_t samples, uint8_t channels, uint32_t sample_rate);
+//int32_t toxav_groupchat_enable_av(Tox *tox, uint32_t groupnumber, toxav_audio_data_cb *audio_callback, void *userdata);
+//int32_t toxav_groupchat_disable_av(Tox *tox, uint32_t groupnumber);
+//bool toxav_groupchat_av_enabled(Tox *tox, uint32_t groupnumber);
+
+};
+