Tha_14/tomato-testing
1
0
Fork 0
forked from Green-Sky/tomato
Code Pull Requests Actions Packages Releases Activity

support direct yuv texture upload (limited) and update dvt

Browse Source
This commit is contained in:
Green Sky 2024-10-05 13:53:55 +02:00
parent 050af74ef2
commit 7886dc5e39
No known key found for this signature in database
5 changed files with 118 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
src/debug_video_tap.cpp
View File

@ -59,7 +59,7 @@ struct DebugVideoTapSink : public FrameStream2SinkI<SDLVideoFrame> {
std::shared_ptr<FrameStream2I<SDLVideoFrame>> subscribe(void) override { std::shared_ptr<FrameStream2I<SDLVideoFrame>> subscribe(void) override {
_writers.emplace_back(Writer{ _writers.emplace_back(Writer{
Writer::View{_id_counter++}, Writer::View{_id_counter++},
std::make_shared<PushConversionVideoStream<LockedFrameStream2<SDLVideoFrame>>>(SDL_PIXELFORMAT_RGBA32) std::make_shared<PushConversionVideoStream<LockedFrameStream2<SDLVideoFrame>>>(SDL_PIXELFORMAT_IYUV)
}); });
return _writers.back().stream; return _writers.back().stream;
@ -189,7 +189,7 @@ float DebugVideoTap::render(void) {
std::string window_title {"DebugVideoTap #"}; std::string window_title {"DebugVideoTap #"};
window_title += std::to_string(view._id); window_title += std::to_string(view._id);
ImGui::SetNextWindowSize({400, 420}, ImGuiCond_Appearing); ImGui::SetNextWindowSize({400, 420}, ImGuiCond_Appearing);
if (ImGui::Begin(window_title.c_str())) { if (ImGui::Begin(window_title.c_str(), nullptr, ImGuiWindowFlags_NoScrollbar)) {
while (auto new_frame_opt = stream->pop()) { while (auto new_frame_opt = stream->pop()) {
// timing // timing
if (view._v_last_ts == 0) { if (view._v_last_ts == 0) {
@ -209,20 +209,21 @@ float DebugVideoTap::render(void) {
SDL_Surface* new_frame_surf = new_frame_opt.value().surface.get(); SDL_Surface* new_frame_surf = new_frame_opt.value().surface.get();
SDL_Surface* converted_surf = new_frame_surf; SDL_Surface* converted_surf = new_frame_surf;
if (new_frame_surf->format != SDL_PIXELFORMAT_RGBA32) { //if (new_frame_surf->format != SDL_PIXELFORMAT_RGBA32) {
// we need to convert // // we need to convert
//std::cerr << "DVT: need to convert\n"; // //std::cerr << "DVT: need to convert\n";
converted_surf = SDL_ConvertSurfaceAndColorspace(new_frame_surf, SDL_PIXELFORMAT_RGBA32, nullptr, SDL_COLORSPACE_RGB_DEFAULT, 0); // converted_surf = SDL_ConvertSurfaceAndColorspace(new_frame_surf, SDL_PIXELFORMAT_RGBA32, nullptr, SDL_COLORSPACE_RGB_DEFAULT, 0);
assert(converted_surf->format == SDL_PIXELFORMAT_RGBA32); // assert(converted_surf->format == SDL_PIXELFORMAT_RGBA32);
} //}
SDL_LockSurface(converted_surf); SDL_LockSurface(converted_surf);
if (view._tex == 0 || (int)view._tex_w != converted_surf->w || (int)view._tex_h != converted_surf->h) { if (view._tex == 0 || (int)view._tex_w != converted_surf->w || (int)view._tex_h != converted_surf->h) {
_tu.destroy(view._tex); _tu.destroy(view._tex);
view._tex = _tu.uploadRGBA( view._tex = _tu.upload(
static_cast<const uint8_t*>(converted_surf->pixels), static_cast<const uint8_t*>(converted_surf->pixels),
converted_surf->w, converted_surf->w,
converted_surf->h, converted_surf->h,
TextureUploaderI::IYUV, // forced conversion
TextureUploaderI::LINEAR, TextureUploaderI::LINEAR,
TextureUploaderI::STREAMING TextureUploaderI::STREAMING
); );
@ -230,14 +231,16 @@ float DebugVideoTap::render(void) {
view._tex_w = converted_surf->w; view._tex_w = converted_surf->w;
view._tex_h = converted_surf->h; view._tex_h = converted_surf->h;
} else { } else {
_tu.updateRGBA(view._tex, static_cast<const uint8_t*>(converted_surf->pixels), converted_surf->w * converted_surf->h * 4); //_tu.update(view._tex, static_cast<const uint8_t*>(converted_surf->pixels), converted_surf->w * converted_surf->h * 4);
_tu.update(view._tex, static_cast<const uint8_t*>(converted_surf->pixels), converted_surf->w * converted_surf->h * 3/2);
//_tu.updateRGBA(view._tex, static_cast<const uint8_t*>(converted_surf->pixels), converted_surf->w * converted_surf->h * 4);
} }
SDL_UnlockSurface(converted_surf); SDL_UnlockSurface(converted_surf);
if (new_frame_surf != converted_surf) { //if (new_frame_surf != converted_surf) {
// clean up temp // // clean up temp
SDL_DestroySurface(converted_surf); // SDL_DestroySurface(converted_surf);
} //}
} }
ImGui::Checkbox("mirror ", &view._mirror); ImGui::Checkbox("mirror ", &view._mirror);
@ -245,7 +248,7 @@ float DebugVideoTap::render(void) {
// img here // img here
if (view._tex != 0) { if (view._tex != 0) {
ImGui::SameLine(); ImGui::SameLine();
ImGui::Text("%dx%d ~avg interval: %.0fms (%.2ffps)", view._tex_w, view._tex_h, view._v_interval_avg*1000.f, 1.f/view._v_interval_avg); ImGui::Text("%dx%d interval: ~%.0fms (%.2ffps)", view._tex_w, view._tex_h, view._v_interval_avg*1000.f, 1.f/view._v_interval_avg);
const float img_w = ImGui::GetContentRegionAvail().x; const float img_w = ImGui::GetContentRegionAvail().x;
ImGui::Image( ImGui::Image(
reinterpret_cast<ImTextureID>(view._tex), reinterpret_cast<ImTextureID>(view._tex),

9
src/frame_streams/sdl/video_push_converter.hpp
View File

@ -5,6 +5,7 @@
#include <cassert> #include <cassert>
#include <chrono> // meh
#include <iostream> // meh #include <iostream> // meh
template<typename RealStream> template<typename RealStream>
@ -20,6 +21,7 @@ struct PushConversionVideoStream : public RealStream {
SDL_Surface* surf = value.surface.get(); SDL_Surface* surf = value.surface.get();
if (surf->format != _forced_format) { if (surf->format != _forced_format) {
//std::cerr << "PCVS: need to convert from " << SDL_GetPixelFormatName(surf->format) << " to " << SDL_GetPixelFormatName(_forced_format) << "\n"; //std::cerr << "PCVS: need to convert from " << SDL_GetPixelFormatName(surf->format) << " to " << SDL_GetPixelFormatName(_forced_format) << "\n";
//const auto start = std::chrono::steady_clock::now();
if ((surf = SDL_ConvertSurface(surf, _forced_format)) == nullptr) { if ((surf = SDL_ConvertSurface(surf, _forced_format)) == nullptr) {
surf = value.surface.get(); // reset ptr surf = value.surface.get(); // reset ptr
//std::cerr << "PCVS warning: default conversion failed: " << SDL_GetError() << "\n"; //std::cerr << "PCVS warning: default conversion failed: " << SDL_GetError() << "\n";
@ -42,15 +44,16 @@ struct PushConversionVideoStream : public RealStream {
} }
} }
} }
//const auto end = std::chrono::steady_clock::now();
//std::cerr << "PCVS: conversion took " << std::chrono::duration_cast<std::chrono::milliseconds>(end-start).count() << "ms\n";
if (surf == nullptr) { if (surf == nullptr) {
// oh god // oh god
std::cerr << "PCVS error: failed to convert surface to IYUV: " << SDL_GetError() << "\n"; std::cerr << "PCVS error: failed to convert surface to " << SDL_GetPixelFormatName(_forced_format) << ": " << SDL_GetError() << "\n";
return false; return false;
} }
}
assert(surf != nullptr); assert(surf != nullptr);
if (surf != value.surface.get()) {
// TODO: add ctr with uptr // TODO: add ctr with uptr
SDLVideoFrame new_value{value.timestampUS, nullptr}; SDLVideoFrame new_value{value.timestampUS, nullptr};
new_value.surface = { new_value.surface = {

81
src/sdlrenderer_texture_uploader.cpp
View File

@ -3,23 +3,59 @@
#include <cassert> #include <cassert>
#include <cstdint> #include <cstdint>
#include <cstring> #include <cstring>
#include <memory>
#include <iostream> #include <iostream>
static SDL_PixelFormat format2sdl(TextureUploaderI::Format format) {
switch (format) {
case TextureUploaderI::Format::RGBA: return SDL_PIXELFORMAT_RGBA32;
//case TextureUploaderI::Format::RGB: return SDL_PIXELFORMAT_RGB24;
case TextureUploaderI::Format::IYUV: return SDL_PIXELFORMAT_IYUV;
case TextureUploaderI::Format::YV12: return SDL_PIXELFORMAT_YV12;
case TextureUploaderI::Format::NV12: return SDL_PIXELFORMAT_NV12;
case TextureUploaderI::Format::NV21: return SDL_PIXELFORMAT_NV21;
case TextureUploaderI::Format::MAX: return SDL_PIXELFORMAT_UNKNOWN;
}
return SDL_PIXELFORMAT_UNKNOWN;
}
SDLRendererTextureUploader::SDLRendererTextureUploader(SDL_Renderer* renderer_) : SDLRendererTextureUploader::SDLRendererTextureUploader(SDL_Renderer* renderer_) :
renderer(renderer_) renderer(renderer_)
{ {
} }
uint64_t SDLRendererTextureUploader::uploadRGBA(const uint8_t* data, uint32_t width, uint32_t height, Filter filter, Access access) { uint64_t SDLRendererTextureUploader::uploadRGBA(const uint8_t* data, uint32_t width, uint32_t height, Filter filter, Access access) {
// TODO: test if pitch is 4 or 4*width return upload(data, width, height, Format::RGBA, filter, access);
SDL_Surface* surf = SDL_CreateSurfaceFrom( }
bool SDLRendererTextureUploader::updateRGBA(uint64_t tex_id, const uint8_t* data, size_t size) {
return update(tex_id, data, size);
}
uint64_t SDLRendererTextureUploader::upload(const uint8_t* data, uint32_t width, uint32_t height, Format format, Filter filter, Access access) {
const auto sdl_format = format2sdl(format);
if (sdl_format == SDL_PIXELFORMAT_UNKNOWN) {
std::cerr << "SDLRTU error: unsupported format '" << format << "'\n";
return 0;
}
// TODO: why do we even create a non owning surface here???
std::unique_ptr<SDL_Surface, decltype(&SDL_DestroySurface)> surf = {
SDL_CreateSurfaceFrom(
width, height, width, height,
SDL_PIXELFORMAT_RGBA32, // auto big/little sdl_format,
(void*)data, (void*)data,
4*width 4*width // ???
); ),
assert(surf); // TODO: add error reporting &SDL_DestroySurface
};
assert(surf);
if (!surf) {
std::cerr << "SDLRTU error: surf creation failed " << SDL_GetError() << "\n";
return 0;
}
SDL_Texture* tex = SDL_CreateTexture( SDL_Texture* tex = SDL_CreateTexture(
renderer, renderer,
@ -27,12 +63,33 @@ uint64_t SDLRendererTextureUploader::uploadRGBA(const uint8_t* data, uint32_t wi
access == Access::STREAMING ? SDL_TEXTUREACCESS_STREAMING : SDL_TEXTUREACCESS_STATIC, access == Access::STREAMING ? SDL_TEXTUREACCESS_STREAMING : SDL_TEXTUREACCESS_STATIC,
surf->w, surf->h surf->w, surf->h
); );
assert(tex); // TODO: add error reporting
// TODO: error reporting assert(tex);
SDL_UpdateTexture(tex, nullptr, surf->pixels, surf->pitch); if (tex == nullptr) {
std::cerr << "SDLRTU error: tex creation failed " << SDL_GetError() << "\n";
return 0;
}
bool need_to_lock = SDL_MUSTLOCK(surf);
if (need_to_lock) {
if (!SDL_LockSurface(surf.get())) {
std::cerr << "SDLRTU error: failed to lock surface " << SDL_GetError() << "\n";
SDL_DestroyTexture(tex);
return 0;
}
}
if (!SDL_UpdateTexture(tex, nullptr, surf->pixels, surf->pitch)) {
std::cerr << "SDLRTU error: tex update failed " << SDL_GetError() << "\n";
}
if (need_to_lock) {
// error check?
SDL_UnlockSurface(surf.get());
}
SDL_BlendMode surf_blend_mode = SDL_BLENDMODE_NONE; SDL_BlendMode surf_blend_mode = SDL_BLENDMODE_NONE;
if (SDL_GetSurfaceBlendMode(surf, &surf_blend_mode)) { if (SDL_GetSurfaceBlendMode(surf.get(), &surf_blend_mode)) {
SDL_SetTextureBlendMode(tex, surf_blend_mode); SDL_SetTextureBlendMode(tex, surf_blend_mode);
} }
@ -42,12 +99,10 @@ uint64_t SDLRendererTextureUploader::uploadRGBA(const uint8_t* data, uint32_t wi
SDL_SetTextureScaleMode(tex, SDL_SCALEMODE_LINEAR); SDL_SetTextureScaleMode(tex, SDL_SCALEMODE_LINEAR);
} }
SDL_DestroySurface(surf);
return reinterpret_cast<uint64_t>(tex); return reinterpret_cast<uint64_t>(tex);
} }
bool SDLRendererTextureUploader::updateRGBA(uint64_t tex_id, const uint8_t* data, size_t size) { bool SDLRendererTextureUploader::update(uint64_t tex_id, const uint8_t* data, size_t size) {
auto* texture = static_cast<SDL_Texture*>(reinterpret_cast<void*>(tex_id)); auto* texture = static_cast<SDL_Texture*>(reinterpret_cast<void*>(tex_id));
if (texture == nullptr) { if (texture == nullptr) {
return false; return false;

2
src/sdlrenderer_texture_uploader.hpp
View File

@ -12,6 +12,8 @@ struct SDLRendererTextureUploader : public TextureUploaderI {
uint64_t uploadRGBA(const uint8_t* data, uint32_t width, uint32_t height, Filter filter, Access access) override; uint64_t uploadRGBA(const uint8_t* data, uint32_t width, uint32_t height, Filter filter, Access access) override;
bool updateRGBA(uint64_t tex_id, const uint8_t* data, size_t size) override; bool updateRGBA(uint64_t tex_id, const uint8_t* data, size_t size) override;
uint64_t upload(const uint8_t* data, uint32_t width, uint32_t height, Format format, Filter filter, Access access) override;
bool update(uint64_t tex_id, const uint8_t* data, size_t size) override;
void destroy(uint64_t tex_id) override; void destroy(uint64_t tex_id) override;
}; };

23
src/texture_uploader.hpp
View File

@ -4,7 +4,7 @@
#include <cstddef> #include <cstddef>
struct TextureUploaderI { struct TextureUploaderI {
static constexpr const char* version {"2"}; static constexpr const char* version {"3"};
enum Filter { enum Filter {
NEAREST, NEAREST,
@ -17,13 +17,30 @@ struct TextureUploaderI {
// target? // target?
}; };
enum Format {
RGBA,
//RGB,
IYUV,
YV12,
NV12,
NV21,
MAX
};
virtual ~TextureUploaderI(void) {} virtual ~TextureUploaderI(void) {}
virtual uint64_t uploadRGBA(const uint8_t* data, uint32_t width, uint32_t height, Filter filter = LINEAR, Access access = STATIC) = 0; [[deprecated]] virtual uint64_t uploadRGBA(const uint8_t* data, uint32_t width, uint32_t height, Filter filter = LINEAR, Access access = STATIC) = 0;
// keeps width height filter // keeps width height filter
// TODO: wh instead of size? // TODO: wh instead of size?
virtual bool updateRGBA(uint64_t tex_id, const uint8_t* data, size_t size) = 0; [[deprecated]] virtual bool updateRGBA(uint64_t tex_id, const uint8_t* data, size_t size) = 0;
// use upload to create a texture, and update to update existing
virtual uint64_t upload(const uint8_t* data, uint32_t width, uint32_t height, Format format = RGBA, Filter filter = LINEAR, Access access = STATIC) = 0;
virtual bool update(uint64_t tex_id, const uint8_t* data, size_t size) = 0;
virtual void destroy(uint64_t tex_id) = 0; virtual void destroy(uint64_t tex_id) = 0;
}; };