Compare commits
54 Commits
0af404a51c
...
fcd15279c8
Author | SHA1 | Date | |
---|---|---|---|
fcd15279c8 | |||
d19ce9fadd | |||
83e4d90b22 | |||
9f33db9406 | |||
1e0c61ed9f | |||
b547c54d6d | |||
7ae488cc02 | |||
0eade9a36f | |||
ea9bb41bc9 | |||
5f39ae0d29 | |||
e212b4a807 | |||
b66c4b8f36 | |||
50b1b5141a | |||
ad8123045b | |||
e3c1106e5d | |||
111ffcb23a | |||
79f1f906ef | |||
278e18686a | |||
6aa4c63860 | |||
56c9b04aa3 | |||
024b99fcc1 | |||
69c670f82d | |||
bb33cd77dc | |||
221cfcf913 | |||
5f1bde1587 | |||
3c91e75396 | |||
75295fe354 | |||
8e2cb03c2c | |||
7eab00e216 | |||
dcee8d3888 | |||
e27834599d | |||
8edb2c1a18 | |||
97ffb64025 | |||
3822351430 | |||
dda66af486 | |||
d1c3b95da5 | |||
751fc403c3 | |||
9d55d84bdf | |||
0b291da068 | |||
b88c62b0fd | |||
1714abaf9c | |||
bd18ea83fe | |||
3f26385e2a | |||
ad739ce505 | |||
443721de52 | |||
624c672696 | |||
eb588f34b5 | |||
d20e95daba | |||
2a82c5455d | |||
b1d4848b81 | |||
3678301916 | |||
459ccf7d6b | |||
db54afd180 | |||
f4fe94dfe6 |
@ -23,8 +23,8 @@ option(TOMATO_ASAN "Build tomato with asan (gcc/clang/msvc)" OFF)
|
||||
if (TOMATO_ASAN)
|
||||
if (${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU" OR ${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang")
|
||||
if (NOT WIN32) # exclude mingw
|
||||
link_libraries(-fsanitize=address)
|
||||
#link_libraries(-fsanitize=address,undefined)
|
||||
#link_libraries(-fsanitize=address)
|
||||
link_libraries(-fsanitize=address,undefined)
|
||||
#link_libraries(-fsanitize=undefined)
|
||||
message("II enabled ASAN")
|
||||
else()
|
||||
|
33
external/CMakeLists.txt
vendored
33
external/CMakeLists.txt
vendored
@ -1,4 +1,4 @@
|
||||
cmake_minimum_required(VERSION 3.9 FATAL_ERROR)
|
||||
cmake_minimum_required(VERSION 3.14...3.24 FATAL_ERROR)
|
||||
|
||||
add_subdirectory(./entt)
|
||||
|
||||
@ -19,3 +19,34 @@ add_subdirectory(./stb)
|
||||
add_subdirectory(./libwebp)
|
||||
add_subdirectory(./qoi)
|
||||
|
||||
if (NOT TARGET nlohmann_json::nlohmann_json)
|
||||
FetchContent_Declare(json
|
||||
URL https://github.com/nlohmann/json/releases/download/v3.11.3/json.tar.xz
|
||||
URL_HASH SHA256=d6c65aca6b1ed68e7a182f4757257b107ae403032760ed6ef121c9d55e81757d
|
||||
EXCLUDE_FROM_ALL
|
||||
)
|
||||
FetchContent_MakeAvailable(json)
|
||||
endif()
|
||||
|
||||
if (NOT TARGET zstd::zstd)
|
||||
# TODO: try find_package() first
|
||||
# TODO: try pkg-config next (will work on most distros)
|
||||
|
||||
set(ZSTD_BUILD_STATIC ON)
|
||||
set(ZSTD_BUILD_SHARED OFF)
|
||||
set(ZSTD_BUILD_PROGRAMS OFF)
|
||||
set(ZSTD_BUILD_CONTRIB OFF)
|
||||
set(ZSTD_BUILD_TESTS OFF)
|
||||
FetchContent_Declare(zstd
|
||||
URL "https://github.com/facebook/zstd/releases/download/v1.5.5/zstd-1.5.5.tar.gz"
|
||||
DOWNLOAD_EXTRACT_TIMESTAMP TRUE
|
||||
SOURCE_SUBDIR build/cmake
|
||||
EXCLUDE_FROM_ALL
|
||||
)
|
||||
FetchContent_MakeAvailable(zstd)
|
||||
|
||||
add_library(zstd INTERFACE) # somehow zstd fkd this up
|
||||
target_include_directories(zstd INTERFACE ${zstd_SOURCE_DIR}/lib/)
|
||||
target_link_libraries(zstd INTERFACE libzstd_static)
|
||||
add_library(zstd::zstd ALIAS zstd)
|
||||
endif()
|
||||
|
8
external/qoi/CMakeLists.txt
vendored
8
external/qoi/CMakeLists.txt
vendored
@ -2,12 +2,10 @@ cmake_minimum_required(VERSION 3.13...3.24 FATAL_ERROR)
|
||||
|
||||
project(qoi C CXX)
|
||||
|
||||
# do fetch or subtree
|
||||
|
||||
#add_library(stb INTERFACE)
|
||||
#target_include_directories(stb SYSTEM INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}")
|
||||
add_library(qoi_interface INTERFACE)
|
||||
target_include_directories(qoi_interface SYSTEM INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}")
|
||||
|
||||
# static lib with impl
|
||||
add_library(qoi "qoi.cpp")
|
||||
#target_link_libraries(qoi stb)
|
||||
target_link_libraries(qoi qoi_interface)
|
||||
|
||||
|
4
external/qoi/qoi.cpp
vendored
4
external/qoi/qoi.cpp
vendored
@ -1 +1,3 @@
|
||||
// TODO: include and impl
|
||||
#define QOI_IMPLEMENTATION
|
||||
#include <qoi/qoi.h>
|
||||
|
||||
|
5
external/qoi/qoi/.gitignore
vendored
Normal file
5
external/qoi/qoi/.gitignore
vendored
Normal file
@ -0,0 +1,5 @@
|
||||
images/
|
||||
stb_image.h
|
||||
stb_image_write.h
|
||||
qoibench
|
||||
qoiconv
|
21
external/qoi/qoi/LICENSE
vendored
Normal file
21
external/qoi/qoi/LICENSE
vendored
Normal file
@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2022 Dominic Szablewski
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
22
external/qoi/qoi/Makefile
vendored
Normal file
22
external/qoi/qoi/Makefile
vendored
Normal file
@ -0,0 +1,22 @@
|
||||
CC ?= gcc
|
||||
CFLAGS_BENCH ?= -std=gnu99 -O3
|
||||
LFLAGS_BENCH ?= -lpng
|
||||
CFLAGS_CONV ?= -std=c99 -O3
|
||||
|
||||
TARGET_BENCH ?= qoibench
|
||||
TARGET_CONV ?= qoiconv
|
||||
|
||||
all: $(TARGET_BENCH) $(TARGET_CONV)
|
||||
|
||||
bench: $(TARGET_BENCH)
|
||||
|
||||
$(TARGET_BENCH):$(TARGET_BENCH).c
|
||||
$(CC) $(CFLAGS_BENCH) $(CFLAGS) $(TARGET_BENCH).c -o $(TARGET_BENCH) $(LFLAGS_BENCH)
|
||||
|
||||
conv: $(TARGET_CONV)
|
||||
$(TARGET_CONV):$(TARGET_CONV).c
|
||||
$(CC) $(CFLAGS_CONV) $(CFLAGS) $(TARGET_CONV).c -o $(TARGET_CONV)
|
||||
|
||||
.PHONY: clean
|
||||
clean:
|
||||
$(RM) $(TARGET_BENCH) $(TARGET_CONV)
|
179
external/qoi/qoi/README.md
vendored
Normal file
179
external/qoi/qoi/README.md
vendored
Normal file
@ -0,0 +1,179 @@
|
||||
![QOI Logo](https://qoiformat.org/qoi-logo.svg)
|
||||
|
||||
# QOI - The “Quite OK Image Format” for fast, lossless image compression
|
||||
|
||||
Single-file MIT licensed library for C/C++
|
||||
|
||||
See [qoi.h](https://github.com/phoboslab/qoi/blob/master/qoi.h) for
|
||||
the documentation and format specification.
|
||||
|
||||
More info at https://qoiformat.org
|
||||
|
||||
|
||||
## Why?
|
||||
|
||||
Compared to stb_image and stb_image_write QOI offers 20x-50x faster encoding,
|
||||
3x-4x faster decoding and 20% better compression. It's also stupidly simple and
|
||||
fits in about 300 lines of C.
|
||||
|
||||
|
||||
## Example Usage
|
||||
|
||||
- [qoiconv.c](https://github.com/phoboslab/qoi/blob/master/qoiconv.c)
|
||||
converts between png <> qoi
|
||||
- [qoibench.c](https://github.com/phoboslab/qoi/blob/master/qoibench.c)
|
||||
a simple wrapper to benchmark stbi, libpng and qoi
|
||||
|
||||
|
||||
## MIME Type, File Extension
|
||||
|
||||
The recommended MIME type for QOI images is `image/qoi`. While QOI is not yet
|
||||
officially registered with IANA, I believe QOI has found enough adoption to
|
||||
prevent any future image format from choosing the same name, thus making a
|
||||
MIME type collision highly unlikely ([see #167](https://github.com/phoboslab/qoi/issues/167)).
|
||||
|
||||
The recommended file extension for QOI images is `.qoi`
|
||||
|
||||
|
||||
## Limitations
|
||||
|
||||
The QOI file format allows for huge images with up to 18 exa-pixels. A streaming
|
||||
en-/decoder can handle these with minimal RAM requirements, assuming there is
|
||||
enough storage space.
|
||||
|
||||
This particular implementation of QOI however is limited to images with a
|
||||
maximum size of 400 million pixels. It will safely refuse to en-/decode anything
|
||||
larger than that. This is not a streaming en-/decoder. It loads the whole image
|
||||
file into RAM before doing any work and is not extensively optimized for
|
||||
performance (but it's still very fast).
|
||||
|
||||
If this is a limitation for your use case, please look into any of the other
|
||||
implementations listed below.
|
||||
|
||||
|
||||
## Improvements, New Versions and Contributing
|
||||
|
||||
The QOI format has been finalized. It was a conscious decision to **not** have a
|
||||
version number in the file header. If you have a working QOI implementation today,
|
||||
you can rest assured that it will be compatible with all QOI files tomorrow.
|
||||
|
||||
There are a lot of interesting ideas for a successor of QOI, but none of these will
|
||||
be implemented here. That doesn't mean you shouldn't experiment with QOI, but please
|
||||
be aware that pull requests that change the format will not be accepted.
|
||||
|
||||
Likewise, pull requests for performance improvements will probably not be accepted
|
||||
either, as this "reference implementation" tries to be as easy to read as possible.
|
||||
|
||||
|
||||
## Tools
|
||||
|
||||
- [floooh/qoiview](https://github.com/floooh/qoiview) - native QOI viewer
|
||||
- [pfusik/qoi-fu](https://github.com/pfusik/qoi-fu/releases) - QOI Plugin installer for Windows Explorer, Finder, GNOME, GIMP, Paint.NET and XnView
|
||||
- [iOrange/QoiFileTypeNet](https://github.com/iOrange/QoiFileTypeNet/releases) - QOI Plugin for Paint.NET
|
||||
- [iOrange/QOIThumbnailProvider](https://github.com/iOrange/QOIThumbnailProvider) - Add thumbnails for QOI images in Windows Explorer
|
||||
- [Tom94/tev](https://github.com/Tom94/tev) - another native QOI viewer (allows pixel peeping and comparison with other image formats)
|
||||
- [qoiconverterx](https://apps.apple.com/br/app/qoiconverterx/id1602159820) QOI <=> PNG converter available on the Mac App Store
|
||||
- [kaetemi/qoi-ma](https://github.com/kaetemi/qoi-max) - QOI Bitmap I/O Plugin for 3ds Max
|
||||
- [rtexviewer](https://raylibtech.itch.io/rtexviewer) - texture viewer, supports QOI
|
||||
- [rtexpacker](https://raylibtech.itch.io/rtexpacker) - texture packer, supports QOI
|
||||
- [DmitriySalnikov/godot_qoi](https://github.com/DmitriySalnikov/godot_qoi) - QOI GDNative Addon for Godot Engine
|
||||
- [dan9er/farbfeld-convert-qoi](https://gitlab.com/dan9er/farbfeld-convert-qoi) - QOI <=> farbfeld converter
|
||||
- [LTMX/Unity.QOI](https://github.com/LTMX/Unity.QOI) - QOI Importer and Exporter for the Unity3D Game Engine
|
||||
- [Ben1138/unity-qoi](https://github.com/Ben1138/unity-qoi) - QOI Importer(only) support for the Unity3D Game Engine
|
||||
- [xiaozhuai/jetbrains-qo](https://github.com/xiaozhuai/jetbrains-qoi) - [QOI Support](https://plugins.jetbrains.com/plugin/19352-qoi-support) for Jetbrains' IDE.
|
||||
- [serge-ivamov/QOIql](https://github.com/serge-ivamov/QOIql) - MacOS QuickLook plugin for QOI
|
||||
- [tobozo/kde-thumbnailer-qoi](https://github.com/tobozo/kde-thumbnailer-qoi) - QOI Thumbnailer for KDE
|
||||
- [walksanatora/qoi-thumbnailer-nemo](https://github.com/walksanatora/qoi-thumbnailer-nemo) - QOI Thumbnailer for Nemo
|
||||
- [hzeller/timg](https://github.com/hzeller/timg) - a terminal image viewer with QOI support
|
||||
- [LuisAlfredo92/Super-QOI-converter](https://github.com/LuisAlfredo92/Super-QOI-converter "LuisAlfredo92/Super-QOI-converter") - A program to convert JPG, JPEG, BMP, and PNG to QOI
|
||||
- [Console version](https://github.com/LuisAlfredo92/Super-QOI-converter-Console- "Console version"): Available for Linux, OSX and Windows
|
||||
- [GUI version](https://github.com/LuisAlfredo92/Super-QOI-converter-GUI- "GUI version"): Available only for windows
|
||||
- [tacent view](https://github.com/bluescan/tacentview) - Image and texture viewer, supports QOI
|
||||
- [colemanrgb/qoi2spr](https://github.com/colemanrgb/qoi2spr) - A variety of applications for decoding and encoding of QOI images on [RISC OS](https://www.riscosopen.org/)
|
||||
|
||||
## Implementations & Bindings of QOI
|
||||
|
||||
- [pfusik/qoi-fu](https://github.com/pfusik/qoi-fu) - Fusion, transpiling to C, C++, C#, D, Java, JavaScript, Python, Swift and TypeScript
|
||||
- [kodonnell/qoi](https://github.com/kodonnell/qoi) - Python
|
||||
- [JaffaKetchup/dqoi](https://github.com/JaffaKetchup/dqoi) - Dart, with Flutter support
|
||||
- [Cr4xy/lua-qoi](https://github.com/Cr4xy/lua-qoi) - Lua
|
||||
- [superzazu/SDL_QOI](https://github.com/superzazu/SDL_QOI) - C, SDL2 bindings
|
||||
- [saharNooby/qoi-java](https://github.com/saharNooby/qoi-java) - Java
|
||||
- [MasterQ32/zig-qoi](https://github.com/MasterQ32/zig-qoi) - Zig
|
||||
- [rbino/qoix](https://github.com/rbino/qoix) - Elixir
|
||||
- [NUlliiON/QoiSharp](https://github.com/NUlliiON/QoiSharp) - C#
|
||||
- [aldanor/qoi-rust](https://github.com/aldanor/qoi-rust) - Rust
|
||||
- [zakarumych/rapid-qoi](https://github.com/zakarumych/rapid-qoi) - Rust
|
||||
- [takeyourhatoff/qoi](https://github.com/takeyourhatoff/qoi) - Go
|
||||
- [DosWorld/pasqoi](https://github.com/DosWorld/pasqoi) - Pascal
|
||||
- [elihwyma/Swift-QOI](https://github.com/elihwyma/Swift-QOI) - Swift
|
||||
- [xfmoulet/qoi](https://github.com/xfmoulet/qoi) - Go
|
||||
- [erratique.ch/qoic](https://erratique.ch/software/qoic) - OCaml
|
||||
- [arian/go-qoi](https://github.com/arian/go-qoi) - Go
|
||||
- [kchapelier/qoijs](https://github.com/kchapelier/qoijs) - JavaScript
|
||||
- [KristofferC/QOI.jl](https://github.com/KristofferC/QOI.jl) - Julia
|
||||
- [shadowMitia/libqoi](https://github.com/shadowMitia/libqoi) - C++
|
||||
- [MKCG/php-qoi](https://github.com/MKCG/php-qoi) - PHP
|
||||
- [LightHouseSoftware/qoiformats](https://github.com/LightHouseSoftware/qoiformats) - D
|
||||
- [mhoward540/qoi-nim](https://github.com/mhoward540/qoi-nim) - Nim
|
||||
- [wx257osn2/qoixx](https://github.com/wx257osn2/qoixx) - C++
|
||||
- [Tiefseetauchner/lr-paint](https://github.com/Tiefseetauchner/lr-paint) - Processing
|
||||
- [amstan/qoi-fpga](https://github.com/amstan/qoi-fpga) - FPGA: verilog
|
||||
- [musabkilic/qoi-decoder](https://github.com/musabkilic/qoi-decoder) - Python
|
||||
- [mathpn/py-qoi](https://github.com/mathpn/py-qoi) - Python
|
||||
- [JohannesFriedrich/qoi4R](https://github.com/JohannesFriedrich/qoi4R) - R
|
||||
- [shraiwi/mini-qoi](https://github.com/shraiwi/mini-qoi) - C, streaming decoder
|
||||
- [10maurycy10/libqoi/](https://github.com/10maurycy10/libqoi/) - Rust
|
||||
- [0xd34df00d/hsqoi](https://github.com/0xd34df00d/hsqoi) - Haskell
|
||||
- [418Coffee/qoi-v](https://github.com/418Coffee/qoi-v) - V
|
||||
- [Imagine-Programming/QoiImagePlugin](https://github.com/Imagine-Programming/QoiImagePlugin) - PureBasic
|
||||
- [Fabien-Chouteau/qoi-spark](https://github.com/Fabien-Chouteau/qoi-spark) - Ada/SPARK formally proven
|
||||
- [mzgreen/qoi-kotlin](https://github.com/mzgreen/qoi-kotlin) - Kotlin Multiplatform
|
||||
- [Aftersol/Simplified-QOI-Codec](https://github.com/Aftersol/Simplified-QOI-Codec) - C99, encoder and decoder, freestanding
|
||||
- [AuburnSounds/gamut](https://github.com/AuburnSounds/gamut) - D
|
||||
- [AngusJohnson/TQoiImage](https://github.com/AngusJohnson/TQoiImage) - Delphi
|
||||
- [MarkJeronimus/qoi-java-spi](https://github.com/MarkJeronimus/qoi-java-spi) - Java SPI
|
||||
- [aumouvantsillage/qoi-racket](https://github.com/aumouvantsillage/qoi-racket) - Racket
|
||||
- [rubikscraft/qoi-stream](https://github.com/rubikscraft/qoi-stream) - C99, one byte at a time streaming encoder and decoder
|
||||
- [rubikscraft/qoi-img](https://github.com/rubikscraft/qoi-img) - NodeJS typescript, bindings to both [QOIxx](https://github.com/wx257osn2/qoixx) and [qoi-stream](https://github.com/rubikscraft/qoi-stream)
|
||||
- [grego/hare-qoi](https://git.sr.ht/~grego/hare-qoi) - Hare
|
||||
- [MrNocole/ZTQOI](https://github.com/MrNocole/ZTQOI) - Objective-C
|
||||
- [bpanthi977/qoi](https://github.com/bpanthi977/qoi) - Common Lisp
|
||||
- [Floessie/pam2qoi](https://github.com/Floessie/pam2qoi) - C++
|
||||
- [SpeckyYT/spwn-qoi](https://github.com/SpeckyYT/spwn-qoi) - SPWN
|
||||
- [n00bmind/qoi](https://github.com/n00bmind/qoi) - Jai
|
||||
- [SixLabors/ImageSharp](https://github.com/SixLabors/ImageSharp) - C# image proccesing library
|
||||
- [zertovitch/gid](https://github.com/zertovitch/gid) - Ada
|
||||
- [nazrin/lil](https://codeberg.org/nazrin/lil) - Lua image library
|
||||
|
||||
## QOI Support in Other Software
|
||||
|
||||
- [Amiga OS QOI datatype](https://github.com/dgaw/qoi-datatype) - adds support for decoding QOI images to the Amiga operating system.
|
||||
- [SerenityOS](https://github.com/SerenityOS/serenity) - supports decoding QOI system wide through a custom [cpp implementation in LibGfx](https://github.com/SerenityOS/serenity/blob/master/Userland/Libraries/LibGfx/QOILoader.h)
|
||||
- [Raylib](https://github.com/raysan5/raylib) - supports decoding and encoding QOI textures through its [rtextures module](https://github.com/raysan5/raylib/blob/master/src/rtextures.c)
|
||||
- [Rebol3](https://github.com/Oldes/Rebol3/issues/39) - supports decoding and encoding QOI using a native codec
|
||||
- [c-ray](https://github.com/vkoskiv/c-ray) - supports QOI natively
|
||||
- [SAIL](https://sail.software) - image decoding library, supports decoding and encoding QOI images
|
||||
- [Orx](https://github.com/orx/orx) - 2D game engine, supports QOI natively
|
||||
- [IrfanView](https://www.irfanview.com) - supports decoding and encoding QOI through its Formats plugin
|
||||
- [ImageMagick](https://github.com/ImageMagick/ImageMagick) - supports decoding and encoding QOI, since 7.1.0-20
|
||||
- [barebox](https://barebox.org) - bootloader, supports decoding QOI images for splash logo, since v2022.03.0
|
||||
- [KorGE](https://korge.org) - & KorIM Kotlin 2D game engine and imaging library, supports decoding and encoding QOI natively since 2.7.0
|
||||
- [DOjS](https://github.com/SuperIlu/DOjS) - DOS JavaScript Canvas implementation supports loading QOI files
|
||||
- [XnView MP](https://www.xnview.com/en/xnviewmp/) - supports decoding QOI since 1.00
|
||||
- [ffmpeg](https://ffmpeg.org/) - supports decoding and encoding QOI since 5.1
|
||||
- [JPEGView](https://github.com/sylikc/jpegview) - lightweight Windows image viewer, supports decoding and encoding of QOI natively, since 1.1.44
|
||||
- [darktable](https://github.com/darktable-org/darktable) - photography workflow application and raw developer, supports decoding since 4.4.0
|
||||
- [KDE](https://kde.org) - supports decoding and encoding QOI images. Implemented in [KImageFormats](https://invent.kde.org/frameworks/kimageformats)
|
||||
- [EFL](https://www.enlightenment.org) - supports decoding and encoding QOI images since 1.27.
|
||||
- [Swingland](https://git.sr.ht/~phlash/swingland) - supports QOI decoding/loading via the `ImageIO` API of this Java Swing reimplemenation for Wayland
|
||||
- [Imagine](https://www.nyam.pe.kr/dev/imagine/) - supports decoding and encoding QOI images since 1.3.9
|
||||
- [Uiua](https://uiua.org) - supports decoding and encoding QOI images since 0.8.0
|
||||
|
||||
## Packages
|
||||
|
||||
- [AUR](https://aur.archlinux.org/pkgbase/qoi-git/) - system-wide qoi.h, qoiconv and qoibench install as split packages.
|
||||
- [Debian](https://packages.debian.org/bookworm/source/qoi) - packages for binaries and qoi.h
|
||||
- [Ubuntu](https://launchpad.net/ubuntu/+source/qoi) - packages for binaries and qoi.h
|
||||
|
||||
Packages for other systems [tracked at Repology](https://repology.org/project/qoi/versions).
|
649
external/qoi/qoi/qoi.h
vendored
Normal file
649
external/qoi/qoi/qoi.h
vendored
Normal file
@ -0,0 +1,649 @@
|
||||
/*
|
||||
|
||||
Copyright (c) 2021, Dominic Szablewski - https://phoboslab.org
|
||||
SPDX-License-Identifier: MIT
|
||||
|
||||
|
||||
QOI - The "Quite OK Image" format for fast, lossless image compression
|
||||
|
||||
-- About
|
||||
|
||||
QOI encodes and decodes images in a lossless format. Compared to stb_image and
|
||||
stb_image_write QOI offers 20x-50x faster encoding, 3x-4x faster decoding and
|
||||
20% better compression.
|
||||
|
||||
|
||||
-- Synopsis
|
||||
|
||||
// Define `QOI_IMPLEMENTATION` in *one* C/C++ file before including this
|
||||
// library to create the implementation.
|
||||
|
||||
#define QOI_IMPLEMENTATION
|
||||
#include "qoi.h"
|
||||
|
||||
// Encode and store an RGBA buffer to the file system. The qoi_desc describes
|
||||
// the input pixel data.
|
||||
qoi_write("image_new.qoi", rgba_pixels, &(qoi_desc){
|
||||
.width = 1920,
|
||||
.height = 1080,
|
||||
.channels = 4,
|
||||
.colorspace = QOI_SRGB
|
||||
});
|
||||
|
||||
// Load and decode a QOI image from the file system into a 32bbp RGBA buffer.
|
||||
// The qoi_desc struct will be filled with the width, height, number of channels
|
||||
// and colorspace read from the file header.
|
||||
qoi_desc desc;
|
||||
void *rgba_pixels = qoi_read("image.qoi", &desc, 4);
|
||||
|
||||
|
||||
|
||||
-- Documentation
|
||||
|
||||
This library provides the following functions;
|
||||
- qoi_read -- read and decode a QOI file
|
||||
- qoi_decode -- decode the raw bytes of a QOI image from memory
|
||||
- qoi_write -- encode and write a QOI file
|
||||
- qoi_encode -- encode an rgba buffer into a QOI image in memory
|
||||
|
||||
See the function declaration below for the signature and more information.
|
||||
|
||||
If you don't want/need the qoi_read and qoi_write functions, you can define
|
||||
QOI_NO_STDIO before including this library.
|
||||
|
||||
This library uses malloc() and free(). To supply your own malloc implementation
|
||||
you can define QOI_MALLOC and QOI_FREE before including this library.
|
||||
|
||||
This library uses memset() to zero-initialize the index. To supply your own
|
||||
implementation you can define QOI_ZEROARR before including this library.
|
||||
|
||||
|
||||
-- Data Format
|
||||
|
||||
A QOI file has a 14 byte header, followed by any number of data "chunks" and an
|
||||
8-byte end marker.
|
||||
|
||||
struct qoi_header_t {
|
||||
char magic[4]; // magic bytes "qoif"
|
||||
uint32_t width; // image width in pixels (BE)
|
||||
uint32_t height; // image height in pixels (BE)
|
||||
uint8_t channels; // 3 = RGB, 4 = RGBA
|
||||
uint8_t colorspace; // 0 = sRGB with linear alpha, 1 = all channels linear
|
||||
};
|
||||
|
||||
Images are encoded row by row, left to right, top to bottom. The decoder and
|
||||
encoder start with {r: 0, g: 0, b: 0, a: 255} as the previous pixel value. An
|
||||
image is complete when all pixels specified by width * height have been covered.
|
||||
|
||||
Pixels are encoded as
|
||||
- a run of the previous pixel
|
||||
- an index into an array of previously seen pixels
|
||||
- a difference to the previous pixel value in r,g,b
|
||||
- full r,g,b or r,g,b,a values
|
||||
|
||||
The color channels are assumed to not be premultiplied with the alpha channel
|
||||
("un-premultiplied alpha").
|
||||
|
||||
A running array[64] (zero-initialized) of previously seen pixel values is
|
||||
maintained by the encoder and decoder. Each pixel that is seen by the encoder
|
||||
and decoder is put into this array at the position formed by a hash function of
|
||||
the color value. In the encoder, if the pixel value at the index matches the
|
||||
current pixel, this index position is written to the stream as QOI_OP_INDEX.
|
||||
The hash function for the index is:
|
||||
|
||||
index_position = (r * 3 + g * 5 + b * 7 + a * 11) % 64
|
||||
|
||||
Each chunk starts with a 2- or 8-bit tag, followed by a number of data bits. The
|
||||
bit length of chunks is divisible by 8 - i.e. all chunks are byte aligned. All
|
||||
values encoded in these data bits have the most significant bit on the left.
|
||||
|
||||
The 8-bit tags have precedence over the 2-bit tags. A decoder must check for the
|
||||
presence of an 8-bit tag first.
|
||||
|
||||
The byte stream's end is marked with 7 0x00 bytes followed a single 0x01 byte.
|
||||
|
||||
|
||||
The possible chunks are:
|
||||
|
||||
|
||||
.- QOI_OP_INDEX ----------.
|
||||
| Byte[0] |
|
||||
| 7 6 5 4 3 2 1 0 |
|
||||
|-------+-----------------|
|
||||
| 0 0 | index |
|
||||
`-------------------------`
|
||||
2-bit tag b00
|
||||
6-bit index into the color index array: 0..63
|
||||
|
||||
A valid encoder must not issue 2 or more consecutive QOI_OP_INDEX chunks to the
|
||||
same index. QOI_OP_RUN should be used instead.
|
||||
|
||||
|
||||
.- QOI_OP_DIFF -----------.
|
||||
| Byte[0] |
|
||||
| 7 6 5 4 3 2 1 0 |
|
||||
|-------+-----+-----+-----|
|
||||
| 0 1 | dr | dg | db |
|
||||
`-------------------------`
|
||||
2-bit tag b01
|
||||
2-bit red channel difference from the previous pixel between -2..1
|
||||
2-bit green channel difference from the previous pixel between -2..1
|
||||
2-bit blue channel difference from the previous pixel between -2..1
|
||||
|
||||
The difference to the current channel values are using a wraparound operation,
|
||||
so "1 - 2" will result in 255, while "255 + 1" will result in 0.
|
||||
|
||||
Values are stored as unsigned integers with a bias of 2. E.g. -2 is stored as
|
||||
0 (b00). 1 is stored as 3 (b11).
|
||||
|
||||
The alpha value remains unchanged from the previous pixel.
|
||||
|
||||
|
||||
.- QOI_OP_LUMA -------------------------------------.
|
||||
| Byte[0] | Byte[1] |
|
||||
| 7 6 5 4 3 2 1 0 | 7 6 5 4 3 2 1 0 |
|
||||
|-------+-----------------+-------------+-----------|
|
||||
| 1 0 | green diff | dr - dg | db - dg |
|
||||
`---------------------------------------------------`
|
||||
2-bit tag b10
|
||||
6-bit green channel difference from the previous pixel -32..31
|
||||
4-bit red channel difference minus green channel difference -8..7
|
||||
4-bit blue channel difference minus green channel difference -8..7
|
||||
|
||||
The green channel is used to indicate the general direction of change and is
|
||||
encoded in 6 bits. The red and blue channels (dr and db) base their diffs off
|
||||
of the green channel difference and are encoded in 4 bits. I.e.:
|
||||
dr_dg = (cur_px.r - prev_px.r) - (cur_px.g - prev_px.g)
|
||||
db_dg = (cur_px.b - prev_px.b) - (cur_px.g - prev_px.g)
|
||||
|
||||
The difference to the current channel values are using a wraparound operation,
|
||||
so "10 - 13" will result in 253, while "250 + 7" will result in 1.
|
||||
|
||||
Values are stored as unsigned integers with a bias of 32 for the green channel
|
||||
and a bias of 8 for the red and blue channel.
|
||||
|
||||
The alpha value remains unchanged from the previous pixel.
|
||||
|
||||
|
||||
.- QOI_OP_RUN ------------.
|
||||
| Byte[0] |
|
||||
| 7 6 5 4 3 2 1 0 |
|
||||
|-------+-----------------|
|
||||
| 1 1 | run |
|
||||
`-------------------------`
|
||||
2-bit tag b11
|
||||
6-bit run-length repeating the previous pixel: 1..62
|
||||
|
||||
The run-length is stored with a bias of -1. Note that the run-lengths 63 and 64
|
||||
(b111110 and b111111) are illegal as they are occupied by the QOI_OP_RGB and
|
||||
QOI_OP_RGBA tags.
|
||||
|
||||
|
||||
.- QOI_OP_RGB ------------------------------------------.
|
||||
| Byte[0] | Byte[1] | Byte[2] | Byte[3] |
|
||||
| 7 6 5 4 3 2 1 0 | 7 .. 0 | 7 .. 0 | 7 .. 0 |
|
||||
|-------------------------+---------+---------+---------|
|
||||
| 1 1 1 1 1 1 1 0 | red | green | blue |
|
||||
`-------------------------------------------------------`
|
||||
8-bit tag b11111110
|
||||
8-bit red channel value
|
||||
8-bit green channel value
|
||||
8-bit blue channel value
|
||||
|
||||
The alpha value remains unchanged from the previous pixel.
|
||||
|
||||
|
||||
.- QOI_OP_RGBA ---------------------------------------------------.
|
||||
| Byte[0] | Byte[1] | Byte[2] | Byte[3] | Byte[4] |
|
||||
| 7 6 5 4 3 2 1 0 | 7 .. 0 | 7 .. 0 | 7 .. 0 | 7 .. 0 |
|
||||
|-------------------------+---------+---------+---------+---------|
|
||||
| 1 1 1 1 1 1 1 1 | red | green | blue | alpha |
|
||||
`-----------------------------------------------------------------`
|
||||
8-bit tag b11111111
|
||||
8-bit red channel value
|
||||
8-bit green channel value
|
||||
8-bit blue channel value
|
||||
8-bit alpha channel value
|
||||
|
||||
*/
|
||||
|
||||
|
||||
/* -----------------------------------------------------------------------------
|
||||
Header - Public functions */
|
||||
|
||||
#ifndef QOI_H
|
||||
#define QOI_H
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* A pointer to a qoi_desc struct has to be supplied to all of qoi's functions.
|
||||
It describes either the input format (for qoi_write and qoi_encode), or is
|
||||
filled with the description read from the file header (for qoi_read and
|
||||
qoi_decode).
|
||||
|
||||
The colorspace in this qoi_desc is an enum where
|
||||
0 = sRGB, i.e. gamma scaled RGB channels and a linear alpha channel
|
||||
1 = all channels are linear
|
||||
You may use the constants QOI_SRGB or QOI_LINEAR. The colorspace is purely
|
||||
informative. It will be saved to the file header, but does not affect
|
||||
how chunks are en-/decoded. */
|
||||
|
||||
#define QOI_SRGB 0
|
||||
#define QOI_LINEAR 1
|
||||
|
||||
typedef struct {
|
||||
unsigned int width;
|
||||
unsigned int height;
|
||||
unsigned char channels;
|
||||
unsigned char colorspace;
|
||||
} qoi_desc;
|
||||
|
||||
#ifndef QOI_NO_STDIO
|
||||
|
||||
/* Encode raw RGB or RGBA pixels into a QOI image and write it to the file
|
||||
system. The qoi_desc struct must be filled with the image width, height,
|
||||
number of channels (3 = RGB, 4 = RGBA) and the colorspace.
|
||||
|
||||
The function returns 0 on failure (invalid parameters, or fopen or malloc
|
||||
failed) or the number of bytes written on success. */
|
||||
|
||||
int qoi_write(const char *filename, const void *data, const qoi_desc *desc);
|
||||
|
||||
|
||||
/* Read and decode a QOI image from the file system. If channels is 0, the
|
||||
number of channels from the file header is used. If channels is 3 or 4 the
|
||||
output format will be forced into this number of channels.
|
||||
|
||||
The function either returns NULL on failure (invalid data, or malloc or fopen
|
||||
failed) or a pointer to the decoded pixels. On success, the qoi_desc struct
|
||||
will be filled with the description from the file header.
|
||||
|
||||
The returned pixel data should be free()d after use. */
|
||||
|
||||
void *qoi_read(const char *filename, qoi_desc *desc, int channels);
|
||||
|
||||
#endif /* QOI_NO_STDIO */
|
||||
|
||||
|
||||
/* Encode raw RGB or RGBA pixels into a QOI image in memory.
|
||||
|
||||
The function either returns NULL on failure (invalid parameters or malloc
|
||||
failed) or a pointer to the encoded data on success. On success the out_len
|
||||
is set to the size in bytes of the encoded data.
|
||||
|
||||
The returned qoi data should be free()d after use. */
|
||||
|
||||
void *qoi_encode(const void *data, const qoi_desc *desc, int *out_len);
|
||||
|
||||
|
||||
/* Decode a QOI image from memory.
|
||||
|
||||
The function either returns NULL on failure (invalid parameters or malloc
|
||||
failed) or a pointer to the decoded pixels. On success, the qoi_desc struct
|
||||
is filled with the description from the file header.
|
||||
|
||||
The returned pixel data should be free()d after use. */
|
||||
|
||||
void *qoi_decode(const void *data, int size, qoi_desc *desc, int channels);
|
||||
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
#endif /* QOI_H */
|
||||
|
||||
|
||||
/* -----------------------------------------------------------------------------
|
||||
Implementation */
|
||||
|
||||
#ifdef QOI_IMPLEMENTATION
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#ifndef QOI_MALLOC
|
||||
#define QOI_MALLOC(sz) malloc(sz)
|
||||
#define QOI_FREE(p) free(p)
|
||||
#endif
|
||||
#ifndef QOI_ZEROARR
|
||||
#define QOI_ZEROARR(a) memset((a),0,sizeof(a))
|
||||
#endif
|
||||
|
||||
#define QOI_OP_INDEX 0x00 /* 00xxxxxx */
|
||||
#define QOI_OP_DIFF 0x40 /* 01xxxxxx */
|
||||
#define QOI_OP_LUMA 0x80 /* 10xxxxxx */
|
||||
#define QOI_OP_RUN 0xc0 /* 11xxxxxx */
|
||||
#define QOI_OP_RGB 0xfe /* 11111110 */
|
||||
#define QOI_OP_RGBA 0xff /* 11111111 */
|
||||
|
||||
#define QOI_MASK_2 0xc0 /* 11000000 */
|
||||
|
||||
#define QOI_COLOR_HASH(C) (C.rgba.r*3 + C.rgba.g*5 + C.rgba.b*7 + C.rgba.a*11)
|
||||
#define QOI_MAGIC \
|
||||
(((unsigned int)'q') << 24 | ((unsigned int)'o') << 16 | \
|
||||
((unsigned int)'i') << 8 | ((unsigned int)'f'))
|
||||
#define QOI_HEADER_SIZE 14
|
||||
|
||||
/* 2GB is the max file size that this implementation can safely handle. We guard
|
||||
against anything larger than that, assuming the worst case with 5 bytes per
|
||||
pixel, rounded down to a nice clean value. 400 million pixels ought to be
|
||||
enough for anybody. */
|
||||
#define QOI_PIXELS_MAX ((unsigned int)400000000)
|
||||
|
||||
typedef union {
|
||||
struct { unsigned char r, g, b, a; } rgba;
|
||||
unsigned int v;
|
||||
} qoi_rgba_t;
|
||||
|
||||
static const unsigned char qoi_padding[8] = {0,0,0,0,0,0,0,1};
|
||||
|
||||
static void qoi_write_32(unsigned char *bytes, int *p, unsigned int v) {
|
||||
bytes[(*p)++] = (0xff000000 & v) >> 24;
|
||||
bytes[(*p)++] = (0x00ff0000 & v) >> 16;
|
||||
bytes[(*p)++] = (0x0000ff00 & v) >> 8;
|
||||
bytes[(*p)++] = (0x000000ff & v);
|
||||
}
|
||||
|
||||
static unsigned int qoi_read_32(const unsigned char *bytes, int *p) {
|
||||
unsigned int a = bytes[(*p)++];
|
||||
unsigned int b = bytes[(*p)++];
|
||||
unsigned int c = bytes[(*p)++];
|
||||
unsigned int d = bytes[(*p)++];
|
||||
return a << 24 | b << 16 | c << 8 | d;
|
||||
}
|
||||
|
||||
void *qoi_encode(const void *data, const qoi_desc *desc, int *out_len) {
|
||||
int i, max_size, p, run;
|
||||
int px_len, px_end, px_pos, channels;
|
||||
unsigned char *bytes;
|
||||
const unsigned char *pixels;
|
||||
qoi_rgba_t index[64];
|
||||
qoi_rgba_t px, px_prev;
|
||||
|
||||
if (
|
||||
data == NULL || out_len == NULL || desc == NULL ||
|
||||
desc->width == 0 || desc->height == 0 ||
|
||||
desc->channels < 3 || desc->channels > 4 ||
|
||||
desc->colorspace > 1 ||
|
||||
desc->height >= QOI_PIXELS_MAX / desc->width
|
||||
) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
max_size =
|
||||
desc->width * desc->height * (desc->channels + 1) +
|
||||
QOI_HEADER_SIZE + sizeof(qoi_padding);
|
||||
|
||||
p = 0;
|
||||
bytes = (unsigned char *) QOI_MALLOC(max_size);
|
||||
if (!bytes) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
qoi_write_32(bytes, &p, QOI_MAGIC);
|
||||
qoi_write_32(bytes, &p, desc->width);
|
||||
qoi_write_32(bytes, &p, desc->height);
|
||||
bytes[p++] = desc->channels;
|
||||
bytes[p++] = desc->colorspace;
|
||||
|
||||
|
||||
pixels = (const unsigned char *)data;
|
||||
|
||||
QOI_ZEROARR(index);
|
||||
|
||||
run = 0;
|
||||
px_prev.rgba.r = 0;
|
||||
px_prev.rgba.g = 0;
|
||||
px_prev.rgba.b = 0;
|
||||
px_prev.rgba.a = 255;
|
||||
px = px_prev;
|
||||
|
||||
px_len = desc->width * desc->height * desc->channels;
|
||||
px_end = px_len - desc->channels;
|
||||
channels = desc->channels;
|
||||
|
||||
for (px_pos = 0; px_pos < px_len; px_pos += channels) {
|
||||
px.rgba.r = pixels[px_pos + 0];
|
||||
px.rgba.g = pixels[px_pos + 1];
|
||||
px.rgba.b = pixels[px_pos + 2];
|
||||
|
||||
if (channels == 4) {
|
||||
px.rgba.a = pixels[px_pos + 3];
|
||||
}
|
||||
|
||||
if (px.v == px_prev.v) {
|
||||
run++;
|
||||
if (run == 62 || px_pos == px_end) {
|
||||
bytes[p++] = QOI_OP_RUN | (run - 1);
|
||||
run = 0;
|
||||
}
|
||||
}
|
||||
else {
|
||||
int index_pos;
|
||||
|
||||
if (run > 0) {
|
||||
bytes[p++] = QOI_OP_RUN | (run - 1);
|
||||
run = 0;
|
||||
}
|
||||
|
||||
index_pos = QOI_COLOR_HASH(px) % 64;
|
||||
|
||||
if (index[index_pos].v == px.v) {
|
||||
bytes[p++] = QOI_OP_INDEX | index_pos;
|
||||
}
|
||||
else {
|
||||
index[index_pos] = px;
|
||||
|
||||
if (px.rgba.a == px_prev.rgba.a) {
|
||||
signed char vr = px.rgba.r - px_prev.rgba.r;
|
||||
signed char vg = px.rgba.g - px_prev.rgba.g;
|
||||
signed char vb = px.rgba.b - px_prev.rgba.b;
|
||||
|
||||
signed char vg_r = vr - vg;
|
||||
signed char vg_b = vb - vg;
|
||||
|
||||
if (
|
||||
vr > -3 && vr < 2 &&
|
||||
vg > -3 && vg < 2 &&
|
||||
vb > -3 && vb < 2
|
||||
) {
|
||||
bytes[p++] = QOI_OP_DIFF | (vr + 2) << 4 | (vg + 2) << 2 | (vb + 2);
|
||||
}
|
||||
else if (
|
||||
vg_r > -9 && vg_r < 8 &&
|
||||
vg > -33 && vg < 32 &&
|
||||
vg_b > -9 && vg_b < 8
|
||||
) {
|
||||
bytes[p++] = QOI_OP_LUMA | (vg + 32);
|
||||
bytes[p++] = (vg_r + 8) << 4 | (vg_b + 8);
|
||||
}
|
||||
else {
|
||||
bytes[p++] = QOI_OP_RGB;
|
||||
bytes[p++] = px.rgba.r;
|
||||
bytes[p++] = px.rgba.g;
|
||||
bytes[p++] = px.rgba.b;
|
||||
}
|
||||
}
|
||||
else {
|
||||
bytes[p++] = QOI_OP_RGBA;
|
||||
bytes[p++] = px.rgba.r;
|
||||
bytes[p++] = px.rgba.g;
|
||||
bytes[p++] = px.rgba.b;
|
||||
bytes[p++] = px.rgba.a;
|
||||
}
|
||||
}
|
||||
}
|
||||
px_prev = px;
|
||||
}
|
||||
|
||||
for (i = 0; i < (int)sizeof(qoi_padding); i++) {
|
||||
bytes[p++] = qoi_padding[i];
|
||||
}
|
||||
|
||||
*out_len = p;
|
||||
return bytes;
|
||||
}
|
||||
|
||||
void *qoi_decode(const void *data, int size, qoi_desc *desc, int channels) {
|
||||
const unsigned char *bytes;
|
||||
unsigned int header_magic;
|
||||
unsigned char *pixels;
|
||||
qoi_rgba_t index[64];
|
||||
qoi_rgba_t px;
|
||||
int px_len, chunks_len, px_pos;
|
||||
int p = 0, run = 0;
|
||||
|
||||
if (
|
||||
data == NULL || desc == NULL ||
|
||||
(channels != 0 && channels != 3 && channels != 4) ||
|
||||
size < QOI_HEADER_SIZE + (int)sizeof(qoi_padding)
|
||||
) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
bytes = (const unsigned char *)data;
|
||||
|
||||
header_magic = qoi_read_32(bytes, &p);
|
||||
desc->width = qoi_read_32(bytes, &p);
|
||||
desc->height = qoi_read_32(bytes, &p);
|
||||
desc->channels = bytes[p++];
|
||||
desc->colorspace = bytes[p++];
|
||||
|
||||
if (
|
||||
desc->width == 0 || desc->height == 0 ||
|
||||
desc->channels < 3 || desc->channels > 4 ||
|
||||
desc->colorspace > 1 ||
|
||||
header_magic != QOI_MAGIC ||
|
||||
desc->height >= QOI_PIXELS_MAX / desc->width
|
||||
) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (channels == 0) {
|
||||
channels = desc->channels;
|
||||
}
|
||||
|
||||
px_len = desc->width * desc->height * channels;
|
||||
pixels = (unsigned char *) QOI_MALLOC(px_len);
|
||||
if (!pixels) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
QOI_ZEROARR(index);
|
||||
px.rgba.r = 0;
|
||||
px.rgba.g = 0;
|
||||
px.rgba.b = 0;
|
||||
px.rgba.a = 255;
|
||||
|
||||
chunks_len = size - (int)sizeof(qoi_padding);
|
||||
for (px_pos = 0; px_pos < px_len; px_pos += channels) {
|
||||
if (run > 0) {
|
||||
run--;
|
||||
}
|
||||
else if (p < chunks_len) {
|
||||
int b1 = bytes[p++];
|
||||
|
||||
if (b1 == QOI_OP_RGB) {
|
||||
px.rgba.r = bytes[p++];
|
||||
px.rgba.g = bytes[p++];
|
||||
px.rgba.b = bytes[p++];
|
||||
}
|
||||
else if (b1 == QOI_OP_RGBA) {
|
||||
px.rgba.r = bytes[p++];
|
||||
px.rgba.g = bytes[p++];
|
||||
px.rgba.b = bytes[p++];
|
||||
px.rgba.a = bytes[p++];
|
||||
}
|
||||
else if ((b1 & QOI_MASK_2) == QOI_OP_INDEX) {
|
||||
px = index[b1];
|
||||
}
|
||||
else if ((b1 & QOI_MASK_2) == QOI_OP_DIFF) {
|
||||
px.rgba.r += ((b1 >> 4) & 0x03) - 2;
|
||||
px.rgba.g += ((b1 >> 2) & 0x03) - 2;
|
||||
px.rgba.b += ( b1 & 0x03) - 2;
|
||||
}
|
||||
else if ((b1 & QOI_MASK_2) == QOI_OP_LUMA) {
|
||||
int b2 = bytes[p++];
|
||||
int vg = (b1 & 0x3f) - 32;
|
||||
px.rgba.r += vg - 8 + ((b2 >> 4) & 0x0f);
|
||||
px.rgba.g += vg;
|
||||
px.rgba.b += vg - 8 + (b2 & 0x0f);
|
||||
}
|
||||
else if ((b1 & QOI_MASK_2) == QOI_OP_RUN) {
|
||||
run = (b1 & 0x3f);
|
||||
}
|
||||
|
||||
index[QOI_COLOR_HASH(px) % 64] = px;
|
||||
}
|
||||
|
||||
pixels[px_pos + 0] = px.rgba.r;
|
||||
pixels[px_pos + 1] = px.rgba.g;
|
||||
pixels[px_pos + 2] = px.rgba.b;
|
||||
|
||||
if (channels == 4) {
|
||||
pixels[px_pos + 3] = px.rgba.a;
|
||||
}
|
||||
}
|
||||
|
||||
return pixels;
|
||||
}
|
||||
|
||||
#ifndef QOI_NO_STDIO
|
||||
#include <stdio.h>
|
||||
|
||||
int qoi_write(const char *filename, const void *data, const qoi_desc *desc) {
|
||||
FILE *f = fopen(filename, "wb");
|
||||
int size, err;
|
||||
void *encoded;
|
||||
|
||||
if (!f) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
encoded = qoi_encode(data, desc, &size);
|
||||
if (!encoded) {
|
||||
fclose(f);
|
||||
return 0;
|
||||
}
|
||||
|
||||
fwrite(encoded, 1, size, f);
|
||||
fflush(f);
|
||||
err = ferror(f);
|
||||
fclose(f);
|
||||
|
||||
QOI_FREE(encoded);
|
||||
return err ? 0 : size;
|
||||
}
|
||||
|
||||
void *qoi_read(const char *filename, qoi_desc *desc, int channels) {
|
||||
FILE *f = fopen(filename, "rb");
|
||||
int size, bytes_read;
|
||||
void *pixels, *data;
|
||||
|
||||
if (!f) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
fseek(f, 0, SEEK_END);
|
||||
size = ftell(f);
|
||||
if (size <= 0 || fseek(f, 0, SEEK_SET) != 0) {
|
||||
fclose(f);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
data = QOI_MALLOC(size);
|
||||
if (!data) {
|
||||
fclose(f);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
bytes_read = fread(data, 1, size, f);
|
||||
fclose(f);
|
||||
pixels = (bytes_read != size) ? NULL : qoi_decode(data, bytes_read, desc, channels);
|
||||
QOI_FREE(data);
|
||||
return pixels;
|
||||
}
|
||||
|
||||
#endif /* QOI_NO_STDIO */
|
||||
#endif /* QOI_IMPLEMENTATION */
|
610
external/qoi/qoi/qoibench.c
vendored
Normal file
610
external/qoi/qoi/qoibench.c
vendored
Normal file
@ -0,0 +1,610 @@
|
||||
/*
|
||||
|
||||
Copyright (c) 2021, Dominic Szablewski - https://phoboslab.org
|
||||
SPDX-License-Identifier: MIT
|
||||
|
||||
|
||||
Simple benchmark suite for png, stbi and qoi
|
||||
|
||||
Requires libpng, "stb_image.h" and "stb_image_write.h"
|
||||
Compile with:
|
||||
gcc qoibench.c -std=gnu99 -lpng -O3 -o qoibench
|
||||
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <dirent.h>
|
||||
#include <png.h>
|
||||
|
||||
#define STB_IMAGE_IMPLEMENTATION
|
||||
#define STBI_ONLY_PNG
|
||||
#define STBI_NO_LINEAR
|
||||
#include "stb_image.h"
|
||||
|
||||
#define STB_IMAGE_WRITE_IMPLEMENTATION
|
||||
#include "stb_image_write.h"
|
||||
|
||||
#define QOI_IMPLEMENTATION
|
||||
#include "qoi.h"
|
||||
|
||||
|
||||
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// Cross platform high resolution timer
|
||||
// From https://gist.github.com/ForeverZer0/0a4f80fc02b96e19380ebb7a3debbee5
|
||||
|
||||
#include <stdint.h>
|
||||
#if defined(__linux)
|
||||
#define HAVE_POSIX_TIMER
|
||||
#include <time.h>
|
||||
#ifdef CLOCK_MONOTONIC
|
||||
#define CLOCKID CLOCK_MONOTONIC
|
||||
#else
|
||||
#define CLOCKID CLOCK_REALTIME
|
||||
#endif
|
||||
#elif defined(__APPLE__)
|
||||
#define HAVE_MACH_TIMER
|
||||
#include <mach/mach_time.h>
|
||||
#elif defined(_WIN32)
|
||||
#define WIN32_LEAN_AND_MEAN
|
||||
#include <windows.h>
|
||||
#endif
|
||||
|
||||
static uint64_t ns() {
|
||||
static uint64_t is_init = 0;
|
||||
#if defined(__APPLE__)
|
||||
static mach_timebase_info_data_t info;
|
||||
if (0 == is_init) {
|
||||
mach_timebase_info(&info);
|
||||
is_init = 1;
|
||||
}
|
||||
uint64_t now;
|
||||
now = mach_absolute_time();
|
||||
now *= info.numer;
|
||||
now /= info.denom;
|
||||
return now;
|
||||
#elif defined(__linux)
|
||||
static struct timespec linux_rate;
|
||||
if (0 == is_init) {
|
||||
clock_getres(CLOCKID, &linux_rate);
|
||||
is_init = 1;
|
||||
}
|
||||
uint64_t now;
|
||||
struct timespec spec;
|
||||
clock_gettime(CLOCKID, &spec);
|
||||
now = spec.tv_sec * 1.0e9 + spec.tv_nsec;
|
||||
return now;
|
||||
#elif defined(_WIN32)
|
||||
static LARGE_INTEGER win_frequency;
|
||||
if (0 == is_init) {
|
||||
QueryPerformanceFrequency(&win_frequency);
|
||||
is_init = 1;
|
||||
}
|
||||
LARGE_INTEGER now;
|
||||
QueryPerformanceCounter(&now);
|
||||
return (uint64_t) ((1e9 * now.QuadPart) / win_frequency.QuadPart);
|
||||
#endif
|
||||
}
|
||||
|
||||
#define STRINGIFY(x) #x
|
||||
#define TOSTRING(x) STRINGIFY(x)
|
||||
#define ERROR(...) printf("abort at line " TOSTRING(__LINE__) ": " __VA_ARGS__); printf("\n"); exit(1)
|
||||
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// libpng encode/decode wrappers
|
||||
// Seriously, who thought this was a good abstraction for an API to read/write
|
||||
// images?
|
||||
|
||||
typedef struct {
|
||||
int size;
|
||||
int capacity;
|
||||
unsigned char *data;
|
||||
} libpng_write_t;
|
||||
|
||||
void libpng_encode_callback(png_structp png_ptr, png_bytep data, png_size_t length) {
|
||||
libpng_write_t *write_data = (libpng_write_t*)png_get_io_ptr(png_ptr);
|
||||
if (write_data->size + length >= write_data->capacity) {
|
||||
ERROR("PNG write");
|
||||
}
|
||||
memcpy(write_data->data + write_data->size, data, length);
|
||||
write_data->size += length;
|
||||
}
|
||||
|
||||
void *libpng_encode(void *pixels, int w, int h, int channels, int *out_len) {
|
||||
png_structp png = png_create_write_struct(PNG_LIBPNG_VER_STRING, NULL, NULL, NULL);
|
||||
if (!png) {
|
||||
ERROR("png_create_write_struct");
|
||||
}
|
||||
|
||||
png_infop info = png_create_info_struct(png);
|
||||
if (!info) {
|
||||
ERROR("png_create_info_struct");
|
||||
}
|
||||
|
||||
if (setjmp(png_jmpbuf(png))) {
|
||||
ERROR("png_jmpbuf");
|
||||
}
|
||||
|
||||
// Output is 8bit depth, RGBA format.
|
||||
png_set_IHDR(
|
||||
png,
|
||||
info,
|
||||
w, h,
|
||||
8,
|
||||
channels == 3 ? PNG_COLOR_TYPE_RGB : PNG_COLOR_TYPE_RGBA,
|
||||
PNG_INTERLACE_NONE,
|
||||
PNG_COMPRESSION_TYPE_DEFAULT,
|
||||
PNG_FILTER_TYPE_DEFAULT
|
||||
);
|
||||
|
||||
png_bytep row_pointers[h];
|
||||
for(int y = 0; y < h; y++){
|
||||
row_pointers[y] = ((unsigned char *)pixels + y * w * channels);
|
||||
}
|
||||
|
||||
libpng_write_t write_data = {
|
||||
.size = 0,
|
||||
.capacity = w * h * channels,
|
||||
.data = malloc(w * h * channels)
|
||||
};
|
||||
|
||||
png_set_rows(png, info, row_pointers);
|
||||
png_set_write_fn(png, &write_data, libpng_encode_callback, NULL);
|
||||
png_write_png(png, info, PNG_TRANSFORM_IDENTITY, NULL);
|
||||
|
||||
png_destroy_write_struct(&png, &info);
|
||||
|
||||
*out_len = write_data.size;
|
||||
return write_data.data;
|
||||
}
|
||||
|
||||
|
||||
typedef struct {
|
||||
int pos;
|
||||
int size;
|
||||
unsigned char *data;
|
||||
} libpng_read_t;
|
||||
|
||||
void png_decode_callback(png_structp png, png_bytep data, png_size_t length) {
|
||||
libpng_read_t *read_data = (libpng_read_t*)png_get_io_ptr(png);
|
||||
if (read_data->pos + length > read_data->size) {
|
||||
ERROR("PNG read %ld bytes at pos %d (size: %d)", length, read_data->pos, read_data->size);
|
||||
}
|
||||
memcpy(data, read_data->data + read_data->pos, length);
|
||||
read_data->pos += length;
|
||||
}
|
||||
|
||||
void png_warning_callback(png_structp png_ptr, png_const_charp warning_msg) {
|
||||
// Ignore warnings about sRGB profiles and such.
|
||||
}
|
||||
|
||||
void *libpng_decode(void *data, int size, int *out_w, int *out_h) {
|
||||
png_structp png = png_create_read_struct(PNG_LIBPNG_VER_STRING, NULL, NULL, png_warning_callback);
|
||||
if (!png) {
|
||||
ERROR("png_create_read_struct");
|
||||
}
|
||||
|
||||
png_infop info = png_create_info_struct(png);
|
||||
if (!info) {
|
||||
ERROR("png_create_info_struct");
|
||||
}
|
||||
|
||||
libpng_read_t read_data = {
|
||||
.pos = 0,
|
||||
.size = size,
|
||||
.data = data
|
||||
};
|
||||
|
||||
png_set_read_fn(png, &read_data, png_decode_callback);
|
||||
png_set_sig_bytes(png, 0);
|
||||
png_read_info(png, info);
|
||||
|
||||
png_uint_32 w, h;
|
||||
int bitDepth, colorType, interlaceType;
|
||||
png_get_IHDR(png, info, &w, &h, &bitDepth, &colorType, &interlaceType, NULL, NULL);
|
||||
|
||||
// 16 bit -> 8 bit
|
||||
png_set_strip_16(png);
|
||||
|
||||
// 1, 2, 4 bit -> 8 bit
|
||||
if (bitDepth < 8) {
|
||||
png_set_packing(png);
|
||||
}
|
||||
|
||||
if (colorType & PNG_COLOR_MASK_PALETTE) {
|
||||
png_set_expand(png);
|
||||
}
|
||||
|
||||
if (!(colorType & PNG_COLOR_MASK_COLOR)) {
|
||||
png_set_gray_to_rgb(png);
|
||||
}
|
||||
|
||||
// set paletted or RGB images with transparency to full alpha so we get RGBA
|
||||
if (png_get_valid(png, info, PNG_INFO_tRNS)) {
|
||||
png_set_tRNS_to_alpha(png);
|
||||
}
|
||||
|
||||
// make sure every pixel has an alpha value
|
||||
if (!(colorType & PNG_COLOR_MASK_ALPHA)) {
|
||||
png_set_filler(png, 255, PNG_FILLER_AFTER);
|
||||
}
|
||||
|
||||
png_read_update_info(png, info);
|
||||
|
||||
unsigned char* out = malloc(w * h * 4);
|
||||
*out_w = w;
|
||||
*out_h = h;
|
||||
|
||||
// png_uint_32 rowBytes = png_get_rowbytes(png, info);
|
||||
png_bytep row_pointers[h];
|
||||
for (png_uint_32 row = 0; row < h; row++ ) {
|
||||
row_pointers[row] = (png_bytep)(out + (row * w * 4));
|
||||
}
|
||||
|
||||
png_read_image(png, row_pointers);
|
||||
png_read_end(png, info);
|
||||
png_destroy_read_struct( &png, &info, NULL);
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// stb_image encode callback
|
||||
|
||||
void stbi_write_callback(void *context, void *data, int size) {
|
||||
int *encoded_size = (int *)context;
|
||||
*encoded_size += size;
|
||||
// In theory we'd need to do another malloc(), memcpy() and free() here to
|
||||
// be fair to the other decode functions...
|
||||
}
|
||||
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// function to load a whole file into memory
|
||||
|
||||
void *fload(const char *path, int *out_size) {
|
||||
FILE *fh = fopen(path, "rb");
|
||||
if (!fh) {
|
||||
ERROR("Can't open file");
|
||||
}
|
||||
|
||||
fseek(fh, 0, SEEK_END);
|
||||
int size = ftell(fh);
|
||||
fseek(fh, 0, SEEK_SET);
|
||||
|
||||
void *buffer = malloc(size);
|
||||
if (!buffer) {
|
||||
ERROR("Malloc for %d bytes failed", size);
|
||||
}
|
||||
|
||||
if (!fread(buffer, size, 1, fh)) {
|
||||
ERROR("Can't read file %s", path);
|
||||
}
|
||||
fclose(fh);
|
||||
|
||||
*out_size = size;
|
||||
return buffer;
|
||||
}
|
||||
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// benchmark runner
|
||||
|
||||
|
||||
int opt_runs = 1;
|
||||
int opt_nopng = 0;
|
||||
int opt_nowarmup = 0;
|
||||
int opt_noverify = 0;
|
||||
int opt_nodecode = 0;
|
||||
int opt_noencode = 0;
|
||||
int opt_norecurse = 0;
|
||||
int opt_onlytotals = 0;
|
||||
|
||||
enum {
|
||||
LIBPNG,
|
||||
STBI,
|
||||
QOI,
|
||||
BENCH_COUNT /* must be the last element */
|
||||
};
|
||||
static const char *const lib_names[BENCH_COUNT] = {
|
||||
// NOTE: pad with spaces so everything lines up properly
|
||||
[LIBPNG] = "libpng: ",
|
||||
[STBI] = "stbi: ",
|
||||
[QOI] = "qoi: ",
|
||||
};
|
||||
|
||||
typedef struct {
|
||||
uint64_t size;
|
||||
uint64_t encode_time;
|
||||
uint64_t decode_time;
|
||||
} benchmark_lib_result_t;
|
||||
|
||||
typedef struct {
|
||||
int count;
|
||||
uint64_t raw_size;
|
||||
uint64_t px;
|
||||
int w;
|
||||
int h;
|
||||
benchmark_lib_result_t libs[BENCH_COUNT];
|
||||
} benchmark_result_t;
|
||||
|
||||
|
||||
void benchmark_print_result(benchmark_result_t res) {
|
||||
res.px /= res.count;
|
||||
res.raw_size /= res.count;
|
||||
|
||||
double px = res.px;
|
||||
printf(" decode ms encode ms decode mpps encode mpps size kb rate\n");
|
||||
for (int i = 0; i < BENCH_COUNT; ++i) {
|
||||
if (opt_nopng && (i == LIBPNG || i == STBI)) {
|
||||
continue;
|
||||
}
|
||||
res.libs[i].encode_time /= res.count;
|
||||
res.libs[i].decode_time /= res.count;
|
||||
res.libs[i].size /= res.count;
|
||||
printf(
|
||||
"%s %8.1f %8.1f %8.2f %8.2f %8ld %4.1f%%\n",
|
||||
lib_names[i],
|
||||
(double)res.libs[i].decode_time/1000000.0,
|
||||
(double)res.libs[i].encode_time/1000000.0,
|
||||
(res.libs[i].decode_time > 0 ? px / ((double)res.libs[i].decode_time/1000.0) : 0),
|
||||
(res.libs[i].encode_time > 0 ? px / ((double)res.libs[i].encode_time/1000.0) : 0),
|
||||
res.libs[i].size/1024,
|
||||
((double)res.libs[i].size/(double)res.raw_size) * 100.0
|
||||
);
|
||||
}
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
// Run __VA_ARGS__ a number of times and measure the time taken. The first
|
||||
// run is ignored.
|
||||
#define BENCHMARK_FN(NOWARMUP, RUNS, AVG_TIME, ...) \
|
||||
do { \
|
||||
uint64_t time = 0; \
|
||||
for (int i = NOWARMUP; i <= RUNS; i++) { \
|
||||
uint64_t time_start = ns(); \
|
||||
__VA_ARGS__ \
|
||||
uint64_t time_end = ns(); \
|
||||
if (i > 0) { \
|
||||
time += time_end - time_start; \
|
||||
} \
|
||||
} \
|
||||
AVG_TIME = time / RUNS; \
|
||||
} while (0)
|
||||
|
||||
|
||||
benchmark_result_t benchmark_image(const char *path) {
|
||||
int encoded_png_size;
|
||||
int encoded_qoi_size;
|
||||
int w;
|
||||
int h;
|
||||
int channels;
|
||||
|
||||
// Load the encoded PNG, encoded QOI and raw pixels into memory
|
||||
if(!stbi_info(path, &w, &h, &channels)) {
|
||||
ERROR("Error decoding header %s", path);
|
||||
}
|
||||
|
||||
if (channels != 3) {
|
||||
channels = 4;
|
||||
}
|
||||
|
||||
void *pixels = (void *)stbi_load(path, &w, &h, NULL, channels);
|
||||
void *encoded_png = fload(path, &encoded_png_size);
|
||||
void *encoded_qoi = qoi_encode(pixels, &(qoi_desc){
|
||||
.width = w,
|
||||
.height = h,
|
||||
.channels = channels,
|
||||
.colorspace = QOI_SRGB
|
||||
}, &encoded_qoi_size);
|
||||
|
||||
if (!pixels || !encoded_qoi || !encoded_png) {
|
||||
ERROR("Error encoding %s", path);
|
||||
}
|
||||
|
||||
// Verify QOI Output
|
||||
|
||||
if (!opt_noverify) {
|
||||
qoi_desc dc;
|
||||
void *pixels_qoi = qoi_decode(encoded_qoi, encoded_qoi_size, &dc, channels);
|
||||
if (memcmp(pixels, pixels_qoi, w * h * channels) != 0) {
|
||||
ERROR("QOI roundtrip pixel mismatch for %s", path);
|
||||
}
|
||||
free(pixels_qoi);
|
||||
}
|
||||
|
||||
|
||||
|
||||
benchmark_result_t res = {0};
|
||||
res.count = 1;
|
||||
res.raw_size = w * h * channels;
|
||||
res.px = w * h;
|
||||
res.w = w;
|
||||
res.h = h;
|
||||
|
||||
|
||||
// Decoding
|
||||
|
||||
if (!opt_nodecode) {
|
||||
if (!opt_nopng) {
|
||||
BENCHMARK_FN(opt_nowarmup, opt_runs, res.libs[LIBPNG].decode_time, {
|
||||
int dec_w, dec_h;
|
||||
void *dec_p = libpng_decode(encoded_png, encoded_png_size, &dec_w, &dec_h);
|
||||
free(dec_p);
|
||||
});
|
||||
|
||||
BENCHMARK_FN(opt_nowarmup, opt_runs, res.libs[STBI].decode_time, {
|
||||
int dec_w, dec_h, dec_channels;
|
||||
void *dec_p = stbi_load_from_memory(encoded_png, encoded_png_size, &dec_w, &dec_h, &dec_channels, 4);
|
||||
free(dec_p);
|
||||
});
|
||||
}
|
||||
|
||||
BENCHMARK_FN(opt_nowarmup, opt_runs, res.libs[QOI].decode_time, {
|
||||
qoi_desc desc;
|
||||
void *dec_p = qoi_decode(encoded_qoi, encoded_qoi_size, &desc, 4);
|
||||
free(dec_p);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
// Encoding
|
||||
if (!opt_noencode) {
|
||||
if (!opt_nopng) {
|
||||
BENCHMARK_FN(opt_nowarmup, opt_runs, res.libs[LIBPNG].encode_time, {
|
||||
int enc_size;
|
||||
void *enc_p = libpng_encode(pixels, w, h, channels, &enc_size);
|
||||
res.libs[LIBPNG].size = enc_size;
|
||||
free(enc_p);
|
||||
});
|
||||
|
||||
BENCHMARK_FN(opt_nowarmup, opt_runs, res.libs[STBI].encode_time, {
|
||||
int enc_size = 0;
|
||||
stbi_write_png_to_func(stbi_write_callback, &enc_size, w, h, channels, pixels, 0);
|
||||
res.libs[STBI].size = enc_size;
|
||||
});
|
||||
}
|
||||
|
||||
BENCHMARK_FN(opt_nowarmup, opt_runs, res.libs[QOI].encode_time, {
|
||||
int enc_size;
|
||||
void *enc_p = qoi_encode(pixels, &(qoi_desc){
|
||||
.width = w,
|
||||
.height = h,
|
||||
.channels = channels,
|
||||
.colorspace = QOI_SRGB
|
||||
}, &enc_size);
|
||||
res.libs[QOI].size = enc_size;
|
||||
free(enc_p);
|
||||
});
|
||||
}
|
||||
|
||||
free(pixels);
|
||||
free(encoded_png);
|
||||
free(encoded_qoi);
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
void benchmark_directory(const char *path, benchmark_result_t *grand_total) {
|
||||
DIR *dir = opendir(path);
|
||||
if (!dir) {
|
||||
ERROR("Couldn't open directory %s", path);
|
||||
}
|
||||
|
||||
struct dirent *file;
|
||||
|
||||
if (!opt_norecurse) {
|
||||
for (int i = 0; (file = readdir(dir)) != NULL; i++) {
|
||||
if (
|
||||
file->d_type & DT_DIR &&
|
||||
strcmp(file->d_name, ".") != 0 &&
|
||||
strcmp(file->d_name, "..") != 0
|
||||
) {
|
||||
char subpath[1024];
|
||||
snprintf(subpath, 1024, "%s/%s", path, file->d_name);
|
||||
benchmark_directory(subpath, grand_total);
|
||||
}
|
||||
}
|
||||
rewinddir(dir);
|
||||
}
|
||||
|
||||
benchmark_result_t dir_total = {0};
|
||||
|
||||
int has_shown_head = 0;
|
||||
for (int i = 0; (file = readdir(dir)) != NULL; i++) {
|
||||
if (strcmp(file->d_name + strlen(file->d_name) - 4, ".png") != 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!has_shown_head) {
|
||||
has_shown_head = 1;
|
||||
printf("## Benchmarking %s/*.png -- %d runs\n\n", path, opt_runs);
|
||||
}
|
||||
|
||||
char *file_path = malloc(strlen(file->d_name) + strlen(path)+8);
|
||||
sprintf(file_path, "%s/%s", path, file->d_name);
|
||||
|
||||
benchmark_result_t res = benchmark_image(file_path);
|
||||
|
||||
if (!opt_onlytotals) {
|
||||
printf("## %s size: %dx%d\n", file_path, res.w, res.h);
|
||||
benchmark_print_result(res);
|
||||
}
|
||||
|
||||
free(file_path);
|
||||
|
||||
dir_total.count++;
|
||||
dir_total.raw_size += res.raw_size;
|
||||
dir_total.px += res.px;
|
||||
for (int i = 0; i < BENCH_COUNT; ++i) {
|
||||
dir_total.libs[i].encode_time += res.libs[i].encode_time;
|
||||
dir_total.libs[i].decode_time += res.libs[i].decode_time;
|
||||
dir_total.libs[i].size += res.libs[i].size;
|
||||
}
|
||||
|
||||
grand_total->count++;
|
||||
grand_total->raw_size += res.raw_size;
|
||||
grand_total->px += res.px;
|
||||
for (int i = 0; i < BENCH_COUNT; ++i) {
|
||||
grand_total->libs[i].encode_time += res.libs[i].encode_time;
|
||||
grand_total->libs[i].decode_time += res.libs[i].decode_time;
|
||||
grand_total->libs[i].size += res.libs[i].size;
|
||||
}
|
||||
}
|
||||
closedir(dir);
|
||||
|
||||
if (dir_total.count > 0) {
|
||||
printf("## Total for %s\n", path);
|
||||
benchmark_print_result(dir_total);
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
if (argc < 3) {
|
||||
printf("Usage: qoibench <iterations> <directory> [options]\n");
|
||||
printf("Options:\n");
|
||||
printf(" --nowarmup ... don't perform a warmup run\n");
|
||||
printf(" --nopng ...... don't run png encode/decode\n");
|
||||
printf(" --noverify ... don't verify qoi roundtrip\n");
|
||||
printf(" --noencode ... don't run encoders\n");
|
||||
printf(" --nodecode ... don't run decoders\n");
|
||||
printf(" --norecurse .. don't descend into directories\n");
|
||||
printf(" --onlytotals . don't print individual image results\n");
|
||||
printf("Examples\n");
|
||||
printf(" qoibench 10 images/textures/\n");
|
||||
printf(" qoibench 1 images/textures/ --nopng --nowarmup\n");
|
||||
exit(1);
|
||||
}
|
||||
|
||||
for (int i = 3; i < argc; i++) {
|
||||
if (strcmp(argv[i], "--nowarmup") == 0) { opt_nowarmup = 1; }
|
||||
else if (strcmp(argv[i], "--nopng") == 0) { opt_nopng = 1; }
|
||||
else if (strcmp(argv[i], "--noverify") == 0) { opt_noverify = 1; }
|
||||
else if (strcmp(argv[i], "--noencode") == 0) { opt_noencode = 1; }
|
||||
else if (strcmp(argv[i], "--nodecode") == 0) { opt_nodecode = 1; }
|
||||
else if (strcmp(argv[i], "--norecurse") == 0) { opt_norecurse = 1; }
|
||||
else if (strcmp(argv[i], "--onlytotals") == 0) { opt_onlytotals = 1; }
|
||||
else { ERROR("Unknown option %s", argv[i]); }
|
||||
}
|
||||
|
||||
opt_runs = atoi(argv[1]);
|
||||
if (opt_runs <=0) {
|
||||
ERROR("Invalid number of runs %d", opt_runs);
|
||||
}
|
||||
|
||||
benchmark_result_t grand_total = {0};
|
||||
benchmark_directory(argv[2], &grand_total);
|
||||
|
||||
if (grand_total.count > 0) {
|
||||
printf("# Grand total for %s\n", argv[2]);
|
||||
benchmark_print_result(grand_total);
|
||||
}
|
||||
else {
|
||||
printf("No images found in %s\n", argv[2]);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
91
external/qoi/qoi/qoiconv.c
vendored
Normal file
91
external/qoi/qoi/qoiconv.c
vendored
Normal file
@ -0,0 +1,91 @@
|
||||
/*
|
||||
|
||||
Copyright (c) 2021, Dominic Szablewski - https://phoboslab.org
|
||||
SPDX-License-Identifier: MIT
|
||||
|
||||
|
||||
Command line tool to convert between png <> qoi format
|
||||
|
||||
Requires:
|
||||
-"stb_image.h" (https://github.com/nothings/stb/blob/master/stb_image.h)
|
||||
-"stb_image_write.h" (https://github.com/nothings/stb/blob/master/stb_image_write.h)
|
||||
-"qoi.h" (https://github.com/phoboslab/qoi/blob/master/qoi.h)
|
||||
|
||||
Compile with:
|
||||
gcc qoiconv.c -std=c99 -O3 -o qoiconv
|
||||
|
||||
*/
|
||||
|
||||
|
||||
#define STB_IMAGE_IMPLEMENTATION
|
||||
#define STBI_ONLY_PNG
|
||||
#define STBI_NO_LINEAR
|
||||
#include "stb_image.h"
|
||||
|
||||
#define STB_IMAGE_WRITE_IMPLEMENTATION
|
||||
#include "stb_image_write.h"
|
||||
|
||||
#define QOI_IMPLEMENTATION
|
||||
#include "qoi.h"
|
||||
|
||||
|
||||
#define STR_ENDS_WITH(S, E) (strcmp(S + strlen(S) - (sizeof(E)-1), E) == 0)
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
if (argc < 3) {
|
||||
puts("Usage: qoiconv <infile> <outfile>");
|
||||
puts("Examples:");
|
||||
puts(" qoiconv input.png output.qoi");
|
||||
puts(" qoiconv input.qoi output.png");
|
||||
exit(1);
|
||||
}
|
||||
|
||||
void *pixels = NULL;
|
||||
int w, h, channels;
|
||||
if (STR_ENDS_WITH(argv[1], ".png")) {
|
||||
if(!stbi_info(argv[1], &w, &h, &channels)) {
|
||||
printf("Couldn't read header %s\n", argv[1]);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
// Force all odd encodings to be RGBA
|
||||
if(channels != 3) {
|
||||
channels = 4;
|
||||
}
|
||||
|
||||
pixels = (void *)stbi_load(argv[1], &w, &h, NULL, channels);
|
||||
}
|
||||
else if (STR_ENDS_WITH(argv[1], ".qoi")) {
|
||||
qoi_desc desc;
|
||||
pixels = qoi_read(argv[1], &desc, 0);
|
||||
channels = desc.channels;
|
||||
w = desc.width;
|
||||
h = desc.height;
|
||||
}
|
||||
|
||||
if (pixels == NULL) {
|
||||
printf("Couldn't load/decode %s\n", argv[1]);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
int encoded = 0;
|
||||
if (STR_ENDS_WITH(argv[2], ".png")) {
|
||||
encoded = stbi_write_png(argv[2], w, h, channels, pixels, 0);
|
||||
}
|
||||
else if (STR_ENDS_WITH(argv[2], ".qoi")) {
|
||||
encoded = qoi_write(argv[2], pixels, &(qoi_desc){
|
||||
.width = w,
|
||||
.height = h,
|
||||
.channels = channels,
|
||||
.colorspace = QOI_SRGB
|
||||
});
|
||||
}
|
||||
|
||||
if (!encoded) {
|
||||
printf("Couldn't write/encode %s\n", argv[2]);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
free(pixels);
|
||||
return 0;
|
||||
}
|
32
external/qoi/qoi/qoifuzz.c
vendored
Normal file
32
external/qoi/qoi/qoifuzz.c
vendored
Normal file
@ -0,0 +1,32 @@
|
||||
/*
|
||||
|
||||
Copyright (c) 2021, Dominic Szablewski - https://phoboslab.org
|
||||
SPDX-License-Identifier: MIT
|
||||
|
||||
|
||||
clang fuzzing harness for qoi_decode
|
||||
|
||||
Compile and run with:
|
||||
clang -fsanitize=address,fuzzer -g -O0 qoifuzz.c && ./a.out
|
||||
|
||||
*/
|
||||
|
||||
|
||||
#define QOI_IMPLEMENTATION
|
||||
#include "qoi.h"
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
|
||||
int w, h;
|
||||
if (size < 4) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
qoi_desc desc;
|
||||
void* decoded = qoi_decode((void*)(data + 4), (int)(size - 4), &desc, *((int *)data));
|
||||
if (decoded != NULL) {
|
||||
free(decoded);
|
||||
}
|
||||
return 0;
|
||||
}
|
15
flake.nix
15
flake.nix
@ -12,13 +12,15 @@
|
||||
flake-utils.lib.eachDefaultSystem (system:
|
||||
let
|
||||
pkgs = import nixpkgs { inherit system; };
|
||||
stdenv = (pkgs.stdenvAdapters.keepDebugInfo pkgs.stdenv);
|
||||
in {
|
||||
packages.default = pkgs.stdenv.mkDerivation {
|
||||
#packages.default = pkgs.stdenv.mkDerivation {
|
||||
packages.default = stdenv.mkDerivation {
|
||||
pname = "tomato";
|
||||
version = "0.0.0";
|
||||
|
||||
src = ./.;
|
||||
submodules = 1;
|
||||
submodules = 1; # does nothing
|
||||
|
||||
nativeBuildInputs = with pkgs; [
|
||||
cmake
|
||||
@ -58,6 +60,11 @@
|
||||
cmakeFlags = [
|
||||
"TOMATO_ASAN=1"
|
||||
"CMAKE_BUILD_TYPE=RelWithDebInfo"
|
||||
|
||||
"-DFETCHCONTENT_SOURCE_DIR_JSON=${pkgs.nlohmann_json.src}" # we care less about version here
|
||||
# do we really care less about the version? do we need a stable abi?
|
||||
|
||||
"-DFETCHCONTENT_SOURCE_DIR_ZSTD=${pkgs.zstd.src}"
|
||||
];
|
||||
|
||||
# TODO: replace with install command
|
||||
@ -66,7 +73,7 @@
|
||||
mv bin/tomato $out/bin
|
||||
'';
|
||||
|
||||
dontStrip = true;
|
||||
dontStrip = true; # does nothing
|
||||
|
||||
# copied from nixpkgs's SDL2 default.nix
|
||||
# SDL is weird in that instead of just dynamically linking with
|
||||
@ -93,6 +100,8 @@
|
||||
'';
|
||||
};
|
||||
|
||||
#packages.debug = pkgs.enableDebugging self.packages.${system}.default;
|
||||
|
||||
devShells.${system}.default = pkgs.mkShell {
|
||||
#inputsFrom = with pkgs; [ SDL2 ];
|
||||
buildInputs = [ self.packages.${system}.default ]; # this makes a prebuild tomato available in the shell, do we want this?
|
||||
|
@ -1,5 +1,60 @@
|
||||
cmake_minimum_required(VERSION 3.9 FATAL_ERROR)
|
||||
|
||||
add_library(fragment_store
|
||||
./fragment_store/fragment_store_i.hpp
|
||||
./fragment_store/fragment_store_i.cpp
|
||||
./fragment_store/types.hpp
|
||||
./fragment_store/meta_components.hpp
|
||||
./fragment_store/meta_components_id.inl
|
||||
./fragment_store/serializer.hpp
|
||||
./fragment_store/fragment_store.hpp
|
||||
./fragment_store/fragment_store.cpp
|
||||
|
||||
./json/message_components.hpp # TODO: move
|
||||
./json/tox_message_components.hpp # TODO: move
|
||||
)
|
||||
|
||||
target_link_libraries(fragment_store PUBLIC
|
||||
nlohmann_json::nlohmann_json
|
||||
EnTT::EnTT
|
||||
solanaceae_util
|
||||
|
||||
zstd::zstd
|
||||
|
||||
solanaceae_tox_messages # TODO: move
|
||||
)
|
||||
|
||||
########################################
|
||||
|
||||
add_library(message_fragment_store
|
||||
./fragment_store/message_serializer.hpp
|
||||
./fragment_store/message_serializer.cpp
|
||||
./fragment_store/message_fragment_store.hpp
|
||||
./fragment_store/message_fragment_store.cpp
|
||||
|
||||
./fragment_store/register_mfs_json_message_components.hpp
|
||||
./fragment_store/register_mfs_json_message_components.cpp
|
||||
./fragment_store/register_mfs_json_tox_message_components.hpp
|
||||
./fragment_store/register_mfs_json_tox_message_components.cpp
|
||||
)
|
||||
target_compile_features(message_fragment_store PRIVATE cxx_std_20)
|
||||
target_link_libraries(message_fragment_store PUBLIC
|
||||
fragment_store
|
||||
solanaceae_message3
|
||||
)
|
||||
|
||||
########################################
|
||||
|
||||
add_executable(fragment_store_test
|
||||
fragment_store/test_fragstore.cpp
|
||||
)
|
||||
|
||||
target_link_libraries(fragment_store_test PUBLIC
|
||||
fragment_store
|
||||
)
|
||||
|
||||
########################################
|
||||
|
||||
add_executable(tomato
|
||||
./main.cpp
|
||||
./icon.rc
|
||||
@ -25,6 +80,8 @@ add_executable(tomato
|
||||
./image_loader_stb.cpp
|
||||
./image_loader_webp.hpp
|
||||
./image_loader_webp.cpp
|
||||
./image_loader_qoi.hpp
|
||||
./image_loader_qoi.cpp
|
||||
|
||||
./texture_uploader.hpp
|
||||
./sdlrenderer_texture_uploader.hpp
|
||||
@ -80,6 +137,9 @@ target_link_libraries(tomato PUBLIC
|
||||
solanaceae_tox_contacts
|
||||
solanaceae_tox_messages
|
||||
|
||||
fragment_store
|
||||
message_fragment_store
|
||||
|
||||
SDL3::SDL3
|
||||
|
||||
imgui
|
||||
@ -90,5 +150,6 @@ target_link_libraries(tomato PUBLIC
|
||||
stb_image_write
|
||||
webpdemux
|
||||
libwebpmux # the f why (needed for anim encode)
|
||||
qoi
|
||||
)
|
||||
|
||||
|
@ -37,6 +37,18 @@ namespace Components {
|
||||
|
||||
} // Components
|
||||
|
||||
namespace Context {
|
||||
|
||||
// TODO: move back to chat log window and keep per window instead of per contact
|
||||
struct CGView {
|
||||
// set to the ts of the newest rendered msg
|
||||
Message3Handle begin{};
|
||||
// set to the ts of the oldest rendered msg
|
||||
Message3Handle end{};
|
||||
};
|
||||
|
||||
} // Context
|
||||
|
||||
static constexpr float lerp(float a, float b, float t) {
|
||||
return a + t * (b - a);
|
||||
}
|
||||
@ -259,28 +271,6 @@ float ChatGui4::render(float time_delta) {
|
||||
|
||||
auto* msg_reg_ptr = _rmm.get(*_selected_contact);
|
||||
|
||||
if (msg_reg_ptr != nullptr) {
|
||||
const auto& mm = *msg_reg_ptr;
|
||||
//const auto& unread_storage = mm.storage<Message::Components::TagUnread>();
|
||||
if (const auto* unread_storage = mm.storage<Message::Components::TagUnread>(); unread_storage != nullptr && !unread_storage->empty()) {
|
||||
//assert(unread_storage->size() == 0);
|
||||
//assert(unread_storage.cbegin() == unread_storage.cend());
|
||||
|
||||
#if 0
|
||||
std::cout << "UNREAD ";
|
||||
Message3 prev_ent = entt::null;
|
||||
for (const Message3 e : mm.view<Message::Components::TagUnread>()) {
|
||||
std::cout << entt::to_integral(e) << " ";
|
||||
if (prev_ent == e) {
|
||||
assert(false && "dup");
|
||||
}
|
||||
prev_ent = e;
|
||||
}
|
||||
std::cout << "\n";
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
constexpr ImGuiTableFlags table_flags =
|
||||
ImGuiTableFlags_BordersInnerV |
|
||||
ImGuiTableFlags_RowBg |
|
||||
@ -293,6 +283,9 @@ float ChatGui4::render(float time_delta) {
|
||||
ImGui::TableSetupColumn("timestamp");
|
||||
ImGui::TableSetupColumn("extra_info", _show_chat_extra_info ? ImGuiTableColumnFlags_None : ImGuiTableColumnFlags_Disabled);
|
||||
|
||||
Message3Handle message_view_oldest; // oldest visible message
|
||||
Message3Handle message_view_newest; // last visible message
|
||||
|
||||
// very hacky, and we have variable hight entries
|
||||
//ImGuiListClipper clipper;
|
||||
|
||||
@ -381,12 +374,26 @@ float ChatGui4::render(float time_delta) {
|
||||
}
|
||||
|
||||
// use username as visibility test
|
||||
if (ImGui::IsItemVisible() && msg_reg.all_of<Message::Components::TagUnread>(e)) {
|
||||
if (ImGui::IsItemVisible()) {
|
||||
if (msg_reg.all_of<Message::Components::TagUnread>(e)) {
|
||||
// get time now
|
||||
const uint64_t ts_now = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
|
||||
msg_reg.emplace_or_replace<Message::Components::Read>(e, ts_now);
|
||||
msg_reg.remove<Message::Components::TagUnread>(e);
|
||||
msg_reg.emplace_or_replace<Components::UnreadFade>(e, 1.f);
|
||||
|
||||
// we remove the unread tag here
|
||||
_rmm.throwEventUpdate(msg_reg, e);
|
||||
}
|
||||
|
||||
// track view
|
||||
if (!static_cast<bool>(message_view_oldest)) {
|
||||
message_view_oldest = {msg_reg, e};
|
||||
message_view_newest = {msg_reg, e};
|
||||
} else if (static_cast<bool>(message_view_newest)) {
|
||||
// update to latest
|
||||
message_view_newest = {msg_reg, e};
|
||||
}
|
||||
}
|
||||
|
||||
// highlight self
|
||||
@ -539,9 +546,90 @@ float ChatGui4::render(float time_delta) {
|
||||
//ImGui::TableNextRow(0, TEXT_BASE_HEIGHT);
|
||||
//ImGui::TableNextRow(0, TEXT_BASE_HEIGHT);
|
||||
|
||||
{ // update view cursers
|
||||
if (!msg_reg.ctx().contains<Context::CGView>()) {
|
||||
msg_reg.ctx().emplace<Context::CGView>();
|
||||
}
|
||||
|
||||
auto& cg_view = msg_reg.ctx().get<Context::CGView>();
|
||||
|
||||
// any message in view
|
||||
if (!static_cast<bool>(message_view_oldest)) {
|
||||
// no message in view, we setup a view at current time, so the next frags are loaded
|
||||
if (!static_cast<bool>(cg_view.begin) || !static_cast<bool>(cg_view.end)) {
|
||||
// fix invalid state
|
||||
if (static_cast<bool>(cg_view.begin)) {
|
||||
cg_view.begin.destroy();
|
||||
_rmm.throwEventDestroy(cg_view.begin);
|
||||
}
|
||||
if (static_cast<bool>(cg_view.end)) {
|
||||
cg_view.end.destroy();
|
||||
_rmm.throwEventDestroy(cg_view.end);
|
||||
}
|
||||
|
||||
// create new
|
||||
cg_view.begin = {msg_reg, msg_reg.create()};
|
||||
cg_view.end = {msg_reg, msg_reg.create()};
|
||||
|
||||
cg_view.begin.emplace_or_replace<Message::Components::ViewCurserBegin>(cg_view.end);
|
||||
cg_view.end.emplace_or_replace<Message::Components::ViewCurserEnd>(cg_view.begin);
|
||||
|
||||
cg_view.begin.get_or_emplace<Message::Components::Timestamp>().ts = Message::getTimeMS();
|
||||
cg_view.end.get_or_emplace<Message::Components::Timestamp>().ts = Message::getTimeMS();
|
||||
|
||||
std::cout << "CG: created view FRONT begin ts\n";
|
||||
_rmm.throwEventConstruct(cg_view.begin);
|
||||
std::cout << "CG: created view FRONT end ts\n";
|
||||
_rmm.throwEventConstruct(cg_view.end);
|
||||
} // else? we do nothing?
|
||||
} else {
|
||||
bool begin_created {false};
|
||||
if (!static_cast<bool>(cg_view.begin)) {
|
||||
cg_view.begin = {msg_reg, msg_reg.create()};
|
||||
begin_created = true;
|
||||
}
|
||||
bool end_created {false};
|
||||
if (!static_cast<bool>(cg_view.end)) {
|
||||
cg_view.end = {msg_reg, msg_reg.create()};
|
||||
end_created = true;
|
||||
}
|
||||
cg_view.begin.emplace_or_replace<Message::Components::ViewCurserBegin>(cg_view.end);
|
||||
cg_view.end.emplace_or_replace<Message::Components::ViewCurserEnd>(cg_view.begin);
|
||||
|
||||
{
|
||||
auto& old_begin_ts = cg_view.begin.get_or_emplace<Message::Components::Timestamp>().ts;
|
||||
if (old_begin_ts != message_view_newest.get<Message::Components::Timestamp>().ts) {
|
||||
old_begin_ts = message_view_newest.get<Message::Components::Timestamp>().ts;
|
||||
if (begin_created) {
|
||||
std::cout << "CG: created view begin ts with " << old_begin_ts << "\n";
|
||||
_rmm.throwEventConstruct(cg_view.begin);
|
||||
} else {
|
||||
//std::cout << "CG: updated view begin ts to " << old_begin_ts << "\n";
|
||||
_rmm.throwEventUpdate(cg_view.begin);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
auto& old_end_ts = cg_view.end.get_or_emplace<Message::Components::Timestamp>().ts;
|
||||
if (old_end_ts != message_view_oldest.get<Message::Components::Timestamp>().ts) {
|
||||
old_end_ts = message_view_oldest.get<Message::Components::Timestamp>().ts;
|
||||
if (end_created) {
|
||||
std::cout << "CG: created view end ts with " << old_end_ts << "\n";
|
||||
_rmm.throwEventConstruct(cg_view.end);
|
||||
} else {
|
||||
//std::cout << "CG: updated view end ts to " << old_end_ts << "\n";
|
||||
_rmm.throwEventUpdate(cg_view.end);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ImGui::EndTable();
|
||||
}
|
||||
|
||||
|
||||
if (ImGui::GetScrollY() >= ImGui::GetScrollMaxY()) {
|
||||
ImGui::SetScrollHereY(1.f);
|
||||
}
|
||||
@ -613,6 +701,7 @@ float ChatGui4::render(float time_delta) {
|
||||
"image/gif",
|
||||
"image/jpeg",
|
||||
"image/bmp",
|
||||
"image/qoi",
|
||||
};
|
||||
|
||||
for (const char* mime_type : image_mime_types) {
|
||||
|
@ -10,6 +10,9 @@
|
||||
#include "./file_selector.hpp"
|
||||
#include "./send_image_popup.hpp"
|
||||
|
||||
// HACK: move to public msg api?
|
||||
#include "./fragment_store/message_fragment_store.hpp"
|
||||
|
||||
#include <entt/container/dense_map.hpp>
|
||||
|
||||
#include <cstdint>
|
||||
@ -32,6 +35,7 @@ class ChatGui4 {
|
||||
FileSelector _fss;
|
||||
SendImagePopup _sip;
|
||||
|
||||
// TODO: refactor this to allow multiple open contacts
|
||||
std::optional<Contact3> _selected_contact;
|
||||
|
||||
// TODO: per contact
|
||||
|
@ -25,41 +25,41 @@ Just keeps the Fragments in memory.
|
||||
|
||||
# File formats
|
||||
|
||||
Files can be compressed and encrypted. Since compression needs the data structure to funcion, it is applied before it is encrypted.
|
||||
Files can be compressed and encrypted. Since compression needs the data's structure to work properly, it is applied before it is encrypted.
|
||||
|
||||
### Text Json
|
||||
|
||||
Text json only makes sense for metadata if it's neither compressed nor encrypted. (otherwise its binary on disk anyway, so why waste bytes).
|
||||
Since the content of data is not looked at, nothing stops you from using text json and ecrypt it, but atleast basic compression is advised.
|
||||
|
||||
A Metadata json object has the following keys:
|
||||
- `enc` (uint) Encryption type of the data, if any
|
||||
- `comp` (uint) Compression type of the data, if any
|
||||
- `metadata` (obj) the
|
||||
A Metadata json object can have arbitrary keys, some are predefined:
|
||||
- `FragComp::DataEncryptionType` (uint) Encryption type of the data, if any
|
||||
- `FragComp::DataCompressionType` (uint) Compression type of the data, if any
|
||||
|
||||
## Binary file headers
|
||||
|
||||
### Split Metadata
|
||||
|
||||
file magic bytes `SOLMET` (6 bytes)
|
||||
msgpack array:
|
||||
|
||||
1 byte encryption type (`0x00` is none)
|
||||
|
||||
1 byte compression type (`0x00` is none)
|
||||
|
||||
...metadata here...
|
||||
- `[0]`: file magic string `SOLMET` (6 bytes)
|
||||
- `[1]`: uint8 encryption type (`0x00` is none)
|
||||
- `[2]`: uint8 compression type (`0x00` is none, `0x01` is zstd)
|
||||
- `[3]`: binary metadata (optionally compressed and encrypted)
|
||||
|
||||
note that the encryption and compression are for the metadata only.
|
||||
The metadata itself contains encryption and compression info about the data.
|
||||
|
||||
### Split Data
|
||||
|
||||
(none) all the data is in the metadata file.
|
||||
All the metadata is in the metadata file. (like encryption and compression)
|
||||
This is mostly to allow direct storage for files in the Fragment store without excessive duplication.
|
||||
Keep in mind to not use the actual file name as the data/meta file name.
|
||||
|
||||
### Single fragment
|
||||
|
||||
Note: this format is unused for now
|
||||
|
||||
file magic bytes `SOLFIL` (6 bytes)
|
||||
|
||||
1 byte encryption type (`0x00` is none)
|
||||
@ -70,3 +70,7 @@ file magic bytes `SOLFIL` (6 bytes)
|
||||
|
||||
...data here...
|
||||
|
||||
## Compression types
|
||||
|
||||
- `0x00` none
|
||||
- `0x01` zstd (without dict)
|
||||
|
869
src/fragment_store/fragment_store.cpp
Normal file
869
src/fragment_store/fragment_store.cpp
Normal file
@ -0,0 +1,869 @@
|
||||
#include "./fragment_store.hpp"
|
||||
|
||||
#include <solanaceae/util/utils.hpp>
|
||||
|
||||
#include <entt/entity/handle.hpp>
|
||||
#include <entt/container/dense_set.hpp>
|
||||
#include <entt/core/hashed_string.hpp>
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
#include <zstd.h>
|
||||
|
||||
#include <cstdint>
|
||||
#include <fstream>
|
||||
#include <filesystem>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <type_traits>
|
||||
#include <utility>
|
||||
#include <algorithm>
|
||||
|
||||
#include <iostream>
|
||||
#include <vector>
|
||||
|
||||
static const char* metaFileTypeSuffix(MetaFileType mft) {
|
||||
switch (mft) {
|
||||
case MetaFileType::TEXT_JSON: return ".json";
|
||||
//case MetaFileType::BINARY_ARB: return ".bin";
|
||||
case MetaFileType::BINARY_MSGPACK: return ".msgpack";
|
||||
}
|
||||
return ""; // .unk?
|
||||
}
|
||||
|
||||
FragmentStore::FragmentStore(void) {
|
||||
{ // random namespace
|
||||
const auto num0 = _rng();
|
||||
const auto num1 = _rng();
|
||||
const auto num2 = _rng();
|
||||
const auto num3 = _rng();
|
||||
|
||||
_session_uuid_namespace[0+0] = (num0 >> 0) & 0xff;
|
||||
_session_uuid_namespace[0+1] = (num0 >> 8) & 0xff;
|
||||
_session_uuid_namespace[0+2] = (num0 >> 16) & 0xff;
|
||||
_session_uuid_namespace[0+3] = (num0 >> 24) & 0xff;
|
||||
|
||||
_session_uuid_namespace[4+0] = (num1 >> 0) & 0xff;
|
||||
_session_uuid_namespace[4+1] = (num1 >> 8) & 0xff;
|
||||
_session_uuid_namespace[4+2] = (num1 >> 16) & 0xff;
|
||||
_session_uuid_namespace[4+3] = (num1 >> 24) & 0xff;
|
||||
|
||||
_session_uuid_namespace[8+0] = (num2 >> 0) & 0xff;
|
||||
_session_uuid_namespace[8+1] = (num2 >> 8) & 0xff;
|
||||
_session_uuid_namespace[8+2] = (num2 >> 16) & 0xff;
|
||||
_session_uuid_namespace[8+3] = (num2 >> 24) & 0xff;
|
||||
|
||||
_session_uuid_namespace[12+0] = (num3 >> 0) & 0xff;
|
||||
_session_uuid_namespace[12+1] = (num3 >> 8) & 0xff;
|
||||
_session_uuid_namespace[12+2] = (num3 >> 16) & 0xff;
|
||||
_session_uuid_namespace[12+3] = (num3 >> 24) & 0xff;
|
||||
}
|
||||
registerSerializers();
|
||||
}
|
||||
|
||||
FragmentStore::FragmentStore(
|
||||
std::array<uint8_t, 16> session_uuid_namespace
|
||||
) : _session_uuid_namespace(std::move(session_uuid_namespace)) {
|
||||
registerSerializers();
|
||||
}
|
||||
|
||||
FragmentHandle FragmentStore::fragmentHandle(FragmentID fid) {
|
||||
return {_reg, fid};
|
||||
}
|
||||
|
||||
std::vector<uint8_t> FragmentStore::generateNewUID(std::array<uint8_t, 16>& uuid_namespace) {
|
||||
std::vector<uint8_t> new_uid(uuid_namespace.cbegin(), uuid_namespace.cend());
|
||||
new_uid.resize(new_uid.size() + 16);
|
||||
|
||||
const auto num0 = _rng();
|
||||
const auto num1 = _rng();
|
||||
const auto num2 = _rng();
|
||||
const auto num3 = _rng();
|
||||
|
||||
new_uid[uuid_namespace.size()+0] = (num0 >> 0) & 0xff;
|
||||
new_uid[uuid_namespace.size()+1] = (num0 >> 8) & 0xff;
|
||||
new_uid[uuid_namespace.size()+2] = (num0 >> 16) & 0xff;
|
||||
new_uid[uuid_namespace.size()+3] = (num0 >> 24) & 0xff;
|
||||
|
||||
new_uid[uuid_namespace.size()+4+0] = (num1 >> 0) & 0xff;
|
||||
new_uid[uuid_namespace.size()+4+1] = (num1 >> 8) & 0xff;
|
||||
new_uid[uuid_namespace.size()+4+2] = (num1 >> 16) & 0xff;
|
||||
new_uid[uuid_namespace.size()+4+3] = (num1 >> 24) & 0xff;
|
||||
|
||||
new_uid[uuid_namespace.size()+8+0] = (num2 >> 0) & 0xff;
|
||||
new_uid[uuid_namespace.size()+8+1] = (num2 >> 8) & 0xff;
|
||||
new_uid[uuid_namespace.size()+8+2] = (num2 >> 16) & 0xff;
|
||||
new_uid[uuid_namespace.size()+8+3] = (num2 >> 24) & 0xff;
|
||||
|
||||
new_uid[uuid_namespace.size()+12+0] = (num3 >> 0) & 0xff;
|
||||
new_uid[uuid_namespace.size()+12+1] = (num3 >> 8) & 0xff;
|
||||
new_uid[uuid_namespace.size()+12+2] = (num3 >> 16) & 0xff;
|
||||
new_uid[uuid_namespace.size()+12+3] = (num3 >> 24) & 0xff;
|
||||
|
||||
return new_uid;
|
||||
}
|
||||
|
||||
std::vector<uint8_t> FragmentStore::generateNewUID(void) {
|
||||
return generateNewUID(_session_uuid_namespace);
|
||||
}
|
||||
|
||||
FragmentID FragmentStore::newFragmentMemoryOwned(
|
||||
const std::vector<uint8_t>& id,
|
||||
size_t initial_size
|
||||
) {
|
||||
{ // first check if id is already used
|
||||
auto exising_id = getFragmentByID(id);
|
||||
if (_reg.valid(exising_id)) {
|
||||
return entt::null;
|
||||
}
|
||||
}
|
||||
|
||||
{ // next check if space in memory budget
|
||||
const auto free_memory = _memory_budget - _memory_usage;
|
||||
if (initial_size > free_memory) {
|
||||
return entt::null;
|
||||
}
|
||||
}
|
||||
|
||||
// actually allocate and create
|
||||
auto new_data = std::make_unique<std::vector<uint8_t>>(initial_size);
|
||||
if (!static_cast<bool>(new_data)) {
|
||||
// allocation failure
|
||||
return entt::null;
|
||||
}
|
||||
_memory_usage += initial_size;
|
||||
|
||||
const auto new_frag = _reg.create();
|
||||
|
||||
_reg.emplace<FragComp::ID>(new_frag, id);
|
||||
// TODO: memory comp
|
||||
_reg.emplace<std::unique_ptr<std::vector<uint8_t>>>(new_frag) = std::move(new_data);
|
||||
|
||||
throwEventConstruct(new_frag);
|
||||
|
||||
return new_frag;
|
||||
}
|
||||
|
||||
FragmentID FragmentStore::newFragmentFile(
|
||||
std::string_view store_path,
|
||||
MetaFileType mft,
|
||||
const std::vector<uint8_t>& id
|
||||
) {
|
||||
{ // first check if id is already used
|
||||
const auto exising_id = getFragmentByID(id);
|
||||
if (_reg.valid(exising_id)) {
|
||||
return entt::null;
|
||||
}
|
||||
}
|
||||
|
||||
if (store_path.empty()) {
|
||||
store_path = _default_store_path;
|
||||
}
|
||||
|
||||
std::filesystem::create_directories(store_path);
|
||||
|
||||
const auto id_hex = bin2hex(id);
|
||||
std::filesystem::path fragment_file_path;
|
||||
|
||||
if (id_hex.size() < 6) {
|
||||
fragment_file_path = std::filesystem::path{store_path}/id_hex;
|
||||
} else {
|
||||
// use the first 2hex (1byte) as a subfolder
|
||||
std::filesystem::create_directories(std::string{store_path} + id_hex.substr(0, 2));
|
||||
fragment_file_path = std::filesystem::path{std::string{store_path} + id_hex.substr(0, 2)} / id_hex.substr(2);
|
||||
}
|
||||
|
||||
if (std::filesystem::exists(fragment_file_path)) {
|
||||
return entt::null;
|
||||
}
|
||||
|
||||
const auto new_frag = _reg.create();
|
||||
|
||||
_reg.emplace<FragComp::ID>(new_frag, id);
|
||||
|
||||
// file (info) comp
|
||||
_reg.emplace<FragComp::Ephemeral::FilePath>(new_frag, fragment_file_path.generic_u8string());
|
||||
|
||||
_reg.emplace<FragComp::Ephemeral::MetaFileType>(new_frag, mft);
|
||||
|
||||
// meta needs to be synced to file
|
||||
std::function<write_to_storage_fetch_data_cb> empty_data_cb = [](const uint8_t*, uint64_t) -> uint64_t { return 0; };
|
||||
if (!syncToStorage(new_frag, empty_data_cb)) {
|
||||
std::cerr << "FS error: syncToStorage failed while creating new fragment file\n";
|
||||
_reg.destroy(new_frag);
|
||||
return entt::null;
|
||||
}
|
||||
|
||||
// while new metadata might be created here, making sure the file could be created is more important
|
||||
throwEventConstruct(new_frag);
|
||||
|
||||
return new_frag;
|
||||
}
|
||||
FragmentID FragmentStore::newFragmentFile(
|
||||
std::string_view store_path,
|
||||
MetaFileType mft
|
||||
) {
|
||||
return newFragmentFile(store_path, mft, generateNewUID());
|
||||
}
|
||||
|
||||
FragmentID FragmentStore::getFragmentByID(
|
||||
const std::vector<uint8_t>& id
|
||||
) {
|
||||
// TODO: accelerate
|
||||
// maybe keep it sorted and binary search? hash table lookup?
|
||||
for (const auto& [frag, id_comp] : _reg.view<FragComp::ID>().each()) {
|
||||
if (id == id_comp.v) {
|
||||
return frag;
|
||||
}
|
||||
}
|
||||
|
||||
return entt::null;
|
||||
}
|
||||
|
||||
FragmentID FragmentStore::getFragmentCustomMatcher(
|
||||
std::function<bool(FragmentID)>& fn
|
||||
) {
|
||||
return entt::null;
|
||||
}
|
||||
|
||||
bool FragmentStore::syncToStorage(FragmentID fid, std::function<write_to_storage_fetch_data_cb>& data_cb) {
|
||||
if (!_reg.valid(fid)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!_reg.all_of<FragComp::Ephemeral::FilePath>(fid)) {
|
||||
// not a file fragment?
|
||||
return false;
|
||||
}
|
||||
|
||||
// split object storage
|
||||
|
||||
MetaFileType meta_type = MetaFileType::TEXT_JSON; // TODO: better defaults
|
||||
if (_reg.all_of<FragComp::Ephemeral::MetaFileType>(fid)) {
|
||||
meta_type = _reg.get<FragComp::Ephemeral::MetaFileType>(fid).type;
|
||||
}
|
||||
|
||||
Encryption meta_enc = Encryption::NONE; // TODO: better defaults
|
||||
Compression meta_comp = Compression::NONE; // TODO: better defaults
|
||||
|
||||
if (meta_type != MetaFileType::TEXT_JSON) {
|
||||
if (_reg.all_of<FragComp::Ephemeral::MetaEncryptionType>(fid)) {
|
||||
meta_enc = _reg.get<FragComp::Ephemeral::MetaEncryptionType>(fid).enc;
|
||||
}
|
||||
|
||||
if (_reg.all_of<FragComp::Ephemeral::MetaCompressionType>(fid)) {
|
||||
meta_comp = _reg.get<FragComp::Ephemeral::MetaCompressionType>(fid).comp;
|
||||
}
|
||||
} else {
|
||||
// we cant have encryption or compression
|
||||
|
||||
// TODO: warning/error?
|
||||
|
||||
// TODO: forcing for testing
|
||||
//if (_reg.all_of<Components::Ephemeral::MetaEncryptionType>(fid)) {
|
||||
_reg.emplace_or_replace<FragComp::Ephemeral::MetaEncryptionType>(fid, Encryption::NONE);
|
||||
//}
|
||||
//if (_reg.all_of<Components::Ephemeral::MetaCompressionType>(fid)) {
|
||||
_reg.emplace_or_replace<FragComp::Ephemeral::MetaCompressionType>(fid, Compression::NONE);
|
||||
//}
|
||||
}
|
||||
|
||||
std::filesystem::path meta_tmp_path = _reg.get<FragComp::Ephemeral::FilePath>(fid).path + ".meta" + metaFileTypeSuffix(meta_type) + ".tmp";
|
||||
meta_tmp_path.replace_filename("." + meta_tmp_path.filename().generic_u8string());
|
||||
std::ofstream meta_file{
|
||||
meta_tmp_path,
|
||||
std::ios::out | std::ios::trunc | std::ios::binary // always binary, also for text
|
||||
};
|
||||
|
||||
if (!meta_file.is_open()) {
|
||||
std::cerr << "FS error: failed to create temporary meta file\n";
|
||||
return false;
|
||||
}
|
||||
|
||||
Compression data_comp = Compression::NONE; // TODO: better defaults
|
||||
if (_reg.all_of<FragComp::DataCompressionType>(fid)) {
|
||||
data_comp = _reg.get<FragComp::DataCompressionType>(fid).comp;
|
||||
}
|
||||
|
||||
std::filesystem::path data_tmp_path = _reg.get<FragComp::Ephemeral::FilePath>(fid).path + ".tmp";
|
||||
data_tmp_path.replace_filename("." + data_tmp_path.filename().generic_u8string());
|
||||
std::ofstream data_file{
|
||||
data_tmp_path,
|
||||
std::ios::out | std::ios::trunc | std::ios::binary // always binary, also for text
|
||||
};
|
||||
|
||||
if (!data_file.is_open()) {
|
||||
meta_file.close();
|
||||
std::filesystem::remove(meta_tmp_path);
|
||||
std::cerr << "FS error: failed to create temporary data file\n";
|
||||
return false;
|
||||
}
|
||||
|
||||
// sharing code between binary msgpack and text json for now
|
||||
nlohmann::json meta_data_j = nlohmann::json::object(); // metadata needs to be an object, null not allowed
|
||||
// metadata file
|
||||
|
||||
for (const auto& [type_id, storage] : _reg.storage()) {
|
||||
if (!storage.contains(fid)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
//std::cout << "storage type: type_id:" << type_id << " name:" << storage.type().name() << "\n";
|
||||
|
||||
// use type_id to find serializer
|
||||
auto s_cb_it = _sc._serl_json.find(type_id);
|
||||
if (s_cb_it == _sc._serl_json.end()) {
|
||||
// could not find serializer, not saving
|
||||
continue;
|
||||
}
|
||||
|
||||
// noooo, why cant numbers be keys
|
||||
//if (meta_type == MetaFileType::BINARY_MSGPACK) { // msgpack uses the hash id instead
|
||||
//s_cb_it->second(storage.value(fid), meta_data[storage.type().hash()]);
|
||||
//} else if (meta_type == MetaFileType::TEXT_JSON) {
|
||||
s_cb_it->second({_reg, fid}, meta_data_j[storage.type().name()]);
|
||||
//}
|
||||
}
|
||||
|
||||
if (meta_type == MetaFileType::BINARY_MSGPACK) { // binary metadata file
|
||||
const std::vector<uint8_t> meta_data = nlohmann::json::to_msgpack(meta_data_j);
|
||||
std::vector<uint8_t> meta_data_compressed; // empty if none
|
||||
//std::vector<uint8_t> meta_data_encrypted; // empty if none
|
||||
|
||||
if (meta_comp == Compression::ZSTD) {
|
||||
meta_data_compressed.resize(ZSTD_compressBound(meta_data.size()));
|
||||
|
||||
size_t const cSize = ZSTD_compress(meta_data_compressed.data(), meta_data_compressed.size(), meta_data.data(), meta_data.size(), 0); // 0 is default is probably 3
|
||||
if (ZSTD_isError(cSize)) {
|
||||
std::cerr << "FS error: compressing meta failed\n";
|
||||
meta_data_compressed.clear();
|
||||
meta_comp = Compression::NONE;
|
||||
} else {
|
||||
meta_data_compressed.resize(cSize);
|
||||
}
|
||||
} else if (meta_comp == Compression::NONE) {
|
||||
// do nothing
|
||||
} else {
|
||||
assert(false && "implement me");
|
||||
}
|
||||
|
||||
// TODO: encryption
|
||||
|
||||
// the meta file is itself msgpack data
|
||||
nlohmann::json meta_header_j = nlohmann::json::array();
|
||||
meta_header_j.emplace_back() = "SOLMET";
|
||||
meta_header_j.push_back(meta_enc);
|
||||
meta_header_j.push_back(meta_comp);
|
||||
|
||||
if (false) { // TODO: encryption
|
||||
} else if (!meta_data_compressed.empty()) {
|
||||
meta_header_j.push_back(nlohmann::json::binary(meta_data_compressed));
|
||||
} else {
|
||||
meta_header_j.push_back(nlohmann::json::binary(meta_data));
|
||||
}
|
||||
|
||||
const auto meta_header_data = nlohmann::json::to_msgpack(meta_header_j);
|
||||
meta_file.write(reinterpret_cast<const char*>(meta_header_data.data()), meta_header_data.size());
|
||||
} else if (meta_type == MetaFileType::TEXT_JSON) {
|
||||
// cant be compressed or encrypted
|
||||
meta_file << meta_data_j.dump(2, ' ', true);
|
||||
}
|
||||
|
||||
// now data
|
||||
if (data_comp == Compression::NONE) {
|
||||
std::array<uint8_t, 1024> buffer;
|
||||
uint64_t buffer_actual_size {0};
|
||||
do {
|
||||
buffer_actual_size = data_cb(buffer.data(), buffer.size());
|
||||
if (buffer_actual_size == 0) {
|
||||
break;
|
||||
}
|
||||
if (buffer_actual_size > buffer.size()) {
|
||||
// wtf
|
||||
break;
|
||||
}
|
||||
|
||||
data_file.write(reinterpret_cast<const char*>(buffer.data()), buffer_actual_size);
|
||||
} while (buffer_actual_size == buffer.size());
|
||||
} else if (data_comp == Compression::ZSTD) {
|
||||
std::vector<uint8_t> buffer(ZSTD_CStreamInSize());
|
||||
std::vector<uint8_t> compressed_buffer(ZSTD_CStreamOutSize());
|
||||
uint64_t buffer_actual_size {0};
|
||||
|
||||
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
|
||||
ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 0); // default (3)
|
||||
ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1); // add extra checksums (to frames?)
|
||||
do {
|
||||
buffer_actual_size = data_cb(buffer.data(), buffer.size());
|
||||
//if (buffer_actual_size == 0) {
|
||||
//break;
|
||||
//}
|
||||
if (buffer_actual_size > buffer.size()) {
|
||||
// wtf
|
||||
break;
|
||||
}
|
||||
bool const lastChunk = (buffer_actual_size < buffer.size());
|
||||
|
||||
ZSTD_EndDirective const mode = lastChunk ? ZSTD_e_end : ZSTD_e_continue;
|
||||
ZSTD_inBuffer input = { buffer.data(), buffer_actual_size, 0 };
|
||||
|
||||
while (input.pos < input.size) {
|
||||
ZSTD_outBuffer output = { compressed_buffer.data(), compressed_buffer.size(), 0 };
|
||||
|
||||
size_t const remaining = ZSTD_compressStream2(cctx, &output , &input, mode);
|
||||
if (ZSTD_isError(remaining)) {
|
||||
std::cerr << "FS error: compressing data failed\n";
|
||||
break;
|
||||
}
|
||||
|
||||
data_file.write(reinterpret_cast<const char*>(compressed_buffer.data()), output.pos);
|
||||
|
||||
if (remaining == 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
// same as if lastChunk break;
|
||||
} while (buffer_actual_size == buffer.size());
|
||||
} else {
|
||||
assert(false && "implement me");
|
||||
}
|
||||
|
||||
meta_file.flush();
|
||||
meta_file.close();
|
||||
data_file.flush();
|
||||
data_file.close();
|
||||
|
||||
std::filesystem::rename(
|
||||
meta_tmp_path,
|
||||
_reg.get<FragComp::Ephemeral::FilePath>(fid).path + ".meta" + metaFileTypeSuffix(meta_type)
|
||||
);
|
||||
|
||||
std::filesystem::rename(
|
||||
data_tmp_path,
|
||||
_reg.get<FragComp::Ephemeral::FilePath>(fid).path
|
||||
);
|
||||
|
||||
// TODO: check return value of renames
|
||||
|
||||
if (_reg.all_of<FragComp::Ephemeral::DirtyTag>(fid)) {
|
||||
_reg.remove<FragComp::Ephemeral::DirtyTag>(fid);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool FragmentStore::syncToStorage(FragmentID fid, const uint8_t* data, const uint64_t data_size) {
|
||||
std::function<FragmentStore::write_to_storage_fetch_data_cb> fn_cb = [read = 0ull, data, data_size](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t {
|
||||
uint64_t i = 0;
|
||||
for (; i+read < data_size && i < buffer_size; i++) {
|
||||
request_buffer[i] = data[i+read];
|
||||
}
|
||||
read += i;
|
||||
|
||||
return i;
|
||||
};
|
||||
return syncToStorage(fid, fn_cb);
|
||||
}
|
||||
|
||||
bool FragmentStore::loadFromStorage(FragmentID fid, std::function<read_from_storage_put_data_cb>& data_cb) {
|
||||
if (!_reg.valid(fid)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!_reg.all_of<FragComp::Ephemeral::FilePath>(fid)) {
|
||||
// not a file fragment?
|
||||
// TODO: memory fragments
|
||||
return false;
|
||||
}
|
||||
|
||||
const auto& frag_path = _reg.get<FragComp::Ephemeral::FilePath>(fid).path;
|
||||
|
||||
// TODO: check if metadata dirty?
|
||||
// TODO: what if file changed on disk?
|
||||
|
||||
std::cout << "FS: loading fragment '" << frag_path << "'\n";
|
||||
|
||||
std::ifstream data_file{
|
||||
frag_path,
|
||||
std::ios::in | std::ios::binary // always binary, also for text
|
||||
};
|
||||
|
||||
if (!data_file.is_open()) {
|
||||
std::cerr << "FS error: fragment data file failed to open '" << frag_path << "'\n";
|
||||
// error
|
||||
return false;
|
||||
}
|
||||
|
||||
Compression data_comp = Compression::NONE;
|
||||
if (_reg.all_of<FragComp::DataCompressionType>(fid)) {
|
||||
data_comp = _reg.get<FragComp::DataCompressionType>(fid).comp;
|
||||
}
|
||||
|
||||
if (data_comp == Compression::NONE) {
|
||||
std::array<uint8_t, 1024> buffer;
|
||||
uint64_t buffer_actual_size {0};
|
||||
do {
|
||||
data_file.read(reinterpret_cast<char*>(buffer.data()), buffer.size());
|
||||
buffer_actual_size = data_file.gcount();
|
||||
|
||||
if (buffer_actual_size == 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
data_cb(buffer.data(), buffer_actual_size);
|
||||
} while (buffer_actual_size == buffer.size() && !data_file.eof());
|
||||
} else if (data_comp == Compression::ZSTD) {
|
||||
std::vector<uint8_t> in_buffer(ZSTD_DStreamInSize());
|
||||
std::vector<uint8_t> out_buffer(ZSTD_DStreamOutSize());
|
||||
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
|
||||
|
||||
uint64_t buffer_actual_size {0};
|
||||
do {
|
||||
data_file.read(reinterpret_cast<char*>(in_buffer.data()), in_buffer.size());
|
||||
buffer_actual_size = data_file.gcount();
|
||||
if (buffer_actual_size == 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
ZSTD_inBuffer input {in_buffer.data(), buffer_actual_size, 0 };
|
||||
do {
|
||||
ZSTD_outBuffer output = { out_buffer.data(), out_buffer.size(), 0 };
|
||||
size_t const ret = ZSTD_decompressStream(dctx, &output , &input);
|
||||
if (ZSTD_isError(ret)) {
|
||||
// error <.<
|
||||
std::cerr << "FS error: decompression error\n";
|
||||
break;
|
||||
}
|
||||
|
||||
data_cb(out_buffer.data(), output.pos);
|
||||
} while (input.pos < input.size);
|
||||
} while (buffer_actual_size == in_buffer.size() && !data_file.eof());
|
||||
|
||||
ZSTD_freeDCtx(dctx);
|
||||
} else {
|
||||
assert(false && "implement me");
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
nlohmann::json FragmentStore::loadFromStorageNJ(FragmentID fid) {
|
||||
std::vector<uint8_t> tmp_buffer;
|
||||
std::function<read_from_storage_put_data_cb> cb = [&tmp_buffer](const uint8_t* buffer, const uint64_t buffer_size) {
|
||||
tmp_buffer.insert(tmp_buffer.end(), buffer, buffer+buffer_size);
|
||||
};
|
||||
|
||||
if (!loadFromStorage(fid, cb)) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
return nlohmann::json::parse(tmp_buffer);
|
||||
}
|
||||
|
||||
size_t FragmentStore::scanStoragePath(std::string_view path) {
|
||||
if (path.empty()) {
|
||||
path = _default_store_path;
|
||||
}
|
||||
// TODO: extract so async can work (or/and make iteratable generator)
|
||||
|
||||
if (!std::filesystem::is_directory(path)) {
|
||||
std::cerr << "FS error: scan path not a directory '" << path << "'\n";
|
||||
return 0;
|
||||
}
|
||||
|
||||
// step 1: make snapshot of files, validate metafiles and save id/path+meta.ext
|
||||
// can be extra thread (if non vfs)
|
||||
struct FragFileEntry {
|
||||
std::string id_str;
|
||||
std::filesystem::path frag_path;
|
||||
std::string meta_ext;
|
||||
|
||||
bool operator==(const FragFileEntry& other) const {
|
||||
// only compare by id
|
||||
return id_str == other.id_str;
|
||||
}
|
||||
};
|
||||
struct FragFileEntryHash {
|
||||
size_t operator()(const FragFileEntry& it) const {
|
||||
return entt::hashed_string(it.id_str.data(), it.id_str.size());
|
||||
}
|
||||
};
|
||||
entt::dense_set<FragFileEntry, FragFileEntryHash> file_frag_list;
|
||||
|
||||
std::filesystem::path storage_path{path};
|
||||
|
||||
auto handle_file = [&](const std::filesystem::path& file_path) {
|
||||
if (!std::filesystem::is_regular_file(file_path)) {
|
||||
return;
|
||||
}
|
||||
// handle file
|
||||
|
||||
if (file_path.has_extension()) {
|
||||
// skip over metadata, assuming only metafiles have extentions (might be wrong?)
|
||||
// also skips temps
|
||||
return;
|
||||
}
|
||||
|
||||
auto relative_path = std::filesystem::proximate(file_path, storage_path);
|
||||
std::string id_str = relative_path.generic_u8string();
|
||||
// delete all '/'
|
||||
id_str.erase(std::remove(id_str.begin(), id_str.end(), '/'), id_str.end());
|
||||
if (id_str.size() % 2 != 0) {
|
||||
std::cerr << "FS error: non hex fragment uid detected: '" << id_str << "'\n";
|
||||
}
|
||||
|
||||
if (file_frag_list.contains(FragFileEntry{id_str, {}, ""})) {
|
||||
std::cerr << "FS error: fragment duplicate detected: '" << id_str << "'\n";
|
||||
return; // skip
|
||||
}
|
||||
|
||||
const char* meta_ext = ".meta.msgpack";
|
||||
{ // find meta
|
||||
// TODO: this as to know all possible extentions
|
||||
bool has_meta_msgpack = std::filesystem::is_regular_file(file_path.generic_u8string() + ".meta.msgpack");
|
||||
bool has_meta_json = std::filesystem::is_regular_file(file_path.generic_u8string() + ".meta.json");
|
||||
const size_t meta_sum =
|
||||
(has_meta_msgpack?1:0) +
|
||||
(has_meta_json?1:0)
|
||||
;
|
||||
|
||||
if (meta_sum > 1) { // has multiple
|
||||
std::cerr << "FS error: fragment with multiple meta files detected: " << id_str << "\n";
|
||||
return; // skip
|
||||
}
|
||||
|
||||
if (meta_sum == 0) {
|
||||
std::cerr << "FS error: fragment missing meta file detected: " << id_str << "\n";
|
||||
return; // skip
|
||||
}
|
||||
|
||||
if (has_meta_json) {
|
||||
meta_ext = ".meta.json";
|
||||
}
|
||||
}
|
||||
|
||||
file_frag_list.emplace(FragFileEntry{
|
||||
std::move(id_str),
|
||||
file_path,
|
||||
meta_ext
|
||||
});
|
||||
};
|
||||
|
||||
for (const auto& outer_path : std::filesystem::directory_iterator(storage_path)) {
|
||||
if (std::filesystem::is_regular_file(outer_path)) {
|
||||
handle_file(outer_path);
|
||||
} else if (std::filesystem::is_directory(outer_path)) {
|
||||
// subdir, part of id
|
||||
for (const auto& inner_path : std::filesystem::directory_iterator(outer_path)) {
|
||||
//if (std::filesystem::is_regular_file(inner_path)) {
|
||||
|
||||
//// handle file
|
||||
//} // TODO: support deeper recursion?
|
||||
handle_file(inner_path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::cout << "FS: scan found:\n";
|
||||
for (const auto& it : file_frag_list) {
|
||||
std::cout << " " << it.id_str << "\n";
|
||||
}
|
||||
|
||||
// step 2: check if files preexist in reg
|
||||
// main thread
|
||||
// (merge into step 3 and do more error checking?)
|
||||
for (auto it = file_frag_list.begin(); it != file_frag_list.end();) {
|
||||
auto id = hex2bin(it->id_str);
|
||||
auto fid = getFragmentByID(id);
|
||||
if (_reg.valid(fid)) {
|
||||
// pre exising (handle differently??)
|
||||
// check if store differs?
|
||||
it = file_frag_list.erase(it);
|
||||
} else {
|
||||
it++;
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<FragmentID> scanned_frags;
|
||||
// step 3: parse meta and insert into reg of non preexising
|
||||
// main thread
|
||||
// TODO: check timestamps of preexisting and reload? mark external/remote dirty?
|
||||
for (const auto& it : file_frag_list) {
|
||||
nlohmann::json j;
|
||||
if (it.meta_ext == ".meta.msgpack") {
|
||||
std::ifstream file(it.frag_path.generic_u8string() + it.meta_ext, std::ios::in | std::ios::binary);
|
||||
if (!file.is_open()) {
|
||||
std::cout << "FS error: failed opening meta " << it.frag_path << "\n";
|
||||
continue;
|
||||
}
|
||||
|
||||
// file is a msgpack within a msgpack
|
||||
|
||||
std::vector<uint8_t> full_meta_data;
|
||||
{ // read meta file
|
||||
// figure out size
|
||||
file.seekg(0, file.end);
|
||||
uint64_t file_size = file.tellg();
|
||||
file.seekg(0, file.beg);
|
||||
|
||||
full_meta_data.resize(file_size);
|
||||
|
||||
file.read(reinterpret_cast<char*>(full_meta_data.data()), full_meta_data.size());
|
||||
}
|
||||
|
||||
const auto meta_header_j = nlohmann::json::from_msgpack(full_meta_data);
|
||||
|
||||
if (!meta_header_j.is_array() || meta_header_j.size() < 4) {
|
||||
std::cerr << "FS error: broken binary meta " << it.frag_path << "\n";
|
||||
continue;
|
||||
}
|
||||
|
||||
if (meta_header_j.at(0) != "SOLMET") {
|
||||
std::cerr << "FS error: wrong magic '" << meta_header_j.at(0) << "' in meta " << it.frag_path << "\n";
|
||||
continue;
|
||||
}
|
||||
|
||||
Encryption meta_enc = meta_header_j.at(1);
|
||||
if (meta_enc != Encryption::NONE) {
|
||||
std::cerr << "FS error: unknown encryption " << it.frag_path << "\n";
|
||||
continue;
|
||||
}
|
||||
|
||||
Compression meta_comp = meta_header_j.at(2);
|
||||
if (meta_comp != Compression::NONE && meta_comp != Compression::ZSTD) {
|
||||
std::cerr << "FS error: unknown compression " << it.frag_path << "\n";
|
||||
continue;
|
||||
}
|
||||
|
||||
//const auto& meta_data_ref = meta_header_j.at(3).is_binary()?meta_header_j.at(3):meta_header_j.at(3).at("data");
|
||||
if (!meta_header_j.at(3).is_binary()) {
|
||||
std::cerr << "FS error: meta data not binary " << it.frag_path << "\n";
|
||||
continue;
|
||||
}
|
||||
const nlohmann::json::binary_t& meta_data_ref = meta_header_j.at(3);
|
||||
|
||||
std::vector<uint8_t> meta_data_decomp;
|
||||
if (meta_comp == Compression::NONE) {
|
||||
// do nothing
|
||||
} else if (meta_comp == Compression::ZSTD) {
|
||||
meta_data_decomp.resize(ZSTD_DStreamOutSize());
|
||||
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
|
||||
|
||||
ZSTD_inBuffer input {meta_data_ref.data(), meta_data_ref.size(), 0};
|
||||
ZSTD_outBuffer output = {meta_data_decomp.data(), meta_data_decomp.size(), 0};
|
||||
do {
|
||||
size_t const ret = ZSTD_decompressStream(dctx, &output , &input);
|
||||
if (ZSTD_isError(ret)) {
|
||||
// error <.<
|
||||
std::cerr << "FS error: decompression error\n";
|
||||
meta_data_decomp.clear();
|
||||
break;
|
||||
}
|
||||
} while (input.pos < input.size);
|
||||
meta_data_decomp.resize(output.pos);
|
||||
|
||||
ZSTD_freeDCtx(dctx);
|
||||
} else {
|
||||
assert(false && "implement me");
|
||||
}
|
||||
|
||||
// TODO: enc
|
||||
|
||||
if (!meta_data_decomp.empty()) {
|
||||
j = nlohmann::json::from_msgpack(meta_data_decomp);
|
||||
} else {
|
||||
j = nlohmann::json::from_msgpack(meta_data_ref);
|
||||
}
|
||||
} else if (it.meta_ext == ".meta.json") {
|
||||
std::ifstream file(it.frag_path.generic_u8string() + it.meta_ext, std::ios::in | std::ios::binary);
|
||||
if (!file.is_open()) {
|
||||
std::cerr << "FS error: failed opening meta " << it.frag_path << "\n";
|
||||
continue;
|
||||
}
|
||||
|
||||
file >> j;
|
||||
} else {
|
||||
assert(false);
|
||||
}
|
||||
|
||||
if (!j.is_object()) {
|
||||
std::cerr << "FS error: json in meta is broken " << it.id_str << "\n";
|
||||
continue;
|
||||
}
|
||||
|
||||
// TODO: existing fragment file
|
||||
//newFragmentFile();
|
||||
FragmentHandle fh{_reg, _reg.create()};
|
||||
fh.emplace<FragComp::ID>(hex2bin(it.id_str));
|
||||
|
||||
fh.emplace<FragComp::Ephemeral::FilePath>(it.frag_path.generic_u8string());
|
||||
|
||||
for (const auto& [k, v] : j.items()) {
|
||||
// type id from string hash
|
||||
const auto type_id = entt::hashed_string(k.data(), k.size());
|
||||
const auto deserl_fn_it = _sc._deserl_json.find(type_id);
|
||||
if (deserl_fn_it != _sc._deserl_json.cend()) {
|
||||
// TODO: check return value
|
||||
deserl_fn_it->second(fh, v);
|
||||
} else {
|
||||
std::cerr << "FS warning: missing deserializer for meta key '" << k << "'\n";
|
||||
}
|
||||
}
|
||||
scanned_frags.push_back(fh);
|
||||
}
|
||||
|
||||
// TODO: mutex and move code to async and return this list ?
|
||||
|
||||
// throw new frag event here, after loading them all
|
||||
for (const FragmentID fid : scanned_frags) {
|
||||
throwEventConstruct(fid);
|
||||
}
|
||||
|
||||
return scanned_frags.size();
|
||||
}
|
||||
|
||||
void FragmentStore::scanStoragePathAsync(std::string path) {
|
||||
// add path to queue
|
||||
// HACK: if path is known/any fragment is in the path, this operation blocks (non async)
|
||||
scanStoragePath(path); // TODO: make async and post result
|
||||
}
|
||||
|
||||
static bool serl_json_data_enc_type(const FragmentHandle fh, nlohmann::json& out) {
|
||||
out = static_cast<std::underlying_type_t<Encryption>>(
|
||||
fh.get<FragComp::DataEncryptionType>().enc
|
||||
);
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool deserl_json_data_enc_type(FragmentHandle fh, const nlohmann::json& in) {
|
||||
fh.emplace_or_replace<FragComp::DataEncryptionType>(
|
||||
static_cast<Encryption>(
|
||||
static_cast<std::underlying_type_t<Encryption>>(in)
|
||||
)
|
||||
);
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool serl_json_data_comp_type(const FragmentHandle fh, nlohmann::json& out) {
|
||||
out = static_cast<std::underlying_type_t<Compression>>(
|
||||
fh.get<FragComp::DataCompressionType>().comp
|
||||
);
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool deserl_json_data_comp_type(FragmentHandle fh, const nlohmann::json& in) {
|
||||
fh.emplace_or_replace<FragComp::DataCompressionType>(
|
||||
static_cast<Compression>(
|
||||
static_cast<std::underlying_type_t<Compression>>(in)
|
||||
)
|
||||
);
|
||||
return true;
|
||||
}
|
||||
|
||||
void FragmentStore::registerSerializers(void) {
|
||||
_sc.registerSerializerJson<FragComp::DataEncryptionType>(serl_json_data_enc_type);
|
||||
_sc.registerDeSerializerJson<FragComp::DataEncryptionType>(deserl_json_data_enc_type);
|
||||
_sc.registerSerializerJson<FragComp::DataCompressionType>(serl_json_data_comp_type);
|
||||
_sc.registerDeSerializerJson<FragComp::DataCompressionType>(deserl_json_data_comp_type);
|
||||
}
|
||||
|
103
src/fragment_store/fragment_store.hpp
Normal file
103
src/fragment_store/fragment_store.hpp
Normal file
@ -0,0 +1,103 @@
|
||||
#pragma once
|
||||
|
||||
#include "./fragment_store_i.hpp"
|
||||
|
||||
#include "./types.hpp"
|
||||
#include "./meta_components.hpp"
|
||||
|
||||
#include "./serializer.hpp"
|
||||
|
||||
#include <entt/core/type_info.hpp>
|
||||
#include <entt/entity/registry.hpp>
|
||||
|
||||
#include <nlohmann/json_fwd.hpp>
|
||||
|
||||
#include <vector>
|
||||
#include <array>
|
||||
#include <cstdint>
|
||||
#include <random>
|
||||
|
||||
struct FragmentStore : public FragmentStoreI {
|
||||
std::minstd_rand _rng{std::random_device{}()};
|
||||
std::array<uint8_t, 16> _session_uuid_namespace;
|
||||
|
||||
std::string _default_store_path;
|
||||
|
||||
uint64_t _memory_budget {10u*1024u*1024u};
|
||||
uint64_t _memory_usage {0u};
|
||||
|
||||
SerializerCallbacks<FragmentID> _sc;
|
||||
|
||||
FragmentStore(void);
|
||||
FragmentStore(std::array<uint8_t, 16> session_uuid_namespace);
|
||||
|
||||
// HACK: get access to the reg
|
||||
FragmentHandle fragmentHandle(FragmentID fid);
|
||||
|
||||
// TODO: make the frags ref counted
|
||||
|
||||
// TODO: check for exising
|
||||
std::vector<uint8_t> generateNewUID(std::array<uint8_t, 16>& uuid_namespace);
|
||||
std::vector<uint8_t> generateNewUID(void);
|
||||
|
||||
// ========== new fragment ==========
|
||||
|
||||
// memory backed owned
|
||||
FragmentID newFragmentMemoryOwned(
|
||||
const std::vector<uint8_t>& id,
|
||||
size_t initial_size
|
||||
);
|
||||
|
||||
// memory backed view (can only be added? not new?)
|
||||
|
||||
// file backed (rw...)
|
||||
// needs to know which store path to put into
|
||||
FragmentID newFragmentFile(
|
||||
std::string_view store_path,
|
||||
MetaFileType mft,
|
||||
const std::vector<uint8_t>& id
|
||||
);
|
||||
// this variant generate a new, mostly unique, id for us
|
||||
FragmentID newFragmentFile(
|
||||
std::string_view store_path,
|
||||
MetaFileType mft
|
||||
);
|
||||
|
||||
// ========== add fragment ==========
|
||||
|
||||
// ========== get fragment ==========
|
||||
FragmentID getFragmentByID(
|
||||
const std::vector<uint8_t>& id
|
||||
);
|
||||
FragmentID getFragmentCustomMatcher(
|
||||
std::function<bool(FragmentID)>& fn
|
||||
);
|
||||
|
||||
// remove fragment?
|
||||
// unload?
|
||||
|
||||
// ========== sync fragment to storage ==========
|
||||
using write_to_storage_fetch_data_cb = uint64_t(uint8_t* request_buffer, uint64_t buffer_size);
|
||||
// calls data_cb with a buffer to be filled in, cb returns actual count of data. if returned < max, its the last buffer.
|
||||
bool syncToStorage(FragmentID fid, std::function<write_to_storage_fetch_data_cb>& data_cb);
|
||||
bool syncToStorage(FragmentID fid, const uint8_t* data, const uint64_t data_size);
|
||||
|
||||
// ========== load fragment data from storage ==========
|
||||
using read_from_storage_put_data_cb = void(const uint8_t* buffer, const uint64_t buffer_size);
|
||||
bool loadFromStorage(FragmentID fid, std::function<read_from_storage_put_data_cb>& data_cb);
|
||||
// convenience function
|
||||
nlohmann::json loadFromStorageNJ(FragmentID fid);
|
||||
|
||||
// fragment discovery?
|
||||
// returns number of new fragments
|
||||
size_t scanStoragePath(std::string_view path);
|
||||
void scanStoragePathAsync(std::string path);
|
||||
|
||||
private:
|
||||
void registerSerializers(void); // internal comps
|
||||
// internal actual backends
|
||||
// TODO: seperate out
|
||||
bool syncToMemory(FragmentID fid, std::function<write_to_storage_fetch_data_cb>& data_cb);
|
||||
bool syncToFile(FragmentID fid, std::function<write_to_storage_fetch_data_cb>& data_cb);
|
||||
};
|
||||
|
23
src/fragment_store/fragment_store_i.cpp
Normal file
23
src/fragment_store/fragment_store_i.cpp
Normal file
@ -0,0 +1,23 @@
|
||||
#include "./fragment_store_i.hpp"
|
||||
|
||||
#include <iostream>
|
||||
|
||||
void FragmentStoreI::throwEventConstruct(const FragmentID fid) {
|
||||
std::cout << "FSI debug: event construct " << entt::to_integral(fid) << "\n";
|
||||
dispatch(
|
||||
FragmentStore_Event::fragment_construct,
|
||||
Fragment::Events::FragmentConstruct{
|
||||
FragmentHandle{_reg, fid}
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
void FragmentStoreI::throwEventUpdate(const FragmentID fid) {
|
||||
std::cout << "FSI debug: event updated " << entt::to_integral(fid) << "\n";
|
||||
dispatch(
|
||||
FragmentStore_Event::fragment_updated,
|
||||
Fragment::Events::FragmentUpdated{
|
||||
FragmentHandle{_reg, fid}
|
||||
}
|
||||
);
|
||||
}
|
63
src/fragment_store/fragment_store_i.hpp
Normal file
63
src/fragment_store/fragment_store_i.hpp
Normal file
@ -0,0 +1,63 @@
|
||||
#pragma once
|
||||
|
||||
#include <solanaceae/util/event_provider.hpp>
|
||||
|
||||
#include <entt/entity/registry.hpp>
|
||||
#include <entt/entity/handle.hpp>
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
// internal id
|
||||
enum class FragmentID : uint32_t {};
|
||||
using FragmentRegistry = entt::basic_registry<FragmentID>;
|
||||
using FragmentHandle = entt::basic_handle<FragmentRegistry>;
|
||||
|
||||
namespace Fragment::Events {
|
||||
|
||||
struct FragmentConstruct {
|
||||
const FragmentHandle e;
|
||||
};
|
||||
struct FragmentUpdated {
|
||||
const FragmentHandle e;
|
||||
};
|
||||
//struct MessageDestory {
|
||||
//const Message3Handle e;
|
||||
//};
|
||||
|
||||
} // Fragment::Events
|
||||
|
||||
enum class FragmentStore_Event : uint32_t {
|
||||
fragment_construct,
|
||||
fragment_updated,
|
||||
//message_destroy,
|
||||
|
||||
MAX
|
||||
};
|
||||
|
||||
struct FragmentStoreEventI {
|
||||
using enumType = FragmentStore_Event;
|
||||
|
||||
virtual ~FragmentStoreEventI(void) {}
|
||||
|
||||
virtual bool onEvent(const Fragment::Events::FragmentConstruct&) { return false; }
|
||||
virtual bool onEvent(const Fragment::Events::FragmentUpdated&) { return false; }
|
||||
//virtual bool onEvent(const Fragment::Events::MessageDestory&) { return false; }
|
||||
|
||||
// mm3
|
||||
// send text
|
||||
// send file path
|
||||
};
|
||||
using FragmentStoreEventProviderI = EventProviderI<FragmentStoreEventI>;
|
||||
|
||||
struct FragmentStoreI : public FragmentStoreEventProviderI {
|
||||
static constexpr const char* version {"1"};
|
||||
|
||||
FragmentRegistry _reg;
|
||||
|
||||
virtual ~FragmentStoreI(void) {}
|
||||
|
||||
void throwEventConstruct(const FragmentID fid);
|
||||
void throwEventUpdate(const FragmentID fid);
|
||||
//void throwEventDestroy();
|
||||
};
|
||||
|
BIN
src/fragment_store/fs_binary_msgpack1.png
Normal file
BIN
src/fragment_store/fs_binary_msgpack1.png
Normal file
Binary file not shown.
After Width: | Height: | Size: 36 KiB |
BIN
src/fragment_store/fs_binary_msgpack2.png
Normal file
BIN
src/fragment_store/fs_binary_msgpack2.png
Normal file
Binary file not shown.
After Width: | Height: | Size: 46 KiB |
900
src/fragment_store/message_fragment_store.cpp
Normal file
900
src/fragment_store/message_fragment_store.cpp
Normal file
@ -0,0 +1,900 @@
|
||||
#include "./message_fragment_store.hpp"
|
||||
|
||||
#include "../json/message_components.hpp"
|
||||
|
||||
#include <solanaceae/util/utils.hpp>
|
||||
|
||||
#include <solanaceae/contact/components.hpp>
|
||||
#include <solanaceae/message3/components.hpp>
|
||||
#include <solanaceae/message3/contact_components.hpp>
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
#include <algorithm>
|
||||
#include <string>
|
||||
#include <cstdint>
|
||||
#include <cassert>
|
||||
#include <iostream>
|
||||
|
||||
// https://youtu.be/CU2exyhYPfA
|
||||
|
||||
// everything assumes a single fragment registry
|
||||
|
||||
namespace Message::Components {
|
||||
|
||||
// ctx
|
||||
struct OpenFragments {
|
||||
//struct OpenFrag final {
|
||||
////std::vector<uint8_t> uid;
|
||||
//FragmentID id;
|
||||
//};
|
||||
// only contains fragments with <1024 messages and <28h tsrage (or whatever)
|
||||
entt::dense_set<FragmentID> fid_open;
|
||||
};
|
||||
|
||||
// all message fragments of this contact
|
||||
struct ContactFragments final {
|
||||
// kept up-to-date by events
|
||||
struct InternalEntry {
|
||||
// indecies into the sorted arrays
|
||||
size_t i_b;
|
||||
size_t i_e;
|
||||
};
|
||||
entt::dense_map<FragmentID, InternalEntry> frags;
|
||||
|
||||
// add 2 sorted contact lists for both range begin and end
|
||||
// TODO: adding and removing becomes expensive with enough frags, consider splitting or heap
|
||||
std::vector<FragmentID> sorted_begin;
|
||||
std::vector<FragmentID> sorted_end;
|
||||
|
||||
// api
|
||||
// return true if it was actually inserted
|
||||
bool insert(FragmentHandle frag);
|
||||
bool erase(FragmentID frag);
|
||||
// update? (just erase() + insert())
|
||||
|
||||
// uses range begin to go back in time
|
||||
FragmentID prev(FragmentID frag) const;
|
||||
// uses range end to go forward in time
|
||||
FragmentID next(FragmentID frag) const;
|
||||
};
|
||||
|
||||
// all LOADED message fragments
|
||||
// TODO: merge into ContactFragments (and pull in openfrags)
|
||||
struct LoadedContactFragments final {
|
||||
// kept up-to-date by events
|
||||
entt::dense_set<FragmentID> frags;
|
||||
};
|
||||
|
||||
} // Message::Components
|
||||
|
||||
namespace Fragment::Components {
|
||||
NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessagesTSRange, begin, end)
|
||||
NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessagesContact, id)
|
||||
} // Fragment::Components
|
||||
|
||||
void MessageFragmentStore::handleMessage(const Message3Handle& m) {
|
||||
if (_fs_ignore_event) {
|
||||
// message event because of us loading a fragment, ignore
|
||||
// TODO: this barely makes a difference
|
||||
return;
|
||||
}
|
||||
|
||||
if (!static_cast<bool>(m)) {
|
||||
return; // huh?
|
||||
}
|
||||
|
||||
if (!m.all_of<Message::Components::Timestamp>()) {
|
||||
return; // we only handle msg with ts
|
||||
}
|
||||
|
||||
_potentially_dirty_contacts.emplace(m.registry()->ctx().get<Contact3>()); // always mark dirty here
|
||||
if (m.any_of<Message::Components::ViewCurserBegin, Message::Components::ViewCurserEnd>()) {
|
||||
// not an actual message, but we probalby need to check and see if we need to load fragments
|
||||
//std::cout << "MFS: new or updated curser\n";
|
||||
return;
|
||||
}
|
||||
|
||||
// TODO: use fid, seving full fuid for every message consumes alot of memory (and heap frag)
|
||||
if (!m.all_of<Message::Components::FID>()) {
|
||||
std::cout << "MFS: new msg missing FID\n";
|
||||
if (!m.registry()->ctx().contains<Message::Components::OpenFragments>()) {
|
||||
m.registry()->ctx().emplace<Message::Components::OpenFragments>();
|
||||
}
|
||||
|
||||
auto& fid_open = m.registry()->ctx().get<Message::Components::OpenFragments>().fid_open;
|
||||
|
||||
const auto msg_ts = m.get<Message::Components::Timestamp>().ts;
|
||||
// missing fuid
|
||||
// find closesed non-sealed off fragment
|
||||
|
||||
FragmentID fragment_id{entt::null};
|
||||
|
||||
// first search for fragment where the ts falls into the range
|
||||
for (const auto& fid : fid_open) {
|
||||
auto fh = _fs.fragmentHandle(fid);
|
||||
assert(static_cast<bool>(fh));
|
||||
|
||||
// assuming ts range exists
|
||||
auto& fts_comp = fh.get<FragComp::MessagesTSRange>();
|
||||
|
||||
if (fts_comp.begin <= msg_ts && fts_comp.end >= msg_ts) {
|
||||
fragment_id = fid;
|
||||
// TODO: check conditions for open here
|
||||
// TODO: mark msg (and frag?) dirty
|
||||
}
|
||||
}
|
||||
|
||||
// if it did not fit into an existing fragment, we next look for fragments that could be extended
|
||||
if (!_fs._reg.valid(fragment_id)) {
|
||||
for (const auto& fid : fid_open) {
|
||||
auto fh = _fs.fragmentHandle(fid);
|
||||
assert(static_cast<bool>(fh));
|
||||
|
||||
// assuming ts range exists
|
||||
auto& fts_comp = fh.get<FragComp::MessagesTSRange>();
|
||||
|
||||
const int64_t frag_range = int64_t(fts_comp.end) - int64_t(fts_comp.begin);
|
||||
constexpr static int64_t max_frag_ts_extent {1000*60*60};
|
||||
//constexpr static int64_t max_frag_ts_extent {1000*60*3}; // 3min for testing
|
||||
const int64_t possible_extention = max_frag_ts_extent - frag_range;
|
||||
|
||||
// which direction
|
||||
if ((fts_comp.begin - possible_extention) <= msg_ts && fts_comp.begin > msg_ts) {
|
||||
fragment_id = fid;
|
||||
|
||||
std::cout << "MFS: extended begin from " << fts_comp.begin << " to " << msg_ts << "\n";
|
||||
|
||||
// assuming ts range exists
|
||||
fts_comp.begin = msg_ts; // extend into the past
|
||||
|
||||
if (m.registry()->ctx().contains<Message::Components::ContactFragments>()) {
|
||||
// should be the case
|
||||
m.registry()->ctx().get<Message::Components::ContactFragments>().erase(fh);
|
||||
m.registry()->ctx().get<Message::Components::ContactFragments>().insert(fh);
|
||||
}
|
||||
|
||||
|
||||
// TODO: check conditions for open here
|
||||
// TODO: mark msg (and frag?) dirty
|
||||
} else if ((fts_comp.end + possible_extention) >= msg_ts && fts_comp.end < msg_ts) {
|
||||
fragment_id = fid;
|
||||
|
||||
std::cout << "MFS: extended end from " << fts_comp.end << " to " << msg_ts << "\n";
|
||||
|
||||
// assuming ts range exists
|
||||
fts_comp.end = msg_ts; // extend into the future
|
||||
|
||||
if (m.registry()->ctx().contains<Message::Components::ContactFragments>()) {
|
||||
// should be the case
|
||||
m.registry()->ctx().get<Message::Components::ContactFragments>().erase(fh);
|
||||
m.registry()->ctx().get<Message::Components::ContactFragments>().insert(fh);
|
||||
}
|
||||
|
||||
// TODO: check conditions for open here
|
||||
// TODO: mark msg (and frag?) dirty
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// if its still not found, we need a new fragment
|
||||
if (!_fs._reg.valid(fragment_id)) {
|
||||
const auto new_fid = _fs.newFragmentFile("test_message_store/", MetaFileType::BINARY_MSGPACK);
|
||||
auto fh = _fs.fragmentHandle(new_fid);
|
||||
if (!static_cast<bool>(fh)) {
|
||||
std::cout << "MFS error: failed to create new fragment for message\n";
|
||||
return;
|
||||
}
|
||||
|
||||
fragment_id = fh;
|
||||
|
||||
fh.emplace_or_replace<FragComp::Ephemeral::MetaCompressionType>().comp = Compression::ZSTD;
|
||||
fh.emplace_or_replace<FragComp::DataCompressionType>().comp = Compression::ZSTD;
|
||||
|
||||
auto& new_ts_range = fh.emplace_or_replace<FragComp::MessagesTSRange>();
|
||||
new_ts_range.begin = msg_ts;
|
||||
new_ts_range.end = msg_ts;
|
||||
|
||||
{
|
||||
const auto msg_reg_contact = m.registry()->ctx().get<Contact3>();
|
||||
if (_cr.all_of<Contact::Components::ID>(msg_reg_contact)) {
|
||||
fh.emplace<FragComp::MessagesContact>(_cr.get<Contact::Components::ID>(msg_reg_contact).data);
|
||||
} else {
|
||||
// ? rage quit?
|
||||
}
|
||||
}
|
||||
|
||||
// contact frag
|
||||
if (!m.registry()->ctx().contains<Message::Components::ContactFragments>()) {
|
||||
m.registry()->ctx().emplace<Message::Components::ContactFragments>();
|
||||
}
|
||||
m.registry()->ctx().get<Message::Components::ContactFragments>().insert(fh);
|
||||
|
||||
// loaded contact frag
|
||||
if (!m.registry()->ctx().contains<Message::Components::LoadedContactFragments>()) {
|
||||
m.registry()->ctx().emplace<Message::Components::LoadedContactFragments>();
|
||||
}
|
||||
m.registry()->ctx().get<Message::Components::LoadedContactFragments>().frags.emplace(fh);
|
||||
|
||||
fid_open.emplace(fragment_id);
|
||||
|
||||
std::cout << "MFS: created new fragment " << bin2hex(fh.get<FragComp::ID>().v) << "\n";
|
||||
|
||||
_fs_ignore_event = true;
|
||||
_fs.throwEventConstruct(fh);
|
||||
_fs_ignore_event = false;
|
||||
}
|
||||
|
||||
// if this is still empty, something is very wrong and we exit here
|
||||
if (!_fs._reg.valid(fragment_id)) {
|
||||
std::cout << "MFS error: failed to find/create fragment for message\n";
|
||||
return;
|
||||
}
|
||||
|
||||
m.emplace_or_replace<Message::Components::FID>(fragment_id);
|
||||
|
||||
// in this case we know the fragment needs an update
|
||||
_fuid_save_queue.push({Message::getTimeMS(), fragment_id, m.registry()});
|
||||
return; // done
|
||||
}
|
||||
|
||||
const auto msg_fh = _fs.fragmentHandle(m.get<Message::Components::FID>().fid);
|
||||
if (!static_cast<bool>(msg_fh)) {
|
||||
std::cerr << "MFS error: fid in message is invalid\n";
|
||||
return; // TODO: properly handle this case
|
||||
}
|
||||
|
||||
if (!m.registry()->ctx().contains<Message::Components::OpenFragments>()) {
|
||||
m.registry()->ctx().emplace<Message::Components::OpenFragments>();
|
||||
}
|
||||
|
||||
auto& fid_open = m.registry()->ctx().get<Message::Components::OpenFragments>().fid_open;
|
||||
|
||||
if (fid_open.contains(msg_fh)) {
|
||||
// TODO: dedup events
|
||||
// TODO: cooldown per fragsave
|
||||
_fuid_save_queue.push({Message::getTimeMS(), msg_fh, m.registry()});
|
||||
return;
|
||||
}
|
||||
|
||||
// TODO: save updates to old fragments, but writing them to a new fragment that would overwrite on merge
|
||||
// new fragment?, since we dont write to others fragments?
|
||||
|
||||
|
||||
// on new message: assign fuid
|
||||
// on new and update: mark as fragment dirty
|
||||
}
|
||||
|
||||
// assumes not loaded frag
|
||||
// need update from frag
|
||||
void MessageFragmentStore::loadFragment(Message3Registry& reg, FragmentHandle fh) {
|
||||
std::cout << "MFS: loadFragment\n";
|
||||
const auto j = _fs.loadFromStorageNJ(fh);
|
||||
|
||||
if (!j.is_array()) {
|
||||
// wrong data
|
||||
return;
|
||||
}
|
||||
|
||||
// TODO: this should probably never be the case, since we already know here that it is a msg frag
|
||||
if (!reg.ctx().contains<Message::Components::ContactFragments>()) {
|
||||
reg.ctx().emplace<Message::Components::ContactFragments>();
|
||||
}
|
||||
reg.ctx().get<Message::Components::ContactFragments>().insert(fh);
|
||||
|
||||
// mark loaded
|
||||
if (!reg.ctx().contains<Message::Components::LoadedContactFragments>()) {
|
||||
reg.ctx().emplace<Message::Components::LoadedContactFragments>();
|
||||
}
|
||||
reg.ctx().get<Message::Components::LoadedContactFragments>().frags.emplace(fh);
|
||||
|
||||
for (const auto& j_entry : j) {
|
||||
auto new_real_msg = Message3Handle{reg, reg.create()};
|
||||
// load into staging reg
|
||||
for (const auto& [k, v] : j_entry.items()) {
|
||||
//std::cout << "K:" << k << " V:" << v.dump() << "\n";
|
||||
const auto type_id = entt::hashed_string(k.data(), k.size());
|
||||
const auto deserl_fn_it = _sc._deserl_json.find(type_id);
|
||||
if (deserl_fn_it != _sc._deserl_json.cend()) {
|
||||
try {
|
||||
if (!deserl_fn_it->second(_sc, new_real_msg, v)) {
|
||||
std::cerr << "MFS error: failed deserializing '" << k << "'\n";
|
||||
}
|
||||
} catch(...) {
|
||||
std::cerr << "MFS error: failed deserializing (threw) '" << k << "'\n";
|
||||
}
|
||||
} else {
|
||||
std::cerr << "MFS warning: missing deserializer for meta key '" << k << "'\n";
|
||||
}
|
||||
}
|
||||
|
||||
new_real_msg.emplace_or_replace<Message::Components::FID>(fh);
|
||||
|
||||
// dup check (hacky, specific to protocols)
|
||||
Message3 dup_msg {entt::null};
|
||||
{
|
||||
// get comparator from contact
|
||||
if (reg.ctx().contains<Contact3>()) {
|
||||
const auto c = reg.ctx().get<Contact3>();
|
||||
if (_cr.all_of<Contact::Components::MessageIsSame>(c)) {
|
||||
auto& comp = _cr.get<Contact::Components::MessageIsSame>(c).comp;
|
||||
// walking EVERY existing message OOF
|
||||
// this needs optimizing
|
||||
for (const Message3 other_msg : reg.view<Message::Components::Timestamp, Message::Components::ContactFrom, Message::Components::ContactTo>()) {
|
||||
if (other_msg == new_real_msg) {
|
||||
continue; // skip self
|
||||
}
|
||||
|
||||
if (comp({reg, other_msg}, new_real_msg)) {
|
||||
// dup
|
||||
dup_msg = other_msg;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (reg.valid(dup_msg)) {
|
||||
// -> merge with preexisting (needs to be order independent)
|
||||
// -> throw update
|
||||
reg.destroy(new_real_msg);
|
||||
//_rmm.throwEventUpdate(reg, new_real_msg);
|
||||
} else {
|
||||
if (!new_real_msg.all_of<Message::Components::Timestamp, Message::Components::ContactFrom, Message::Components::ContactTo>()) {
|
||||
// does not have needed components to be stand alone
|
||||
reg.destroy(new_real_msg);
|
||||
std::cerr << "MFS warning: message with missing basic compoments\n";
|
||||
continue;
|
||||
}
|
||||
|
||||
// -> throw create
|
||||
_rmm.throwEventConstruct(reg, new_real_msg);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool MessageFragmentStore::syncFragToStorage(FragmentHandle fh, Message3Registry& reg) {
|
||||
auto& ftsrange = fh.get_or_emplace<FragComp::MessagesTSRange>(Message::getTimeMS(), Message::getTimeMS());
|
||||
|
||||
auto j = nlohmann::json::array();
|
||||
|
||||
// TODO: does every message have ts?
|
||||
auto msg_view = reg.view<Message::Components::Timestamp>();
|
||||
// we also assume all messages have fid
|
||||
for (auto it = msg_view.rbegin(), it_end = msg_view.rend(); it != it_end; it++) {
|
||||
const Message3 m = *it;
|
||||
|
||||
if (!reg.all_of<Message::Components::FID, Message::Components::ContactFrom, Message::Components::ContactTo>(m)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// require msg for now
|
||||
if (!reg.any_of<Message::Components::MessageText/*, Message::Components::Transfer::FileInfo*/>(m)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (_fuid_save_queue.front().id != reg.get<Message::Components::FID>(m).fid) {
|
||||
continue; // not ours
|
||||
}
|
||||
|
||||
{ // potentially adjust tsrange (some external processes can change timestamps)
|
||||
const auto msg_ts = msg_view.get<Message::Components::Timestamp>(m).ts;
|
||||
if (ftsrange.begin > msg_ts) {
|
||||
ftsrange.begin = msg_ts;
|
||||
} else if (ftsrange.end < msg_ts) {
|
||||
ftsrange.end = msg_ts;
|
||||
}
|
||||
}
|
||||
|
||||
auto& j_entry = j.emplace_back(nlohmann::json::object());
|
||||
|
||||
for (const auto& [type_id, storage] : reg.storage()) {
|
||||
if (!storage.contains(m)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
//std::cout << "storage type: type_id:" << type_id << " name:" << storage.type().name() << "\n";
|
||||
|
||||
// use type_id to find serializer
|
||||
auto s_cb_it = _sc._serl_json.find(type_id);
|
||||
if (s_cb_it == _sc._serl_json.end()) {
|
||||
// could not find serializer, not saving
|
||||
//std::cout << "missing " << storage.type().name() << "(" << type_id << ")\n";
|
||||
continue;
|
||||
}
|
||||
|
||||
s_cb_it->second(_sc, {reg, m}, j_entry[storage.type().name()]);
|
||||
}
|
||||
}
|
||||
|
||||
// we cant skip if array is empty (in theory it will not be empty later on)
|
||||
|
||||
// if save as binary
|
||||
//nlohmann::json::to_msgpack(j);
|
||||
auto j_dump = j.dump(2, ' ', true);
|
||||
if (_fs.syncToStorage(fh, reinterpret_cast<const uint8_t*>(j_dump.data()), j_dump.size())) {
|
||||
//std::cout << "MFS: dumped " << j_dump << "\n";
|
||||
// succ
|
||||
return true;
|
||||
}
|
||||
|
||||
// TODO: error
|
||||
return false;
|
||||
}
|
||||
|
||||
MessageFragmentStore::MessageFragmentStore(
|
||||
Contact3Registry& cr,
|
||||
RegistryMessageModel& rmm,
|
||||
FragmentStore& fs
|
||||
) : _cr(cr), _rmm(rmm), _fs(fs), _sc{_cr, {}, {}} {
|
||||
_rmm.subscribe(this, RegistryMessageModel_Event::message_construct);
|
||||
_rmm.subscribe(this, RegistryMessageModel_Event::message_updated);
|
||||
_rmm.subscribe(this, RegistryMessageModel_Event::message_destroy);
|
||||
|
||||
_fs._sc.registerSerializerJson<FragComp::MessagesTSRange>();
|
||||
_fs._sc.registerDeSerializerJson<FragComp::MessagesTSRange>();
|
||||
_fs._sc.registerSerializerJson<FragComp::MessagesContact>();
|
||||
_fs._sc.registerDeSerializerJson<FragComp::MessagesContact>();
|
||||
|
||||
_fs.subscribe(this, FragmentStore_Event::fragment_construct);
|
||||
}
|
||||
|
||||
MessageFragmentStore::~MessageFragmentStore(void) {
|
||||
while (!_fuid_save_queue.empty()) {
|
||||
auto fh = _fs.fragmentHandle(_fuid_save_queue.front().id);
|
||||
auto* reg = _fuid_save_queue.front().reg;
|
||||
assert(reg != nullptr);
|
||||
syncFragToStorage(fh, *reg);
|
||||
_fuid_save_queue.pop(); // pop unconditionally
|
||||
}
|
||||
}
|
||||
|
||||
MessageSerializerCallbacks& MessageFragmentStore::getMSC(void) {
|
||||
return _sc;
|
||||
}
|
||||
|
||||
// checks range against all cursers in msgreg
|
||||
static bool rangeVisible(uint64_t range_begin, uint64_t range_end, const Message3Registry& msg_reg) {
|
||||
// 1D collision checks:
|
||||
// - for range vs range:
|
||||
// r1 rhs >= r0 lhs AND r1 lhs <= r0 rhs
|
||||
// - for range vs point:
|
||||
// p >= r0 lhs AND p <= r0 rhs
|
||||
// NOTE: directions for us are reversed (begin has larger values as end)
|
||||
|
||||
auto c_b_view = msg_reg.view<Message::Components::Timestamp, Message::Components::ViewCurserBegin>();
|
||||
c_b_view.use<Message::Components::ViewCurserBegin>();
|
||||
for (const auto& [m, ts_begin_comp, vcb] : c_b_view.each()) {
|
||||
// p and r1 rhs can be seen as the same
|
||||
// but first we need to know if a curser begin is a point or a range
|
||||
|
||||
// TODO: margin?
|
||||
auto ts_begin = ts_begin_comp.ts;
|
||||
auto ts_end = ts_begin_comp.ts; // simplyfy code by making a single begin curser act as an infinitly small range
|
||||
if (msg_reg.valid(vcb.curser_end) && msg_reg.all_of<Message::Components::ViewCurserEnd>(vcb.curser_end)) {
|
||||
// TODO: respect curser end's begin?
|
||||
// TODO: remember which ends we checked and check remaining
|
||||
ts_end = msg_reg.get<Message::Components::Timestamp>(vcb.curser_end).ts;
|
||||
|
||||
// sanity check curser order
|
||||
if (ts_end > ts_begin) {
|
||||
std::cerr << "MFS warning: begin curser and end curser of view swapped!!\n";
|
||||
std::swap(ts_begin, ts_end);
|
||||
}
|
||||
}
|
||||
|
||||
// perform both checks here
|
||||
if (ts_begin < range_end || ts_end > range_begin) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// range hits a view
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool isLess(const std::vector<uint8_t>& lhs, const std::vector<uint8_t>& rhs) {
|
||||
size_t i = 0;
|
||||
for (; i < lhs.size() && i < rhs.size(); i++) {
|
||||
if (lhs[i] < rhs[i]) {
|
||||
return true;
|
||||
} else if (lhs[i] > rhs[i]) {
|
||||
return false;
|
||||
}
|
||||
// else continue
|
||||
}
|
||||
|
||||
// here we have equality of common lenths
|
||||
|
||||
// we define smaller arrays to be less
|
||||
return lhs.size() < rhs.size();
|
||||
}
|
||||
|
||||
float MessageFragmentStore::tick(float time_delta) {
|
||||
// sync dirty fragments here
|
||||
if (!_fuid_save_queue.empty()) {
|
||||
auto fh = _fs.fragmentHandle(_fuid_save_queue.front().id);
|
||||
auto* reg = _fuid_save_queue.front().reg;
|
||||
assert(reg != nullptr);
|
||||
if (syncFragToStorage(fh, *reg)) {
|
||||
_fuid_save_queue.pop();
|
||||
}
|
||||
}
|
||||
|
||||
// load needed fragments here
|
||||
|
||||
// last check event frags
|
||||
// only checks if it collides with ranges, not adjacent
|
||||
// bc ~range~ msgreg will be marked dirty and checked next tick
|
||||
const bool had_events = !_event_check_queue.empty();
|
||||
for (size_t i = 0; i < 10 && !_event_check_queue.empty(); i++) {
|
||||
std::cout << "MFS: event check\n";
|
||||
auto fh = _fs.fragmentHandle(_event_check_queue.front().fid);
|
||||
auto c = _event_check_queue.front().c;
|
||||
_event_check_queue.pop();
|
||||
|
||||
if (!static_cast<bool>(fh)) {
|
||||
return 0.05f;
|
||||
}
|
||||
|
||||
if (!fh.all_of<FragComp::MessagesTSRange>()) {
|
||||
return 0.05f;
|
||||
}
|
||||
|
||||
// get ts range of frag and collide with all curser(s/ranges)
|
||||
const auto& frag_range = fh.get<FragComp::MessagesTSRange>();
|
||||
|
||||
auto* msg_reg = _rmm.get(c);
|
||||
if (msg_reg == nullptr) {
|
||||
return 0.05f;
|
||||
}
|
||||
|
||||
if (rangeVisible(frag_range.begin, frag_range.end, !msg_reg)) {
|
||||
loadFragment(*msg_reg, fh);
|
||||
_potentially_dirty_contacts.emplace(c);
|
||||
return 0.05f; // only one but soon again
|
||||
}
|
||||
}
|
||||
if (had_events) {
|
||||
std::cout << "MFS: event check none\n";
|
||||
return 0.05f; // only check events, even if non where hit
|
||||
}
|
||||
|
||||
if (!_potentially_dirty_contacts.empty()) {
|
||||
std::cout << "MFS: pdc\n";
|
||||
// here we check if any view of said contact needs frag loading
|
||||
// only once per tick tho
|
||||
|
||||
// TODO: this makes order depend on internal order and is not fair
|
||||
auto it = _potentially_dirty_contacts.cbegin();
|
||||
|
||||
auto* msg_reg = _rmm.get(*it);
|
||||
|
||||
// first do collision check agains every contact associated fragment
|
||||
// that is not already loaded !!
|
||||
if (msg_reg->ctx().contains<Message::Components::ContactFragments>()) {
|
||||
const auto& cf = msg_reg->ctx().get<Message::Components::ContactFragments>();
|
||||
if (!cf.frags.empty()) {
|
||||
if (!msg_reg->ctx().contains<Message::Components::LoadedContactFragments>()) {
|
||||
msg_reg->ctx().emplace<Message::Components::LoadedContactFragments>();
|
||||
}
|
||||
const auto& loaded_frags = msg_reg->ctx().get<Message::Components::LoadedContactFragments>().frags;
|
||||
|
||||
for (const auto& [fid, si] : msg_reg->ctx().get<Message::Components::ContactFragments>().frags) {
|
||||
if (loaded_frags.contains(fid)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
auto fh = _fs.fragmentHandle(fid);
|
||||
|
||||
if (!static_cast<bool>(fh)) {
|
||||
std::cerr << "MFS error: frag is invalid\n";
|
||||
// WHAT
|
||||
msg_reg->ctx().get<Message::Components::ContactFragments>().erase(fid);
|
||||
return 0.05f;
|
||||
}
|
||||
|
||||
if (!fh.all_of<FragComp::MessagesTSRange>()) {
|
||||
std::cerr << "MFS error: frag has no range\n";
|
||||
// ????
|
||||
msg_reg->ctx().get<Message::Components::ContactFragments>().erase(fid);
|
||||
return 0.05f;
|
||||
}
|
||||
|
||||
// get ts range of frag and collide with all curser(s/ranges)
|
||||
const auto& [range_begin, range_end] = fh.get<FragComp::MessagesTSRange>();
|
||||
|
||||
if (rangeVisible(range_begin, range_end, *msg_reg)) {
|
||||
std::cout << "MFS: frag hit by vis range\n";
|
||||
loadFragment(*msg_reg, fh);
|
||||
return 0.05f;
|
||||
}
|
||||
}
|
||||
// no new visible fragment
|
||||
std::cout << "MFS: no new frag directly visible\n";
|
||||
|
||||
// now, finally, check for adjecent fragments that need to be loaded
|
||||
// we do this by finding the outermost fragment in a rage, and extend it by one
|
||||
|
||||
// TODO: rewrite using some bounding range tree to perform collision checks !!!
|
||||
// (this is now performing better, but still)
|
||||
|
||||
|
||||
// for each view
|
||||
auto c_b_view = msg_reg->view<Message::Components::Timestamp, Message::Components::ViewCurserBegin>();
|
||||
c_b_view.use<Message::Components::ViewCurserBegin>();
|
||||
for (const auto& [_, ts_begin_comp, vcb] : c_b_view.each()) {
|
||||
// aka "scroll down"
|
||||
{ // find newest(-ish) frag in range
|
||||
// or in reverse frag end <= range begin
|
||||
|
||||
|
||||
// lower bound of frag end and range begin
|
||||
const auto right = std::lower_bound(
|
||||
cf.sorted_end.crbegin(),
|
||||
cf.sorted_end.crend(),
|
||||
ts_begin_comp.ts,
|
||||
[&](const FragmentID element, const auto& value) -> bool {
|
||||
return _fs._reg.get<FragComp::MessagesTSRange>(element).end >= value;
|
||||
}
|
||||
);
|
||||
|
||||
FragmentID next_frag{entt::null};
|
||||
if (right != cf.sorted_end.crend()) {
|
||||
next_frag = cf.next(*right);
|
||||
}
|
||||
// we checked earlier that cf is not empty
|
||||
if (!_fs._reg.valid(next_frag)) {
|
||||
// fall back to closest, cf is not empty
|
||||
next_frag = cf.sorted_end.front();
|
||||
}
|
||||
|
||||
// a single adjacent frag is often not enough
|
||||
// only ok bc next is cheap
|
||||
for (size_t i = 0; i < 5 && _fs._reg.valid(next_frag); i++) {
|
||||
if (!loaded_frags.contains(next_frag)) {
|
||||
std::cout << "MFS: next frag of range\n";
|
||||
loadFragment(*msg_reg, {_fs._reg, next_frag});
|
||||
return 0.05f;
|
||||
}
|
||||
|
||||
next_frag = cf.next(next_frag);
|
||||
}
|
||||
}
|
||||
|
||||
// curser end
|
||||
if (!msg_reg->valid(vcb.curser_end) || !msg_reg->all_of<Message::Components::Timestamp>(vcb.curser_end)) {
|
||||
continue;
|
||||
}
|
||||
const auto ts_end = msg_reg->get<Message::Components::Timestamp>(vcb.curser_end).ts;
|
||||
|
||||
// aka "scroll up"
|
||||
{ // find oldest(-ish) frag in range
|
||||
// frag begin >= range end
|
||||
|
||||
// lower bound of frag begin and range end
|
||||
const auto left = std::lower_bound(
|
||||
cf.sorted_begin.cbegin(),
|
||||
cf.sorted_begin.cend(),
|
||||
ts_end,
|
||||
[&](const FragmentID element, const auto& value) -> bool {
|
||||
return _fs._reg.get<FragComp::MessagesTSRange>(element).begin < value;
|
||||
}
|
||||
);
|
||||
|
||||
FragmentID prev_frag{entt::null};
|
||||
if (left != cf.sorted_begin.cend()) {
|
||||
prev_frag = cf.prev(*left);
|
||||
}
|
||||
// we checked earlier that cf is not empty
|
||||
if (!_fs._reg.valid(prev_frag)) {
|
||||
// fall back to closest, cf is not empty
|
||||
prev_frag = cf.sorted_begin.back();
|
||||
}
|
||||
|
||||
// a single adjacent frag is often not enough
|
||||
// only ok bc next is cheap
|
||||
for (size_t i = 0; i < 5 && _fs._reg.valid(prev_frag); i++) {
|
||||
if (!loaded_frags.contains(prev_frag)) {
|
||||
std::cout << "MFS: prev frag of range\n";
|
||||
loadFragment(*msg_reg, {_fs._reg, prev_frag});
|
||||
return 0.05f;
|
||||
}
|
||||
|
||||
prev_frag = cf.prev(prev_frag);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// contact has no fragments, skip
|
||||
}
|
||||
|
||||
_potentially_dirty_contacts.erase(it);
|
||||
|
||||
return 0.05f;
|
||||
}
|
||||
|
||||
|
||||
return 1000.f*60.f*60.f;
|
||||
}
|
||||
|
||||
void MessageFragmentStore::triggerScan(void) {
|
||||
_fs.scanStoragePath("test_message_store/");
|
||||
}
|
||||
|
||||
bool MessageFragmentStore::onEvent(const Message::Events::MessageConstruct& e) {
|
||||
handleMessage(e.e);
|
||||
return false;
|
||||
}
|
||||
|
||||
bool MessageFragmentStore::onEvent(const Message::Events::MessageUpdated& e) {
|
||||
handleMessage(e.e);
|
||||
return false;
|
||||
}
|
||||
|
||||
// TODO: handle deletes? diff between unload?
|
||||
|
||||
bool MessageFragmentStore::onEvent(const Fragment::Events::FragmentConstruct& e) {
|
||||
if (_fs_ignore_event) {
|
||||
return false; // skip self
|
||||
}
|
||||
|
||||
if (!e.e.all_of<FragComp::MessagesTSRange, FragComp::MessagesContact>()) {
|
||||
return false; // not for us
|
||||
}
|
||||
|
||||
// TODO: are we sure it is a *new* fragment?
|
||||
|
||||
Contact3 frag_contact = entt::null;
|
||||
{ // get contact
|
||||
const auto& frag_contact_id = e.e.get<FragComp::MessagesContact>().id;
|
||||
// TODO: id lookup table, this is very inefficent
|
||||
for (const auto& [c_it, id_it] : _cr.view<Contact::Components::ID>().each()) {
|
||||
if (frag_contact_id == id_it.data) {
|
||||
frag_contact = c_it;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!_cr.valid(frag_contact)) {
|
||||
// unkown contact
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// create if not exist
|
||||
auto* msg_reg = _rmm.get(frag_contact);
|
||||
if (msg_reg == nullptr) {
|
||||
// msg reg not created yet
|
||||
// TODO: this is an erroious path
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!msg_reg->ctx().contains<Message::Components::ContactFragments>()) {
|
||||
msg_reg->ctx().emplace<Message::Components::ContactFragments>();
|
||||
}
|
||||
msg_reg->ctx().get<Message::Components::ContactFragments>().erase(e.e); // TODO: check/update/fragment update
|
||||
msg_reg->ctx().get<Message::Components::ContactFragments>().insert(e.e);
|
||||
|
||||
_event_check_queue.push(ECQueueEntry{e.e, frag_contact});
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
bool Message::Components::ContactFragments::insert(FragmentHandle frag) {
|
||||
if (frags.contains(frag)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// both sorted arrays are sorted ascending
|
||||
// so for insertion we search for the last index that is <= and insert after it
|
||||
// or we search for the first > (or end) and insert before it <---
|
||||
// since equal fragments are UB, we can assume they are only > or <
|
||||
|
||||
size_t begin_index {0};
|
||||
{ // begin
|
||||
const auto pos = std::find_if(
|
||||
sorted_begin.cbegin(),
|
||||
sorted_begin.cend(),
|
||||
[frag](const FragmentID a) -> bool {
|
||||
const auto begin_a = frag.registry()->get<FragComp::MessagesTSRange>(a).begin;
|
||||
const auto begin_frag = frag.get<FragComp::MessagesTSRange>().begin;
|
||||
if (begin_a > begin_frag) {
|
||||
return true;
|
||||
} else if (begin_a < begin_frag) {
|
||||
return false;
|
||||
} else {
|
||||
// equal ts, we need to fall back to id (id can not be equal)
|
||||
return isLess(frag.get<FragComp::ID>().v, frag.registry()->get<FragComp::ID>(a).v);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
begin_index = std::distance(sorted_begin.cbegin(), pos);
|
||||
|
||||
// we need to insert before pos (end is valid here)
|
||||
sorted_begin.insert(pos, frag);
|
||||
}
|
||||
|
||||
size_t end_index {0};
|
||||
{ // end
|
||||
const auto pos = std::find_if_not(
|
||||
sorted_end.cbegin(),
|
||||
sorted_end.cend(),
|
||||
[frag](const FragmentID a) -> bool {
|
||||
const auto end_a = frag.registry()->get<FragComp::MessagesTSRange>(a).end;
|
||||
const auto end_frag = frag.get<FragComp::MessagesTSRange>().end;
|
||||
if (end_a > end_frag) {
|
||||
return true;
|
||||
} else if (end_a < end_frag) {
|
||||
return false;
|
||||
} else {
|
||||
// equal ts, we need to fall back to id (id can not be equal)
|
||||
return isLess(frag.get<FragComp::ID>().v, frag.registry()->get<FragComp::ID>(a).v);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
end_index = std::distance(sorted_end.cbegin(), pos);
|
||||
|
||||
// we need to insert before pos (end is valid here)
|
||||
sorted_end.insert(pos, frag);
|
||||
}
|
||||
|
||||
frags.emplace(frag, InternalEntry{begin_index, end_index});
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Message::Components::ContactFragments::erase(FragmentID frag) {
|
||||
auto frags_it = frags.find(frag);
|
||||
if (frags_it == frags.end()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
assert(sorted_begin.size() == sorted_end.size());
|
||||
assert(sorted_begin.size() > frags_it->second.i_b);
|
||||
|
||||
sorted_begin.erase(sorted_begin.begin() + frags_it->second.i_b);
|
||||
sorted_end.erase(sorted_end.begin() + frags_it->second.i_e);
|
||||
|
||||
frags.erase(frags_it);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
FragmentID Message::Components::ContactFragments::prev(FragmentID frag) const {
|
||||
// uses range begin to go back in time
|
||||
|
||||
auto it = frags.find(frag);
|
||||
if (it == frags.end()) {
|
||||
return entt::null;
|
||||
}
|
||||
|
||||
const auto src_i = it->second.i_b;
|
||||
if (src_i > 0) {
|
||||
return sorted_begin[src_i-1];
|
||||
}
|
||||
|
||||
return entt::null;
|
||||
}
|
||||
|
||||
FragmentID Message::Components::ContactFragments::next(FragmentID frag) const {
|
||||
// uses range end to go forward in time
|
||||
|
||||
auto it = frags.find(frag);
|
||||
if (it == frags.end()) {
|
||||
return entt::null;
|
||||
}
|
||||
|
||||
const auto src_i = it->second.i_e;
|
||||
if (src_i+1 < sorted_end.size()) {
|
||||
return sorted_end[src_i+1];
|
||||
}
|
||||
|
||||
return entt::null;
|
||||
}
|
||||
|
132
src/fragment_store/message_fragment_store.hpp
Normal file
132
src/fragment_store/message_fragment_store.hpp
Normal file
@ -0,0 +1,132 @@
|
||||
#pragma once
|
||||
|
||||
#include "./meta_components.hpp"
|
||||
#include "./fragment_store_i.hpp"
|
||||
#include "./fragment_store.hpp"
|
||||
|
||||
#include "./message_serializer.hpp"
|
||||
|
||||
#include <entt/entity/registry.hpp>
|
||||
#include <entt/container/dense_map.hpp>
|
||||
#include <entt/container/dense_set.hpp>
|
||||
|
||||
#include <solanaceae/contact/contact_model3.hpp>
|
||||
#include <solanaceae/message3/registry_message_model.hpp>
|
||||
|
||||
#include <queue>
|
||||
#include <vector>
|
||||
#include <cstdint>
|
||||
|
||||
namespace Message::Components {
|
||||
|
||||
// unused, consumes too much memory (highly compressable)
|
||||
//using FUID = FragComp::ID;
|
||||
|
||||
struct FID {
|
||||
FragmentID fid {entt::null};
|
||||
};
|
||||
|
||||
// points to the front/newer message
|
||||
// together they define a range that is,
|
||||
// eg the first(end) and last(begin) message being rendered
|
||||
// MFS requires there to be atleast one other fragment after/before,
|
||||
// if not loaded fragment with fitting tsrange(direction) available
|
||||
// uses fragmentAfter/Before()
|
||||
// they can exist standalone
|
||||
// if they are a pair, the inside is filled first
|
||||
// cursers require a timestamp ???
|
||||
struct ViewCurserBegin {
|
||||
Message3 curser_end{entt::null};
|
||||
};
|
||||
struct ViewCurserEnd {
|
||||
Message3 curser_begin{entt::null};
|
||||
};
|
||||
|
||||
// TODO: add adjacency range comp or inside curser
|
||||
|
||||
// TODO: unused
|
||||
// mfs will only load a limited number of fragments per tick (1),
|
||||
// so this tag will be set if we loaded a fragment and
|
||||
// every tick we check all cursers for this tag and continue
|
||||
// and remove once no fragment could be loaded anymore
|
||||
// (internal)
|
||||
struct TagCurserUnsatisfied {};
|
||||
|
||||
} // Message::Components
|
||||
|
||||
namespace Fragment::Components {
|
||||
struct MessagesTSRange {
|
||||
// timestamp range within the fragment
|
||||
uint64_t begin {0}; // newer msg -> higher number
|
||||
uint64_t end {0};
|
||||
};
|
||||
|
||||
struct MessagesContact {
|
||||
std::vector<uint8_t> id;
|
||||
};
|
||||
|
||||
// TODO: add src contact (self id)
|
||||
|
||||
} // Fragment::Components
|
||||
|
||||
// handles fragments for messages
|
||||
// on new message: assign fuid
|
||||
// on new and update: mark as fragment dirty
|
||||
// on delete: mark as fragment dirty?
|
||||
class MessageFragmentStore : public RegistryMessageModelEventI, public FragmentStoreEventI {
|
||||
protected:
|
||||
Contact3Registry& _cr;
|
||||
RegistryMessageModel& _rmm;
|
||||
FragmentStore& _fs;
|
||||
bool _fs_ignore_event {false};
|
||||
|
||||
// for message components only
|
||||
MessageSerializerCallbacks _sc;
|
||||
|
||||
void handleMessage(const Message3Handle& m);
|
||||
|
||||
void loadFragment(Message3Registry& reg, FragmentHandle fh);
|
||||
|
||||
bool syncFragToStorage(FragmentHandle fh, Message3Registry& reg);
|
||||
|
||||
struct SaveQueueEntry final {
|
||||
uint64_t ts_since_dirty{0};
|
||||
//std::vector<uint8_t> id;
|
||||
FragmentID id;
|
||||
Message3Registry* reg{nullptr};
|
||||
};
|
||||
std::queue<SaveQueueEntry> _fuid_save_queue;
|
||||
|
||||
struct ECQueueEntry final {
|
||||
FragmentID fid;
|
||||
Contact3 c;
|
||||
};
|
||||
std::queue<ECQueueEntry> _event_check_queue;
|
||||
|
||||
// range changed or fragment loaded.
|
||||
// we only load a limited number of fragments at once,
|
||||
// so we need to keep them dirty until nothing was loaded.
|
||||
entt::dense_set<Contact3> _potentially_dirty_contacts;
|
||||
|
||||
public:
|
||||
MessageFragmentStore(
|
||||
Contact3Registry& cr,
|
||||
RegistryMessageModel& rmm,
|
||||
FragmentStore& fs
|
||||
);
|
||||
virtual ~MessageFragmentStore(void);
|
||||
|
||||
MessageSerializerCallbacks& getMSC(void);
|
||||
|
||||
float tick(float time_delta);
|
||||
|
||||
void triggerScan(void);
|
||||
|
||||
protected: // rmm
|
||||
bool onEvent(const Message::Events::MessageConstruct& e) override;
|
||||
bool onEvent(const Message::Events::MessageUpdated& e) override;
|
||||
|
||||
protected: // fs
|
||||
bool onEvent(const Fragment::Events::FragmentConstruct& e) override;
|
||||
};
|
||||
|
107
src/fragment_store/message_serializer.cpp
Normal file
107
src/fragment_store/message_serializer.cpp
Normal file
@ -0,0 +1,107 @@
|
||||
#include "./message_serializer.hpp"
|
||||
|
||||
#include <solanaceae/message3/components.hpp>
|
||||
#include <solanaceae/contact/components.hpp>
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
#include <iostream>
|
||||
|
||||
static Contact3 findContactByID(Contact3Registry& cr, const std::vector<uint8_t>& id) {
|
||||
// TODO: id lookup table, this is very inefficent
|
||||
for (const auto& [c_it, id_it] : cr.view<Contact::Components::ID>().each()) {
|
||||
if (id == id_it.data) {
|
||||
return c_it;
|
||||
}
|
||||
}
|
||||
|
||||
return entt::null;
|
||||
}
|
||||
|
||||
template<>
|
||||
bool MessageSerializerCallbacks::component_get_json<Message::Components::ContactFrom>(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& j) {
|
||||
const Contact3 c = h.get<Message::Components::ContactFrom>().c;
|
||||
if (!msc.cr.valid(c)) {
|
||||
// while this is invalid registry state, it is valid serialization
|
||||
j = nullptr;
|
||||
std::cerr << "MSC warning: encountered invalid contact\n";
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!msc.cr.all_of<Contact::Components::ID>(c)) {
|
||||
// unlucky, this contact is purely ephemeral
|
||||
j = nullptr;
|
||||
std::cerr << "MSC warning: encountered contact without ID\n";
|
||||
return true;
|
||||
}
|
||||
|
||||
j = nlohmann::json::binary(msc.cr.get<Contact::Components::ID>(c).data);
|
||||
|
||||
return true;
|
||||
}
|
||||
template<>
|
||||
bool MessageSerializerCallbacks::component_emplace_or_replace_json<Message::Components::ContactFrom>(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& j) {
|
||||
if (j.is_null()) {
|
||||
std::cerr << "MSC warning: encountered null contact\n";
|
||||
h.emplace_or_replace<Message::Components::ContactFrom>();
|
||||
return true;
|
||||
}
|
||||
|
||||
const std::vector<uint8_t> id = j.is_binary()?j:j["bytes"];
|
||||
|
||||
Contact3 other_c = findContactByID(msc.cr, id);
|
||||
if (!msc.cr.valid(other_c)) {
|
||||
// create sparse contact with id only
|
||||
other_c = msc.cr.create();
|
||||
msc.cr.emplace_or_replace<Contact::Components::ID>(other_c, id);
|
||||
}
|
||||
|
||||
h.emplace_or_replace<Message::Components::ContactFrom>(other_c);
|
||||
|
||||
// TODO: should we return false if the contact is unknown??
|
||||
return true;
|
||||
}
|
||||
|
||||
template<>
|
||||
bool MessageSerializerCallbacks::component_get_json<Message::Components::ContactTo>(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& j) {
|
||||
const Contact3 c = h.get<Message::Components::ContactTo>().c;
|
||||
if (!msc.cr.valid(c)) {
|
||||
// while this is invalid registry state, it is valid serialization
|
||||
j = nullptr;
|
||||
std::cerr << "MSC warning: encountered invalid contact\n";
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!msc.cr.all_of<Contact::Components::ID>(c)) {
|
||||
// unlucky, this contact is purely ephemeral
|
||||
j = nullptr;
|
||||
std::cerr << "MSC warning: encountered contact without ID\n";
|
||||
return true;
|
||||
}
|
||||
|
||||
j = nlohmann::json::binary(msc.cr.get<Contact::Components::ID>(c).data);
|
||||
|
||||
return true;
|
||||
}
|
||||
template<>
|
||||
bool MessageSerializerCallbacks::component_emplace_or_replace_json<Message::Components::ContactTo>(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& j) {
|
||||
if (j.is_null()) {
|
||||
std::cerr << "MSC warning: encountered null contact\n";
|
||||
h.emplace_or_replace<Message::Components::ContactTo>();
|
||||
return true;
|
||||
}
|
||||
|
||||
const std::vector<uint8_t> id = j.is_binary()?j:j["bytes"];
|
||||
|
||||
Contact3 other_c = findContactByID(msc.cr, id);
|
||||
if (!msc.cr.valid(other_c)) {
|
||||
// create sparse contact with id only
|
||||
other_c = msc.cr.create();
|
||||
msc.cr.emplace_or_replace<Contact::Components::ID>(other_c, id);
|
||||
}
|
||||
|
||||
h.emplace_or_replace<Message::Components::ContactTo>(other_c);
|
||||
|
||||
// TODO: should we return false if the contact is unknown??
|
||||
return true;
|
||||
}
|
85
src/fragment_store/message_serializer.hpp
Normal file
85
src/fragment_store/message_serializer.hpp
Normal file
@ -0,0 +1,85 @@
|
||||
#pragma once
|
||||
|
||||
#include <entt/core/type_info.hpp>
|
||||
#include <entt/container/dense_map.hpp>
|
||||
|
||||
#include <solanaceae/message3/registry_message_model.hpp>
|
||||
|
||||
#include <nlohmann/json_fwd.hpp>
|
||||
|
||||
struct MessageSerializerCallbacks {
|
||||
using Registry = Message3Registry;
|
||||
using Handle = Message3Handle;
|
||||
|
||||
Contact3Registry& cr;
|
||||
|
||||
// nlohmann
|
||||
// json/msgpack
|
||||
using serialize_json_fn = bool(*)(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& out);
|
||||
entt::dense_map<entt::id_type, serialize_json_fn> _serl_json;
|
||||
|
||||
using deserialize_json_fn = bool(*)(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& in);
|
||||
entt::dense_map<entt::id_type, deserialize_json_fn> _deserl_json;
|
||||
|
||||
template<typename T>
|
||||
static bool component_get_json(MessageSerializerCallbacks&, const Handle h, nlohmann::json& j) {
|
||||
if (h.template all_of<T>()) {
|
||||
if constexpr (!std::is_empty_v<T>) {
|
||||
j = h.template get<T>();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
static bool component_emplace_or_replace_json(MessageSerializerCallbacks&, Handle h, const nlohmann::json& j) {
|
||||
if constexpr (std::is_empty_v<T>) {
|
||||
h.template emplace_or_replace<T>(); // assert empty json?
|
||||
} else {
|
||||
h.template emplace_or_replace<T>(static_cast<T>(j));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info) {
|
||||
_serl_json[type_info.hash()] = fn;
|
||||
}
|
||||
|
||||
template<typename CompType>
|
||||
void registerSerializerJson(
|
||||
serialize_json_fn fn = component_get_json<CompType>,
|
||||
const entt::type_info& type_info = entt::type_id<CompType>()
|
||||
) {
|
||||
registerSerializerJson(fn, type_info);
|
||||
}
|
||||
|
||||
void registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info) {
|
||||
_deserl_json[type_info.hash()] = fn;
|
||||
}
|
||||
|
||||
template<typename CompType>
|
||||
void registerDeSerializerJson(
|
||||
deserialize_json_fn fn = component_emplace_or_replace_json<CompType>,
|
||||
const entt::type_info& type_info = entt::type_id<CompType>()
|
||||
) {
|
||||
registerDeSerializerJson(fn, type_info);
|
||||
}
|
||||
};
|
||||
|
||||
// fwd
|
||||
namespace Message::Components {
|
||||
struct ContactFrom;
|
||||
struct ContactTo;
|
||||
}
|
||||
|
||||
// make specializations known
|
||||
template<>
|
||||
bool MessageSerializerCallbacks::component_get_json<Message::Components::ContactFrom>(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& j);
|
||||
template<>
|
||||
bool MessageSerializerCallbacks::component_emplace_or_replace_json<Message::Components::ContactFrom>(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& j);
|
||||
template<>
|
||||
bool MessageSerializerCallbacks::component_get_json<Message::Components::ContactTo>(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& j);
|
||||
template<>
|
||||
bool MessageSerializerCallbacks::component_emplace_or_replace_json<Message::Components::ContactTo>(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& j);
|
60
src/fragment_store/meta_components.hpp
Normal file
60
src/fragment_store/meta_components.hpp
Normal file
@ -0,0 +1,60 @@
|
||||
#pragma once
|
||||
|
||||
#include "./types.hpp"
|
||||
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include <cstdint>
|
||||
|
||||
namespace Fragment::Components {
|
||||
|
||||
// TODO: is this special and should this be saved to meta or not (its already in the file name on disk)
|
||||
struct ID {
|
||||
std::vector<uint8_t> v;
|
||||
};
|
||||
|
||||
struct DataEncryptionType {
|
||||
Encryption enc {Encryption::NONE};
|
||||
};
|
||||
|
||||
struct DataCompressionType {
|
||||
Compression comp {Compression::NONE};
|
||||
};
|
||||
|
||||
|
||||
// meta that is not written to (meta-)file
|
||||
namespace Ephemeral {
|
||||
|
||||
// excluded from file meta
|
||||
struct FilePath {
|
||||
// contains store path, if any
|
||||
std::string path;
|
||||
};
|
||||
|
||||
// TODO: seperate into remote and local?
|
||||
// (remote meaning eg. the file on disk was changed by another program)
|
||||
struct DirtyTag {};
|
||||
|
||||
|
||||
// type as comp
|
||||
struct MetaFileType {
|
||||
::MetaFileType type {::MetaFileType::TEXT_JSON};
|
||||
};
|
||||
|
||||
struct MetaEncryptionType {
|
||||
Encryption enc {Encryption::NONE};
|
||||
};
|
||||
|
||||
struct MetaCompressionType {
|
||||
Compression comp {Compression::NONE};
|
||||
};
|
||||
|
||||
} // Ephemeral
|
||||
|
||||
} // Components
|
||||
|
||||
// shortened to save bytes (until I find a way to save by ID in msgpack)
|
||||
namespace FragComp = Fragment::Components;
|
||||
|
||||
#include "./meta_components_id.inl"
|
||||
|
26
src/fragment_store/meta_components_id.inl
Normal file
26
src/fragment_store/meta_components_id.inl
Normal file
@ -0,0 +1,26 @@
|
||||
#pragma once
|
||||
|
||||
#include "./meta_components.hpp"
|
||||
|
||||
#include <entt/core/type_info.hpp>
|
||||
|
||||
// TODO: move more central
|
||||
#define DEFINE_COMP_ID(x) \
|
||||
template<> \
|
||||
constexpr entt::id_type entt::type_hash<x>::value() noexcept { \
|
||||
using namespace entt::literals; \
|
||||
return #x##_hs; \
|
||||
} \
|
||||
template<> \
|
||||
constexpr std::string_view entt::type_name<x>::value() noexcept { \
|
||||
return #x; \
|
||||
}
|
||||
|
||||
// cross compiler stable ids
|
||||
|
||||
DEFINE_COMP_ID(FragComp::DataEncryptionType)
|
||||
DEFINE_COMP_ID(FragComp::DataCompressionType)
|
||||
|
||||
#undef DEFINE_COMP_ID
|
||||
|
||||
|
35
src/fragment_store/register_mfs_json_message_components.cpp
Normal file
35
src/fragment_store/register_mfs_json_message_components.cpp
Normal file
@ -0,0 +1,35 @@
|
||||
#include "./register_mfs_json_message_components.hpp"
|
||||
|
||||
#include "./message_serializer.hpp"
|
||||
#include "../json/message_components.hpp"
|
||||
|
||||
void registerMFSJsonMessageComponents(MessageSerializerCallbacks& msc) {
|
||||
msc.registerSerializerJson<Message::Components::Timestamp>();
|
||||
msc.registerDeSerializerJson<Message::Components::Timestamp>();
|
||||
msc.registerSerializerJson<Message::Components::TimestampProcessed>();
|
||||
msc.registerDeSerializerJson<Message::Components::TimestampProcessed>();
|
||||
msc.registerSerializerJson<Message::Components::TimestampWritten>();
|
||||
msc.registerDeSerializerJson<Message::Components::TimestampWritten>();
|
||||
msc.registerSerializerJson<Message::Components::ContactFrom>();
|
||||
msc.registerDeSerializerJson<Message::Components::ContactFrom>();
|
||||
msc.registerSerializerJson<Message::Components::ContactTo>();
|
||||
msc.registerDeSerializerJson<Message::Components::ContactTo>();
|
||||
msc.registerSerializerJson<Message::Components::TagUnread>();
|
||||
msc.registerDeSerializerJson<Message::Components::TagUnread>();
|
||||
msc.registerSerializerJson<Message::Components::Read>();
|
||||
msc.registerDeSerializerJson<Message::Components::Read>();
|
||||
msc.registerSerializerJson<Message::Components::MessageText>();
|
||||
msc.registerDeSerializerJson<Message::Components::MessageText>();
|
||||
msc.registerSerializerJson<Message::Components::TagMessageIsAction>();
|
||||
msc.registerDeSerializerJson<Message::Components::TagMessageIsAction>();
|
||||
|
||||
// files
|
||||
//_sc.registerSerializerJson<Message::Components::Transfer::FileID>()
|
||||
//_sc.registerSerializerJson<Message::Components::Transfer::FileInfo>();
|
||||
//_sc.registerDeSerializerJson<Message::Components::Transfer::FileInfo>();
|
||||
//_sc.registerSerializerJson<Message::Components::Transfer::FileInfoLocal>();
|
||||
//_sc.registerDeSerializerJson<Message::Components::Transfer::FileInfoLocal>();
|
||||
//_sc.registerSerializerJson<Message::Components::Transfer::TagHaveAll>();
|
||||
//_sc.registerDeSerializerJson<Message::Components::Transfer::TagHaveAll>();
|
||||
}
|
||||
|
@ -0,0 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include "./message_serializer.hpp"
|
||||
|
||||
void registerMFSJsonMessageComponents(MessageSerializerCallbacks& msc);
|
||||
|
@ -0,0 +1,10 @@
|
||||
#include "./register_mfs_json_message_components.hpp"
|
||||
|
||||
#include "./message_serializer.hpp"
|
||||
#include "../json/tox_message_components.hpp"
|
||||
|
||||
void registerMFSJsonToxMessageComponents(MessageSerializerCallbacks& msc) {
|
||||
msc.registerSerializerJson<Message::Components::ToxGroupMessageID>();
|
||||
msc.registerDeSerializerJson<Message::Components::ToxGroupMessageID>();
|
||||
}
|
||||
|
@ -0,0 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include "./message_serializer.hpp"
|
||||
|
||||
void registerMFSJsonToxMessageComponents(MessageSerializerCallbacks& msc);
|
||||
|
68
src/fragment_store/serializer.hpp
Normal file
68
src/fragment_store/serializer.hpp
Normal file
@ -0,0 +1,68 @@
|
||||
#pragma once
|
||||
|
||||
#include <entt/core/type_info.hpp>
|
||||
#include <entt/container/dense_map.hpp>
|
||||
#include <entt/entity/handle.hpp>
|
||||
|
||||
#include <nlohmann/json_fwd.hpp>
|
||||
|
||||
template<typename EntityType = entt::entity>
|
||||
struct SerializerCallbacks {
|
||||
using Registry = entt::basic_registry<EntityType>;
|
||||
using Handle = entt::basic_handle<Registry>;
|
||||
|
||||
// nlohmann
|
||||
// json/msgpack
|
||||
using serialize_json_fn = bool(*)(const Handle h, nlohmann::json& out);
|
||||
entt::dense_map<entt::id_type, serialize_json_fn> _serl_json;
|
||||
|
||||
using deserialize_json_fn = bool(*)(Handle h, const nlohmann::json& in);
|
||||
entt::dense_map<entt::id_type, deserialize_json_fn> _deserl_json;
|
||||
|
||||
template<typename T>
|
||||
static bool component_get_json(const Handle h, nlohmann::json& j) {
|
||||
if (h.template all_of<T>()) {
|
||||
if constexpr (!std::is_empty_v<T>) {
|
||||
j = h.template get<T>();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
static bool component_emplace_or_replace_json(Handle h, const nlohmann::json& j) {
|
||||
if constexpr (std::is_empty_v<T>) {
|
||||
h.template emplace_or_replace<T>(); // assert empty json?
|
||||
} else {
|
||||
h.template emplace_or_replace<T>(static_cast<T>(j));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info) {
|
||||
_serl_json[type_info.hash()] = fn;
|
||||
}
|
||||
|
||||
template<typename CompType>
|
||||
void registerSerializerJson(
|
||||
serialize_json_fn fn = component_get_json<CompType>,
|
||||
const entt::type_info& type_info = entt::type_id<CompType>()
|
||||
) {
|
||||
registerSerializerJson(fn, type_info);
|
||||
}
|
||||
|
||||
void registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info) {
|
||||
_deserl_json[type_info.hash()] = fn;
|
||||
}
|
||||
|
||||
template<typename CompType>
|
||||
void registerDeSerializerJson(
|
||||
deserialize_json_fn fn = component_emplace_or_replace_json<CompType>,
|
||||
const entt::type_info& type_info = entt::type_id<CompType>()
|
||||
) {
|
||||
registerDeSerializerJson(fn, type_info);
|
||||
}
|
||||
};
|
||||
|
80
src/fragment_store/test_fragstore.cpp
Normal file
80
src/fragment_store/test_fragstore.cpp
Normal file
@ -0,0 +1,80 @@
|
||||
#include <cstdint>
|
||||
#include <iostream>
|
||||
|
||||
#include "./fragment_store.hpp"
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
#include <entt/entity/handle.hpp>
|
||||
|
||||
namespace Components {
|
||||
struct MessagesTimestampRange {
|
||||
uint64_t begin {0};
|
||||
uint64_t end {1000};
|
||||
};
|
||||
|
||||
NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessagesTimestampRange, begin, end)
|
||||
} // Components
|
||||
|
||||
|
||||
int main(void) {
|
||||
FragmentStore fs;
|
||||
fs._default_store_path = "test_store/";
|
||||
fs._sc.registerSerializerJson<Components::MessagesTimestampRange>();
|
||||
fs._sc.registerDeSerializerJson<Components::MessagesTimestampRange>();
|
||||
|
||||
const auto frag0 = fs.newFragmentFile("", MetaFileType::TEXT_JSON, {0xff, 0xf1, 0xf2, 0xf0, 0xff, 0xff, 0xff, 0xf9});
|
||||
|
||||
const auto frag1 = fs.newFragmentFile("", MetaFileType::BINARY_MSGPACK);
|
||||
|
||||
const auto frag2 = fs.newFragmentFile("", MetaFileType::BINARY_MSGPACK);
|
||||
|
||||
{
|
||||
auto frag0h = fs.fragmentHandle(frag0);
|
||||
|
||||
frag0h.emplace_or_replace<FragComp::DataCompressionType>();
|
||||
frag0h.emplace_or_replace<FragComp::DataEncryptionType>();
|
||||
frag0h.emplace_or_replace<Components::MessagesTimestampRange>();
|
||||
|
||||
std::function<FragmentStore::write_to_storage_fetch_data_cb> fn_cb = [read = 0ul](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t {
|
||||
uint64_t i = 0;
|
||||
for (; i+read < 3000 && i < buffer_size; i++) {
|
||||
request_buffer[i] = uint8_t((i+read) & 0xff);
|
||||
}
|
||||
read += i;
|
||||
|
||||
return i;
|
||||
};
|
||||
fs.syncToStorage(frag0, fn_cb);
|
||||
}
|
||||
|
||||
{
|
||||
auto frag1h = fs.fragmentHandle(frag1);
|
||||
|
||||
frag1h.emplace_or_replace<FragComp::DataCompressionType>();
|
||||
frag1h.emplace_or_replace<FragComp::DataEncryptionType>();
|
||||
|
||||
std::function<FragmentStore::write_to_storage_fetch_data_cb> fn_cb = [read = 0ul](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t {
|
||||
static constexpr std::string_view text = "This is some random data";
|
||||
uint64_t i = 0;
|
||||
for (; i+read < text.size() && i < buffer_size; i++) {
|
||||
request_buffer[i] = text[i+read];
|
||||
}
|
||||
read += i;
|
||||
|
||||
return i;
|
||||
};
|
||||
fs.syncToStorage(frag1, fn_cb);
|
||||
}
|
||||
|
||||
{
|
||||
auto frag2h = fs.fragmentHandle(frag2);
|
||||
|
||||
frag2h.emplace_or_replace<FragComp::DataCompressionType>();
|
||||
frag2h.emplace_or_replace<FragComp::DataEncryptionType>();
|
||||
|
||||
static constexpr std::string_view text = "This is more random data";
|
||||
fs.syncToStorage(frag2, reinterpret_cast<const uint8_t*>(text.data()), text.size());
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
19
src/fragment_store/types.hpp
Normal file
19
src/fragment_store/types.hpp
Normal file
@ -0,0 +1,19 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
enum class Encryption : uint8_t {
|
||||
NONE = 0x00,
|
||||
};
|
||||
enum class Compression : uint8_t {
|
||||
NONE = 0x00,
|
||||
ZSTD = 0x01,
|
||||
// TODO: zstd without magic
|
||||
// TODO: zstd meta dict
|
||||
// TODO: zstd data(message) dict
|
||||
};
|
||||
enum class MetaFileType : uint8_t {
|
||||
TEXT_JSON,
|
||||
BINARY_MSGPACK, // msgpacked msgpack
|
||||
};
|
||||
|
91
src/image_loader_qoi.cpp
Normal file
91
src/image_loader_qoi.cpp
Normal file
@ -0,0 +1,91 @@
|
||||
#include "./image_loader_qoi.hpp"
|
||||
|
||||
#include <cstdint>
|
||||
#include <qoi/qoi.h>
|
||||
|
||||
#include <iostream>
|
||||
|
||||
ImageLoaderQOI::ImageInfo ImageLoaderQOI::loadInfoFromMemory(const uint8_t* data, uint64_t data_size) {
|
||||
ImageInfo res;
|
||||
|
||||
qoi_desc desc;
|
||||
// TODO: only read the header
|
||||
auto* ret = qoi_decode(data, data_size, &desc, 4);
|
||||
if (ret == nullptr) {
|
||||
return res;
|
||||
}
|
||||
free(ret);
|
||||
|
||||
res.width = desc.width;
|
||||
res.height = desc.height;
|
||||
//desc.colorspace;
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
ImageLoaderQOI::ImageResult ImageLoaderQOI::loadFromMemoryRGBA(const uint8_t* data, uint64_t data_size) {
|
||||
ImageResult res;
|
||||
|
||||
qoi_desc desc;
|
||||
|
||||
uint8_t* img_data = static_cast<uint8_t*>(
|
||||
qoi_decode(data, data_size, &desc, 4)
|
||||
);
|
||||
if (img_data == nullptr) {
|
||||
// not readable
|
||||
return res;
|
||||
}
|
||||
|
||||
res.width = desc.width;
|
||||
res.height = desc.height;
|
||||
|
||||
auto& new_frame = res.frames.emplace_back();
|
||||
new_frame.ms = 0;
|
||||
new_frame.data.insert(new_frame.data.cbegin(), img_data, img_data+(desc.width*desc.height*4));
|
||||
|
||||
free(img_data);
|
||||
return res;
|
||||
}
|
||||
|
||||
std::vector<uint8_t> ImageEncoderQOI::encodeToMemoryRGBA(const ImageResult& input_image, const std::map<std::string, float>&) {
|
||||
if (input_image.frames.empty()) {
|
||||
std::cerr << "IEQOI error: empty image\n";
|
||||
return {};
|
||||
}
|
||||
|
||||
if (input_image.frames.size() > 1) {
|
||||
std::cerr << "IEQOI warning: image with animation, only first frame will be encoded!\n";
|
||||
return {};
|
||||
}
|
||||
|
||||
// TODO: look into RDO (eg https://github.com/richgel999/rdopng)
|
||||
//int png_compression_level = 8;
|
||||
//if (extra_options.count("png_compression_level")) {
|
||||
//png_compression_level = extra_options.at("png_compression_level");
|
||||
//}
|
||||
|
||||
qoi_desc desc;
|
||||
desc.width = input_image.width;
|
||||
desc.height = input_image.height;
|
||||
desc.channels = 4;
|
||||
desc.colorspace = QOI_SRGB; // TODO: decide
|
||||
|
||||
int out_len {0};
|
||||
uint8_t* enc_data = static_cast<uint8_t*>(qoi_encode(
|
||||
input_image.frames.front().data.data(),
|
||||
&desc,
|
||||
&out_len
|
||||
));
|
||||
|
||||
if (enc_data == nullptr) {
|
||||
std::cerr << "IEQOI error: qoi_encode failed!\n";
|
||||
return {};
|
||||
}
|
||||
|
||||
std::vector<uint8_t> new_data(enc_data, enc_data+out_len);
|
||||
|
||||
free(enc_data); // TODO: a streaming encoder would be better
|
||||
|
||||
return new_data;
|
||||
}
|
||||
|
13
src/image_loader_qoi.hpp
Normal file
13
src/image_loader_qoi.hpp
Normal file
@ -0,0 +1,13 @@
|
||||
#pragma once
|
||||
|
||||
#include "./image_loader.hpp"
|
||||
|
||||
struct ImageLoaderQOI : public ImageLoaderI {
|
||||
ImageInfo loadInfoFromMemory(const uint8_t* data, uint64_t data_size) override;
|
||||
ImageResult loadFromMemoryRGBA(const uint8_t* data, uint64_t data_size) override;
|
||||
};
|
||||
|
||||
struct ImageEncoderQOI : public ImageEncoderI {
|
||||
std::vector<uint8_t> encodeToMemoryRGBA(const ImageResult& input_image, const std::map<std::string, float>& extra_options = {}) override;
|
||||
};
|
||||
|
27
src/json/message_components.hpp
Normal file
27
src/json/message_components.hpp
Normal file
@ -0,0 +1,27 @@
|
||||
#pragma once
|
||||
|
||||
#include <solanaceae/util/utils.hpp>
|
||||
|
||||
#include <solanaceae/message3/components.hpp>
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
namespace Message::Components {
|
||||
|
||||
NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Timestamp, ts)
|
||||
NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(TimestampProcessed, ts)
|
||||
NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(TimestampWritten, ts)
|
||||
NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(ContactFrom, c)
|
||||
NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(ContactTo, c)
|
||||
NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Read, ts)
|
||||
// TODO: SyncedBy
|
||||
NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessageText, text)
|
||||
|
||||
namespace Transfer {
|
||||
NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(FileInfo::FileDirEntry, file_name, file_size)
|
||||
NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(FileInfo, file_list, total_size)
|
||||
NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(FileInfoLocal, file_list)
|
||||
} // Transfer
|
||||
|
||||
} // Message::Components
|
||||
|
16
src/json/tox_message_components.hpp
Normal file
16
src/json/tox_message_components.hpp
Normal file
@ -0,0 +1,16 @@
|
||||
#pragma once
|
||||
|
||||
#include <solanaceae/util/utils.hpp>
|
||||
|
||||
#include <solanaceae/tox_messages/components.hpp>
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
namespace Message::Components {
|
||||
|
||||
// TODO: friend msg id, does not have the same qualities
|
||||
NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(ToxGroupMessageID, id)
|
||||
// TODO: transfer stuff, needs content rewrite
|
||||
|
||||
} // Message::Components
|
||||
|
@ -51,6 +51,12 @@ int main(int argc, char** argv) {
|
||||
std::cerr << "SDL_CreateRenderer failed (" << SDL_GetError() << ")\n";
|
||||
return 1;
|
||||
}
|
||||
{
|
||||
SDL_RendererInfo ri;
|
||||
if (SDL_GetRendererInfo(renderer.get(), &ri) == 0) {
|
||||
std::cout << "SDL Renderer: " << ri.name << "(f:" << ri.flags << ")\n";
|
||||
}
|
||||
}
|
||||
|
||||
IMGUI_CHECKVERSION();
|
||||
ImGui::CreateContext();
|
||||
|
@ -1,5 +1,8 @@
|
||||
#include "./main_screen.hpp"
|
||||
|
||||
#include "./fragment_store/register_mfs_json_message_components.hpp"
|
||||
#include "./fragment_store/register_mfs_json_tox_message_components.hpp"
|
||||
|
||||
#include <solanaceae/contact/components.hpp>
|
||||
|
||||
#include <imgui/imgui.h>
|
||||
@ -13,6 +16,7 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri
|
||||
renderer(renderer_),
|
||||
rmm(cr),
|
||||
mts(rmm),
|
||||
mfs(cr, rmm, fs),
|
||||
tc(save_path, save_password),
|
||||
tpi(tc.getTox()),
|
||||
ad(tc),
|
||||
@ -33,6 +37,8 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri
|
||||
tdch(tpi)
|
||||
{
|
||||
tel.subscribeAll(tc);
|
||||
registerMFSJsonMessageComponents(mfs.getMSC());
|
||||
registerMFSJsonToxMessageComponents(mfs.getMSC());
|
||||
|
||||
conf.set("tox", "save_file_path", save_path);
|
||||
|
||||
@ -49,6 +55,10 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri
|
||||
std::cout << "own address: " << tc.toxSelfGetAddressStr() << "\n";
|
||||
|
||||
{ // setup plugin instances
|
||||
// TODO: make interface useful
|
||||
g_provideInstance<FragmentStoreI>("FragmentStoreI", "host", &fs);
|
||||
g_provideInstance<FragmentStore>("FragmentStore", "host", &fs);
|
||||
|
||||
g_provideInstance<ConfigModelI>("ConfigModelI", "host", &conf);
|
||||
g_provideInstance<Contact3Registry>("Contact3Registry", "1", "host", &cr);
|
||||
g_provideInstance<RegistryMessageModel>("RegistryMessageModel", "host", &rmm);
|
||||
@ -74,6 +84,8 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri
|
||||
}
|
||||
|
||||
conf.dump();
|
||||
|
||||
mfs.triggerScan(); // HACK: after plugins and tox contacts got loaded
|
||||
}
|
||||
|
||||
MainScreen::~MainScreen(void) {
|
||||
@ -388,7 +400,8 @@ Screen* MainScreen::tick(float time_delta, bool& quit) {
|
||||
|
||||
tdch.tick(time_delta); // compute
|
||||
|
||||
mts.iterate(); // compute
|
||||
const float mfs_interval = mfs.tick(time_delta);
|
||||
mts.iterate(); // compute (after mfs)
|
||||
|
||||
_min_tick_interval = std::min<float>(
|
||||
// HACK: pow by 1.6 to increase 50 -> ~500 (~522)
|
||||
@ -400,6 +413,10 @@ Screen* MainScreen::tick(float time_delta, bool& quit) {
|
||||
_min_tick_interval,
|
||||
fo_interval
|
||||
);
|
||||
_min_tick_interval = std::min<float>(
|
||||
_min_tick_interval,
|
||||
mfs_interval
|
||||
);
|
||||
|
||||
//std::cout << "MS: min tick interval: " << _min_tick_interval << "\n";
|
||||
|
||||
|
@ -2,10 +2,12 @@
|
||||
|
||||
#include "./screen.hpp"
|
||||
|
||||
#include "./fragment_store/fragment_store.hpp"
|
||||
#include <solanaceae/util/simple_config_model.hpp>
|
||||
#include <solanaceae/contact/contact_model3.hpp>
|
||||
#include <solanaceae/message3/registry_message_model.hpp>
|
||||
#include <solanaceae/message3/message_time_sort.hpp>
|
||||
#include "./fragment_store/message_fragment_store.hpp"
|
||||
#include <solanaceae/plugin/plugin_manager.hpp>
|
||||
#include <solanaceae/toxcore/tox_event_logger.hpp>
|
||||
#include "./tox_private_impl.hpp"
|
||||
@ -43,10 +45,13 @@ extern "C" {
|
||||
struct MainScreen final : public Screen {
|
||||
SDL_Renderer* renderer;
|
||||
|
||||
FragmentStore fs;
|
||||
|
||||
SimpleConfigModel conf;
|
||||
Contact3Registry cr;
|
||||
RegistryMessageModel rmm;
|
||||
MessageTimeSort mts;
|
||||
MessageFragmentStore mfs;
|
||||
|
||||
PluginManager pm;
|
||||
|
||||
|
@ -2,6 +2,7 @@
|
||||
|
||||
#include "./image_loader_webp.hpp"
|
||||
#include "./image_loader_sdl_bmp.hpp"
|
||||
#include "./image_loader_qoi.hpp"
|
||||
#include "./image_loader_stb.hpp"
|
||||
|
||||
#include <solanaceae/message3/components.hpp>
|
||||
@ -77,6 +78,7 @@ MediaMetaInfoLoader::MediaMetaInfoLoader(RegistryMessageModel& rmm) : _rmm(rmm)
|
||||
// HACK: make them be added externally?
|
||||
_image_loaders.push_back(std::make_unique<ImageLoaderWebP>());
|
||||
_image_loaders.push_back(std::make_unique<ImageLoaderSDLBMP>());
|
||||
_image_loaders.push_back(std::make_unique<ImageLoaderQOI>());
|
||||
_image_loaders.push_back(std::make_unique<ImageLoaderSTB>());
|
||||
|
||||
_rmm.subscribe(this, RegistryMessageModel_Event::message_construct);
|
||||
|
@ -1,6 +1,7 @@
|
||||
#include "./message_image_loader.hpp"
|
||||
|
||||
#include "./image_loader_sdl_bmp.hpp"
|
||||
#include "./image_loader_qoi.hpp"
|
||||
#include "./image_loader_stb.hpp"
|
||||
#include "./image_loader_webp.hpp"
|
||||
#include "./media_meta_info_loader.hpp"
|
||||
@ -19,6 +20,7 @@ uint64_t getTimeMS(void);
|
||||
|
||||
MessageImageLoader::MessageImageLoader(void) {
|
||||
_image_loaders.push_back(std::make_unique<ImageLoaderSDLBMP>());
|
||||
_image_loaders.push_back(std::make_unique<ImageLoaderQOI>());
|
||||
_image_loaders.push_back(std::make_unique<ImageLoaderWebP>());
|
||||
_image_loaders.push_back(std::make_unique<ImageLoaderSTB>());
|
||||
}
|
||||
|
@ -3,6 +3,7 @@
|
||||
#include "./image_loader_sdl_bmp.hpp"
|
||||
#include "./image_loader_stb.hpp"
|
||||
#include "./image_loader_webp.hpp"
|
||||
#include "./image_loader_qoi.hpp"
|
||||
|
||||
#include <imgui/imgui.h>
|
||||
|
||||
@ -13,6 +14,7 @@ uint64_t getTimeMS(void);
|
||||
|
||||
SendImagePopup::SendImagePopup(TextureUploaderI& tu) : _tu(tu) {
|
||||
_image_loaders.push_back(std::make_unique<ImageLoaderSDLBMP>());
|
||||
_image_loaders.push_back(std::make_unique<ImageLoaderQOI>());
|
||||
_image_loaders.push_back(std::make_unique<ImageLoaderWebP>());
|
||||
_image_loaders.push_back(std::make_unique<ImageLoaderSTB>());
|
||||
}
|
||||
@ -421,7 +423,7 @@ void SendImagePopup::render(float time_delta) {
|
||||
|
||||
if (compress) {
|
||||
ImGui::SameLine();
|
||||
ImGui::Combo("##compression_type", ¤t_compressor, "webp\0jpeg\0png\n");
|
||||
ImGui::Combo("##compression_type", ¤t_compressor, "webp\0jpeg\0png\0qoi\0");
|
||||
|
||||
ImGui::Indent();
|
||||
// combo "webp""webp-lossless""png""jpg?"
|
||||
@ -486,6 +488,11 @@ void SendImagePopup::render(float time_delta) {
|
||||
if (!new_data.empty()) {
|
||||
_on_send(new_data, ".png");
|
||||
}
|
||||
} else if (current_compressor == 3) {
|
||||
new_data = ImageEncoderQOI{}.encodeToMemoryRGBA(tmp_img, {});;
|
||||
if (!new_data.empty()) {
|
||||
_on_send(new_data, ".qoi");
|
||||
}
|
||||
}
|
||||
// error
|
||||
|
||||
|
@ -1,6 +1,7 @@
|
||||
#include "./tox_avatar_loader.hpp"
|
||||
|
||||
#include "./image_loader_sdl_bmp.hpp"
|
||||
#include "./image_loader_qoi.hpp"
|
||||
#include "./image_loader_stb.hpp"
|
||||
#include "./image_loader_webp.hpp"
|
||||
|
||||
@ -21,6 +22,7 @@ uint64_t getTimeMS(void);
|
||||
|
||||
ToxAvatarLoader::ToxAvatarLoader(Contact3Registry& cr) : _cr(cr) {
|
||||
_image_loaders.push_back(std::make_unique<ImageLoaderSDLBMP>());
|
||||
_image_loaders.push_back(std::make_unique<ImageLoaderQOI>());
|
||||
_image_loaders.push_back(std::make_unique<ImageLoaderWebP>());
|
||||
_image_loaders.push_back(std::make_unique<ImageLoaderSTB>());
|
||||
}
|
||||
|
@ -120,7 +120,8 @@ ToxFriendFauxOfflineMessaging::dfmc_Ret ToxFriendFauxOfflineMessaging::doFriendM
|
||||
// require
|
||||
if (!mr->all_of<
|
||||
Message::Components::MessageText, // text only for now
|
||||
Message::Components::ContactTo
|
||||
Message::Components::ContactTo,
|
||||
Message::Components::ToxFriendMessageID // yes, needs fake ids
|
||||
>(msg)
|
||||
) {
|
||||
continue; // skip
|
||||
|
Loading…
Reference in New Issue
Block a user