refactor saving and save on exit

save alot of memory by using fid instead of fuid

CMakeLists.txt

@@ -23,8 +23,8 @@ option(TOMATO_ASAN "Build tomato with asan (gcc/clang/msvc)" OFF)
 if (TOMATO_ASAN)
 	if (${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU" OR ${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang")
 		if (NOT WIN32) # exclude mingw
-			link_libraries(-fsanitize=address)
+			#link_libraries(-fsanitize=address)
-			#link_libraries(-fsanitize=address,undefined)
+			link_libraries(-fsanitize=address,undefined)
 			#link_libraries(-fsanitize=undefined)
 			message("II enabled ASAN")
 		else()

external/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.9 FATAL_ERROR)
+cmake_minimum_required(VERSION 3.14...3.24 FATAL_ERROR)
 
 add_subdirectory(./entt)
 
@@ -17,4 +17,36 @@ add_subdirectory(./imgui)
 
 add_subdirectory(./stb)
 add_subdirectory(./libwebp)
+add_subdirectory(./qoi)
 
+if (NOT TARGET nlohmann_json::nlohmann_json)
+	FetchContent_Declare(json
+		URL https://github.com/nlohmann/json/releases/download/v3.11.3/json.tar.xz
+		URL_HASH SHA256=d6c65aca6b1ed68e7a182f4757257b107ae403032760ed6ef121c9d55e81757d
+		EXCLUDE_FROM_ALL
+	)
+	FetchContent_MakeAvailable(json)
+endif()
+
+if (NOT TARGET zstd::zstd)
+	# TODO: try find_package() first
+	# TODO: try pkg-config next (will work on most distros)
+
+	set(ZSTD_BUILD_STATIC ON)
+	set(ZSTD_BUILD_SHARED OFF)
+	set(ZSTD_BUILD_PROGRAMS OFF)
+	set(ZSTD_BUILD_CONTRIB OFF)
+	set(ZSTD_BUILD_TESTS OFF)
+	FetchContent_Declare(zstd
+		URL "https://github.com/facebook/zstd/releases/download/v1.5.5/zstd-1.5.5.tar.gz"
+		DOWNLOAD_EXTRACT_TIMESTAMP TRUE
+		SOURCE_SUBDIR build/cmake
+		EXCLUDE_FROM_ALL
+	)
+	FetchContent_MakeAvailable(zstd)
+
+	add_library(zstd INTERFACE) # somehow zstd fkd this up
+	target_include_directories(zstd INTERFACE ${zstd_SOURCE_DIR}/lib/)
+	target_link_libraries(zstd INTERFACE libzstd_static)
+	add_library(zstd::zstd ALIAS zstd)
+endif()

external/qoi/CMakeLists.txt

@@ -0,0 +1,11 @@
+cmake_minimum_required(VERSION 3.13...3.24 FATAL_ERROR)
+
+project(qoi C CXX)
+
+add_library(qoi_interface INTERFACE)
+target_include_directories(qoi_interface SYSTEM INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}")
+
+# static lib with impl
+add_library(qoi "qoi.cpp")
+target_link_libraries(qoi qoi_interface)
+

external/qoi/qoi.cpp

@@ -0,0 +1,3 @@
+#define QOI_IMPLEMENTATION
+#include <qoi/qoi.h>
+

external/qoi/qoi/.gitignore

@@ -0,0 +1,5 @@
+images/
+stb_image.h
+stb_image_write.h
+qoibench
+qoiconv

external/qoi/qoi/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Dominic Szablewski
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

external/qoi/qoi/Makefile

@@ -0,0 +1,22 @@
+CC ?= gcc
+CFLAGS_BENCH ?= -std=gnu99 -O3
+LFLAGS_BENCH ?= -lpng
+CFLAGS_CONV ?= -std=c99 -O3
+
+TARGET_BENCH ?= qoibench
+TARGET_CONV ?= qoiconv
+
+all: $(TARGET_BENCH) $(TARGET_CONV)
+
+bench: $(TARGET_BENCH)
+
+$(TARGET_BENCH):$(TARGET_BENCH).c
+	$(CC) $(CFLAGS_BENCH) $(CFLAGS) $(TARGET_BENCH).c -o $(TARGET_BENCH) $(LFLAGS_BENCH)
+
+conv: $(TARGET_CONV)
+$(TARGET_CONV):$(TARGET_CONV).c
+	$(CC) $(CFLAGS_CONV) $(CFLAGS) $(TARGET_CONV).c -o $(TARGET_CONV)
+
+.PHONY: clean
+clean:
+	$(RM) $(TARGET_BENCH) $(TARGET_CONV)

external/qoi/qoi/README.md

@@ -0,0 +1,179 @@
+![QOI Logo](https://qoiformat.org/qoi-logo.svg)
+
+# QOI - The “Quite OK Image Format” for fast, lossless image compression
+
+Single-file MIT licensed library for C/C++
+
+See [qoi.h](https://github.com/phoboslab/qoi/blob/master/qoi.h) for
+the documentation and format specification.
+
+More info at https://qoiformat.org
+
+
+## Why?
+
+Compared to stb_image and stb_image_write QOI offers 20x-50x faster encoding,
+3x-4x faster decoding and 20% better compression. It's also stupidly simple and
+fits in about 300 lines of C.
+
+
+## Example Usage
+
+- [qoiconv.c](https://github.com/phoboslab/qoi/blob/master/qoiconv.c)
+converts between png <> qoi
+ - [qoibench.c](https://github.com/phoboslab/qoi/blob/master/qoibench.c)
+a simple wrapper to benchmark stbi, libpng and qoi
+
+
+## MIME Type, File Extension
+
+The recommended MIME type for QOI images is `image/qoi`. While QOI is not yet
+officially registered with IANA, I believe QOI has found enough adoption to
+prevent any future image format from choosing the same name, thus making a 
+MIME type collision highly unlikely ([see #167](https://github.com/phoboslab/qoi/issues/167)).
+
+The recommended file extension for QOI images is `.qoi`
+
+
+## Limitations
+
+The QOI file format allows for huge images with up to 18 exa-pixels. A streaming 
+en-/decoder can handle these with minimal RAM requirements, assuming there is 
+enough storage space.
+
+This particular implementation of QOI however is limited to images with a 
+maximum size of 400 million pixels. It will safely refuse to en-/decode anything
+larger than that. This is not a streaming en-/decoder. It loads the whole image
+file into RAM before doing any work and is not extensively optimized for 
+performance (but it's still very fast).
+
+If this is a limitation for your use case, please look into any of the other 
+implementations listed below.
+
+
+## Improvements, New Versions and Contributing
+
+The QOI format has been finalized. It was a conscious decision to **not** have a
+version number in the file header. If you have a working QOI implementation today, 
+you can rest assured that it will be compatible with all QOI files tomorrow.
+
+There are a lot of interesting ideas for a successor of QOI, but none of these will 
+be implemented here. That doesn't mean you shouldn't experiment with QOI, but please
+be aware that pull requests that change the format will not be accepted.
+
+Likewise, pull requests for performance improvements will probably not be accepted
+either, as this "reference implementation" tries to be as easy to read as possible.
+
+
+## Tools
+
+- [floooh/qoiview](https://github.com/floooh/qoiview) - native QOI viewer
+- [pfusik/qoi-fu](https://github.com/pfusik/qoi-fu/releases) - QOI Plugin installer for Windows Explorer, Finder, GNOME, GIMP, Paint.NET and XnView
+- [iOrange/QoiFileTypeNet](https://github.com/iOrange/QoiFileTypeNet/releases) - QOI Plugin for Paint.NET
+- [iOrange/QOIThumbnailProvider](https://github.com/iOrange/QOIThumbnailProvider) - Add thumbnails for QOI images in Windows Explorer
+- [Tom94/tev](https://github.com/Tom94/tev) - another native QOI viewer (allows pixel peeping and comparison with other image formats)
+- [qoiconverterx](https://apps.apple.com/br/app/qoiconverterx/id1602159820) QOI <=> PNG converter available on the Mac App Store
+- [kaetemi/qoi-ma](https://github.com/kaetemi/qoi-max) - QOI Bitmap I/O Plugin for 3ds Max
+- [rtexviewer](https://raylibtech.itch.io/rtexviewer) - texture viewer, supports QOI
+- [rtexpacker](https://raylibtech.itch.io/rtexpacker) - texture packer, supports QOI
+- [DmitriySalnikov/godot_qoi](https://github.com/DmitriySalnikov/godot_qoi) - QOI GDNative Addon for Godot Engine
+- [dan9er/farbfeld-convert-qoi](https://gitlab.com/dan9er/farbfeld-convert-qoi) - QOI <=> farbfeld converter
+- [LTMX/Unity.QOI](https://github.com/LTMX/Unity.QOI) - QOI Importer and Exporter for the Unity3D Game Engine
+- [Ben1138/unity-qoi](https://github.com/Ben1138/unity-qoi) - QOI Importer(only) support for the Unity3D Game Engine
+- [xiaozhuai/jetbrains-qo](https://github.com/xiaozhuai/jetbrains-qoi) - [QOI Support](https://plugins.jetbrains.com/plugin/19352-qoi-support) for Jetbrains' IDE.
+- [serge-ivamov/QOIql](https://github.com/serge-ivamov/QOIql) - MacOS QuickLook plugin for QOI
+- [tobozo/kde-thumbnailer-qoi](https://github.com/tobozo/kde-thumbnailer-qoi) - QOI Thumbnailer for KDE
+- [walksanatora/qoi-thumbnailer-nemo](https://github.com/walksanatora/qoi-thumbnailer-nemo) - QOI Thumbnailer for Nemo
+- [hzeller/timg](https://github.com/hzeller/timg) - a terminal image viewer with QOI support
+- [LuisAlfredo92/Super-QOI-converter](https://github.com/LuisAlfredo92/Super-QOI-converter "LuisAlfredo92/Super-QOI-converter") - A program to convert JPG, JPEG, BMP, and PNG to QOI
+	- [Console version](https://github.com/LuisAlfredo92/Super-QOI-converter-Console- "Console version"): Available for Linux, OSX and Windows
+	- [GUI version](https://github.com/LuisAlfredo92/Super-QOI-converter-GUI- "GUI version"): Available only for windows
+- [tacent view](https://github.com/bluescan/tacentview) - Image and texture viewer, supports QOI
+- [colemanrgb/qoi2spr](https://github.com/colemanrgb/qoi2spr) - A variety of applications for decoding and encoding of QOI images on [RISC OS](https://www.riscosopen.org/)
+
+## Implementations & Bindings of QOI
+
+- [pfusik/qoi-fu](https://github.com/pfusik/qoi-fu) - Fusion, transpiling to C, C++, C#, D, Java, JavaScript, Python, Swift and TypeScript
+- [kodonnell/qoi](https://github.com/kodonnell/qoi) - Python
+- [JaffaKetchup/dqoi](https://github.com/JaffaKetchup/dqoi) - Dart, with Flutter support
+- [Cr4xy/lua-qoi](https://github.com/Cr4xy/lua-qoi) - Lua
+- [superzazu/SDL_QOI](https://github.com/superzazu/SDL_QOI) - C, SDL2 bindings
+- [saharNooby/qoi-java](https://github.com/saharNooby/qoi-java) - Java
+- [MasterQ32/zig-qoi](https://github.com/MasterQ32/zig-qoi) - Zig
+- [rbino/qoix](https://github.com/rbino/qoix) - Elixir
+- [NUlliiON/QoiSharp](https://github.com/NUlliiON/QoiSharp) - C#
+- [aldanor/qoi-rust](https://github.com/aldanor/qoi-rust) - Rust
+- [zakarumych/rapid-qoi](https://github.com/zakarumych/rapid-qoi) - Rust
+- [takeyourhatoff/qoi](https://github.com/takeyourhatoff/qoi) - Go
+- [DosWorld/pasqoi](https://github.com/DosWorld/pasqoi) - Pascal
+- [elihwyma/Swift-QOI](https://github.com/elihwyma/Swift-QOI) - Swift
+- [xfmoulet/qoi](https://github.com/xfmoulet/qoi) - Go
+- [erratique.ch/qoic](https://erratique.ch/software/qoic) - OCaml
+- [arian/go-qoi](https://github.com/arian/go-qoi) - Go
+- [kchapelier/qoijs](https://github.com/kchapelier/qoijs) - JavaScript
+- [KristofferC/QOI.jl](https://github.com/KristofferC/QOI.jl) - Julia
+- [shadowMitia/libqoi](https://github.com/shadowMitia/libqoi) - C++
+- [MKCG/php-qoi](https://github.com/MKCG/php-qoi) - PHP
+- [LightHouseSoftware/qoiformats](https://github.com/LightHouseSoftware/qoiformats) - D
+- [mhoward540/qoi-nim](https://github.com/mhoward540/qoi-nim) - Nim
+- [wx257osn2/qoixx](https://github.com/wx257osn2/qoixx) - C++
+- [Tiefseetauchner/lr-paint](https://github.com/Tiefseetauchner/lr-paint) - Processing
+- [amstan/qoi-fpga](https://github.com/amstan/qoi-fpga) - FPGA: verilog
+- [musabkilic/qoi-decoder](https://github.com/musabkilic/qoi-decoder) - Python
+- [mathpn/py-qoi](https://github.com/mathpn/py-qoi) - Python
+- [JohannesFriedrich/qoi4R](https://github.com/JohannesFriedrich/qoi4R) - R
+- [shraiwi/mini-qoi](https://github.com/shraiwi/mini-qoi) - C, streaming decoder
+- [10maurycy10/libqoi/](https://github.com/10maurycy10/libqoi/) - Rust
+- [0xd34df00d/hsqoi](https://github.com/0xd34df00d/hsqoi) - Haskell
+- [418Coffee/qoi-v](https://github.com/418Coffee/qoi-v) - V
+- [Imagine-Programming/QoiImagePlugin](https://github.com/Imagine-Programming/QoiImagePlugin) - PureBasic
+- [Fabien-Chouteau/qoi-spark](https://github.com/Fabien-Chouteau/qoi-spark) - Ada/SPARK formally proven
+- [mzgreen/qoi-kotlin](https://github.com/mzgreen/qoi-kotlin) - Kotlin Multiplatform
+- [Aftersol/Simplified-QOI-Codec](https://github.com/Aftersol/Simplified-QOI-Codec) - C99, encoder and decoder, freestanding
+- [AuburnSounds/gamut](https://github.com/AuburnSounds/gamut) - D
+- [AngusJohnson/TQoiImage](https://github.com/AngusJohnson/TQoiImage) - Delphi
+- [MarkJeronimus/qoi-java-spi](https://github.com/MarkJeronimus/qoi-java-spi) - Java SPI
+- [aumouvantsillage/qoi-racket](https://github.com/aumouvantsillage/qoi-racket) - Racket
+- [rubikscraft/qoi-stream](https://github.com/rubikscraft/qoi-stream) - C99, one byte at a time streaming encoder and decoder
+- [rubikscraft/qoi-img](https://github.com/rubikscraft/qoi-img) - NodeJS typescript, bindings to both [QOIxx](https://github.com/wx257osn2/qoixx) and [qoi-stream](https://github.com/rubikscraft/qoi-stream)
+- [grego/hare-qoi](https://git.sr.ht/~grego/hare-qoi) - Hare
+- [MrNocole/ZTQOI](https://github.com/MrNocole/ZTQOI) - Objective-C
+- [bpanthi977/qoi](https://github.com/bpanthi977/qoi) - Common Lisp
+- [Floessie/pam2qoi](https://github.com/Floessie/pam2qoi) - C++
+- [SpeckyYT/spwn-qoi](https://github.com/SpeckyYT/spwn-qoi) - SPWN
+- [n00bmind/qoi](https://github.com/n00bmind/qoi) - Jai
+- [SixLabors/ImageSharp](https://github.com/SixLabors/ImageSharp) - C# image proccesing library
+- [zertovitch/gid](https://github.com/zertovitch/gid) - Ada
+- [nazrin/lil](https://codeberg.org/nazrin/lil) - Lua image library
+
+## QOI Support in Other Software
+
+- [Amiga OS QOI datatype](https://github.com/dgaw/qoi-datatype) - adds support for decoding QOI images to the Amiga operating system.
+- [SerenityOS](https://github.com/SerenityOS/serenity) - supports decoding QOI system wide through a custom [cpp implementation in LibGfx](https://github.com/SerenityOS/serenity/blob/master/Userland/Libraries/LibGfx/QOILoader.h)
+- [Raylib](https://github.com/raysan5/raylib) - supports decoding and encoding QOI textures through its [rtextures module](https://github.com/raysan5/raylib/blob/master/src/rtextures.c)
+- [Rebol3](https://github.com/Oldes/Rebol3/issues/39) - supports decoding and encoding QOI using a native codec
+- [c-ray](https://github.com/vkoskiv/c-ray) - supports QOI natively
+- [SAIL](https://sail.software) - image decoding library, supports decoding and encoding QOI images
+- [Orx](https://github.com/orx/orx) - 2D game engine, supports QOI natively
+- [IrfanView](https://www.irfanview.com) - supports decoding and encoding QOI through its Formats plugin
+- [ImageMagick](https://github.com/ImageMagick/ImageMagick) - supports decoding and encoding QOI, since 7.1.0-20
+- [barebox](https://barebox.org) - bootloader, supports decoding QOI images for splash logo, since v2022.03.0
+- [KorGE](https://korge.org) - & KorIM Kotlin 2D game engine and imaging library, supports decoding and encoding QOI natively since 2.7.0
+- [DOjS](https://github.com/SuperIlu/DOjS) - DOS JavaScript Canvas implementation supports loading QOI files
+- [XnView MP](https://www.xnview.com/en/xnviewmp/) - supports decoding QOI since 1.00
+- [ffmpeg](https://ffmpeg.org/) - supports decoding and encoding QOI since 5.1
+- [JPEGView](https://github.com/sylikc/jpegview) - lightweight Windows image viewer, supports decoding and encoding of QOI natively, since 1.1.44
+- [darktable](https://github.com/darktable-org/darktable) - photography workflow application and raw developer, supports decoding since 4.4.0
+- [KDE](https://kde.org) - supports decoding and encoding QOI images. Implemented in [KImageFormats](https://invent.kde.org/frameworks/kimageformats)
+- [EFL](https://www.enlightenment.org) - supports decoding and encoding QOI images since 1.27.
+- [Swingland](https://git.sr.ht/~phlash/swingland) - supports QOI decoding/loading via the `ImageIO` API of this Java Swing reimplemenation for Wayland
+- [Imagine](https://www.nyam.pe.kr/dev/imagine/) - supports decoding and encoding QOI images since 1.3.9
+- [Uiua](https://uiua.org) - supports decoding and encoding QOI images since 0.8.0
+
+## Packages
+
+- [AUR](https://aur.archlinux.org/pkgbase/qoi-git/) - system-wide qoi.h, qoiconv and qoibench install as split packages.
+- [Debian](https://packages.debian.org/bookworm/source/qoi) - packages for binaries and qoi.h
+- [Ubuntu](https://launchpad.net/ubuntu/+source/qoi) - packages for binaries and qoi.h
+
+Packages for other systems [tracked at Repology](https://repology.org/project/qoi/versions).

external/qoi/qoi/qoi.h

@@ -0,0 +1,649 @@
+/*
+
+Copyright (c) 2021, Dominic Szablewski - https://phoboslab.org
+SPDX-License-Identifier: MIT
+
+
+QOI - The "Quite OK Image" format for fast, lossless image compression
+
+-- About
+
+QOI encodes and decodes images in a lossless format. Compared to stb_image and
+stb_image_write QOI offers 20x-50x faster encoding, 3x-4x faster decoding and
+20% better compression.
+
+
+-- Synopsis
+
+// Define `QOI_IMPLEMENTATION` in *one* C/C++ file before including this
+// library to create the implementation.
+
+#define QOI_IMPLEMENTATION
+#include "qoi.h"
+
+// Encode and store an RGBA buffer to the file system. The qoi_desc describes
+// the input pixel data.
+qoi_write("image_new.qoi", rgba_pixels, &(qoi_desc){
+	.width = 1920,
+	.height = 1080,
+	.channels = 4,
+	.colorspace = QOI_SRGB
+});
+
+// Load and decode a QOI image from the file system into a 32bbp RGBA buffer.
+// The qoi_desc struct will be filled with the width, height, number of channels
+// and colorspace read from the file header.
+qoi_desc desc;
+void *rgba_pixels = qoi_read("image.qoi", &desc, 4);
+
+
+
+-- Documentation
+
+This library provides the following functions;
+- qoi_read    -- read and decode a QOI file
+- qoi_decode  -- decode the raw bytes of a QOI image from memory
+- qoi_write   -- encode and write a QOI file
+- qoi_encode  -- encode an rgba buffer into a QOI image in memory
+
+See the function declaration below for the signature and more information.
+
+If you don't want/need the qoi_read and qoi_write functions, you can define
+QOI_NO_STDIO before including this library.
+
+This library uses malloc() and free(). To supply your own malloc implementation
+you can define QOI_MALLOC and QOI_FREE before including this library.
+
+This library uses memset() to zero-initialize the index. To supply your own
+implementation you can define QOI_ZEROARR before including this library.
+
+
+-- Data Format
+
+A QOI file has a 14 byte header, followed by any number of data "chunks" and an
+8-byte end marker.
+
+struct qoi_header_t {
+	char     magic[4];   // magic bytes "qoif"
+	uint32_t width;      // image width in pixels (BE)
+	uint32_t height;     // image height in pixels (BE)
+	uint8_t  channels;   // 3 = RGB, 4 = RGBA
+	uint8_t  colorspace; // 0 = sRGB with linear alpha, 1 = all channels linear
+};
+
+Images are encoded row by row, left to right, top to bottom. The decoder and
+encoder start with {r: 0, g: 0, b: 0, a: 255} as the previous pixel value. An
+image is complete when all pixels specified by width * height have been covered.
+
+Pixels are encoded as
+ - a run of the previous pixel
+ - an index into an array of previously seen pixels
+ - a difference to the previous pixel value in r,g,b
+ - full r,g,b or r,g,b,a values
+
+The color channels are assumed to not be premultiplied with the alpha channel
+("un-premultiplied alpha").
+
+A running array[64] (zero-initialized) of previously seen pixel values is
+maintained by the encoder and decoder. Each pixel that is seen by the encoder
+and decoder is put into this array at the position formed by a hash function of
+the color value. In the encoder, if the pixel value at the index matches the
+current pixel, this index position is written to the stream as QOI_OP_INDEX.
+The hash function for the index is:
+
+	index_position = (r * 3 + g * 5 + b * 7 + a * 11) % 64
+
+Each chunk starts with a 2- or 8-bit tag, followed by a number of data bits. The
+bit length of chunks is divisible by 8 - i.e. all chunks are byte aligned. All
+values encoded in these data bits have the most significant bit on the left.
+
+The 8-bit tags have precedence over the 2-bit tags. A decoder must check for the
+presence of an 8-bit tag first.
+
+The byte stream's end is marked with 7 0x00 bytes followed a single 0x01 byte.
+
+
+The possible chunks are:
+
+
+.- QOI_OP_INDEX ----------.
+|         Byte[0]         |
+|  7  6  5  4  3  2  1  0 |
+|-------+-----------------|
+|  0  0 |     index       |
+`-------------------------`
+2-bit tag b00
+6-bit index into the color index array: 0..63
+
+A valid encoder must not issue 2 or more consecutive QOI_OP_INDEX chunks to the
+same index. QOI_OP_RUN should be used instead.
+
+
+.- QOI_OP_DIFF -----------.
+|         Byte[0]         |
+|  7  6  5  4  3  2  1  0 |
+|-------+-----+-----+-----|
+|  0  1 |  dr |  dg |  db |
+`-------------------------`
+2-bit tag b01
+2-bit   red channel difference from the previous pixel between -2..1
+2-bit green channel difference from the previous pixel between -2..1
+2-bit  blue channel difference from the previous pixel between -2..1
+
+The difference to the current channel values are using a wraparound operation,
+so "1 - 2" will result in 255, while "255 + 1" will result in 0.
+
+Values are stored as unsigned integers with a bias of 2. E.g. -2 is stored as
+0 (b00). 1 is stored as 3 (b11).
+
+The alpha value remains unchanged from the previous pixel.
+
+
+.- QOI_OP_LUMA -------------------------------------.
+|         Byte[0]         |         Byte[1]         |
+|  7  6  5  4  3  2  1  0 |  7  6  5  4  3  2  1  0 |
+|-------+-----------------+-------------+-----------|
+|  1  0 |  green diff     |   dr - dg   |  db - dg  |
+`---------------------------------------------------`
+2-bit tag b10
+6-bit green channel difference from the previous pixel -32..31
+4-bit   red channel difference minus green channel difference -8..7
+4-bit  blue channel difference minus green channel difference -8..7
+
+The green channel is used to indicate the general direction of change and is
+encoded in 6 bits. The red and blue channels (dr and db) base their diffs off
+of the green channel difference and are encoded in 4 bits. I.e.:
+	dr_dg = (cur_px.r - prev_px.r) - (cur_px.g - prev_px.g)
+	db_dg = (cur_px.b - prev_px.b) - (cur_px.g - prev_px.g)
+
+The difference to the current channel values are using a wraparound operation,
+so "10 - 13" will result in 253, while "250 + 7" will result in 1.
+
+Values are stored as unsigned integers with a bias of 32 for the green channel
+and a bias of 8 for the red and blue channel.
+
+The alpha value remains unchanged from the previous pixel.
+
+
+.- QOI_OP_RUN ------------.
+|         Byte[0]         |
+|  7  6  5  4  3  2  1  0 |
+|-------+-----------------|
+|  1  1 |       run       |
+`-------------------------`
+2-bit tag b11
+6-bit run-length repeating the previous pixel: 1..62
+
+The run-length is stored with a bias of -1. Note that the run-lengths 63 and 64
+(b111110 and b111111) are illegal as they are occupied by the QOI_OP_RGB and
+QOI_OP_RGBA tags.
+
+
+.- QOI_OP_RGB ------------------------------------------.
+|         Byte[0]         | Byte[1] | Byte[2] | Byte[3] |
+|  7  6  5  4  3  2  1  0 | 7 .. 0  | 7 .. 0  | 7 .. 0  |
+|-------------------------+---------+---------+---------|
+|  1  1  1  1  1  1  1  0 |   red   |  green  |  blue   |
+`-------------------------------------------------------`
+8-bit tag b11111110
+8-bit   red channel value
+8-bit green channel value
+8-bit  blue channel value
+
+The alpha value remains unchanged from the previous pixel.
+
+
+.- QOI_OP_RGBA ---------------------------------------------------.
+|         Byte[0]         | Byte[1] | Byte[2] | Byte[3] | Byte[4] |
+|  7  6  5  4  3  2  1  0 | 7 .. 0  | 7 .. 0  | 7 .. 0  | 7 .. 0  |
+|-------------------------+---------+---------+---------+---------|
+|  1  1  1  1  1  1  1  1 |   red   |  green  |  blue   |  alpha  |
+`-----------------------------------------------------------------`
+8-bit tag b11111111
+8-bit   red channel value
+8-bit green channel value
+8-bit  blue channel value
+8-bit alpha channel value
+
+*/
+
+
+/* -----------------------------------------------------------------------------
+Header - Public functions */
+
+#ifndef QOI_H
+#define QOI_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* A pointer to a qoi_desc struct has to be supplied to all of qoi's functions.
+It describes either the input format (for qoi_write and qoi_encode), or is
+filled with the description read from the file header (for qoi_read and
+qoi_decode).
+
+The colorspace in this qoi_desc is an enum where
+	0 = sRGB, i.e. gamma scaled RGB channels and a linear alpha channel
+	1 = all channels are linear
+You may use the constants QOI_SRGB or QOI_LINEAR. The colorspace is purely
+informative. It will be saved to the file header, but does not affect
+how chunks are en-/decoded. */
+
+#define QOI_SRGB   0
+#define QOI_LINEAR 1
+
+typedef struct {
+	unsigned int width;
+	unsigned int height;
+	unsigned char channels;
+	unsigned char colorspace;
+} qoi_desc;
+
+#ifndef QOI_NO_STDIO
+
+/* Encode raw RGB or RGBA pixels into a QOI image and write it to the file
+system. The qoi_desc struct must be filled with the image width, height,
+number of channels (3 = RGB, 4 = RGBA) and the colorspace.
+
+The function returns 0 on failure (invalid parameters, or fopen or malloc
+failed) or the number of bytes written on success. */
+
+int qoi_write(const char *filename, const void *data, const qoi_desc *desc);
+
+
+/* Read and decode a QOI image from the file system. If channels is 0, the
+number of channels from the file header is used. If channels is 3 or 4 the
+output format will be forced into this number of channels.
+
+The function either returns NULL on failure (invalid data, or malloc or fopen
+failed) or a pointer to the decoded pixels. On success, the qoi_desc struct
+will be filled with the description from the file header.
+
+The returned pixel data should be free()d after use. */
+
+void *qoi_read(const char *filename, qoi_desc *desc, int channels);
+
+#endif /* QOI_NO_STDIO */
+
+
+/* Encode raw RGB or RGBA pixels into a QOI image in memory.
+
+The function either returns NULL on failure (invalid parameters or malloc
+failed) or a pointer to the encoded data on success. On success the out_len
+is set to the size in bytes of the encoded data.
+
+The returned qoi data should be free()d after use. */
+
+void *qoi_encode(const void *data, const qoi_desc *desc, int *out_len);
+
+
+/* Decode a QOI image from memory.
+
+The function either returns NULL on failure (invalid parameters or malloc
+failed) or a pointer to the decoded pixels. On success, the qoi_desc struct
+is filled with the description from the file header.
+
+The returned pixel data should be free()d after use. */
+
+void *qoi_decode(const void *data, int size, qoi_desc *desc, int channels);
+
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* QOI_H */
+
+
+/* -----------------------------------------------------------------------------
+Implementation */
+
+#ifdef QOI_IMPLEMENTATION
+#include <stdlib.h>
+#include <string.h>
+
+#ifndef QOI_MALLOC
+	#define QOI_MALLOC(sz) malloc(sz)
+	#define QOI_FREE(p)    free(p)
+#endif
+#ifndef QOI_ZEROARR
+	#define QOI_ZEROARR(a) memset((a),0,sizeof(a))
+#endif
+
+#define QOI_OP_INDEX  0x00 /* 00xxxxxx */
+#define QOI_OP_DIFF   0x40 /* 01xxxxxx */
+#define QOI_OP_LUMA   0x80 /* 10xxxxxx */
+#define QOI_OP_RUN    0xc0 /* 11xxxxxx */
+#define QOI_OP_RGB    0xfe /* 11111110 */
+#define QOI_OP_RGBA   0xff /* 11111111 */
+
+#define QOI_MASK_2    0xc0 /* 11000000 */
+
+#define QOI_COLOR_HASH(C) (C.rgba.r*3 + C.rgba.g*5 + C.rgba.b*7 + C.rgba.a*11)
+#define QOI_MAGIC \
+	(((unsigned int)'q') << 24 | ((unsigned int)'o') << 16 | \
+	 ((unsigned int)'i') <<  8 | ((unsigned int)'f'))
+#define QOI_HEADER_SIZE 14
+
+/* 2GB is the max file size that this implementation can safely handle. We guard
+against anything larger than that, assuming the worst case with 5 bytes per
+pixel, rounded down to a nice clean value. 400 million pixels ought to be
+enough for anybody. */
+#define QOI_PIXELS_MAX ((unsigned int)400000000)
+
+typedef union {
+	struct { unsigned char r, g, b, a; } rgba;
+	unsigned int v;
+} qoi_rgba_t;
+
+static const unsigned char qoi_padding[8] = {0,0,0,0,0,0,0,1};
+
+static void qoi_write_32(unsigned char *bytes, int *p, unsigned int v) {
+	bytes[(*p)++] = (0xff000000 & v) >> 24;
+	bytes[(*p)++] = (0x00ff0000 & v) >> 16;
+	bytes[(*p)++] = (0x0000ff00 & v) >> 8;
+	bytes[(*p)++] = (0x000000ff & v);
+}
+
+static unsigned int qoi_read_32(const unsigned char *bytes, int *p) {
+	unsigned int a = bytes[(*p)++];
+	unsigned int b = bytes[(*p)++];
+	unsigned int c = bytes[(*p)++];
+	unsigned int d = bytes[(*p)++];
+	return a << 24 | b << 16 | c << 8 | d;
+}
+
+void *qoi_encode(const void *data, const qoi_desc *desc, int *out_len) {
+	int i, max_size, p, run;
+	int px_len, px_end, px_pos, channels;
+	unsigned char *bytes;
+	const unsigned char *pixels;
+	qoi_rgba_t index[64];
+	qoi_rgba_t px, px_prev;
+
+	if (
+		data == NULL || out_len == NULL || desc == NULL ||
+		desc->width == 0 || desc->height == 0 ||
+		desc->channels < 3 || desc->channels > 4 ||
+		desc->colorspace > 1 ||
+		desc->height >= QOI_PIXELS_MAX / desc->width
+	) {
+		return NULL;
+	}
+
+	max_size =
+		desc->width * desc->height * (desc->channels + 1) +
+		QOI_HEADER_SIZE + sizeof(qoi_padding);
+
+	p = 0;
+	bytes = (unsigned char *) QOI_MALLOC(max_size);
+	if (!bytes) {
+		return NULL;
+	}
+
+	qoi_write_32(bytes, &p, QOI_MAGIC);
+	qoi_write_32(bytes, &p, desc->width);
+	qoi_write_32(bytes, &p, desc->height);
+	bytes[p++] = desc->channels;
+	bytes[p++] = desc->colorspace;
+
+
+	pixels = (const unsigned char *)data;
+
+	QOI_ZEROARR(index);
+
+	run = 0;
+	px_prev.rgba.r = 0;
+	px_prev.rgba.g = 0;
+	px_prev.rgba.b = 0;
+	px_prev.rgba.a = 255;
+	px = px_prev;
+
+	px_len = desc->width * desc->height * desc->channels;
+	px_end = px_len - desc->channels;
+	channels = desc->channels;
+
+	for (px_pos = 0; px_pos < px_len; px_pos += channels) {
+		px.rgba.r = pixels[px_pos + 0];
+		px.rgba.g = pixels[px_pos + 1];
+		px.rgba.b = pixels[px_pos + 2];
+
+		if (channels == 4) {
+			px.rgba.a = pixels[px_pos + 3];
+		}
+
+		if (px.v == px_prev.v) {
+			run++;
+			if (run == 62 || px_pos == px_end) {
+				bytes[p++] = QOI_OP_RUN | (run - 1);
+				run = 0;
+			}
+		}
+		else {
+			int index_pos;
+
+			if (run > 0) {
+				bytes[p++] = QOI_OP_RUN | (run - 1);
+				run = 0;
+			}
+
+			index_pos = QOI_COLOR_HASH(px) % 64;
+
+			if (index[index_pos].v == px.v) {
+				bytes[p++] = QOI_OP_INDEX | index_pos;
+			}
+			else {
+				index[index_pos] = px;
+
+				if (px.rgba.a == px_prev.rgba.a) {
+					signed char vr = px.rgba.r - px_prev.rgba.r;
+					signed char vg = px.rgba.g - px_prev.rgba.g;
+					signed char vb = px.rgba.b - px_prev.rgba.b;
+
+					signed char vg_r = vr - vg;
+					signed char vg_b = vb - vg;
+
+					if (
+						vr > -3 && vr < 2 &&
+						vg > -3 && vg < 2 &&
+						vb > -3 && vb < 2
+					) {
+						bytes[p++] = QOI_OP_DIFF | (vr + 2) << 4 | (vg + 2) << 2 | (vb + 2);
+					}
+					else if (
+						vg_r >  -9 && vg_r <  8 &&
+						vg   > -33 && vg   < 32 &&
+						vg_b >  -9 && vg_b <  8
+					) {
+						bytes[p++] = QOI_OP_LUMA     | (vg   + 32);
+						bytes[p++] = (vg_r + 8) << 4 | (vg_b +  8);
+					}
+					else {
+						bytes[p++] = QOI_OP_RGB;
+						bytes[p++] = px.rgba.r;
+						bytes[p++] = px.rgba.g;
+						bytes[p++] = px.rgba.b;
+					}
+				}
+				else {
+					bytes[p++] = QOI_OP_RGBA;
+					bytes[p++] = px.rgba.r;
+					bytes[p++] = px.rgba.g;
+					bytes[p++] = px.rgba.b;
+					bytes[p++] = px.rgba.a;
+				}
+			}
+		}
+		px_prev = px;
+	}
+
+	for (i = 0; i < (int)sizeof(qoi_padding); i++) {
+		bytes[p++] = qoi_padding[i];
+	}
+
+	*out_len = p;
+	return bytes;
+}
+
+void *qoi_decode(const void *data, int size, qoi_desc *desc, int channels) {
+	const unsigned char *bytes;
+	unsigned int header_magic;
+	unsigned char *pixels;
+	qoi_rgba_t index[64];
+	qoi_rgba_t px;
+	int px_len, chunks_len, px_pos;
+	int p = 0, run = 0;
+
+	if (
+		data == NULL || desc == NULL ||
+		(channels != 0 && channels != 3 && channels != 4) ||
+		size < QOI_HEADER_SIZE + (int)sizeof(qoi_padding)
+	) {
+		return NULL;
+	}
+
+	bytes = (const unsigned char *)data;
+
+	header_magic = qoi_read_32(bytes, &p);
+	desc->width = qoi_read_32(bytes, &p);
+	desc->height = qoi_read_32(bytes, &p);
+	desc->channels = bytes[p++];
+	desc->colorspace = bytes[p++];
+
+	if (
+		desc->width == 0 || desc->height == 0 ||
+		desc->channels < 3 || desc->channels > 4 ||
+		desc->colorspace > 1 ||
+		header_magic != QOI_MAGIC ||
+		desc->height >= QOI_PIXELS_MAX / desc->width
+	) {
+		return NULL;
+	}
+
+	if (channels == 0) {
+		channels = desc->channels;
+	}
+
+	px_len = desc->width * desc->height * channels;
+	pixels = (unsigned char *) QOI_MALLOC(px_len);
+	if (!pixels) {
+		return NULL;
+	}
+
+	QOI_ZEROARR(index);
+	px.rgba.r = 0;
+	px.rgba.g = 0;
+	px.rgba.b = 0;
+	px.rgba.a = 255;
+
+	chunks_len = size - (int)sizeof(qoi_padding);
+	for (px_pos = 0; px_pos < px_len; px_pos += channels) {
+		if (run > 0) {
+			run--;
+		}
+		else if (p < chunks_len) {
+			int b1 = bytes[p++];
+
+			if (b1 == QOI_OP_RGB) {
+				px.rgba.r = bytes[p++];
+				px.rgba.g = bytes[p++];
+				px.rgba.b = bytes[p++];
+			}
+			else if (b1 == QOI_OP_RGBA) {
+				px.rgba.r = bytes[p++];
+				px.rgba.g = bytes[p++];
+				px.rgba.b = bytes[p++];
+				px.rgba.a = bytes[p++];
+			}
+			else if ((b1 & QOI_MASK_2) == QOI_OP_INDEX) {
+				px = index[b1];
+			}
+			else if ((b1 & QOI_MASK_2) == QOI_OP_DIFF) {
+				px.rgba.r += ((b1 >> 4) & 0x03) - 2;
+				px.rgba.g += ((b1 >> 2) & 0x03) - 2;
+				px.rgba.b += ( b1       & 0x03) - 2;
+			}
+			else if ((b1 & QOI_MASK_2) == QOI_OP_LUMA) {
+				int b2 = bytes[p++];
+				int vg = (b1 & 0x3f) - 32;
+				px.rgba.r += vg - 8 + ((b2 >> 4) & 0x0f);
+				px.rgba.g += vg;
+				px.rgba.b += vg - 8 +  (b2       & 0x0f);
+			}
+			else if ((b1 & QOI_MASK_2) == QOI_OP_RUN) {
+				run = (b1 & 0x3f);
+			}
+
+			index[QOI_COLOR_HASH(px) % 64] = px;
+		}
+
+		pixels[px_pos + 0] = px.rgba.r;
+		pixels[px_pos + 1] = px.rgba.g;
+		pixels[px_pos + 2] = px.rgba.b;
+		
+		if (channels == 4) {
+			pixels[px_pos + 3] = px.rgba.a;
+		}
+	}
+
+	return pixels;
+}
+
+#ifndef QOI_NO_STDIO
+#include <stdio.h>
+
+int qoi_write(const char *filename, const void *data, const qoi_desc *desc) {
+	FILE *f = fopen(filename, "wb");
+	int size, err;
+	void *encoded;
+
+	if (!f) {
+		return 0;
+	}
+
+	encoded = qoi_encode(data, desc, &size);
+	if (!encoded) {
+		fclose(f);
+		return 0;
+	}
+
+	fwrite(encoded, 1, size, f);
+	fflush(f);
+	err = ferror(f);
+	fclose(f);
+
+	QOI_FREE(encoded);
+	return err ? 0 : size;
+}
+
+void *qoi_read(const char *filename, qoi_desc *desc, int channels) {
+	FILE *f = fopen(filename, "rb");
+	int size, bytes_read;
+	void *pixels, *data;
+
+	if (!f) {
+		return NULL;
+	}
+
+	fseek(f, 0, SEEK_END);
+	size = ftell(f);
+	if (size <= 0 || fseek(f, 0, SEEK_SET) != 0) {
+		fclose(f);
+		return NULL;
+	}
+
+	data = QOI_MALLOC(size);
+	if (!data) {
+		fclose(f);
+		return NULL;
+	}
+
+	bytes_read = fread(data, 1, size, f);
+	fclose(f);
+	pixels = (bytes_read != size) ? NULL : qoi_decode(data, bytes_read, desc, channels);
+	QOI_FREE(data);
+	return pixels;
+}
+
+#endif /* QOI_NO_STDIO */
+#endif /* QOI_IMPLEMENTATION */

external/qoi/qoi/qoibench.c

@@ -0,0 +1,610 @@
+/*
+
+Copyright (c) 2021, Dominic Szablewski - https://phoboslab.org
+SPDX-License-Identifier: MIT
+
+
+Simple benchmark suite for png, stbi and qoi
+
+Requires libpng, "stb_image.h" and "stb_image_write.h"
+Compile with: 
+	gcc qoibench.c -std=gnu99 -lpng -O3 -o qoibench 
+
+*/
+
+#include <stdio.h>
+#include <dirent.h>
+#include <png.h>
+
+#define STB_IMAGE_IMPLEMENTATION
+#define STBI_ONLY_PNG
+#define STBI_NO_LINEAR
+#include "stb_image.h"
+
+#define STB_IMAGE_WRITE_IMPLEMENTATION
+#include "stb_image_write.h"
+
+#define QOI_IMPLEMENTATION
+#include "qoi.h"
+
+
+
+
+// -----------------------------------------------------------------------------
+// Cross platform high resolution timer
+// From https://gist.github.com/ForeverZer0/0a4f80fc02b96e19380ebb7a3debbee5
+
+#include <stdint.h>
+#if defined(__linux)
+	#define HAVE_POSIX_TIMER
+	#include <time.h>
+	#ifdef CLOCK_MONOTONIC
+		#define CLOCKID CLOCK_MONOTONIC
+	#else
+		#define CLOCKID CLOCK_REALTIME
+	#endif
+#elif defined(__APPLE__)
+	#define HAVE_MACH_TIMER
+	#include <mach/mach_time.h>
+#elif defined(_WIN32)
+	#define WIN32_LEAN_AND_MEAN
+	#include <windows.h>
+#endif
+
+static uint64_t ns() {
+	static uint64_t is_init = 0;
+#if defined(__APPLE__)
+		static mach_timebase_info_data_t info;
+		if (0 == is_init) {
+			mach_timebase_info(&info);
+			is_init = 1;
+		}
+		uint64_t now;
+		now = mach_absolute_time();
+		now *= info.numer;
+		now /= info.denom;
+		return now;
+#elif defined(__linux)
+		static struct timespec linux_rate;
+		if (0 == is_init) {
+			clock_getres(CLOCKID, &linux_rate);
+			is_init = 1;
+		}
+		uint64_t now;
+		struct timespec spec;
+		clock_gettime(CLOCKID, &spec);
+		now = spec.tv_sec * 1.0e9 + spec.tv_nsec;
+		return now;
+#elif defined(_WIN32)
+		static LARGE_INTEGER win_frequency;
+		if (0 == is_init) {
+			QueryPerformanceFrequency(&win_frequency);
+			is_init = 1;
+		}
+		LARGE_INTEGER now;
+		QueryPerformanceCounter(&now);
+		return (uint64_t) ((1e9 * now.QuadPart)	/ win_frequency.QuadPart);
+#endif
+}
+
+#define STRINGIFY(x) #x
+#define TOSTRING(x) STRINGIFY(x)
+#define ERROR(...) printf("abort at line " TOSTRING(__LINE__) ": " __VA_ARGS__); printf("\n"); exit(1)
+
+
+// -----------------------------------------------------------------------------
+// libpng encode/decode wrappers
+// Seriously, who thought this was a good abstraction for an API to read/write
+// images?
+
+typedef struct {
+	int size;
+	int capacity;
+	unsigned char *data;
+} libpng_write_t;
+
+void libpng_encode_callback(png_structp png_ptr, png_bytep data, png_size_t length) {
+	libpng_write_t *write_data = (libpng_write_t*)png_get_io_ptr(png_ptr);
+	if (write_data->size + length >= write_data->capacity) {
+		ERROR("PNG write");
+	}
+	memcpy(write_data->data + write_data->size, data, length);
+	write_data->size += length;
+}
+
+void *libpng_encode(void *pixels, int w, int h, int channels, int *out_len) {
+	png_structp png = png_create_write_struct(PNG_LIBPNG_VER_STRING, NULL, NULL, NULL);
+	if (!png) {
+		ERROR("png_create_write_struct");
+	}
+
+	png_infop info = png_create_info_struct(png);
+	if (!info) {
+		ERROR("png_create_info_struct");
+	}
+
+	if (setjmp(png_jmpbuf(png))) {
+		ERROR("png_jmpbuf");
+	}
+
+	// Output is 8bit depth, RGBA format.
+	png_set_IHDR(
+		png,
+		info,
+		w, h,
+		8,
+		channels == 3 ? PNG_COLOR_TYPE_RGB : PNG_COLOR_TYPE_RGBA,
+		PNG_INTERLACE_NONE,
+		PNG_COMPRESSION_TYPE_DEFAULT,
+		PNG_FILTER_TYPE_DEFAULT
+	);
+
+	png_bytep row_pointers[h];
+	for(int y = 0; y < h; y++){
+		row_pointers[y] = ((unsigned char *)pixels + y * w * channels);
+	}
+
+	libpng_write_t write_data = {
+		.size = 0,
+		.capacity = w * h * channels,
+		.data = malloc(w * h * channels)
+	};
+
+	png_set_rows(png, info, row_pointers);
+	png_set_write_fn(png, &write_data, libpng_encode_callback, NULL);
+	png_write_png(png, info, PNG_TRANSFORM_IDENTITY, NULL);
+
+	png_destroy_write_struct(&png, &info);
+
+	*out_len = write_data.size;
+	return write_data.data;
+}
+
+
+typedef struct {
+	int pos;
+	int size;
+	unsigned char *data;
+} libpng_read_t;
+
+void png_decode_callback(png_structp png, png_bytep data, png_size_t length) {
+	libpng_read_t *read_data = (libpng_read_t*)png_get_io_ptr(png);
+	if (read_data->pos + length > read_data->size) {
+		ERROR("PNG read %ld bytes at pos %d (size: %d)", length, read_data->pos, read_data->size);
+	}
+	memcpy(data, read_data->data + read_data->pos, length);
+	read_data->pos += length;
+}
+
+void png_warning_callback(png_structp png_ptr, png_const_charp warning_msg) {
+	// Ignore warnings about sRGB profiles and such.
+}
+
+void *libpng_decode(void *data, int size, int *out_w, int *out_h) {	
+	png_structp png = png_create_read_struct(PNG_LIBPNG_VER_STRING, NULL, NULL, png_warning_callback);
+	if (!png) {
+		ERROR("png_create_read_struct");
+	}
+
+	png_infop info = png_create_info_struct(png);
+	if (!info) {
+		ERROR("png_create_info_struct");
+	}
+
+	libpng_read_t read_data = {
+		.pos = 0,
+		.size = size,
+		.data = data
+	};
+	
+	png_set_read_fn(png, &read_data, png_decode_callback);
+	png_set_sig_bytes(png, 0);
+	png_read_info(png, info);
+	
+	png_uint_32 w, h;
+	int bitDepth, colorType, interlaceType;
+	png_get_IHDR(png, info, &w, &h, &bitDepth, &colorType, &interlaceType, NULL, NULL);
+	
+	// 16 bit -> 8 bit
+	png_set_strip_16(png);
+	
+	// 1, 2, 4 bit -> 8 bit
+	if (bitDepth < 8) {
+		png_set_packing(png);
+	}
+
+	if (colorType & PNG_COLOR_MASK_PALETTE) {
+		png_set_expand(png);
+	}
+	
+	if (!(colorType & PNG_COLOR_MASK_COLOR)) {
+		png_set_gray_to_rgb(png);
+	}
+
+	// set paletted or RGB images with transparency to full alpha so we get RGBA
+	if (png_get_valid(png, info, PNG_INFO_tRNS)) {
+		png_set_tRNS_to_alpha(png);
+	}
+	
+	// make sure every pixel has an alpha value
+	if (!(colorType & PNG_COLOR_MASK_ALPHA)) {
+		png_set_filler(png, 255, PNG_FILLER_AFTER);
+	}
+	
+	png_read_update_info(png, info);
+
+	unsigned char* out = malloc(w * h * 4);
+	*out_w = w;
+	*out_h = h;
+	
+	// png_uint_32 rowBytes = png_get_rowbytes(png, info);
+	png_bytep row_pointers[h];
+	for (png_uint_32 row = 0; row < h; row++ ) {
+		row_pointers[row] = (png_bytep)(out + (row * w * 4));
+	}
+	
+	png_read_image(png, row_pointers);
+	png_read_end(png, info);
+	png_destroy_read_struct( &png, &info, NULL);
+	
+	return out;
+}
+
+
+// -----------------------------------------------------------------------------
+// stb_image encode callback
+
+void stbi_write_callback(void *context, void *data, int size) {
+	int *encoded_size = (int *)context;
+	*encoded_size += size;
+	// In theory we'd need to do another malloc(), memcpy() and free() here to 
+	// be fair to the other decode functions...
+}
+
+
+// -----------------------------------------------------------------------------
+// function to load a whole file into memory
+
+void *fload(const char *path, int *out_size) {
+	FILE *fh = fopen(path, "rb");
+	if (!fh) {
+		ERROR("Can't open file");
+	}
+
+	fseek(fh, 0, SEEK_END);
+	int size = ftell(fh);
+	fseek(fh, 0, SEEK_SET);
+
+	void *buffer = malloc(size);
+	if (!buffer) {
+		ERROR("Malloc for %d bytes failed", size);
+	}
+
+	if (!fread(buffer, size, 1, fh)) {
+		ERROR("Can't read file %s", path);
+	}
+	fclose(fh);
+
+	*out_size = size;
+	return buffer;
+}
+
+
+// -----------------------------------------------------------------------------
+// benchmark runner
+
+
+int opt_runs = 1;
+int opt_nopng = 0;
+int opt_nowarmup = 0;
+int opt_noverify = 0;
+int opt_nodecode = 0;
+int opt_noencode = 0;
+int opt_norecurse = 0;
+int opt_onlytotals = 0;
+
+enum {
+	LIBPNG,
+	STBI,
+	QOI,
+	BENCH_COUNT /* must be the last element */
+};
+static const char *const lib_names[BENCH_COUNT] = {
+	// NOTE: pad with spaces so everything lines up properly
+	[LIBPNG] =  "libpng: ",
+	[STBI]   =  "stbi:   ",
+	[QOI]    =  "qoi:    ",
+};
+
+typedef struct {
+	uint64_t size;
+	uint64_t encode_time;
+	uint64_t decode_time;
+} benchmark_lib_result_t;
+
+typedef struct {
+	int count;
+	uint64_t raw_size;
+	uint64_t px;
+	int w;
+	int h;
+	benchmark_lib_result_t libs[BENCH_COUNT];
+} benchmark_result_t;
+
+
+void benchmark_print_result(benchmark_result_t res) {
+	res.px /= res.count;
+	res.raw_size /= res.count;
+
+	double px = res.px;
+	printf("          decode ms   encode ms   decode mpps   encode mpps   size kb    rate\n");
+	for (int i = 0; i < BENCH_COUNT; ++i) {
+		if (opt_nopng && (i == LIBPNG || i == STBI)) {
+			continue;
+		}
+		res.libs[i].encode_time /= res.count;
+		res.libs[i].decode_time /= res.count;
+		res.libs[i].size /= res.count;
+		printf(
+			"%s   %8.1f    %8.1f      %8.2f      %8.2f  %8ld   %4.1f%%\n",
+			lib_names[i],
+			(double)res.libs[i].decode_time/1000000.0,
+			(double)res.libs[i].encode_time/1000000.0,
+			(res.libs[i].decode_time > 0 ? px / ((double)res.libs[i].decode_time/1000.0) : 0),
+			(res.libs[i].encode_time > 0 ? px / ((double)res.libs[i].encode_time/1000.0) : 0),
+			res.libs[i].size/1024,
+			((double)res.libs[i].size/(double)res.raw_size) * 100.0
+		);
+	}
+	printf("\n");
+}
+
+// Run __VA_ARGS__ a number of times and measure the time taken. The first
+// run is ignored.
+#define BENCHMARK_FN(NOWARMUP, RUNS, AVG_TIME, ...) \
+	do { \
+		uint64_t time = 0; \
+		for (int i = NOWARMUP; i <= RUNS; i++) { \
+			uint64_t time_start = ns(); \
+			__VA_ARGS__ \
+			uint64_t time_end = ns(); \
+			if (i > 0) { \
+				time += time_end - time_start; \
+			} \
+		} \
+		AVG_TIME = time / RUNS; \
+	} while (0)
+
+
+benchmark_result_t benchmark_image(const char *path) {
+	int encoded_png_size;
+	int encoded_qoi_size;
+	int w;
+	int h;
+	int channels;
+
+	// Load the encoded PNG, encoded QOI and raw pixels into memory
+	if(!stbi_info(path, &w, &h, &channels)) {
+		ERROR("Error decoding header %s", path);
+	}
+
+	if (channels != 3) {
+		channels = 4;
+	}
+
+	void *pixels = (void *)stbi_load(path, &w, &h, NULL, channels);
+	void *encoded_png = fload(path, &encoded_png_size);
+	void *encoded_qoi = qoi_encode(pixels, &(qoi_desc){
+			.width = w,
+			.height = h, 
+			.channels = channels,
+			.colorspace = QOI_SRGB
+		}, &encoded_qoi_size);
+
+	if (!pixels || !encoded_qoi || !encoded_png) {
+		ERROR("Error encoding %s", path);
+	}
+
+	// Verify QOI Output
+
+	if (!opt_noverify) {
+		qoi_desc dc;
+		void *pixels_qoi = qoi_decode(encoded_qoi, encoded_qoi_size, &dc, channels);
+		if (memcmp(pixels, pixels_qoi, w * h * channels) != 0) {
+			ERROR("QOI roundtrip pixel mismatch for %s", path);
+		}
+		free(pixels_qoi);
+	}
+
+
+
+	benchmark_result_t res = {0};
+	res.count = 1;
+	res.raw_size = w * h * channels;
+	res.px = w * h;
+	res.w = w;
+	res.h = h;
+
+
+	// Decoding
+
+	if (!opt_nodecode) {
+		if (!opt_nopng) {
+			BENCHMARK_FN(opt_nowarmup, opt_runs, res.libs[LIBPNG].decode_time, {
+				int dec_w, dec_h;
+				void *dec_p = libpng_decode(encoded_png, encoded_png_size, &dec_w, &dec_h);
+				free(dec_p);
+			});
+
+			BENCHMARK_FN(opt_nowarmup, opt_runs, res.libs[STBI].decode_time, {
+				int dec_w, dec_h, dec_channels;
+				void *dec_p = stbi_load_from_memory(encoded_png, encoded_png_size, &dec_w, &dec_h, &dec_channels, 4);
+				free(dec_p);
+			});
+		}
+
+		BENCHMARK_FN(opt_nowarmup, opt_runs, res.libs[QOI].decode_time, {
+			qoi_desc desc;
+			void *dec_p = qoi_decode(encoded_qoi, encoded_qoi_size, &desc, 4);
+			free(dec_p);
+		});
+	}
+
+
+	// Encoding
+	if (!opt_noencode) {
+		if (!opt_nopng) {
+			BENCHMARK_FN(opt_nowarmup, opt_runs, res.libs[LIBPNG].encode_time, {
+				int enc_size;
+				void *enc_p = libpng_encode(pixels, w, h, channels, &enc_size);
+				res.libs[LIBPNG].size = enc_size;
+				free(enc_p);
+			});
+
+			BENCHMARK_FN(opt_nowarmup, opt_runs, res.libs[STBI].encode_time, {
+				int enc_size = 0;
+				stbi_write_png_to_func(stbi_write_callback, &enc_size, w, h, channels, pixels, 0);
+				res.libs[STBI].size = enc_size;
+			});
+		}
+
+		BENCHMARK_FN(opt_nowarmup, opt_runs, res.libs[QOI].encode_time, {
+			int enc_size;
+			void *enc_p = qoi_encode(pixels, &(qoi_desc){
+				.width = w,
+				.height = h, 
+				.channels = channels,
+				.colorspace = QOI_SRGB
+			}, &enc_size);
+			res.libs[QOI].size = enc_size;
+			free(enc_p);
+		});
+	}
+
+	free(pixels);
+	free(encoded_png);
+	free(encoded_qoi);
+
+	return res;
+}
+
+void benchmark_directory(const char *path, benchmark_result_t *grand_total) {
+	DIR *dir = opendir(path);
+	if (!dir) {
+		ERROR("Couldn't open directory %s", path);
+	}
+
+	struct dirent *file;
+
+	if (!opt_norecurse) {
+		for (int i = 0; (file = readdir(dir)) != NULL; i++) {
+			if (
+				file->d_type & DT_DIR &&
+				strcmp(file->d_name, ".") != 0 &&
+				strcmp(file->d_name, "..") != 0
+			) {
+				char subpath[1024];
+				snprintf(subpath, 1024, "%s/%s", path, file->d_name);
+				benchmark_directory(subpath, grand_total);
+			}
+		}
+		rewinddir(dir);
+	}
+
+	benchmark_result_t dir_total = {0};
+	
+	int has_shown_head = 0;
+	for (int i = 0; (file = readdir(dir)) != NULL; i++) {
+		if (strcmp(file->d_name + strlen(file->d_name) - 4, ".png") != 0) {
+			continue;
+		}
+
+		if (!has_shown_head) {
+			has_shown_head = 1;
+			printf("## Benchmarking %s/*.png -- %d runs\n\n", path, opt_runs);
+		}
+
+		char *file_path = malloc(strlen(file->d_name) + strlen(path)+8);
+		sprintf(file_path, "%s/%s", path, file->d_name);
+		
+		benchmark_result_t res = benchmark_image(file_path);
+
+		if (!opt_onlytotals) {
+			printf("## %s size: %dx%d\n", file_path, res.w, res.h);
+			benchmark_print_result(res);
+		}
+
+		free(file_path);
+		
+		dir_total.count++;
+		dir_total.raw_size += res.raw_size;
+		dir_total.px += res.px;
+		for (int i = 0; i < BENCH_COUNT; ++i) {
+			dir_total.libs[i].encode_time += res.libs[i].encode_time;
+			dir_total.libs[i].decode_time += res.libs[i].decode_time;
+			dir_total.libs[i].size += res.libs[i].size;
+		}
+
+		grand_total->count++;
+		grand_total->raw_size += res.raw_size;
+		grand_total->px += res.px;
+		for (int i = 0; i < BENCH_COUNT; ++i) {
+			grand_total->libs[i].encode_time += res.libs[i].encode_time;
+			grand_total->libs[i].decode_time += res.libs[i].decode_time;
+			grand_total->libs[i].size += res.libs[i].size;
+		}
+	}
+	closedir(dir);
+
+	if (dir_total.count > 0) {
+		printf("## Total for %s\n", path);
+		benchmark_print_result(dir_total);
+	}
+}
+
+int main(int argc, char **argv) {
+	if (argc < 3) {
+		printf("Usage: qoibench <iterations> <directory> [options]\n");
+		printf("Options:\n");
+		printf("    --nowarmup ... don't perform a warmup run\n");
+		printf("    --nopng ...... don't run png encode/decode\n");
+		printf("    --noverify ... don't verify qoi roundtrip\n");
+		printf("    --noencode ... don't run encoders\n");
+		printf("    --nodecode ... don't run decoders\n");
+		printf("    --norecurse .. don't descend into directories\n");
+		printf("    --onlytotals . don't print individual image results\n");
+		printf("Examples\n");
+		printf("    qoibench 10 images/textures/\n");
+		printf("    qoibench 1 images/textures/ --nopng --nowarmup\n");
+		exit(1);
+	}
+
+	for (int i = 3; i < argc; i++) {
+		if (strcmp(argv[i], "--nowarmup") == 0) { opt_nowarmup = 1; }
+		else if (strcmp(argv[i], "--nopng") == 0) { opt_nopng = 1; }
+		else if (strcmp(argv[i], "--noverify") == 0) { opt_noverify = 1; }
+		else if (strcmp(argv[i], "--noencode") == 0) { opt_noencode = 1; }
+		else if (strcmp(argv[i], "--nodecode") == 0) { opt_nodecode = 1; }
+		else if (strcmp(argv[i], "--norecurse") == 0) { opt_norecurse = 1; }
+		else if (strcmp(argv[i], "--onlytotals") == 0) { opt_onlytotals = 1; }
+		else { ERROR("Unknown option %s", argv[i]); }
+	}
+
+	opt_runs = atoi(argv[1]);
+	if (opt_runs <=0) {
+		ERROR("Invalid number of runs %d", opt_runs);
+	}
+
+	benchmark_result_t grand_total = {0};
+	benchmark_directory(argv[2], &grand_total);
+
+	if (grand_total.count > 0) {
+		printf("# Grand total for %s\n", argv[2]);
+		benchmark_print_result(grand_total);
+	}
+	else {
+		printf("No images found in %s\n", argv[2]);
+	}
+
+	return 0;
+}

external/qoi/qoi/qoiconv.c

@@ -0,0 +1,91 @@
+/*
+
+Copyright (c) 2021, Dominic Szablewski - https://phoboslab.org
+SPDX-License-Identifier: MIT
+
+
+Command line tool to convert between png <> qoi format
+
+Requires:
+	-"stb_image.h" (https://github.com/nothings/stb/blob/master/stb_image.h)
+	-"stb_image_write.h" (https://github.com/nothings/stb/blob/master/stb_image_write.h)
+	-"qoi.h" (https://github.com/phoboslab/qoi/blob/master/qoi.h)
+
+Compile with: 
+	gcc qoiconv.c -std=c99 -O3 -o qoiconv
+
+*/
+
+
+#define STB_IMAGE_IMPLEMENTATION
+#define STBI_ONLY_PNG
+#define STBI_NO_LINEAR
+#include "stb_image.h"
+
+#define STB_IMAGE_WRITE_IMPLEMENTATION
+#include "stb_image_write.h"
+
+#define QOI_IMPLEMENTATION
+#include "qoi.h"
+
+
+#define STR_ENDS_WITH(S, E) (strcmp(S + strlen(S) - (sizeof(E)-1), E) == 0)
+
+int main(int argc, char **argv) {
+	if (argc < 3) {
+		puts("Usage: qoiconv <infile> <outfile>");
+		puts("Examples:");
+		puts("  qoiconv input.png output.qoi");
+		puts("  qoiconv input.qoi output.png");
+		exit(1);
+	}
+
+	void *pixels = NULL;
+	int w, h, channels;
+	if (STR_ENDS_WITH(argv[1], ".png")) {
+		if(!stbi_info(argv[1], &w, &h, &channels)) {
+			printf("Couldn't read header %s\n", argv[1]);
+			exit(1);
+		}
+
+		// Force all odd encodings to be RGBA
+		if(channels != 3) {
+			channels = 4;
+		}
+
+		pixels = (void *)stbi_load(argv[1], &w, &h, NULL, channels);
+	}
+	else if (STR_ENDS_WITH(argv[1], ".qoi")) {
+		qoi_desc desc;
+		pixels = qoi_read(argv[1], &desc, 0);
+		channels = desc.channels;
+		w = desc.width;
+		h = desc.height;
+	}
+
+	if (pixels == NULL) {
+		printf("Couldn't load/decode %s\n", argv[1]);
+		exit(1);
+	}
+
+	int encoded = 0;
+	if (STR_ENDS_WITH(argv[2], ".png")) {
+		encoded = stbi_write_png(argv[2], w, h, channels, pixels, 0);
+	}
+	else if (STR_ENDS_WITH(argv[2], ".qoi")) {
+		encoded = qoi_write(argv[2], pixels, &(qoi_desc){
+			.width = w,
+			.height = h, 
+			.channels = channels,
+			.colorspace = QOI_SRGB
+		});
+	}
+
+	if (!encoded) {
+		printf("Couldn't write/encode %s\n", argv[2]);
+		exit(1);
+	}
+
+	free(pixels);
+	return 0;
+}

external/qoi/qoi/qoifuzz.c

@@ -0,0 +1,32 @@
+/*
+
+Copyright (c) 2021, Dominic Szablewski - https://phoboslab.org
+SPDX-License-Identifier: MIT
+
+
+clang fuzzing harness for qoi_decode
+
+Compile and run with: 
+	clang -fsanitize=address,fuzzer -g -O0 qoifuzz.c && ./a.out
+
+*/
+
+
+#define QOI_IMPLEMENTATION
+#include "qoi.h"
+#include <stddef.h>
+#include <stdint.h>
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+	int w, h;
+	if (size < 4) {
+		return 0;
+	}
+
+	qoi_desc desc;
+	void* decoded = qoi_decode((void*)(data + 4), (int)(size - 4), &desc, *((int *)data));
+	if (decoded != NULL) {
+		free(decoded);
+	}
+	return 0;
+}

external/solanaceae_contact

@@ -1 +1 @@
-Subproject commit 6f3f9ef1911ce8189609ba9d6c7a5931ab1efc69
+Subproject commit e40271670b4df96a8d02f32a1ba61a838419db48

external/solanaceae_message3

@@ -1 +1 @@
-Subproject commit c73f727adc99b2b73d09b7ebaa696b882ce535b6
+Subproject commit b8893b1c5ca49178f6ab96db18bf72a50f6b6653

external/solanaceae_tox

@@ -1 +1 @@
-Subproject commit b2a3cb7052041bea2d4793b8446fb2a00411773a
+Subproject commit eeb57e137d08798bb08dc8de1f69abcabf0e0611

flake.nix

@@ -12,13 +12,15 @@
     flake-utils.lib.eachDefaultSystem (system:
     let
       pkgs = import nixpkgs { inherit system; };
+      stdenv = (pkgs.stdenvAdapters.keepDebugInfo pkgs.stdenv);
     in {
-      packages.default = pkgs.stdenv.mkDerivation {
+      #packages.default = pkgs.stdenv.mkDerivation {
+      packages.default = stdenv.mkDerivation {
         pname = "tomato";
         version = "0.0.0";
 
         src = ./.;
-        submodules = 1;
+        submodules = 1; # does nothing
 
         nativeBuildInputs = with pkgs; [
           cmake
@@ -58,6 +60,11 @@
         cmakeFlags = [
           "TOMATO_ASAN=1"
           "CMAKE_BUILD_TYPE=RelWithDebInfo"
+
+          "-DFETCHCONTENT_SOURCE_DIR_JSON=${pkgs.nlohmann_json.src}" # we care less about version here
+          # do we really care less about the version? do we need a stable abi?
+
+          "-DFETCHCONTENT_SOURCE_DIR_ZSTD=${pkgs.zstd.src}"
         ];
 
         # TODO: replace with install command
@@ -66,7 +73,7 @@
           mv bin/tomato $out/bin
         '';
 
-        dontStrip = true;
+        dontStrip = true; # does nothing
 
         # copied from nixpkgs's SDL2 default.nix
         # SDL is weird in that instead of just dynamically linking with
@@ -93,6 +100,8 @@
           '';
       };
 
+      #packages.debug = pkgs.enableDebugging self.packages.${system}.default;
+
       devShells.${system}.default = pkgs.mkShell {
         #inputsFrom = with pkgs; [ SDL2 ];
         buildInputs = [ self.packages.${system}.default ]; # this makes a prebuild tomato available in the shell, do we want this?

src/CMakeLists.txt

@@ -1,5 +1,60 @@
 cmake_minimum_required(VERSION 3.9 FATAL_ERROR)
 
+add_library(fragment_store
+	./fragment_store/fragment_store_i.hpp
+	./fragment_store/fragment_store_i.cpp
+	./fragment_store/types.hpp
+	./fragment_store/meta_components.hpp
+	./fragment_store/meta_components_id.inl
+	./fragment_store/serializer.hpp
+	./fragment_store/fragment_store.hpp
+	./fragment_store/fragment_store.cpp
+
+	./json/message_components.hpp # TODO: move
+	./json/tox_message_components.hpp # TODO: move
+)
+
+target_link_libraries(fragment_store PUBLIC
+	nlohmann_json::nlohmann_json
+	EnTT::EnTT
+	solanaceae_util
+
+	zstd::zstd
+
+	solanaceae_tox_messages # TODO: move
+)
+
+########################################
+
+add_library(message_fragment_store
+	./fragment_store/message_serializer.hpp
+	./fragment_store/message_serializer.cpp
+	./fragment_store/message_fragment_store.hpp
+	./fragment_store/message_fragment_store.cpp
+
+	./fragment_store/register_mfs_json_message_components.hpp
+	./fragment_store/register_mfs_json_message_components.cpp
+	./fragment_store/register_mfs_json_tox_message_components.hpp
+	./fragment_store/register_mfs_json_tox_message_components.cpp
+)
+target_compile_features(message_fragment_store PRIVATE cxx_std_20)
+target_link_libraries(message_fragment_store PUBLIC
+	fragment_store
+	solanaceae_message3
+)
+
+########################################
+
+add_executable(fragment_store_test
+	fragment_store/test_fragstore.cpp
+)
+
+target_link_libraries(fragment_store_test PUBLIC
+	fragment_store
+)
+
+########################################
+
 add_executable(tomato
 	./main.cpp
 	./icon.rc
@@ -25,6 +80,8 @@ add_executable(tomato
 	./image_loader_stb.cpp
 	./image_loader_webp.hpp
 	./image_loader_webp.cpp
+	./image_loader_qoi.hpp
+	./image_loader_qoi.cpp
 
 	./texture_uploader.hpp
 	./sdlrenderer_texture_uploader.hpp
@@ -80,6 +137,9 @@ target_link_libraries(tomato PUBLIC
 	solanaceae_tox_contacts
 	solanaceae_tox_messages
 
+	fragment_store
+	message_fragment_store
+
 	SDL3::SDL3
 
 	imgui
@@ -90,5 +150,6 @@ target_link_libraries(tomato PUBLIC
 	stb_image_write
 	webpdemux
 	libwebpmux # the f why (needed for anim encode)
+	qoi
 )
 

src/chat_gui4.cpp

@@ -37,6 +37,18 @@ namespace Components {
 
 } // Components
 
+namespace Context {
+
+	// TODO: move back to chat log window and keep per window instead of per contact
+	struct CGView {
+		// set to the ts of the newest rendered msg
+		Message3Handle begin{};
+		// set to the ts of the oldest rendered msg
+		Message3Handle end{};
+	};
+
+} // Context
+
 static constexpr float lerp(float a, float b, float t) {
 	return a + t * (b - a);
 }
@@ -259,28 +271,6 @@ float ChatGui4::render(float time_delta) {
 
 					auto* msg_reg_ptr = _rmm.get(*_selected_contact);
 
-					if (msg_reg_ptr != nullptr) {
-						const auto& mm = *msg_reg_ptr;
-						//const auto& unread_storage = mm.storage<Message::Components::TagUnread>();
-						if (const auto* unread_storage = mm.storage<Message::Components::TagUnread>(); unread_storage != nullptr && !unread_storage->empty()) {
-							//assert(unread_storage->size() == 0);
-							//assert(unread_storage.cbegin() == unread_storage.cend());
-
-#if 0
-							std::cout << "UNREAD ";
-							Message3 prev_ent = entt::null;
-							for (const Message3 e : mm.view<Message::Components::TagUnread>()) {
-								std::cout << entt::to_integral(e) << " ";
-								if (prev_ent == e) {
-									assert(false && "dup");
-								}
-								prev_ent = e;
-							}
-							std::cout << "\n";
-#endif
-						}
-					}
-
 					constexpr ImGuiTableFlags table_flags =
 						ImGuiTableFlags_BordersInnerV |
 						ImGuiTableFlags_RowBg |
@@ -293,6 +283,9 @@ float ChatGui4::render(float time_delta) {
 						ImGui::TableSetupColumn("timestamp");
 						ImGui::TableSetupColumn("extra_info", _show_chat_extra_info ? ImGuiTableColumnFlags_None : ImGuiTableColumnFlags_Disabled);
 
+						Message3Handle message_view_oldest; // oldest visible message
+						Message3Handle message_view_newest; // last visible message
+
 						// very hacky, and we have variable hight entries
 						//ImGuiListClipper clipper;
 
@@ -381,12 +374,26 @@ float ChatGui4::render(float time_delta) {
 								}
 
 								// use username as visibility test
-								if (ImGui::IsItemVisible() && msg_reg.all_of<Message::Components::TagUnread>(e)) {
+								if (ImGui::IsItemVisible()) {
+									if (msg_reg.all_of<Message::Components::TagUnread>(e)) {
 										// get time now
 										const uint64_t ts_now = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
 										msg_reg.emplace_or_replace<Message::Components::Read>(e, ts_now);
 										msg_reg.remove<Message::Components::TagUnread>(e);
 										msg_reg.emplace_or_replace<Components::UnreadFade>(e, 1.f);
+
+										// we remove the unread tag here
+										_rmm.throwEventUpdate(msg_reg, e);
+									}
+
+									// track view
+									if (!static_cast<bool>(message_view_oldest)) {
+										message_view_oldest = {msg_reg, e};
+										message_view_newest = {msg_reg, e};
+									} else if (static_cast<bool>(message_view_newest)) {
+										// update to latest
+										message_view_newest = {msg_reg, e};
+									}
 								}
 
 								// highlight self
@@ -539,9 +546,90 @@ float ChatGui4::render(float time_delta) {
 						//ImGui::TableNextRow(0, TEXT_BASE_HEIGHT);
 						//ImGui::TableNextRow(0, TEXT_BASE_HEIGHT);
 
+						{ // update view cursers
+							if (!msg_reg.ctx().contains<Context::CGView>()) {
+								msg_reg.ctx().emplace<Context::CGView>();
+							}
+
+							auto& cg_view = msg_reg.ctx().get<Context::CGView>();
+
+							// any message in view
+							if (!static_cast<bool>(message_view_oldest)) {
+								// no message in view, we setup a view at current time, so the next frags are loaded
+								if (!static_cast<bool>(cg_view.begin) || !static_cast<bool>(cg_view.end)) {
+									// fix invalid state
+									if (static_cast<bool>(cg_view.begin)) {
+										cg_view.begin.destroy();
+										_rmm.throwEventDestroy(cg_view.begin);
+									}
+									if (static_cast<bool>(cg_view.end)) {
+										cg_view.end.destroy();
+										_rmm.throwEventDestroy(cg_view.end);
+									}
+
+									// create new
+									cg_view.begin = {msg_reg, msg_reg.create()};
+									cg_view.end = {msg_reg, msg_reg.create()};
+
+									cg_view.begin.emplace_or_replace<Message::Components::ViewCurserBegin>(cg_view.end);
+									cg_view.end.emplace_or_replace<Message::Components::ViewCurserEnd>(cg_view.begin);
+
+									cg_view.begin.get_or_emplace<Message::Components::Timestamp>().ts = Message::getTimeMS();
+									cg_view.end.get_or_emplace<Message::Components::Timestamp>().ts = Message::getTimeMS();
+
+									std::cout << "CG: created view FRONT begin ts\n";
+									_rmm.throwEventConstruct(cg_view.begin);
+									std::cout << "CG: created view FRONT end ts\n";
+									_rmm.throwEventConstruct(cg_view.end);
+								} // else? we do nothing?
+							} else {
+								bool begin_created {false};
+								if (!static_cast<bool>(cg_view.begin)) {
+									cg_view.begin = {msg_reg, msg_reg.create()};
+									begin_created = true;
+								}
+								bool end_created {false};
+								if (!static_cast<bool>(cg_view.end)) {
+									cg_view.end = {msg_reg, msg_reg.create()};
+									end_created = true;
+								}
+								cg_view.begin.emplace_or_replace<Message::Components::ViewCurserBegin>(cg_view.end);
+								cg_view.end.emplace_or_replace<Message::Components::ViewCurserEnd>(cg_view.begin);
+
+								{
+									auto& old_begin_ts = cg_view.begin.get_or_emplace<Message::Components::Timestamp>().ts;
+									if (old_begin_ts != message_view_newest.get<Message::Components::Timestamp>().ts) {
+										old_begin_ts = message_view_newest.get<Message::Components::Timestamp>().ts;
+										if (begin_created) {
+											std::cout << "CG: created view begin ts with " << old_begin_ts << "\n";
+											_rmm.throwEventConstruct(cg_view.begin);
+										} else {
+											//std::cout << "CG: updated view begin ts to " << old_begin_ts << "\n";
+											_rmm.throwEventUpdate(cg_view.begin);
+										}
+									}
+								}
+
+								{
+									auto& old_end_ts = cg_view.end.get_or_emplace<Message::Components::Timestamp>().ts;
+									if (old_end_ts != message_view_oldest.get<Message::Components::Timestamp>().ts) {
+										old_end_ts = message_view_oldest.get<Message::Components::Timestamp>().ts;
+										if (end_created) {
+											std::cout << "CG: created view end ts with " << old_end_ts << "\n";
+											_rmm.throwEventConstruct(cg_view.end);
+										} else {
+											//std::cout << "CG: updated view end ts to " << old_end_ts << "\n";
+											_rmm.throwEventUpdate(cg_view.end);
+										}
+									}
+								}
+							}
+						}
+
 						ImGui::EndTable();
 					}
 
+
 					if (ImGui::GetScrollY() >= ImGui::GetScrollMaxY()) {
 						ImGui::SetScrollHereY(1.f);
 					}
@@ -613,6 +701,7 @@ float ChatGui4::render(float time_delta) {
 								"image/gif",
 								"image/jpeg",
 								"image/bmp",
+								"image/qoi",
 							};
 
 							for (const char* mime_type : image_mime_types) {

src/chat_gui4.hpp

@@ -10,6 +10,9 @@
 #include "./file_selector.hpp"
 #include "./send_image_popup.hpp"
 
+// HACK: move to public msg api?
+#include "./fragment_store/message_fragment_store.hpp"
+
 #include <entt/container/dense_map.hpp>
 
 #include <cstdint>
@@ -32,6 +35,7 @@ class ChatGui4 {
 	FileSelector _fss;
 	SendImagePopup _sip;
 
+	// TODO: refactor this to allow multiple open contacts
 	std::optional<Contact3> _selected_contact;
 
 	// TODO: per contact

src/fragment_store/README.md

@@ -25,41 +25,41 @@ Just keeps the Fragments in memory.
 
 # File formats
 
-Files can be compressed and encrypted. Since compression needs the data structure to funcion, it is applied before it is encrypted.
+Files can be compressed and encrypted. Since compression needs the data's structure to work properly, it is applied before it is encrypted.
 
 ### Text Json
 
 Text json only makes sense for metadata if it's neither compressed nor encrypted. (otherwise its binary on disk anyway, so why waste bytes).
 Since the content of data is not looked at, nothing stops you from using text json and ecrypt it, but atleast basic compression is advised.
 
-A Metadata json object has the following keys:
+A Metadata json object can have arbitrary keys, some are predefined:
-- `enc` (uint) Encryption type of the data, if any
+- `FragComp::DataEncryptionType` (uint) Encryption type of the data, if any
-- `comp` (uint) Compression type of the data, if any
+- `FragComp::DataCompressionType` (uint) Compression type of the data, if any
-- `metadata` (obj) the 
 
 ## Binary file headers
 
 ### Split Metadata
 
-file magic bytes `SOLMET` (6 bytes)
+msgpack array:
 
-1 byte encryption type (`0x00` is none)
+- `[0]`: file magic string `SOLMET` (6 bytes)
-
+- `[1]`: uint8 encryption type (`0x00` is none)
-1 byte compression type (`0x00` is none)
+- `[2]`: uint8 compression type (`0x00` is none, `0x01` is zstd)
-
+- `[3]`: binary metadata (optionally compressed and encrypted)
-...metadata here...
 
 note that the encryption and compression are for the metadata only.
 The metadata itself contains encryption and compression info about the data.
 
 ### Split Data
 
-(none) all the data is in the metadata file.
+All the metadata is in the metadata file. (like encryption and compression)
 This is mostly to allow direct storage for files in the Fragment store without excessive duplication.
 Keep in mind to not use the actual file name as the data/meta file name.
 
 ### Single fragment
 
+Note: this format is unused for now
+
 file magic bytes `SOLFIL` (6 bytes)
 
 1 byte encryption type (`0x00` is none)
@@ -70,3 +70,7 @@ file magic bytes `SOLFIL` (6 bytes)
 
 ...data here...
 
+## Compression types
+
+- `0x00` none
+- `0x01` zstd (without dict)

src/fragment_store/fragment_store.cpp

@@ -0,0 +1,869 @@
+#include "./fragment_store.hpp"
+
+#include <solanaceae/util/utils.hpp>
+
+#include <entt/entity/handle.hpp>
+#include <entt/container/dense_set.hpp>
+#include <entt/core/hashed_string.hpp>
+
+#include <nlohmann/json.hpp>
+
+#include <zstd.h>
+
+#include <cstdint>
+#include <fstream>
+#include <filesystem>
+#include <memory>
+#include <mutex>
+#include <type_traits>
+#include <utility>
+#include <algorithm>
+
+#include <iostream>
+#include <vector>
+
+static const char* metaFileTypeSuffix(MetaFileType mft) {
+	switch (mft) {
+		case MetaFileType::TEXT_JSON: return ".json";
+		//case MetaFileType::BINARY_ARB: return ".bin";
+		case MetaFileType::BINARY_MSGPACK: return ".msgpack";
+	}
+	return ""; // .unk?
+}
+
+FragmentStore::FragmentStore(void) {
+	{ // random namespace
+		const auto num0 = _rng();
+		const auto num1 = _rng();
+		const auto num2 = _rng();
+		const auto num3 = _rng();
+
+		_session_uuid_namespace[0+0] = (num0 >> 0) & 0xff;
+		_session_uuid_namespace[0+1] = (num0 >> 8) & 0xff;
+		_session_uuid_namespace[0+2] = (num0 >> 16) & 0xff;
+		_session_uuid_namespace[0+3] = (num0 >> 24) & 0xff;
+
+		_session_uuid_namespace[4+0] = (num1 >> 0) & 0xff;
+		_session_uuid_namespace[4+1] = (num1 >> 8) & 0xff;
+		_session_uuid_namespace[4+2] = (num1 >> 16) & 0xff;
+		_session_uuid_namespace[4+3] = (num1 >> 24) & 0xff;
+
+		_session_uuid_namespace[8+0] = (num2 >> 0) & 0xff;
+		_session_uuid_namespace[8+1] = (num2 >> 8) & 0xff;
+		_session_uuid_namespace[8+2] = (num2 >> 16) & 0xff;
+		_session_uuid_namespace[8+3] = (num2 >> 24) & 0xff;
+
+		_session_uuid_namespace[12+0] = (num3 >> 0) & 0xff;
+		_session_uuid_namespace[12+1] = (num3 >> 8) & 0xff;
+		_session_uuid_namespace[12+2] = (num3 >> 16) & 0xff;
+		_session_uuid_namespace[12+3] = (num3 >> 24) & 0xff;
+	}
+	registerSerializers();
+}
+
+FragmentStore::FragmentStore(
+	std::array<uint8_t, 16> session_uuid_namespace
+) : _session_uuid_namespace(std::move(session_uuid_namespace)) {
+	registerSerializers();
+}
+
+FragmentHandle FragmentStore::fragmentHandle(FragmentID fid) {
+	return {_reg, fid};
+}
+
+std::vector<uint8_t> FragmentStore::generateNewUID(std::array<uint8_t, 16>& uuid_namespace) {
+	std::vector<uint8_t> new_uid(uuid_namespace.cbegin(), uuid_namespace.cend());
+	new_uid.resize(new_uid.size() + 16);
+
+	const auto num0 = _rng();
+	const auto num1 = _rng();
+	const auto num2 = _rng();
+	const auto num3 = _rng();
+
+	new_uid[uuid_namespace.size()+0] = (num0 >> 0) & 0xff;
+	new_uid[uuid_namespace.size()+1] = (num0 >> 8) & 0xff;
+	new_uid[uuid_namespace.size()+2] = (num0 >> 16) & 0xff;
+	new_uid[uuid_namespace.size()+3] = (num0 >> 24) & 0xff;
+
+	new_uid[uuid_namespace.size()+4+0] = (num1 >> 0) & 0xff;
+	new_uid[uuid_namespace.size()+4+1] = (num1 >> 8) & 0xff;
+	new_uid[uuid_namespace.size()+4+2] = (num1 >> 16) & 0xff;
+	new_uid[uuid_namespace.size()+4+3] = (num1 >> 24) & 0xff;
+
+	new_uid[uuid_namespace.size()+8+0] = (num2 >> 0) & 0xff;
+	new_uid[uuid_namespace.size()+8+1] = (num2 >> 8) & 0xff;
+	new_uid[uuid_namespace.size()+8+2] = (num2 >> 16) & 0xff;
+	new_uid[uuid_namespace.size()+8+3] = (num2 >> 24) & 0xff;
+
+	new_uid[uuid_namespace.size()+12+0] = (num3 >> 0) & 0xff;
+	new_uid[uuid_namespace.size()+12+1] = (num3 >> 8) & 0xff;
+	new_uid[uuid_namespace.size()+12+2] = (num3 >> 16) & 0xff;
+	new_uid[uuid_namespace.size()+12+3] = (num3 >> 24) & 0xff;
+
+	return new_uid;
+}
+
+std::vector<uint8_t> FragmentStore::generateNewUID(void) {
+	return generateNewUID(_session_uuid_namespace);
+}
+
+FragmentID FragmentStore::newFragmentMemoryOwned(
+	const std::vector<uint8_t>& id,
+	size_t initial_size
+) {
+	{ // first check if id is already used
+		auto exising_id = getFragmentByID(id);
+		if (_reg.valid(exising_id)) {
+			return entt::null;
+		}
+	}
+
+	{ // next check if space in memory budget
+		const auto free_memory = _memory_budget - _memory_usage;
+		if (initial_size > free_memory) {
+			return entt::null;
+		}
+	}
+
+	// actually allocate and create
+	auto new_data = std::make_unique<std::vector<uint8_t>>(initial_size);
+	if (!static_cast<bool>(new_data)) {
+		// allocation failure
+		return entt::null;
+	}
+	_memory_usage += initial_size;
+
+	const auto new_frag = _reg.create();
+
+	_reg.emplace<FragComp::ID>(new_frag, id);
+	// TODO: memory comp
+	_reg.emplace<std::unique_ptr<std::vector<uint8_t>>>(new_frag) = std::move(new_data);
+
+	throwEventConstruct(new_frag);
+
+	return new_frag;
+}
+
+FragmentID FragmentStore::newFragmentFile(
+	std::string_view store_path,
+	MetaFileType mft,
+	const std::vector<uint8_t>& id
+) {
+	{ // first check if id is already used
+		const auto exising_id = getFragmentByID(id);
+		if (_reg.valid(exising_id)) {
+			return entt::null;
+		}
+	}
+
+	if (store_path.empty()) {
+		store_path = _default_store_path;
+	}
+
+	std::filesystem::create_directories(store_path);
+
+	const auto id_hex = bin2hex(id);
+	std::filesystem::path fragment_file_path;
+
+	if (id_hex.size() < 6) {
+		fragment_file_path = std::filesystem::path{store_path}/id_hex;
+	} else {
+		// use the first 2hex (1byte) as a subfolder
+		std::filesystem::create_directories(std::string{store_path} + id_hex.substr(0, 2));
+		fragment_file_path = std::filesystem::path{std::string{store_path} + id_hex.substr(0, 2)} / id_hex.substr(2);
+	}
+
+	if (std::filesystem::exists(fragment_file_path)) {
+		return entt::null;
+	}
+
+	const auto new_frag = _reg.create();
+
+	_reg.emplace<FragComp::ID>(new_frag, id);
+
+	// file (info) comp
+	_reg.emplace<FragComp::Ephemeral::FilePath>(new_frag, fragment_file_path.generic_u8string());
+
+	_reg.emplace<FragComp::Ephemeral::MetaFileType>(new_frag, mft);
+
+	// meta needs to be synced to file
+	std::function<write_to_storage_fetch_data_cb> empty_data_cb = [](const uint8_t*, uint64_t) -> uint64_t { return 0; };
+	if (!syncToStorage(new_frag, empty_data_cb)) {
+		std::cerr << "FS error: syncToStorage failed while creating new fragment file\n";
+		_reg.destroy(new_frag);
+		return entt::null;
+	}
+
+	// while new metadata might be created here, making sure the file could be created is more important
+	throwEventConstruct(new_frag);
+
+	return new_frag;
+}
+FragmentID FragmentStore::newFragmentFile(
+	std::string_view store_path,
+	MetaFileType mft
+) {
+	return newFragmentFile(store_path, mft, generateNewUID());
+}
+
+FragmentID FragmentStore::getFragmentByID(
+	const std::vector<uint8_t>& id
+) {
+	// TODO: accelerate
+	// maybe keep it sorted and binary search? hash table lookup?
+	for (const auto& [frag, id_comp] : _reg.view<FragComp::ID>().each()) {
+		if (id == id_comp.v) {
+			return frag;
+		}
+	}
+
+	return entt::null;
+}
+
+FragmentID FragmentStore::getFragmentCustomMatcher(
+	std::function<bool(FragmentID)>& fn
+) {
+	return entt::null;
+}
+
+bool FragmentStore::syncToStorage(FragmentID fid, std::function<write_to_storage_fetch_data_cb>& data_cb) {
+	if (!_reg.valid(fid)) {
+		return false;
+	}
+
+	if (!_reg.all_of<FragComp::Ephemeral::FilePath>(fid)) {
+		// not a file fragment?
+		return false;
+	}
+
+	// split object storage
+
+	MetaFileType meta_type = MetaFileType::TEXT_JSON; // TODO: better defaults
+	if (_reg.all_of<FragComp::Ephemeral::MetaFileType>(fid)) {
+		meta_type = _reg.get<FragComp::Ephemeral::MetaFileType>(fid).type;
+	}
+
+	Encryption meta_enc = Encryption::NONE; // TODO: better defaults
+	Compression meta_comp = Compression::NONE; // TODO: better defaults
+
+	if (meta_type != MetaFileType::TEXT_JSON) {
+		if (_reg.all_of<FragComp::Ephemeral::MetaEncryptionType>(fid)) {
+			meta_enc = _reg.get<FragComp::Ephemeral::MetaEncryptionType>(fid).enc;
+		}
+
+		if (_reg.all_of<FragComp::Ephemeral::MetaCompressionType>(fid)) {
+			meta_comp = _reg.get<FragComp::Ephemeral::MetaCompressionType>(fid).comp;
+		}
+	} else {
+		// we cant have encryption or compression
+
+		// TODO: warning/error?
+
+		// TODO: forcing for testing
+		//if (_reg.all_of<Components::Ephemeral::MetaEncryptionType>(fid)) {
+			_reg.emplace_or_replace<FragComp::Ephemeral::MetaEncryptionType>(fid, Encryption::NONE);
+		//}
+		//if (_reg.all_of<Components::Ephemeral::MetaCompressionType>(fid)) {
+			_reg.emplace_or_replace<FragComp::Ephemeral::MetaCompressionType>(fid, Compression::NONE);
+		//}
+	}
+
+	std::filesystem::path meta_tmp_path = _reg.get<FragComp::Ephemeral::FilePath>(fid).path + ".meta" + metaFileTypeSuffix(meta_type) + ".tmp";
+	meta_tmp_path.replace_filename("." + meta_tmp_path.filename().generic_u8string());
+	std::ofstream meta_file{
+		meta_tmp_path,
+		std::ios::out | std::ios::trunc | std::ios::binary // always binary, also for text
+	};
+
+	if (!meta_file.is_open()) {
+		std::cerr << "FS error: failed to create temporary meta file\n";
+		return false;
+	}
+
+	Compression data_comp = Compression::NONE; // TODO: better defaults
+	if (_reg.all_of<FragComp::DataCompressionType>(fid)) {
+		data_comp = _reg.get<FragComp::DataCompressionType>(fid).comp;
+	}
+
+	std::filesystem::path data_tmp_path = _reg.get<FragComp::Ephemeral::FilePath>(fid).path + ".tmp";
+	data_tmp_path.replace_filename("." + data_tmp_path.filename().generic_u8string());
+	std::ofstream data_file{
+		data_tmp_path,
+		std::ios::out | std::ios::trunc | std::ios::binary // always binary, also for text
+	};
+
+	if (!data_file.is_open()) {
+		meta_file.close();
+		std::filesystem::remove(meta_tmp_path);
+		std::cerr << "FS error: failed to create temporary data file\n";
+		return false;
+	}
+
+	// sharing code between binary msgpack and text json for now
+	nlohmann::json meta_data_j = nlohmann::json::object(); // metadata needs to be an object, null not allowed
+	// metadata file
+
+	for (const auto& [type_id, storage] : _reg.storage()) {
+		if (!storage.contains(fid)) {
+			continue;
+		}
+
+		//std::cout << "storage type: type_id:" << type_id << " name:" << storage.type().name() << "\n";
+
+		// use type_id to find serializer
+		auto s_cb_it = _sc._serl_json.find(type_id);
+		if (s_cb_it == _sc._serl_json.end()) {
+			// could not find serializer, not saving
+			continue;
+		}
+
+		// noooo, why cant numbers be keys
+		//if (meta_type == MetaFileType::BINARY_MSGPACK) { // msgpack uses the hash id instead
+			//s_cb_it->second(storage.value(fid), meta_data[storage.type().hash()]);
+		//} else if (meta_type == MetaFileType::TEXT_JSON) {
+		s_cb_it->second({_reg, fid}, meta_data_j[storage.type().name()]);
+		//}
+	}
+
+	if (meta_type == MetaFileType::BINARY_MSGPACK) { // binary metadata file
+		const std::vector<uint8_t> meta_data = nlohmann::json::to_msgpack(meta_data_j);
+		std::vector<uint8_t> meta_data_compressed; // empty if none
+		//std::vector<uint8_t> meta_data_encrypted; // empty if none
+
+		if (meta_comp == Compression::ZSTD) {
+			meta_data_compressed.resize(ZSTD_compressBound(meta_data.size()));
+
+			size_t const cSize = ZSTD_compress(meta_data_compressed.data(), meta_data_compressed.size(), meta_data.data(), meta_data.size(), 0); // 0 is default is probably 3
+			if (ZSTD_isError(cSize)) {
+				std::cerr << "FS error: compressing meta failed\n";
+				meta_data_compressed.clear();
+				meta_comp = Compression::NONE;
+			} else {
+				meta_data_compressed.resize(cSize);
+			}
+		} else if (meta_comp == Compression::NONE) {
+			// do nothing
+		} else {
+			assert(false && "implement me");
+		}
+
+		// TODO: encryption
+
+		// the meta file is itself msgpack data
+		nlohmann::json meta_header_j = nlohmann::json::array();
+		meta_header_j.emplace_back() = "SOLMET";
+		meta_header_j.push_back(meta_enc);
+		meta_header_j.push_back(meta_comp);
+
+		if (false) { // TODO: encryption
+		} else if (!meta_data_compressed.empty()) {
+			meta_header_j.push_back(nlohmann::json::binary(meta_data_compressed));
+		} else {
+			meta_header_j.push_back(nlohmann::json::binary(meta_data));
+		}
+
+		const auto meta_header_data = nlohmann::json::to_msgpack(meta_header_j);
+		meta_file.write(reinterpret_cast<const char*>(meta_header_data.data()), meta_header_data.size());
+	} else if (meta_type == MetaFileType::TEXT_JSON) {
+		// cant be compressed or encrypted
+		meta_file << meta_data_j.dump(2, ' ', true);
+	}
+
+	// now data
+	if (data_comp == Compression::NONE) {
+		std::array<uint8_t, 1024> buffer;
+		uint64_t buffer_actual_size {0};
+		do {
+			buffer_actual_size = data_cb(buffer.data(), buffer.size());
+			if (buffer_actual_size == 0) {
+				break;
+			}
+			if (buffer_actual_size > buffer.size()) {
+				// wtf
+				break;
+			}
+
+			data_file.write(reinterpret_cast<const char*>(buffer.data()), buffer_actual_size);
+		} while (buffer_actual_size == buffer.size());
+	} else if (data_comp == Compression::ZSTD) {
+		std::vector<uint8_t> buffer(ZSTD_CStreamInSize());
+		std::vector<uint8_t> compressed_buffer(ZSTD_CStreamOutSize());
+		uint64_t buffer_actual_size {0};
+
+		ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+		ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 0); // default (3)
+		ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1); // add extra checksums (to frames?)
+		do {
+			buffer_actual_size = data_cb(buffer.data(), buffer.size());
+			//if (buffer_actual_size == 0) {
+				//break;
+			//}
+			if (buffer_actual_size > buffer.size()) {
+				// wtf
+				break;
+			}
+			bool const lastChunk = (buffer_actual_size < buffer.size());
+
+			ZSTD_EndDirective const mode = lastChunk ? ZSTD_e_end : ZSTD_e_continue;
+			ZSTD_inBuffer input = { buffer.data(), buffer_actual_size, 0 };
+
+			while (input.pos < input.size) {
+				ZSTD_outBuffer output = { compressed_buffer.data(), compressed_buffer.size(), 0 };
+
+				size_t const remaining = ZSTD_compressStream2(cctx, &output , &input, mode);
+				if (ZSTD_isError(remaining)) {
+					std::cerr << "FS error: compressing data failed\n";
+					break;
+				}
+
+				data_file.write(reinterpret_cast<const char*>(compressed_buffer.data()), output.pos);
+
+				if (remaining == 0) {
+					break;
+				}
+			}
+			// same as if lastChunk break;
+		} while (buffer_actual_size == buffer.size());
+	} else {
+		assert(false && "implement me");
+	}
+
+	meta_file.flush();
+	meta_file.close();
+	data_file.flush();
+	data_file.close();
+
+	std::filesystem::rename(
+		meta_tmp_path,
+		_reg.get<FragComp::Ephemeral::FilePath>(fid).path + ".meta" + metaFileTypeSuffix(meta_type)
+	);
+
+	std::filesystem::rename(
+		data_tmp_path,
+		_reg.get<FragComp::Ephemeral::FilePath>(fid).path
+	);
+
+	// TODO: check return value of renames
+
+	if (_reg.all_of<FragComp::Ephemeral::DirtyTag>(fid)) {
+		_reg.remove<FragComp::Ephemeral::DirtyTag>(fid);
+	}
+
+	return true;
+}
+
+bool FragmentStore::syncToStorage(FragmentID fid, const uint8_t* data, const uint64_t data_size) {
+	std::function<FragmentStore::write_to_storage_fetch_data_cb> fn_cb = [read = 0ull, data, data_size](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t {
+		uint64_t i = 0;
+		for (; i+read < data_size && i < buffer_size; i++) {
+			request_buffer[i] = data[i+read];
+		}
+		read += i;
+
+		return i;
+	};
+	return syncToStorage(fid, fn_cb);
+}
+
+bool FragmentStore::loadFromStorage(FragmentID fid, std::function<read_from_storage_put_data_cb>& data_cb) {
+	if (!_reg.valid(fid)) {
+		return false;
+	}
+
+	if (!_reg.all_of<FragComp::Ephemeral::FilePath>(fid)) {
+		// not a file fragment?
+		// TODO: memory fragments
+		return false;
+	}
+
+	const auto& frag_path = _reg.get<FragComp::Ephemeral::FilePath>(fid).path;
+
+	// TODO: check if metadata dirty?
+	// TODO: what if file changed on disk?
+
+	std::cout << "FS: loading fragment '" << frag_path << "'\n";
+
+	std::ifstream data_file{
+		frag_path,
+		std::ios::in | std::ios::binary // always binary, also for text
+	};
+
+	if (!data_file.is_open()) {
+		std::cerr << "FS error: fragment data file failed to open '" << frag_path << "'\n";
+		// error
+		return false;
+	}
+
+	Compression data_comp = Compression::NONE;
+	if (_reg.all_of<FragComp::DataCompressionType>(fid)) {
+		data_comp = _reg.get<FragComp::DataCompressionType>(fid).comp;
+	}
+
+	if (data_comp == Compression::NONE) {
+		std::array<uint8_t, 1024> buffer;
+		uint64_t buffer_actual_size {0};
+		do {
+			data_file.read(reinterpret_cast<char*>(buffer.data()), buffer.size());
+			buffer_actual_size = data_file.gcount();
+
+			if (buffer_actual_size == 0) {
+				break;
+			}
+
+			data_cb(buffer.data(), buffer_actual_size);
+		} while (buffer_actual_size == buffer.size() && !data_file.eof());
+	} else if (data_comp == Compression::ZSTD) {
+		std::vector<uint8_t> in_buffer(ZSTD_DStreamInSize());
+		std::vector<uint8_t> out_buffer(ZSTD_DStreamOutSize());
+		ZSTD_DCtx* const dctx = ZSTD_createDCtx();
+
+		uint64_t buffer_actual_size {0};
+		do {
+			data_file.read(reinterpret_cast<char*>(in_buffer.data()), in_buffer.size());
+			buffer_actual_size = data_file.gcount();
+			if (buffer_actual_size == 0) {
+				break;
+			}
+
+			ZSTD_inBuffer input {in_buffer.data(), buffer_actual_size, 0 };
+			do {
+				ZSTD_outBuffer output = { out_buffer.data(), out_buffer.size(), 0 };
+				size_t const ret = ZSTD_decompressStream(dctx, &output , &input);
+				if (ZSTD_isError(ret)) {
+					// error <.<
+					std::cerr << "FS error: decompression error\n";
+					break;
+				}
+
+				data_cb(out_buffer.data(), output.pos);
+			} while (input.pos < input.size);
+		} while (buffer_actual_size == in_buffer.size() && !data_file.eof());
+
+		ZSTD_freeDCtx(dctx);
+	} else {
+		assert(false && "implement me");
+	}
+
+	return true;
+}
+
+nlohmann::json FragmentStore::loadFromStorageNJ(FragmentID fid) {
+	std::vector<uint8_t> tmp_buffer;
+	std::function<read_from_storage_put_data_cb> cb = [&tmp_buffer](const uint8_t* buffer, const uint64_t buffer_size) {
+		tmp_buffer.insert(tmp_buffer.end(), buffer, buffer+buffer_size);
+	};
+
+	if (!loadFromStorage(fid, cb)) {
+		return nullptr;
+	}
+
+	return nlohmann::json::parse(tmp_buffer);
+}
+
+size_t FragmentStore::scanStoragePath(std::string_view path) {
+	if (path.empty()) {
+		path = _default_store_path;
+	}
+	// TODO: extract so async can work (or/and make iteratable generator)
+
+	if (!std::filesystem::is_directory(path)) {
+		std::cerr << "FS error: scan path not a directory '" << path << "'\n";
+		return 0;
+	}
+
+	// step 1: make snapshot of files, validate metafiles and save id/path+meta.ext
+	// can be extra thread (if non vfs)
+	struct FragFileEntry {
+		std::string id_str;
+		std::filesystem::path frag_path;
+		std::string meta_ext;
+
+		bool operator==(const FragFileEntry& other) const {
+			// only compare by id
+			return id_str == other.id_str;
+		}
+	};
+	struct FragFileEntryHash {
+		size_t operator()(const FragFileEntry& it) const {
+			return entt::hashed_string(it.id_str.data(), it.id_str.size());
+		}
+	};
+	entt::dense_set<FragFileEntry, FragFileEntryHash> file_frag_list;
+
+	std::filesystem::path storage_path{path};
+
+	auto handle_file = [&](const std::filesystem::path& file_path) {
+		if (!std::filesystem::is_regular_file(file_path)) {
+			return;
+		}
+		// handle file
+
+		if (file_path.has_extension()) {
+			// skip over metadata, assuming only metafiles have extentions (might be wrong?)
+			// also skips temps
+			return;
+		}
+
+		auto relative_path = std::filesystem::proximate(file_path, storage_path);
+		std::string id_str = relative_path.generic_u8string();
+		// delete all '/'
+		id_str.erase(std::remove(id_str.begin(), id_str.end(), '/'), id_str.end());
+		if (id_str.size() % 2 != 0) {
+			std::cerr << "FS error: non hex fragment uid detected: '" << id_str << "'\n";
+		}
+
+		if (file_frag_list.contains(FragFileEntry{id_str, {}, ""})) {
+			std::cerr << "FS error: fragment duplicate detected: '" << id_str << "'\n";
+			return; // skip
+		}
+
+		const char* meta_ext = ".meta.msgpack";
+		{ // find meta
+			// TODO: this as to know all possible extentions
+			bool has_meta_msgpack = std::filesystem::is_regular_file(file_path.generic_u8string() + ".meta.msgpack");
+			bool has_meta_json = std::filesystem::is_regular_file(file_path.generic_u8string() + ".meta.json");
+			const size_t meta_sum =
+				(has_meta_msgpack?1:0) +
+				(has_meta_json?1:0)
+			;
+
+			if (meta_sum > 1) { // has multiple
+				std::cerr << "FS error: fragment with multiple meta files detected: " << id_str << "\n";
+				return; // skip
+			}
+
+			if (meta_sum == 0) {
+				std::cerr << "FS error: fragment missing meta file detected: " << id_str << "\n";
+				return; // skip
+			}
+
+			if (has_meta_json) {
+				meta_ext = ".meta.json";
+			}
+		}
+
+		file_frag_list.emplace(FragFileEntry{
+			std::move(id_str),
+			file_path,
+			meta_ext
+		});
+	};
+
+	for (const auto& outer_path : std::filesystem::directory_iterator(storage_path)) {
+		if (std::filesystem::is_regular_file(outer_path)) {
+			handle_file(outer_path);
+		} else if (std::filesystem::is_directory(outer_path)) {
+			// subdir, part of id
+			for (const auto& inner_path : std::filesystem::directory_iterator(outer_path)) {
+				//if (std::filesystem::is_regular_file(inner_path)) {
+
+					//// handle file
+				//} // TODO: support deeper recursion?
+				handle_file(inner_path);
+			}
+		}
+	}
+
+	std::cout << "FS: scan found:\n";
+	for (const auto& it : file_frag_list) {
+		std::cout << "  " << it.id_str << "\n";
+	}
+
+	// step 2: check if files preexist in reg
+	// main thread
+	// (merge into step 3 and do more error checking?)
+	for (auto it = file_frag_list.begin(); it != file_frag_list.end();) {
+		auto id = hex2bin(it->id_str);
+		auto fid = getFragmentByID(id);
+		if (_reg.valid(fid)) {
+			// pre exising (handle differently??)
+			// check if store differs?
+			it = file_frag_list.erase(it);
+		} else {
+			it++;
+		}
+	}
+
+	std::vector<FragmentID> scanned_frags;
+	// step 3: parse meta and insert into reg of non preexising
+	// main thread
+	// TODO: check timestamps of preexisting and reload? mark external/remote dirty?
+	for (const auto& it : file_frag_list) {
+		nlohmann::json j;
+		if (it.meta_ext == ".meta.msgpack") {
+			std::ifstream file(it.frag_path.generic_u8string() + it.meta_ext, std::ios::in | std::ios::binary);
+			if (!file.is_open()) {
+				std::cout << "FS error: failed opening meta " << it.frag_path << "\n";
+				continue;
+			}
+
+			// file is a msgpack within a msgpack
+
+			std::vector<uint8_t> full_meta_data;
+			{ // read meta file
+				// figure out size
+				file.seekg(0, file.end);
+				uint64_t file_size = file.tellg();
+				file.seekg(0, file.beg);
+
+				full_meta_data.resize(file_size);
+
+				file.read(reinterpret_cast<char*>(full_meta_data.data()), full_meta_data.size());
+			}
+
+			const auto meta_header_j = nlohmann::json::from_msgpack(full_meta_data);
+
+			if (!meta_header_j.is_array() || meta_header_j.size() < 4) {
+				std::cerr << "FS error: broken binary meta " << it.frag_path << "\n";
+				continue;
+			}
+
+			if (meta_header_j.at(0) != "SOLMET") {
+				std::cerr << "FS error: wrong magic '" << meta_header_j.at(0) << "' in meta " << it.frag_path << "\n";
+				continue;
+			}
+
+			Encryption meta_enc = meta_header_j.at(1);
+			if (meta_enc != Encryption::NONE) {
+				std::cerr << "FS error: unknown encryption " << it.frag_path << "\n";
+				continue;
+			}
+
+			Compression meta_comp = meta_header_j.at(2);
+			if (meta_comp != Compression::NONE && meta_comp != Compression::ZSTD) {
+				std::cerr << "FS error: unknown compression " << it.frag_path << "\n";
+				continue;
+			}
+
+			//const auto& meta_data_ref = meta_header_j.at(3).is_binary()?meta_header_j.at(3):meta_header_j.at(3).at("data");
+			if (!meta_header_j.at(3).is_binary()) {
+				std::cerr << "FS error: meta data not binary " << it.frag_path << "\n";
+				continue;
+			}
+			const nlohmann::json::binary_t& meta_data_ref = meta_header_j.at(3);
+
+			std::vector<uint8_t> meta_data_decomp;
+			if (meta_comp == Compression::NONE) {
+				// do nothing
+			} else if (meta_comp == Compression::ZSTD) {
+				meta_data_decomp.resize(ZSTD_DStreamOutSize());
+				ZSTD_DCtx* const dctx = ZSTD_createDCtx();
+
+				ZSTD_inBuffer input {meta_data_ref.data(), meta_data_ref.size(), 0};
+				ZSTD_outBuffer output = {meta_data_decomp.data(), meta_data_decomp.size(), 0};
+				do {
+					size_t const ret = ZSTD_decompressStream(dctx, &output , &input);
+					if (ZSTD_isError(ret)) {
+						// error <.<
+						std::cerr << "FS error: decompression error\n";
+						meta_data_decomp.clear();
+						break;
+					}
+				} while (input.pos < input.size);
+				meta_data_decomp.resize(output.pos);
+
+				ZSTD_freeDCtx(dctx);
+			} else {
+				assert(false && "implement me");
+			}
+
+			// TODO: enc
+
+			if (!meta_data_decomp.empty()) {
+				j = nlohmann::json::from_msgpack(meta_data_decomp);
+			} else {
+				j = nlohmann::json::from_msgpack(meta_data_ref);
+			}
+		} else if (it.meta_ext == ".meta.json") {
+			std::ifstream file(it.frag_path.generic_u8string() + it.meta_ext, std::ios::in | std::ios::binary);
+			if (!file.is_open()) {
+				std::cerr << "FS error: failed opening meta " << it.frag_path << "\n";
+				continue;
+			}
+
+			file >> j;
+		} else {
+			assert(false);
+		}
+
+		if (!j.is_object()) {
+			std::cerr << "FS error: json in meta is broken " << it.id_str << "\n";
+			continue;
+		}
+
+		// TODO: existing fragment file
+		//newFragmentFile();
+		FragmentHandle fh{_reg, _reg.create()};
+		fh.emplace<FragComp::ID>(hex2bin(it.id_str));
+
+		fh.emplace<FragComp::Ephemeral::FilePath>(it.frag_path.generic_u8string());
+
+		for (const auto& [k, v] : j.items()) {
+			// type id from string hash
+			const auto type_id = entt::hashed_string(k.data(), k.size());
+			const auto deserl_fn_it = _sc._deserl_json.find(type_id);
+			if (deserl_fn_it != _sc._deserl_json.cend()) {
+				// TODO: check return value
+				deserl_fn_it->second(fh, v);
+			} else {
+				std::cerr << "FS warning: missing deserializer for meta key '" << k << "'\n";
+			}
+		}
+		scanned_frags.push_back(fh);
+	}
+
+	// TODO: mutex and move code to async and return this list ?
+
+	// throw new frag event here, after loading them all
+	for (const FragmentID fid : scanned_frags) {
+		throwEventConstruct(fid);
+	}
+
+	return scanned_frags.size();
+}
+
+void FragmentStore::scanStoragePathAsync(std::string path) {
+	// add path to queue
+	// HACK: if path is known/any fragment is in the path, this operation blocks (non async)
+	scanStoragePath(path); // TODO: make async and post result
+}
+
+static bool serl_json_data_enc_type(const FragmentHandle fh, nlohmann::json& out) {
+	out = static_cast<std::underlying_type_t<Encryption>>(
+		fh.get<FragComp::DataEncryptionType>().enc
+	);
+	return true;
+}
+
+static bool deserl_json_data_enc_type(FragmentHandle fh, const nlohmann::json& in) {
+	fh.emplace_or_replace<FragComp::DataEncryptionType>(
+		static_cast<Encryption>(
+			static_cast<std::underlying_type_t<Encryption>>(in)
+		)
+	);
+	return true;
+}
+
+static bool serl_json_data_comp_type(const FragmentHandle fh, nlohmann::json& out) {
+	out = static_cast<std::underlying_type_t<Compression>>(
+		fh.get<FragComp::DataCompressionType>().comp
+	);
+	return true;
+}
+
+static bool deserl_json_data_comp_type(FragmentHandle fh, const nlohmann::json& in) {
+	fh.emplace_or_replace<FragComp::DataCompressionType>(
+		static_cast<Compression>(
+			static_cast<std::underlying_type_t<Compression>>(in)
+		)
+	);
+	return true;
+}
+
+void FragmentStore::registerSerializers(void) {
+	_sc.registerSerializerJson<FragComp::DataEncryptionType>(serl_json_data_enc_type);
+	_sc.registerDeSerializerJson<FragComp::DataEncryptionType>(deserl_json_data_enc_type);
+	_sc.registerSerializerJson<FragComp::DataCompressionType>(serl_json_data_comp_type);
+	_sc.registerDeSerializerJson<FragComp::DataCompressionType>(deserl_json_data_comp_type);
+}
+

src/fragment_store/fragment_store.hpp

@@ -0,0 +1,103 @@
+#pragma once
+
+#include "./fragment_store_i.hpp"
+
+#include "./types.hpp"
+#include "./meta_components.hpp"
+
+#include "./serializer.hpp"
+
+#include <entt/core/type_info.hpp>
+#include <entt/entity/registry.hpp>
+
+#include <nlohmann/json_fwd.hpp>
+
+#include <vector>
+#include <array>
+#include <cstdint>
+#include <random>
+
+struct FragmentStore : public FragmentStoreI {
+	std::minstd_rand _rng{std::random_device{}()};
+	std::array<uint8_t, 16> _session_uuid_namespace;
+
+	std::string _default_store_path;
+
+	uint64_t _memory_budget {10u*1024u*1024u};
+	uint64_t _memory_usage {0u};
+
+	SerializerCallbacks<FragmentID> _sc;
+
+	FragmentStore(void);
+	FragmentStore(std::array<uint8_t, 16> session_uuid_namespace);
+
+	// HACK: get access to the reg
+	FragmentHandle fragmentHandle(FragmentID fid);
+
+	// TODO: make the frags ref counted
+
+	// TODO: check for exising
+	std::vector<uint8_t> generateNewUID(std::array<uint8_t, 16>& uuid_namespace);
+	std::vector<uint8_t> generateNewUID(void);
+
+	// ========== new fragment ==========
+
+	// memory backed owned
+	FragmentID newFragmentMemoryOwned(
+		const std::vector<uint8_t>& id,
+		size_t initial_size
+	);
+
+	// memory backed view (can only be added? not new?)
+
+	// file backed (rw...)
+	// needs to know which store path to put into
+	FragmentID newFragmentFile(
+		std::string_view store_path,
+		MetaFileType mft,
+		const std::vector<uint8_t>& id
+	);
+	// this variant generate a new, mostly unique, id for us
+	FragmentID newFragmentFile(
+		std::string_view store_path,
+		MetaFileType mft
+	);
+
+	// ========== add fragment ==========
+
+	// ========== get fragment ==========
+	FragmentID getFragmentByID(
+		const std::vector<uint8_t>& id
+	);
+	FragmentID getFragmentCustomMatcher(
+		std::function<bool(FragmentID)>& fn
+	);
+
+	// remove fragment?
+	// unload?
+
+	// ========== sync fragment to storage ==========
+	using write_to_storage_fetch_data_cb = uint64_t(uint8_t* request_buffer, uint64_t buffer_size);
+	// calls data_cb with a buffer to be filled in, cb returns actual count of data. if returned < max, its the last buffer.
+	bool syncToStorage(FragmentID fid, std::function<write_to_storage_fetch_data_cb>& data_cb);
+	bool syncToStorage(FragmentID fid, const uint8_t* data, const uint64_t data_size);
+
+	// ========== load fragment data from storage ==========
+	using read_from_storage_put_data_cb = void(const uint8_t* buffer, const uint64_t buffer_size);
+	bool loadFromStorage(FragmentID fid, std::function<read_from_storage_put_data_cb>& data_cb);
+	// convenience function
+	nlohmann::json loadFromStorageNJ(FragmentID fid);
+
+	// fragment discovery?
+	// returns number of new fragments
+	size_t scanStoragePath(std::string_view path);
+	void scanStoragePathAsync(std::string path);
+
+	private:
+		void registerSerializers(void); // internal comps
+		// internal actual backends
+		// TODO: seperate out
+		bool syncToMemory(FragmentID fid, std::function<write_to_storage_fetch_data_cb>& data_cb);
+		bool syncToFile(FragmentID fid, std::function<write_to_storage_fetch_data_cb>& data_cb);
+};
+

src/fragment_store/fragment_store_i.cpp

@@ -0,0 +1,23 @@
+#include "./fragment_store_i.hpp"
+
+#include <iostream>
+
+void FragmentStoreI::throwEventConstruct(const FragmentID fid) {
+	std::cout << "FSI debug: event construct " << entt::to_integral(fid) << "\n";
+	dispatch(
+		FragmentStore_Event::fragment_construct,
+		Fragment::Events::FragmentConstruct{
+			FragmentHandle{_reg, fid}
+		}
+	);
+}
+
+void FragmentStoreI::throwEventUpdate(const FragmentID fid) {
+	std::cout << "FSI debug: event updated " << entt::to_integral(fid) << "\n";
+	dispatch(
+		FragmentStore_Event::fragment_updated,
+		Fragment::Events::FragmentUpdated{
+			FragmentHandle{_reg, fid}
+		}
+	);
+}

src/fragment_store/fragment_store_i.hpp

@@ -0,0 +1,63 @@
+#pragma once
+
+#include <solanaceae/util/event_provider.hpp>
+
+#include <entt/entity/registry.hpp>
+#include <entt/entity/handle.hpp>
+
+#include <cstdint>
+
+// internal id
+enum class FragmentID : uint32_t {};
+using FragmentRegistry = entt::basic_registry<FragmentID>;
+using FragmentHandle = entt::basic_handle<FragmentRegistry>;
+
+namespace Fragment::Events {
+
+	struct FragmentConstruct {
+		const FragmentHandle e;
+	};
+	struct FragmentUpdated {
+		const FragmentHandle e;
+	};
+	//struct MessageDestory {
+		//const Message3Handle e;
+	//};
+
+} // Fragment::Events
+
+enum class FragmentStore_Event : uint32_t {
+	fragment_construct,
+	fragment_updated,
+	//message_destroy,
+
+	MAX
+};
+
+struct FragmentStoreEventI {
+	using enumType = FragmentStore_Event;
+
+	virtual ~FragmentStoreEventI(void) {}
+
+	virtual bool onEvent(const Fragment::Events::FragmentConstruct&) { return false; }
+	virtual bool onEvent(const Fragment::Events::FragmentUpdated&) { return false; }
+	//virtual bool onEvent(const Fragment::Events::MessageDestory&) { return false; }
+
+	// mm3
+	// send text
+	// send file path
+};
+using FragmentStoreEventProviderI = EventProviderI<FragmentStoreEventI>;
+
+struct FragmentStoreI  : public FragmentStoreEventProviderI {
+	static constexpr const char* version {"1"};
+
+	FragmentRegistry _reg;
+
+	virtual ~FragmentStoreI(void) {}
+
+	void throwEventConstruct(const FragmentID fid);
+	void throwEventUpdate(const FragmentID fid);
+	//void throwEventDestroy();
+};
+

src/fragment_store/message_fragment_store.cpp

@@ -0,0 +1,900 @@
+#include "./message_fragment_store.hpp"
+
+#include "../json/message_components.hpp"
+
+#include <solanaceae/util/utils.hpp>
+
+#include <solanaceae/contact/components.hpp>
+#include <solanaceae/message3/components.hpp>
+#include <solanaceae/message3/contact_components.hpp>
+
+#include <nlohmann/json.hpp>
+
+#include <algorithm>
+#include <string>
+#include <cstdint>
+#include <cassert>
+#include <iostream>
+
+// https://youtu.be/CU2exyhYPfA
+
+// everything assumes a single fragment registry
+
+namespace Message::Components {
+
+	// ctx
+	struct OpenFragments {
+		//struct OpenFrag final {
+			////std::vector<uint8_t> uid;
+			//FragmentID id;
+		//};
+		// only contains fragments with <1024 messages and <28h tsrage (or whatever)
+		entt::dense_set<FragmentID> fid_open;
+	};
+
+	// all message fragments of this contact
+	struct ContactFragments final {
+		// kept up-to-date by events
+		struct InternalEntry {
+			// indecies into the sorted arrays
+			size_t i_b;
+			size_t i_e;
+		};
+		entt::dense_map<FragmentID, InternalEntry> frags;
+
+		// add 2 sorted contact lists for both range begin and end
+		// TODO: adding and removing becomes expensive with enough frags, consider splitting or heap
+		std::vector<FragmentID> sorted_begin;
+		std::vector<FragmentID> sorted_end;
+
+		// api
+		// return true if it was actually inserted
+		bool insert(FragmentHandle frag);
+		bool erase(FragmentID frag);
+		// update? (just erase() + insert())
+
+		// uses range begin to go back in time
+		FragmentID prev(FragmentID frag) const;
+		// uses range end to go forward in time
+		FragmentID next(FragmentID frag) const;
+	};
+
+	// all LOADED message fragments
+	// TODO: merge into ContactFragments (and pull in openfrags)
+	struct LoadedContactFragments final {
+		// kept up-to-date by events
+		entt::dense_set<FragmentID> frags;
+	};
+
+} // Message::Components
+
+namespace Fragment::Components {
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessagesTSRange, begin, end)
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessagesContact, id)
+} // Fragment::Components
+
+void MessageFragmentStore::handleMessage(const Message3Handle& m) {
+	if (_fs_ignore_event) {
+		// message event because of us loading a fragment, ignore
+		// TODO: this barely makes a difference
+		return;
+	}
+
+	if (!static_cast<bool>(m)) {
+		return; // huh?
+	}
+
+	if (!m.all_of<Message::Components::Timestamp>()) {
+		return; // we only handle msg with ts
+	}
+
+	_potentially_dirty_contacts.emplace(m.registry()->ctx().get<Contact3>()); // always mark dirty here
+	if (m.any_of<Message::Components::ViewCurserBegin, Message::Components::ViewCurserEnd>()) {
+		// not an actual message, but we probalby need to check and see if we need to load fragments
+		//std::cout << "MFS: new or updated curser\n";
+		return;
+	}
+
+	// TODO: use fid, seving full fuid for every message consumes alot of memory (and heap frag)
+	if (!m.all_of<Message::Components::FID>()) {
+		std::cout << "MFS: new msg missing FID\n";
+		if (!m.registry()->ctx().contains<Message::Components::OpenFragments>()) {
+			m.registry()->ctx().emplace<Message::Components::OpenFragments>();
+		}
+
+		auto& fid_open = m.registry()->ctx().get<Message::Components::OpenFragments>().fid_open;
+
+		const auto msg_ts = m.get<Message::Components::Timestamp>().ts;
+		// missing fuid
+		// find closesed non-sealed off fragment
+
+		FragmentID fragment_id{entt::null};
+
+		// first search for fragment where the ts falls into the range
+		for (const auto& fid : fid_open) {
+			auto fh = _fs.fragmentHandle(fid);
+			assert(static_cast<bool>(fh));
+
+			// assuming ts range exists
+			auto& fts_comp = fh.get<FragComp::MessagesTSRange>();
+
+			if (fts_comp.begin <= msg_ts && fts_comp.end >= msg_ts) {
+				fragment_id = fid;
+				// TODO: check conditions for open here
+				// TODO: mark msg (and frag?) dirty
+			}
+		}
+
+		// if it did not fit into an existing fragment, we next look for fragments that could be extended
+		if (!_fs._reg.valid(fragment_id)) {
+			for (const auto& fid : fid_open) {
+				auto fh = _fs.fragmentHandle(fid);
+				assert(static_cast<bool>(fh));
+
+				// assuming ts range exists
+				auto& fts_comp = fh.get<FragComp::MessagesTSRange>();
+
+				const int64_t frag_range = int64_t(fts_comp.end) - int64_t(fts_comp.begin);
+				constexpr static int64_t max_frag_ts_extent {1000*60*60};
+				//constexpr static int64_t max_frag_ts_extent {1000*60*3}; // 3min for testing
+				const int64_t possible_extention = max_frag_ts_extent - frag_range;
+
+				// which direction
+				if ((fts_comp.begin - possible_extention) <= msg_ts && fts_comp.begin > msg_ts) {
+					fragment_id = fid;
+
+					std::cout << "MFS: extended begin from " << fts_comp.begin << " to " << msg_ts << "\n";
+
+					// assuming ts range exists
+					fts_comp.begin = msg_ts; // extend into the past
+
+					if (m.registry()->ctx().contains<Message::Components::ContactFragments>()) {
+						// should be the case
+						m.registry()->ctx().get<Message::Components::ContactFragments>().erase(fh);
+						m.registry()->ctx().get<Message::Components::ContactFragments>().insert(fh);
+					}
+
+
+					// TODO: check conditions for open here
+					// TODO: mark msg (and frag?) dirty
+				} else if ((fts_comp.end + possible_extention) >= msg_ts && fts_comp.end < msg_ts) {
+					fragment_id = fid;
+
+					std::cout << "MFS: extended end from " << fts_comp.end << " to " << msg_ts << "\n";
+
+					// assuming ts range exists
+					fts_comp.end = msg_ts; // extend into the future
+
+					if (m.registry()->ctx().contains<Message::Components::ContactFragments>()) {
+						// should be the case
+						m.registry()->ctx().get<Message::Components::ContactFragments>().erase(fh);
+						m.registry()->ctx().get<Message::Components::ContactFragments>().insert(fh);
+					}
+
+					// TODO: check conditions for open here
+					// TODO: mark msg (and frag?) dirty
+				}
+			}
+		}
+
+		// if its still not found, we need a new fragment
+		if (!_fs._reg.valid(fragment_id)) {
+			const auto new_fid = _fs.newFragmentFile("test_message_store/", MetaFileType::BINARY_MSGPACK);
+			auto fh = _fs.fragmentHandle(new_fid);
+			if (!static_cast<bool>(fh)) {
+				std::cout << "MFS error: failed to create new fragment for message\n";
+				return;
+			}
+
+			fragment_id = fh;
+
+			fh.emplace_or_replace<FragComp::Ephemeral::MetaCompressionType>().comp = Compression::ZSTD;
+			fh.emplace_or_replace<FragComp::DataCompressionType>().comp = Compression::ZSTD;
+
+			auto& new_ts_range = fh.emplace_or_replace<FragComp::MessagesTSRange>();
+			new_ts_range.begin = msg_ts;
+			new_ts_range.end = msg_ts;
+
+			{
+				const auto msg_reg_contact = m.registry()->ctx().get<Contact3>();
+				if (_cr.all_of<Contact::Components::ID>(msg_reg_contact)) {
+					fh.emplace<FragComp::MessagesContact>(_cr.get<Contact::Components::ID>(msg_reg_contact).data);
+				} else {
+					// ? rage quit?
+				}
+			}
+
+			// contact frag
+			if (!m.registry()->ctx().contains<Message::Components::ContactFragments>()) {
+				m.registry()->ctx().emplace<Message::Components::ContactFragments>();
+			}
+			m.registry()->ctx().get<Message::Components::ContactFragments>().insert(fh);
+
+			// loaded contact frag
+			if (!m.registry()->ctx().contains<Message::Components::LoadedContactFragments>()) {
+				m.registry()->ctx().emplace<Message::Components::LoadedContactFragments>();
+			}
+			m.registry()->ctx().get<Message::Components::LoadedContactFragments>().frags.emplace(fh);
+
+			fid_open.emplace(fragment_id);
+
+			std::cout << "MFS: created new fragment " << bin2hex(fh.get<FragComp::ID>().v) << "\n";
+
+			_fs_ignore_event = true;
+			_fs.throwEventConstruct(fh);
+			_fs_ignore_event = false;
+		}
+
+		// if this is still empty, something is very wrong and we exit here
+		if (!_fs._reg.valid(fragment_id)) {
+			std::cout << "MFS error: failed to find/create fragment for message\n";
+			return;
+		}
+
+		m.emplace_or_replace<Message::Components::FID>(fragment_id);
+
+		// in this case we know the fragment needs an update
+		_fuid_save_queue.push({Message::getTimeMS(), fragment_id, m.registry()});
+		return; // done
+	}
+
+	const auto msg_fh = _fs.fragmentHandle(m.get<Message::Components::FID>().fid);
+	if (!static_cast<bool>(msg_fh)) {
+		std::cerr << "MFS error: fid in message is invalid\n";
+		return; // TODO: properly handle this case
+	}
+
+	if (!m.registry()->ctx().contains<Message::Components::OpenFragments>()) {
+		m.registry()->ctx().emplace<Message::Components::OpenFragments>();
+	}
+
+	auto& fid_open = m.registry()->ctx().get<Message::Components::OpenFragments>().fid_open;
+
+	if (fid_open.contains(msg_fh)) {
+		// TODO: dedup events
+		// TODO: cooldown per fragsave
+		_fuid_save_queue.push({Message::getTimeMS(), msg_fh, m.registry()});
+		return;
+	}
+
+	// TODO: save updates to old fragments, but writing them to a new fragment that would overwrite on merge
+	// new fragment?, since we dont write to others fragments?
+
+
+	// on new message: assign fuid
+	// on new and update: mark as fragment dirty
+}
+
+// assumes not loaded frag
+// need update from frag
+void MessageFragmentStore::loadFragment(Message3Registry& reg, FragmentHandle fh) {
+	std::cout << "MFS: loadFragment\n";
+	const auto j = _fs.loadFromStorageNJ(fh);
+
+	if (!j.is_array()) {
+		// wrong data
+		return;
+	}
+
+	// TODO: this should probably never be the case, since we already know here that it is a msg frag
+	if (!reg.ctx().contains<Message::Components::ContactFragments>()) {
+		reg.ctx().emplace<Message::Components::ContactFragments>();
+	}
+	reg.ctx().get<Message::Components::ContactFragments>().insert(fh);
+
+	// mark loaded
+	if (!reg.ctx().contains<Message::Components::LoadedContactFragments>()) {
+		reg.ctx().emplace<Message::Components::LoadedContactFragments>();
+	}
+	reg.ctx().get<Message::Components::LoadedContactFragments>().frags.emplace(fh);
+
+	for (const auto& j_entry : j) {
+		auto new_real_msg = Message3Handle{reg, reg.create()};
+		// load into staging reg
+		for (const auto& [k, v] : j_entry.items()) {
+			//std::cout << "K:" << k << " V:" << v.dump() << "\n";
+			const auto type_id = entt::hashed_string(k.data(), k.size());
+			const auto deserl_fn_it = _sc._deserl_json.find(type_id);
+			if (deserl_fn_it != _sc._deserl_json.cend()) {
+				try {
+					if (!deserl_fn_it->second(_sc, new_real_msg, v)) {
+						std::cerr << "MFS error: failed deserializing '" << k << "'\n";
+					}
+				} catch(...) {
+					std::cerr << "MFS error: failed deserializing (threw) '" << k << "'\n";
+				}
+			} else {
+				std::cerr << "MFS warning: missing deserializer for meta key '" << k << "'\n";
+			}
+		}
+
+		new_real_msg.emplace_or_replace<Message::Components::FID>(fh);
+
+		// dup check (hacky, specific to protocols)
+		Message3 dup_msg {entt::null};
+		{
+			// get comparator from contact
+			if (reg.ctx().contains<Contact3>()) {
+				const auto c = reg.ctx().get<Contact3>();
+				if (_cr.all_of<Contact::Components::MessageIsSame>(c)) {
+					auto& comp = _cr.get<Contact::Components::MessageIsSame>(c).comp;
+					// walking EVERY existing message OOF
+					// this needs optimizing
+					for (const Message3 other_msg : reg.view<Message::Components::Timestamp, Message::Components::ContactFrom, Message::Components::ContactTo>()) {
+						if (other_msg == new_real_msg) {
+							continue; // skip self
+						}
+
+						if (comp({reg, other_msg}, new_real_msg)) {
+							// dup
+							dup_msg = other_msg;
+							break;
+						}
+					}
+				}
+			}
+		}
+
+		if (reg.valid(dup_msg)) {
+			//  -> merge with preexisting (needs to be order independent)
+			//  -> throw update
+			reg.destroy(new_real_msg);
+			//_rmm.throwEventUpdate(reg, new_real_msg);
+		} else {
+			if (!new_real_msg.all_of<Message::Components::Timestamp, Message::Components::ContactFrom, Message::Components::ContactTo>()) {
+				// does not have needed components to be stand alone
+				reg.destroy(new_real_msg);
+				std::cerr << "MFS warning: message with missing basic compoments\n";
+				continue;
+			}
+
+			//  -> throw create
+			_rmm.throwEventConstruct(reg, new_real_msg);
+		}
+	}
+}
+
+bool MessageFragmentStore::syncFragToStorage(FragmentHandle fh, Message3Registry& reg) {
+	auto& ftsrange = fh.get_or_emplace<FragComp::MessagesTSRange>(Message::getTimeMS(), Message::getTimeMS());
+
+	auto j = nlohmann::json::array();
+
+	// TODO: does every message have ts?
+	auto msg_view = reg.view<Message::Components::Timestamp>();
+	// we also assume all messages have fid
+	for (auto it = msg_view.rbegin(), it_end = msg_view.rend(); it != it_end; it++) {
+		const Message3 m = *it;
+
+		if (!reg.all_of<Message::Components::FID, Message::Components::ContactFrom, Message::Components::ContactTo>(m)) {
+			continue;
+		}
+
+		// require msg for now
+		if (!reg.any_of<Message::Components::MessageText/*, Message::Components::Transfer::FileInfo*/>(m)) {
+			continue;
+		}
+
+		if (_fuid_save_queue.front().id != reg.get<Message::Components::FID>(m).fid) {
+			continue; // not ours
+		}
+
+		{ // potentially adjust tsrange (some external processes can change timestamps)
+			const auto msg_ts = msg_view.get<Message::Components::Timestamp>(m).ts;
+			if (ftsrange.begin > msg_ts) {
+				ftsrange.begin = msg_ts;
+			} else if (ftsrange.end < msg_ts) {
+				ftsrange.end = msg_ts;
+			}
+		}
+
+		auto& j_entry = j.emplace_back(nlohmann::json::object());
+
+		for (const auto& [type_id, storage] : reg.storage()) {
+			if (!storage.contains(m)) {
+				continue;
+			}
+
+			//std::cout << "storage type: type_id:" << type_id << " name:" << storage.type().name() << "\n";
+
+			// use type_id to find serializer
+			auto s_cb_it = _sc._serl_json.find(type_id);
+			if (s_cb_it == _sc._serl_json.end()) {
+				// could not find serializer, not saving
+				//std::cout << "missing " << storage.type().name() << "(" << type_id << ")\n";
+				continue;
+			}
+
+			s_cb_it->second(_sc, {reg, m}, j_entry[storage.type().name()]);
+		}
+	}
+
+	// we cant skip if array is empty (in theory it will not be empty later on)
+
+	// if save as binary
+	//nlohmann::json::to_msgpack(j);
+	auto j_dump = j.dump(2, ' ', true);
+	if (_fs.syncToStorage(fh, reinterpret_cast<const uint8_t*>(j_dump.data()), j_dump.size())) {
+		//std::cout << "MFS: dumped " << j_dump << "\n";
+		// succ
+		return true;
+	}
+
+	// TODO: error
+	return false;
+}
+
+MessageFragmentStore::MessageFragmentStore(
+	Contact3Registry& cr,
+	RegistryMessageModel& rmm,
+	FragmentStore& fs
+) : _cr(cr), _rmm(rmm), _fs(fs), _sc{_cr, {}, {}} {
+	_rmm.subscribe(this, RegistryMessageModel_Event::message_construct);
+	_rmm.subscribe(this, RegistryMessageModel_Event::message_updated);
+	_rmm.subscribe(this, RegistryMessageModel_Event::message_destroy);
+
+	_fs._sc.registerSerializerJson<FragComp::MessagesTSRange>();
+	_fs._sc.registerDeSerializerJson<FragComp::MessagesTSRange>();
+	_fs._sc.registerSerializerJson<FragComp::MessagesContact>();
+	_fs._sc.registerDeSerializerJson<FragComp::MessagesContact>();
+
+	_fs.subscribe(this, FragmentStore_Event::fragment_construct);
+}
+
+MessageFragmentStore::~MessageFragmentStore(void) {
+	while (!_fuid_save_queue.empty()) {
+		auto fh = _fs.fragmentHandle(_fuid_save_queue.front().id);
+		auto* reg = _fuid_save_queue.front().reg;
+		assert(reg != nullptr);
+		syncFragToStorage(fh, *reg);
+		_fuid_save_queue.pop(); // pop unconditionally
+	}
+}
+
+MessageSerializerCallbacks& MessageFragmentStore::getMSC(void) {
+	return _sc;
+}
+
+// checks range against all cursers in msgreg
+static bool rangeVisible(uint64_t range_begin, uint64_t range_end, const Message3Registry& msg_reg) {
+	// 1D collision checks:
+	//  - for range vs range:
+	//    r1 rhs >= r0 lhs AND r1 lhs <= r0 rhs
+	//  - for range vs point:
+	//    p >= r0 lhs AND p <= r0 rhs
+	// NOTE: directions for us are reversed (begin has larger values as end)
+
+	auto c_b_view = msg_reg.view<Message::Components::Timestamp, Message::Components::ViewCurserBegin>();
+	c_b_view.use<Message::Components::ViewCurserBegin>();
+	for (const auto& [m, ts_begin_comp, vcb] : c_b_view.each()) {
+		// p and r1 rhs can be seen as the same
+		// but first we need to know if a curser begin is a point or a range
+
+		// TODO: margin?
+		auto ts_begin = ts_begin_comp.ts;
+		auto ts_end = ts_begin_comp.ts; // simplyfy code by making a single begin curser act as an infinitly small range
+		if (msg_reg.valid(vcb.curser_end) && msg_reg.all_of<Message::Components::ViewCurserEnd>(vcb.curser_end)) {
+			// TODO: respect curser end's begin?
+			// TODO: remember which ends we checked and check remaining
+			ts_end = msg_reg.get<Message::Components::Timestamp>(vcb.curser_end).ts;
+
+			// sanity check curser order
+			if (ts_end > ts_begin) {
+				std::cerr << "MFS warning: begin curser and end curser of view swapped!!\n";
+				std::swap(ts_begin, ts_end);
+			}
+		}
+
+		// perform both checks here
+		if (ts_begin < range_end || ts_end > range_begin) {
+			continue;
+		}
+
+		// range hits a view
+		return true;
+	}
+
+	return false;
+}
+
+static bool isLess(const std::vector<uint8_t>& lhs, const std::vector<uint8_t>& rhs) {
+	size_t i = 0;
+	for (; i < lhs.size() && i < rhs.size(); i++) {
+		if (lhs[i] < rhs[i]) {
+			return true;
+		} else if (lhs[i] > rhs[i]) {
+			return false;
+		}
+		// else continue
+	}
+
+	// here we have equality of common lenths
+
+	// we define smaller arrays to be less
+	return lhs.size() < rhs.size();
+}
+
+float MessageFragmentStore::tick(float time_delta) {
+	// sync dirty fragments here
+	if (!_fuid_save_queue.empty()) {
+		auto fh = _fs.fragmentHandle(_fuid_save_queue.front().id);
+		auto* reg = _fuid_save_queue.front().reg;
+		assert(reg != nullptr);
+		if (syncFragToStorage(fh, *reg)) {
+			_fuid_save_queue.pop();
+		}
+	}
+
+	// load needed fragments here
+
+	// last check event frags
+	// only checks if it collides with ranges, not adjacent
+	// bc ~range~ msgreg will be marked dirty and checked next tick
+	const bool had_events = !_event_check_queue.empty();
+	for (size_t i = 0; i < 10 && !_event_check_queue.empty(); i++) {
+		std::cout << "MFS: event check\n";
+		auto fh = _fs.fragmentHandle(_event_check_queue.front().fid);
+		auto c = _event_check_queue.front().c;
+		_event_check_queue.pop();
+
+		if (!static_cast<bool>(fh)) {
+			return 0.05f;
+		}
+
+		if (!fh.all_of<FragComp::MessagesTSRange>()) {
+			return 0.05f;
+		}
+
+		// get ts range of frag and collide with all curser(s/ranges)
+		const auto& frag_range = fh.get<FragComp::MessagesTSRange>();
+
+		auto* msg_reg = _rmm.get(c);
+		if (msg_reg == nullptr) {
+			return 0.05f;
+		}
+
+		if (rangeVisible(frag_range.begin, frag_range.end, !msg_reg)) {
+			loadFragment(*msg_reg, fh);
+			_potentially_dirty_contacts.emplace(c);
+			return 0.05f; // only one but soon again
+		}
+	}
+	if (had_events) {
+		std::cout << "MFS: event check none\n";
+		return 0.05f; // only check events, even if non where hit
+	}
+
+	if (!_potentially_dirty_contacts.empty()) {
+		std::cout << "MFS: pdc\n";
+		// here we check if any view of said contact needs frag loading
+		// only once per tick tho
+
+		// TODO: this makes order depend on internal order and is not fair
+		auto it = _potentially_dirty_contacts.cbegin();
+
+		auto* msg_reg = _rmm.get(*it);
+
+		// first do collision check agains every contact associated fragment
+		// that is not already loaded !!
+		if (msg_reg->ctx().contains<Message::Components::ContactFragments>()) {
+			const auto& cf = msg_reg->ctx().get<Message::Components::ContactFragments>();
+			if (!cf.frags.empty()) {
+				if (!msg_reg->ctx().contains<Message::Components::LoadedContactFragments>()) {
+					msg_reg->ctx().emplace<Message::Components::LoadedContactFragments>();
+				}
+				const auto& loaded_frags = msg_reg->ctx().get<Message::Components::LoadedContactFragments>().frags;
+
+				for (const auto& [fid, si] : msg_reg->ctx().get<Message::Components::ContactFragments>().frags) {
+					if (loaded_frags.contains(fid)) {
+						continue;
+					}
+
+					auto fh = _fs.fragmentHandle(fid);
+
+					if (!static_cast<bool>(fh)) {
+						std::cerr << "MFS error: frag is invalid\n";
+						// WHAT
+						msg_reg->ctx().get<Message::Components::ContactFragments>().erase(fid);
+						return 0.05f;
+					}
+
+					if (!fh.all_of<FragComp::MessagesTSRange>()) {
+						std::cerr << "MFS error: frag has no range\n";
+						// ????
+						msg_reg->ctx().get<Message::Components::ContactFragments>().erase(fid);
+						return 0.05f;
+					}
+
+					// get ts range of frag and collide with all curser(s/ranges)
+					const auto& [range_begin, range_end] = fh.get<FragComp::MessagesTSRange>();
+
+					if (rangeVisible(range_begin, range_end, *msg_reg)) {
+						std::cout << "MFS: frag hit by vis range\n";
+						loadFragment(*msg_reg, fh);
+						return 0.05f;
+					}
+				}
+				// no new visible fragment
+				std::cout << "MFS: no new frag directly visible\n";
+
+				// now, finally, check for adjecent fragments that need to be loaded
+				// we do this by finding the outermost fragment in a rage, and extend it by one
+
+				// TODO: rewrite using some bounding range tree to perform collision checks !!!
+				// (this is now performing better, but still)
+
+
+				// for each view
+				auto c_b_view = msg_reg->view<Message::Components::Timestamp, Message::Components::ViewCurserBegin>();
+				c_b_view.use<Message::Components::ViewCurserBegin>();
+				for (const auto& [_, ts_begin_comp, vcb] : c_b_view.each()) {
+					// aka "scroll down"
+					{ // find newest(-ish) frag in range
+						// or in reverse frag end <= range begin
+
+
+						// lower bound of frag end and range begin
+						const auto right = std::lower_bound(
+							cf.sorted_end.crbegin(),
+							cf.sorted_end.crend(),
+							ts_begin_comp.ts,
+							[&](const FragmentID element, const auto& value) -> bool {
+								return _fs._reg.get<FragComp::MessagesTSRange>(element).end >= value;
+							}
+						);
+
+						FragmentID next_frag{entt::null};
+						if (right != cf.sorted_end.crend()) {
+							next_frag = cf.next(*right);
+						}
+						// we checked earlier that cf is not empty
+						if (!_fs._reg.valid(next_frag)) {
+							// fall back to closest, cf is not empty
+							next_frag = cf.sorted_end.front();
+						}
+
+						// a single adjacent frag is often not enough
+						// only ok bc next is cheap
+						for (size_t i = 0; i < 5 && _fs._reg.valid(next_frag); i++) {
+							if (!loaded_frags.contains(next_frag)) {
+								std::cout << "MFS: next frag of range\n";
+								loadFragment(*msg_reg, {_fs._reg, next_frag});
+								return 0.05f;
+							}
+
+							next_frag = cf.next(next_frag);
+						}
+					}
+
+					// curser end
+					if (!msg_reg->valid(vcb.curser_end) || !msg_reg->all_of<Message::Components::Timestamp>(vcb.curser_end)) {
+						continue;
+					}
+					const auto ts_end = msg_reg->get<Message::Components::Timestamp>(vcb.curser_end).ts;
+
+					// aka "scroll up"
+					{ // find oldest(-ish) frag in range
+						// frag begin >= range end
+
+						// lower bound of frag begin and range end
+						const auto left = std::lower_bound(
+							cf.sorted_begin.cbegin(),
+							cf.sorted_begin.cend(),
+							ts_end,
+							[&](const FragmentID element, const auto& value) -> bool {
+								return _fs._reg.get<FragComp::MessagesTSRange>(element).begin < value;
+							}
+						);
+
+						FragmentID prev_frag{entt::null};
+						if (left != cf.sorted_begin.cend()) {
+							prev_frag = cf.prev(*left);
+						}
+						// we checked earlier that cf is not empty
+						if (!_fs._reg.valid(prev_frag)) {
+							// fall back to closest, cf is not empty
+							prev_frag = cf.sorted_begin.back();
+						}
+
+						// a single adjacent frag is often not enough
+						// only ok bc next is cheap
+						for (size_t i = 0; i < 5 && _fs._reg.valid(prev_frag); i++) {
+							if (!loaded_frags.contains(prev_frag)) {
+								std::cout << "MFS: prev frag of range\n";
+								loadFragment(*msg_reg, {_fs._reg, prev_frag});
+								return 0.05f;
+							}
+
+							prev_frag = cf.prev(prev_frag);
+						}
+					}
+				}
+			}
+		} else {
+			// contact has no fragments, skip
+		}
+
+		_potentially_dirty_contacts.erase(it);
+
+		return 0.05f;
+	}
+
+
+	return 1000.f*60.f*60.f;
+}
+
+void MessageFragmentStore::triggerScan(void) {
+	_fs.scanStoragePath("test_message_store/");
+}
+
+bool MessageFragmentStore::onEvent(const Message::Events::MessageConstruct& e) {
+	handleMessage(e.e);
+	return false;
+}
+
+bool MessageFragmentStore::onEvent(const Message::Events::MessageUpdated& e) {
+	handleMessage(e.e);
+	return false;
+}
+
+// TODO: handle deletes? diff between unload?
+
+bool MessageFragmentStore::onEvent(const Fragment::Events::FragmentConstruct& e) {
+	if (_fs_ignore_event) {
+		return false; // skip self
+	}
+
+	if (!e.e.all_of<FragComp::MessagesTSRange, FragComp::MessagesContact>()) {
+		return false; // not for us
+	}
+
+	// TODO: are we sure it is a *new* fragment?
+
+	Contact3 frag_contact = entt::null;
+	{ // get contact
+		const auto& frag_contact_id = e.e.get<FragComp::MessagesContact>().id;
+		// TODO: id lookup table, this is very inefficent
+		for (const auto& [c_it, id_it] : _cr.view<Contact::Components::ID>().each()) {
+			if (frag_contact_id == id_it.data) {
+				frag_contact = c_it;
+				break;
+			}
+		}
+		if (!_cr.valid(frag_contact)) {
+			// unkown contact
+			return false;
+		}
+	}
+
+	// create if not exist
+	auto* msg_reg = _rmm.get(frag_contact);
+	if (msg_reg == nullptr) {
+		// msg reg not created yet
+		// TODO: this is an erroious path
+		return false;
+	}
+
+	if (!msg_reg->ctx().contains<Message::Components::ContactFragments>()) {
+		msg_reg->ctx().emplace<Message::Components::ContactFragments>();
+	}
+	msg_reg->ctx().get<Message::Components::ContactFragments>().erase(e.e); // TODO: check/update/fragment update
+	msg_reg->ctx().get<Message::Components::ContactFragments>().insert(e.e);
+
+	_event_check_queue.push(ECQueueEntry{e.e, frag_contact});
+
+	return false;
+}
+
+bool Message::Components::ContactFragments::insert(FragmentHandle frag) {
+	if (frags.contains(frag)) {
+		return false;
+	}
+
+	// both sorted arrays are sorted ascending
+	// so for insertion we search for the last index that is <= and insert after it
+	// or we search for the first > (or end) and insert before it    <---
+	// since equal fragments are UB, we can assume they are only > or <
+
+	size_t begin_index {0};
+	{ // begin
+		const auto pos = std::find_if(
+			sorted_begin.cbegin(),
+			sorted_begin.cend(),
+			[frag](const FragmentID a) -> bool {
+				const auto begin_a = frag.registry()->get<FragComp::MessagesTSRange>(a).begin;
+				const auto begin_frag = frag.get<FragComp::MessagesTSRange>().begin;
+				if (begin_a > begin_frag) {
+					return true;
+				} else if (begin_a < begin_frag) {
+					return false;
+				} else {
+					// equal ts, we need to fall back to id (id can not be equal)
+					return isLess(frag.get<FragComp::ID>().v, frag.registry()->get<FragComp::ID>(a).v);
+				}
+			}
+		);
+
+		begin_index = std::distance(sorted_begin.cbegin(), pos);
+
+		// we need to insert before pos (end is valid here)
+		sorted_begin.insert(pos, frag);
+	}
+
+	size_t end_index {0};
+	{ // end
+		const auto pos = std::find_if_not(
+			sorted_end.cbegin(),
+			sorted_end.cend(),
+			[frag](const FragmentID a) -> bool {
+				const auto end_a = frag.registry()->get<FragComp::MessagesTSRange>(a).end;
+				const auto end_frag = frag.get<FragComp::MessagesTSRange>().end;
+				if (end_a > end_frag) {
+					return true;
+				} else if (end_a < end_frag) {
+					return false;
+				} else {
+					// equal ts, we need to fall back to id (id can not be equal)
+					return isLess(frag.get<FragComp::ID>().v, frag.registry()->get<FragComp::ID>(a).v);
+				}
+			}
+		);
+
+		end_index = std::distance(sorted_end.cbegin(), pos);
+
+		// we need to insert before pos (end is valid here)
+		sorted_end.insert(pos, frag);
+	}
+
+	frags.emplace(frag, InternalEntry{begin_index, end_index});
+
+	return true;
+}
+
+bool Message::Components::ContactFragments::erase(FragmentID frag) {
+	auto frags_it = frags.find(frag);
+	if (frags_it == frags.end()) {
+		return false;
+	}
+
+	assert(sorted_begin.size() == sorted_end.size());
+	assert(sorted_begin.size() > frags_it->second.i_b);
+
+	sorted_begin.erase(sorted_begin.begin() + frags_it->second.i_b);
+	sorted_end.erase(sorted_end.begin() + frags_it->second.i_e);
+
+	frags.erase(frags_it);
+
+	return true;
+}
+
+FragmentID Message::Components::ContactFragments::prev(FragmentID frag) const {
+	// uses range begin to go back in time
+
+	auto it = frags.find(frag);
+	if (it == frags.end()) {
+		return entt::null;
+	}
+
+	const auto src_i = it->second.i_b;
+	if (src_i > 0) {
+		return sorted_begin[src_i-1];
+	}
+
+	return entt::null;
+}
+
+FragmentID Message::Components::ContactFragments::next(FragmentID frag) const {
+	// uses range end to go forward in time
+
+	auto it = frags.find(frag);
+	if (it == frags.end()) {
+		return entt::null;
+	}
+
+	const auto src_i = it->second.i_e;
+	if (src_i+1 < sorted_end.size()) {
+		return sorted_end[src_i+1];
+	}
+
+	return entt::null;
+}
+

src/fragment_store/message_fragment_store.hpp

@@ -0,0 +1,132 @@
+#pragma once
+
+#include "./meta_components.hpp"
+#include "./fragment_store_i.hpp"
+#include "./fragment_store.hpp"
+
+#include "./message_serializer.hpp"
+
+#include <entt/entity/registry.hpp>
+#include <entt/container/dense_map.hpp>
+#include <entt/container/dense_set.hpp>
+
+#include <solanaceae/contact/contact_model3.hpp>
+#include <solanaceae/message3/registry_message_model.hpp>
+
+#include <queue>
+#include <vector>
+#include <cstdint>
+
+namespace Message::Components {
+
+	// unused, consumes too much memory (highly compressable)
+	//using FUID = FragComp::ID;
+
+	struct FID {
+		FragmentID fid {entt::null};
+	};
+
+	// points to the front/newer message
+	// together they define a range that is,
+	// eg the first(end) and last(begin) message being rendered
+	// MFS requires there to be atleast one other fragment after/before,
+	// if not loaded fragment with fitting tsrange(direction) available
+	// uses fragmentAfter/Before()
+	// they can exist standalone
+	// if they are a pair, the inside is filled first
+	// cursers require a timestamp ???
+	struct ViewCurserBegin {
+		Message3 curser_end{entt::null};
+	};
+	struct ViewCurserEnd {
+		Message3 curser_begin{entt::null};
+	};
+
+	// TODO: add adjacency range comp or inside curser
+
+	// TODO: unused
+	// mfs will only load a limited number of fragments per tick (1),
+	// so this tag will be set if we loaded a fragment and
+	// every tick we check all cursers for this tag and continue
+	// and remove once no fragment could be loaded anymore
+	// (internal)
+	struct TagCurserUnsatisfied {};
+
+} // Message::Components
+
+namespace Fragment::Components {
+	struct MessagesTSRange {
+		// timestamp range within the fragment
+		uint64_t begin {0}; // newer msg -> higher number
+		uint64_t end {0};
+	};
+
+	struct MessagesContact {
+		std::vector<uint8_t> id;
+	};
+
+	// TODO: add src contact (self id)
+
+} // Fragment::Components
+
+// handles fragments for messages
+// on new message: assign fuid
+// on new and update: mark as fragment dirty
+// on delete: mark as fragment dirty?
+class MessageFragmentStore : public RegistryMessageModelEventI, public FragmentStoreEventI {
+	protected:
+		Contact3Registry& _cr;
+		RegistryMessageModel& _rmm;
+		FragmentStore& _fs;
+		bool _fs_ignore_event {false};
+
+		// for message components only
+		MessageSerializerCallbacks _sc;
+
+		void handleMessage(const Message3Handle& m);
+
+		void loadFragment(Message3Registry& reg, FragmentHandle fh);
+
+		bool syncFragToStorage(FragmentHandle fh, Message3Registry& reg);
+
+		struct SaveQueueEntry final {
+			uint64_t ts_since_dirty{0};
+			//std::vector<uint8_t> id;
+			FragmentID id;
+			Message3Registry* reg{nullptr};
+		};
+		std::queue<SaveQueueEntry> _fuid_save_queue;
+
+		struct ECQueueEntry final {
+			FragmentID fid;
+			Contact3 c;
+		};
+		std::queue<ECQueueEntry> _event_check_queue;
+
+		// range changed or fragment loaded.
+		// we only load a limited number of fragments at once,
+		// so we need to keep them dirty until nothing was loaded.
+		entt::dense_set<Contact3> _potentially_dirty_contacts;
+
+	public:
+		MessageFragmentStore(
+			Contact3Registry& cr,
+			RegistryMessageModel& rmm,
+			FragmentStore& fs
+		);
+		virtual ~MessageFragmentStore(void);
+
+		MessageSerializerCallbacks& getMSC(void);
+
+		float tick(float time_delta);
+
+		void triggerScan(void);
+
+	protected: // rmm
+		bool onEvent(const Message::Events::MessageConstruct& e) override;
+		bool onEvent(const Message::Events::MessageUpdated& e) override;
+
+	protected: // fs
+		bool onEvent(const Fragment::Events::FragmentConstruct& e) override;
+};
+

src/fragment_store/message_serializer.cpp

@@ -0,0 +1,107 @@
+#include "./message_serializer.hpp"
+
+#include <solanaceae/message3/components.hpp>
+#include <solanaceae/contact/components.hpp>
+
+#include <nlohmann/json.hpp>
+
+#include <iostream>
+
+static Contact3 findContactByID(Contact3Registry& cr, const std::vector<uint8_t>& id) {
+	// TODO: id lookup table, this is very inefficent
+	for (const auto& [c_it, id_it] : cr.view<Contact::Components::ID>().each()) {
+		if (id == id_it.data) {
+			return c_it;
+		}
+	}
+
+	return entt::null;
+}
+
+template<>
+bool MessageSerializerCallbacks::component_get_json<Message::Components::ContactFrom>(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& j) {
+	const Contact3 c = h.get<Message::Components::ContactFrom>().c;
+	if (!msc.cr.valid(c)) {
+		// while this is invalid registry state, it is valid serialization
+		j = nullptr;
+		std::cerr << "MSC warning: encountered invalid contact\n";
+		return true;
+	}
+
+	if (!msc.cr.all_of<Contact::Components::ID>(c)) {
+		// unlucky, this contact is purely ephemeral
+		j = nullptr;
+		std::cerr << "MSC warning: encountered contact without ID\n";
+		return true;
+	}
+
+	j = nlohmann::json::binary(msc.cr.get<Contact::Components::ID>(c).data);
+
+	return true;
+}
+template<>
+bool MessageSerializerCallbacks::component_emplace_or_replace_json<Message::Components::ContactFrom>(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& j) {
+	if (j.is_null()) {
+		std::cerr << "MSC warning: encountered null contact\n";
+		h.emplace_or_replace<Message::Components::ContactFrom>();
+		return true;
+	}
+
+	const std::vector<uint8_t> id = j.is_binary()?j:j["bytes"];
+
+	Contact3 other_c = findContactByID(msc.cr, id);
+	if (!msc.cr.valid(other_c)) {
+		// create sparse contact with id only
+		other_c = msc.cr.create();
+		msc.cr.emplace_or_replace<Contact::Components::ID>(other_c, id);
+	}
+
+	h.emplace_or_replace<Message::Components::ContactFrom>(other_c);
+
+	// TODO: should we return false if the contact is unknown??
+	return true;
+}
+
+template<>
+bool MessageSerializerCallbacks::component_get_json<Message::Components::ContactTo>(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& j) {
+	const Contact3 c = h.get<Message::Components::ContactTo>().c;
+	if (!msc.cr.valid(c)) {
+		// while this is invalid registry state, it is valid serialization
+		j = nullptr;
+		std::cerr << "MSC warning: encountered invalid contact\n";
+		return true;
+	}
+
+	if (!msc.cr.all_of<Contact::Components::ID>(c)) {
+		// unlucky, this contact is purely ephemeral
+		j = nullptr;
+		std::cerr << "MSC warning: encountered contact without ID\n";
+		return true;
+	}
+
+	j = nlohmann::json::binary(msc.cr.get<Contact::Components::ID>(c).data);
+
+	return true;
+}
+template<>
+bool MessageSerializerCallbacks::component_emplace_or_replace_json<Message::Components::ContactTo>(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& j) {
+	if (j.is_null()) {
+		std::cerr << "MSC warning: encountered null contact\n";
+		h.emplace_or_replace<Message::Components::ContactTo>();
+		return true;
+	}
+
+	const std::vector<uint8_t> id = j.is_binary()?j:j["bytes"];
+
+	Contact3 other_c = findContactByID(msc.cr, id);
+	if (!msc.cr.valid(other_c)) {
+		// create sparse contact with id only
+		other_c = msc.cr.create();
+		msc.cr.emplace_or_replace<Contact::Components::ID>(other_c, id);
+	}
+
+	h.emplace_or_replace<Message::Components::ContactTo>(other_c);
+
+	// TODO: should we return false if the contact is unknown??
+	return true;
+}

src/fragment_store/message_serializer.hpp

@@ -0,0 +1,85 @@
+#pragma once
+
+#include <entt/core/type_info.hpp>
+#include <entt/container/dense_map.hpp>
+
+#include <solanaceae/message3/registry_message_model.hpp>
+
+#include <nlohmann/json_fwd.hpp>
+
+struct MessageSerializerCallbacks {
+	using Registry = Message3Registry;
+	using Handle = Message3Handle;
+
+	Contact3Registry& cr;
+
+	// nlohmann
+	// json/msgpack
+	using serialize_json_fn = bool(*)(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& out);
+	entt::dense_map<entt::id_type, serialize_json_fn> _serl_json;
+
+	using deserialize_json_fn = bool(*)(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& in);
+	entt::dense_map<entt::id_type, deserialize_json_fn> _deserl_json;
+
+	template<typename T>
+	static bool component_get_json(MessageSerializerCallbacks&, const Handle h, nlohmann::json& j) {
+		if (h.template all_of<T>()) {
+			if constexpr (!std::is_empty_v<T>) {
+				j = h.template get<T>();
+			}
+			return true;
+		}
+
+		return false;
+	}
+
+	template<typename T>
+	static bool component_emplace_or_replace_json(MessageSerializerCallbacks&, Handle h, const nlohmann::json& j) {
+		if constexpr (std::is_empty_v<T>) {
+			h.template emplace_or_replace<T>(); // assert empty json?
+		} else {
+			h.template emplace_or_replace<T>(static_cast<T>(j));
+		}
+		return true;
+	}
+
+	void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info) {
+		_serl_json[type_info.hash()] = fn;
+	}
+
+	template<typename CompType>
+	void registerSerializerJson(
+		serialize_json_fn fn = component_get_json<CompType>,
+		const entt::type_info& type_info = entt::type_id<CompType>()
+	) {
+		registerSerializerJson(fn, type_info);
+	}
+
+	void registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info) {
+		_deserl_json[type_info.hash()] = fn;
+	}
+
+	template<typename CompType>
+	void registerDeSerializerJson(
+		deserialize_json_fn fn = component_emplace_or_replace_json<CompType>,
+		const entt::type_info& type_info = entt::type_id<CompType>()
+	) {
+		registerDeSerializerJson(fn, type_info);
+	}
+};
+
+// fwd
+namespace Message::Components {
+struct ContactFrom;
+struct ContactTo;
+}
+
+// make specializations known
+template<>
+bool MessageSerializerCallbacks::component_get_json<Message::Components::ContactFrom>(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& j);
+template<>
+bool MessageSerializerCallbacks::component_emplace_or_replace_json<Message::Components::ContactFrom>(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& j);
+template<>
+bool MessageSerializerCallbacks::component_get_json<Message::Components::ContactTo>(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& j);
+template<>
+bool MessageSerializerCallbacks::component_emplace_or_replace_json<Message::Components::ContactTo>(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& j);

src/fragment_store/meta_components.hpp

@@ -0,0 +1,60 @@
+#pragma once
+
+#include "./types.hpp"
+
+#include <vector>
+#include <string>
+#include <cstdint>
+
+namespace Fragment::Components {
+
+	// TODO: is this special and should this be saved to meta or not (its already in the file name on disk)
+	struct ID {
+		std::vector<uint8_t> v;
+	};
+
+	struct DataEncryptionType {
+		Encryption enc {Encryption::NONE};
+	};
+
+	struct DataCompressionType {
+		Compression comp {Compression::NONE};
+	};
+
+
+	// meta that is not written to (meta-)file
+	namespace Ephemeral {
+
+		// excluded from file meta
+		struct FilePath {
+			// contains store path, if any
+			std::string path;
+		};
+
+		// TODO: seperate into remote and local?
+		// (remote meaning eg. the file on disk was changed by another program)
+		struct DirtyTag {};
+
+
+		// type as comp
+		struct MetaFileType {
+			::MetaFileType type {::MetaFileType::TEXT_JSON};
+		};
+
+		struct MetaEncryptionType {
+			Encryption enc {Encryption::NONE};
+		};
+
+		struct MetaCompressionType {
+			Compression comp {Compression::NONE};
+		};
+
+	} // Ephemeral
+
+} // Components
+
+// shortened to save bytes (until I find a way to save by ID in msgpack)
+namespace FragComp = Fragment::Components;
+
+#include "./meta_components_id.inl"
+

src/fragment_store/meta_components_id.inl

@@ -0,0 +1,26 @@
+#pragma once
+
+#include "./meta_components.hpp"
+
+#include <entt/core/type_info.hpp>
+
+// TODO: move more central
+#define DEFINE_COMP_ID(x) \
+template<> \
+constexpr entt::id_type entt::type_hash<x>::value() noexcept { \
+	using namespace entt::literals; \
+	return #x##_hs; \
+} \
+template<> \
+constexpr std::string_view entt::type_name<x>::value() noexcept { \
+	return #x; \
+}
+
+// cross compiler stable ids
+
+DEFINE_COMP_ID(FragComp::DataEncryptionType)
+DEFINE_COMP_ID(FragComp::DataCompressionType)
+
+#undef DEFINE_COMP_ID
+
+

src/fragment_store/register_mfs_json_message_components.cpp

@@ -0,0 +1,35 @@
+#include "./register_mfs_json_message_components.hpp"
+
+#include "./message_serializer.hpp"
+#include "../json/message_components.hpp"
+
+void registerMFSJsonMessageComponents(MessageSerializerCallbacks& msc) {
+	msc.registerSerializerJson<Message::Components::Timestamp>();
+	msc.registerDeSerializerJson<Message::Components::Timestamp>();
+	msc.registerSerializerJson<Message::Components::TimestampProcessed>();
+	msc.registerDeSerializerJson<Message::Components::TimestampProcessed>();
+	msc.registerSerializerJson<Message::Components::TimestampWritten>();
+	msc.registerDeSerializerJson<Message::Components::TimestampWritten>();
+	msc.registerSerializerJson<Message::Components::ContactFrom>();
+	msc.registerDeSerializerJson<Message::Components::ContactFrom>();
+	msc.registerSerializerJson<Message::Components::ContactTo>();
+	msc.registerDeSerializerJson<Message::Components::ContactTo>();
+	msc.registerSerializerJson<Message::Components::TagUnread>();
+	msc.registerDeSerializerJson<Message::Components::TagUnread>();
+	msc.registerSerializerJson<Message::Components::Read>();
+	msc.registerDeSerializerJson<Message::Components::Read>();
+	msc.registerSerializerJson<Message::Components::MessageText>();
+	msc.registerDeSerializerJson<Message::Components::MessageText>();
+	msc.registerSerializerJson<Message::Components::TagMessageIsAction>();
+	msc.registerDeSerializerJson<Message::Components::TagMessageIsAction>();
+
+	// files
+	//_sc.registerSerializerJson<Message::Components::Transfer::FileID>()
+	//_sc.registerSerializerJson<Message::Components::Transfer::FileInfo>();
+	//_sc.registerDeSerializerJson<Message::Components::Transfer::FileInfo>();
+	//_sc.registerSerializerJson<Message::Components::Transfer::FileInfoLocal>();
+	//_sc.registerDeSerializerJson<Message::Components::Transfer::FileInfoLocal>();
+	//_sc.registerSerializerJson<Message::Components::Transfer::TagHaveAll>();
+	//_sc.registerDeSerializerJson<Message::Components::Transfer::TagHaveAll>();
+}
+

src/fragment_store/register_mfs_json_message_components.hpp

@@ -0,0 +1,6 @@
+#pragma once
+
+#include "./message_serializer.hpp"
+
+void registerMFSJsonMessageComponents(MessageSerializerCallbacks& msc);
+

src/fragment_store/register_mfs_json_tox_message_components.cpp

@@ -0,0 +1,10 @@
+#include "./register_mfs_json_message_components.hpp"
+
+#include "./message_serializer.hpp"
+#include "../json/tox_message_components.hpp"
+
+void registerMFSJsonToxMessageComponents(MessageSerializerCallbacks& msc) {
+	msc.registerSerializerJson<Message::Components::ToxGroupMessageID>();
+	msc.registerDeSerializerJson<Message::Components::ToxGroupMessageID>();
+}
+

src/fragment_store/register_mfs_json_tox_message_components.hpp

@@ -0,0 +1,6 @@
+#pragma once
+
+#include "./message_serializer.hpp"
+
+void registerMFSJsonToxMessageComponents(MessageSerializerCallbacks& msc);
+

src/fragment_store/serializer.hpp

@@ -0,0 +1,68 @@
+#pragma once
+
+#include <entt/core/type_info.hpp>
+#include <entt/container/dense_map.hpp>
+#include <entt/entity/handle.hpp>
+
+#include <nlohmann/json_fwd.hpp>
+
+template<typename EntityType = entt::entity>
+struct SerializerCallbacks {
+	using Registry = entt::basic_registry<EntityType>;
+	using Handle = entt::basic_handle<Registry>;
+
+	// nlohmann
+	// json/msgpack
+	using serialize_json_fn = bool(*)(const Handle h, nlohmann::json& out);
+	entt::dense_map<entt::id_type, serialize_json_fn> _serl_json;
+
+	using deserialize_json_fn = bool(*)(Handle h, const nlohmann::json& in);
+	entt::dense_map<entt::id_type, deserialize_json_fn> _deserl_json;
+
+	template<typename T>
+	static bool component_get_json(const Handle h, nlohmann::json& j) {
+		if (h.template all_of<T>()) {
+			if constexpr (!std::is_empty_v<T>) {
+				j = h.template get<T>();
+			}
+			return true;
+		}
+
+		return false;
+	}
+
+	template<typename T>
+	static bool component_emplace_or_replace_json(Handle h, const nlohmann::json& j) {
+		if constexpr (std::is_empty_v<T>) {
+			h.template emplace_or_replace<T>(); // assert empty json?
+		} else {
+			h.template emplace_or_replace<T>(static_cast<T>(j));
+		}
+		return true;
+	}
+
+	void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info) {
+		_serl_json[type_info.hash()] = fn;
+	}
+
+	template<typename CompType>
+	void registerSerializerJson(
+		serialize_json_fn fn = component_get_json<CompType>,
+		const entt::type_info& type_info = entt::type_id<CompType>()
+	) {
+		registerSerializerJson(fn, type_info);
+	}
+
+	void registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info) {
+		_deserl_json[type_info.hash()] = fn;
+	}
+
+	template<typename CompType>
+	void registerDeSerializerJson(
+		deserialize_json_fn fn = component_emplace_or_replace_json<CompType>,
+		const entt::type_info& type_info = entt::type_id<CompType>()
+	) {
+		registerDeSerializerJson(fn, type_info);
+	}
+};
+

src/fragment_store/test_fragstore.cpp

@@ -0,0 +1,80 @@
+#include <cstdint>
+#include <iostream>
+
+#include "./fragment_store.hpp"
+
+#include <nlohmann/json.hpp>
+#include <entt/entity/handle.hpp>
+
+namespace Components {
+	struct MessagesTimestampRange {
+		uint64_t begin {0};
+		uint64_t end {1000};
+	};
+
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessagesTimestampRange, begin, end)
+} // Components
+
+
+int main(void) {
+	FragmentStore fs;
+	fs._default_store_path = "test_store/";
+	fs._sc.registerSerializerJson<Components::MessagesTimestampRange>();
+	fs._sc.registerDeSerializerJson<Components::MessagesTimestampRange>();
+
+	const auto frag0 = fs.newFragmentFile("", MetaFileType::TEXT_JSON, {0xff, 0xf1, 0xf2, 0xf0, 0xff, 0xff, 0xff, 0xf9});
+
+	const auto frag1 = fs.newFragmentFile("", MetaFileType::BINARY_MSGPACK);
+
+	const auto frag2 = fs.newFragmentFile("", MetaFileType::BINARY_MSGPACK);
+
+	{
+		auto frag0h = fs.fragmentHandle(frag0);
+
+		frag0h.emplace_or_replace<FragComp::DataCompressionType>();
+		frag0h.emplace_or_replace<FragComp::DataEncryptionType>();
+		frag0h.emplace_or_replace<Components::MessagesTimestampRange>();
+
+		std::function<FragmentStore::write_to_storage_fetch_data_cb> fn_cb = [read = 0ul](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t {
+			uint64_t i = 0;
+			for (; i+read < 3000 && i < buffer_size; i++) {
+				request_buffer[i] = uint8_t((i+read) & 0xff);
+			}
+			read += i;
+
+			return i;
+		};
+		fs.syncToStorage(frag0, fn_cb);
+	}
+
+	{
+		auto frag1h = fs.fragmentHandle(frag1);
+
+		frag1h.emplace_or_replace<FragComp::DataCompressionType>();
+		frag1h.emplace_or_replace<FragComp::DataEncryptionType>();
+
+		std::function<FragmentStore::write_to_storage_fetch_data_cb> fn_cb = [read = 0ul](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t {
+			static constexpr std::string_view text = "This is some random data";
+			uint64_t i = 0;
+			for (; i+read < text.size() && i < buffer_size; i++) {
+				request_buffer[i] = text[i+read];
+			}
+			read += i;
+
+			return i;
+		};
+		fs.syncToStorage(frag1, fn_cb);
+	}
+
+	{
+		auto frag2h = fs.fragmentHandle(frag2);
+
+		frag2h.emplace_or_replace<FragComp::DataCompressionType>();
+		frag2h.emplace_or_replace<FragComp::DataEncryptionType>();
+
+		static constexpr std::string_view text = "This is more random data";
+		fs.syncToStorage(frag2, reinterpret_cast<const uint8_t*>(text.data()), text.size());
+	}
+	return 0;
+}
+

src/fragment_store/types.hpp

@@ -0,0 +1,19 @@
+#pragma once
+
+#include <cstdint>
+
+enum class Encryption : uint8_t {
+	NONE = 0x00,
+};
+enum class Compression : uint8_t {
+	NONE = 0x00,
+	ZSTD = 0x01,
+	// TODO: zstd without magic
+	// TODO: zstd meta dict
+	// TODO: zstd data(message) dict
+};
+enum class MetaFileType : uint8_t {
+	TEXT_JSON,
+	BINARY_MSGPACK, // msgpacked msgpack
+};
+

src/image_loader_qoi.cpp

@@ -0,0 +1,91 @@
+#include "./image_loader_qoi.hpp"
+
+#include <cstdint>
+#include <qoi/qoi.h>
+
+#include <iostream>
+
+ImageLoaderQOI::ImageInfo ImageLoaderQOI::loadInfoFromMemory(const uint8_t* data, uint64_t data_size) {
+	ImageInfo res;
+
+	qoi_desc desc;
+	// TODO: only read the header
+	auto* ret = qoi_decode(data, data_size, &desc, 4);
+	if (ret == nullptr) {
+		return res;
+	}
+	free(ret);
+
+	res.width = desc.width;
+	res.height = desc.height;
+	//desc.colorspace;
+
+	return res;
+}
+
+ImageLoaderQOI::ImageResult ImageLoaderQOI::loadFromMemoryRGBA(const uint8_t* data, uint64_t data_size) {
+	ImageResult res;
+
+	qoi_desc desc;
+
+	uint8_t* img_data = static_cast<uint8_t*>(
+		qoi_decode(data, data_size, &desc, 4)
+	);
+	if (img_data == nullptr) {
+		// not readable
+		return res;
+	}
+
+	res.width = desc.width;
+	res.height = desc.height;
+
+	auto& new_frame = res.frames.emplace_back();
+	new_frame.ms = 0;
+	new_frame.data.insert(new_frame.data.cbegin(), img_data, img_data+(desc.width*desc.height*4));
+
+	free(img_data);
+	return res;
+}
+
+std::vector<uint8_t> ImageEncoderQOI::encodeToMemoryRGBA(const ImageResult& input_image, const std::map<std::string, float>&) {
+	if (input_image.frames.empty()) {
+		std::cerr << "IEQOI error: empty image\n";
+		return {};
+	}
+
+	if (input_image.frames.size() > 1) {
+		std::cerr << "IEQOI warning: image with animation, only first frame will be encoded!\n";
+		return {};
+	}
+
+	// TODO: look into RDO (eg https://github.com/richgel999/rdopng)
+	//int png_compression_level = 8;
+	//if (extra_options.count("png_compression_level")) {
+		//png_compression_level = extra_options.at("png_compression_level");
+	//}
+
+	qoi_desc desc;
+	desc.width = input_image.width;
+	desc.height = input_image.height;
+	desc.channels = 4;
+	desc.colorspace = QOI_SRGB; // TODO: decide
+
+	int out_len {0};
+	uint8_t* enc_data = static_cast<uint8_t*>(qoi_encode(
+		input_image.frames.front().data.data(),
+		&desc,
+		&out_len
+	));
+
+	if (enc_data == nullptr) {
+		std::cerr << "IEQOI error: qoi_encode failed!\n";
+		return {};
+	}
+
+	std::vector<uint8_t> new_data(enc_data, enc_data+out_len);
+
+	free(enc_data); // TODO: a streaming encoder would be better
+
+	return new_data;
+}
+

src/image_loader_qoi.hpp

@@ -0,0 +1,13 @@
+#pragma once
+
+#include "./image_loader.hpp"
+
+struct ImageLoaderQOI : public ImageLoaderI {
+	ImageInfo loadInfoFromMemory(const uint8_t* data, uint64_t data_size) override;
+	ImageResult loadFromMemoryRGBA(const uint8_t* data, uint64_t data_size) override;
+};
+
+struct ImageEncoderQOI : public ImageEncoderI {
+	std::vector<uint8_t> encodeToMemoryRGBA(const ImageResult& input_image, const std::map<std::string, float>& extra_options = {}) override;
+};
+

src/json/message_components.hpp

@@ -0,0 +1,27 @@
+#pragma once
+
+#include <solanaceae/util/utils.hpp>
+
+#include <solanaceae/message3/components.hpp>
+
+#include <nlohmann/json.hpp>
+
+namespace Message::Components {
+
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Timestamp, ts)
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(TimestampProcessed, ts)
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(TimestampWritten, ts)
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(ContactFrom, c)
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(ContactTo, c)
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Read, ts)
+	// TODO: SyncedBy
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessageText, text)
+
+	namespace Transfer {
+		NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(FileInfo::FileDirEntry, file_name, file_size)
+		NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(FileInfo, file_list, total_size)
+		NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(FileInfoLocal, file_list)
+	} // Transfer
+
+} // Message::Components
+

src/json/tox_message_components.hpp

@@ -0,0 +1,16 @@
+#pragma once
+
+#include <solanaceae/util/utils.hpp>
+
+#include <solanaceae/tox_messages/components.hpp>
+
+#include <nlohmann/json.hpp>
+
+namespace Message::Components {
+
+	// TODO: friend msg id, does not have the same qualities
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(ToxGroupMessageID, id)
+	// TODO: transfer stuff, needs content rewrite
+
+} // Message::Components
+

src/main_screen.cpp

@@ -1,5 +1,8 @@
 #include "./main_screen.hpp"
 
+#include "./fragment_store/register_mfs_json_message_components.hpp"
+#include "./fragment_store/register_mfs_json_tox_message_components.hpp"
+
 #include <solanaceae/contact/components.hpp>
 
 #include <imgui/imgui.h>
@@ -13,6 +16,7 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri
 	renderer(renderer_),
 	rmm(cr),
 	mts(rmm),
+	mfs(cr, rmm, fs),
 	tc(save_path, save_password),
 	tpi(tc.getTox()),
 	ad(tc),
@@ -33,6 +37,8 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri
 	tdch(tpi)
 {
 	tel.subscribeAll(tc);
+	registerMFSJsonMessageComponents(mfs.getMSC());
+	registerMFSJsonToxMessageComponents(mfs.getMSC());
 
 	conf.set("tox", "save_file_path", save_path);
 
@@ -49,6 +55,10 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri
 	std::cout << "own address: " << tc.toxSelfGetAddressStr() << "\n";
 
 	{ // setup plugin instances
+		// TODO: make interface useful
+		g_provideInstance<FragmentStoreI>("FragmentStoreI", "host", &fs);
+		g_provideInstance<FragmentStore>("FragmentStore", "host", &fs);
+
 		g_provideInstance<ConfigModelI>("ConfigModelI", "host", &conf);
 		g_provideInstance<Contact3Registry>("Contact3Registry", "1", "host", &cr);
 		g_provideInstance<RegistryMessageModel>("RegistryMessageModel", "host", &rmm);
@@ -74,6 +84,8 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri
 	}
 
 	conf.dump();
+
+	mfs.triggerScan(); // HACK: after plugins and tox contacts got loaded
 }
 
 MainScreen::~MainScreen(void) {
@@ -388,7 +400,8 @@ Screen* MainScreen::tick(float time_delta, bool& quit) {
 
 	tdch.tick(time_delta); // compute
 
-	mts.iterate(); // compute
+	const float mfs_interval = mfs.tick(time_delta);
+	mts.iterate(); // compute (after mfs)
 
 	_min_tick_interval = std::min<float>(
 		// HACK: pow by 1.6 to increase 50 -> ~500 (~522)
@@ -400,6 +413,10 @@ Screen* MainScreen::tick(float time_delta, bool& quit) {
 		_min_tick_interval,
 		fo_interval
 	);
+	_min_tick_interval = std::min<float>(
+		_min_tick_interval,
+		mfs_interval
+	);
 
 	//std::cout << "MS: min tick interval: " << _min_tick_interval << "\n";
 

src/main_screen.hpp

@@ -2,10 +2,12 @@
 
 #include "./screen.hpp"
 
+#include "./fragment_store/fragment_store.hpp"
 #include <solanaceae/util/simple_config_model.hpp>
 #include <solanaceae/contact/contact_model3.hpp>
 #include <solanaceae/message3/registry_message_model.hpp>
 #include <solanaceae/message3/message_time_sort.hpp>
+#include "./fragment_store/message_fragment_store.hpp"
 #include <solanaceae/plugin/plugin_manager.hpp>
 #include <solanaceae/toxcore/tox_event_logger.hpp>
 #include "./tox_private_impl.hpp"
@@ -43,10 +45,13 @@ extern "C" {
 struct MainScreen final : public Screen {
 	SDL_Renderer* renderer;
 
+	FragmentStore fs;
+
 	SimpleConfigModel conf;
 	Contact3Registry cr;
 	RegistryMessageModel rmm;
 	MessageTimeSort mts;
+	MessageFragmentStore mfs;
 
 	PluginManager pm;
 

src/media_meta_info_loader.cpp

@@ -2,6 +2,7 @@
 
 #include "./image_loader_webp.hpp"
 #include "./image_loader_sdl_bmp.hpp"
+#include "./image_loader_qoi.hpp"
 #include "./image_loader_stb.hpp"
 
 #include <solanaceae/message3/components.hpp>
@@ -77,6 +78,7 @@ MediaMetaInfoLoader::MediaMetaInfoLoader(RegistryMessageModel& rmm) : _rmm(rmm)
 	// HACK: make them be added externally?
 	_image_loaders.push_back(std::make_unique<ImageLoaderWebP>());
 	_image_loaders.push_back(std::make_unique<ImageLoaderSDLBMP>());
+	_image_loaders.push_back(std::make_unique<ImageLoaderQOI>());
 	_image_loaders.push_back(std::make_unique<ImageLoaderSTB>());
 
 	_rmm.subscribe(this, RegistryMessageModel_Event::message_construct);

src/message_image_loader.cpp

@@ -1,6 +1,7 @@
 #include "./message_image_loader.hpp"
 
 #include "./image_loader_sdl_bmp.hpp"
+#include "./image_loader_qoi.hpp"
 #include "./image_loader_stb.hpp"
 #include "./image_loader_webp.hpp"
 #include "./media_meta_info_loader.hpp"
@@ -19,6 +20,7 @@ uint64_t getTimeMS(void);
 
 MessageImageLoader::MessageImageLoader(void) {
 	_image_loaders.push_back(std::make_unique<ImageLoaderSDLBMP>());
+	_image_loaders.push_back(std::make_unique<ImageLoaderQOI>());
 	_image_loaders.push_back(std::make_unique<ImageLoaderWebP>());
 	_image_loaders.push_back(std::make_unique<ImageLoaderSTB>());
 }

src/send_image_popup.cpp

@@ -3,6 +3,7 @@
 #include "./image_loader_sdl_bmp.hpp"
 #include "./image_loader_stb.hpp"
 #include "./image_loader_webp.hpp"
+#include "./image_loader_qoi.hpp"
 
 #include <imgui/imgui.h>
 
@@ -13,6 +14,7 @@ uint64_t getTimeMS(void);
 
 SendImagePopup::SendImagePopup(TextureUploaderI& tu) : _tu(tu) {
 	_image_loaders.push_back(std::make_unique<ImageLoaderSDLBMP>());
+	_image_loaders.push_back(std::make_unique<ImageLoaderQOI>());
 	_image_loaders.push_back(std::make_unique<ImageLoaderWebP>());
 	_image_loaders.push_back(std::make_unique<ImageLoaderSTB>());
 }
@@ -421,7 +423,7 @@ void SendImagePopup::render(float time_delta) {
 
 		if (compress) {
 			ImGui::SameLine();
-			ImGui::Combo("##compression_type", &current_compressor, "webp\0jpeg\0png\n");
+			ImGui::Combo("##compression_type", &current_compressor, "webp\0jpeg\0png\0qoi\0");
 
 			ImGui::Indent();
 			// combo "webp""webp-lossless""png""jpg?"
@@ -486,6 +488,11 @@ void SendImagePopup::render(float time_delta) {
 					if (!new_data.empty()) {
 						_on_send(new_data, ".png");
 					}
+				} else if (current_compressor == 3) {
+					new_data = ImageEncoderQOI{}.encodeToMemoryRGBA(tmp_img, {});;
+					if (!new_data.empty()) {
+						_on_send(new_data, ".qoi");
+					}
 				}
 				// error
 

src/tox_avatar_loader.cpp

@@ -1,6 +1,7 @@
 #include "./tox_avatar_loader.hpp"
 
 #include "./image_loader_sdl_bmp.hpp"
+#include "./image_loader_qoi.hpp"
 #include "./image_loader_stb.hpp"
 #include "./image_loader_webp.hpp"
 
@@ -21,6 +22,7 @@ uint64_t getTimeMS(void);
 
 ToxAvatarLoader::ToxAvatarLoader(Contact3Registry& cr) : _cr(cr) {
 	_image_loaders.push_back(std::make_unique<ImageLoaderSDLBMP>());
+	_image_loaders.push_back(std::make_unique<ImageLoaderQOI>());
 	_image_loaders.push_back(std::make_unique<ImageLoaderWebP>());
 	_image_loaders.push_back(std::make_unique<ImageLoaderSTB>());
 }

src/tox_friend_faux_offline_messaging.cpp

@@ -120,7 +120,8 @@ ToxFriendFauxOfflineMessaging::dfmc_Ret ToxFriendFauxOfflineMessaging::doFriendM
 		// require
 		if (!mr->all_of<
 				Message::Components::MessageText, // text only for now
-				Message::Components::ContactTo
+				Message::Components::ContactTo,
+				Message::Components::ToxFriendMessageID // yes, needs fake ids
 			>(msg)
 		) {
 			continue; // skip