From f27300a8fa2cda354b83604eb67f6b674f4f05cb Mon Sep 17 00:00:00 2001 From: Green Sky Date: Tue, 6 Dec 2022 19:21:20 +0100 Subject: [PATCH] add new serializer based on bitpacking it has the important stuff, but still misses conviniences like buffers and optimizations like varints. only works for little endian but theoretically only needs endian detection to be able to support bigendian --- .vimspector.json | 28 ++ framework/CMakeLists.txt | 1 + framework/s6zer/CMakeLists.txt | 23 + framework/s6zer/src/mm/s6zer/serialize.hpp | 33 ++ framework/s6zer/src/mm/s6zer/stream.hpp | 331 ++++++++++++ framework/s6zer/test/CMakeLists.txt | 18 + framework/s6zer/test/test.cpp | 553 +++++++++++++++++++++ 7 files changed, 987 insertions(+) create mode 100644 .vimspector.json create mode 100644 framework/s6zer/CMakeLists.txt create mode 100644 framework/s6zer/src/mm/s6zer/serialize.hpp create mode 100644 framework/s6zer/src/mm/s6zer/stream.hpp create mode 100644 framework/s6zer/test/CMakeLists.txt create mode 100644 framework/s6zer/test/test.cpp diff --git a/.vimspector.json b/.vimspector.json new file mode 100644 index 0000000..4265e7a --- /dev/null +++ b/.vimspector.json @@ -0,0 +1,28 @@ +{ + "configurations": { + "run_ctest_native": { + "default": true, + "adapter": "CodeLLDB", + "configuration": { + "request": "launch", + "stopOnEntry": true, + "console": "integratedTerminal", + "program": "ctest", + "cwd": "${workspaceRoot}/build" + } + }, + "run_native": { + "adapter": "CodeLLDB", + "variables": { + "Executable": "s6zer_test" + }, + "configuration": { + "request": "launch", + "stopOnEntry": true, + "console": "integratedTerminal", + "program": "${workspaceRoot}/build/bin/${Executable}", + "cwd": "${workspaceRoot}/build" + } + } + } +} diff --git a/framework/CMakeLists.txt b/framework/CMakeLists.txt index 822919f..ecb3725 100644 --- a/framework/CMakeLists.txt +++ b/framework/CMakeLists.txt @@ -5,6 +5,7 @@ project(framework) add_subdirectory(engine) add_subdirectory(logger) add_subdirectory(resource_manager) +add_subdirectory(s6zer) add_subdirectory(common_components) add_subdirectory(std_utils) add_subdirectory(random) diff --git a/framework/s6zer/CMakeLists.txt b/framework/s6zer/CMakeLists.txt new file mode 100644 index 0000000..7e195e7 --- /dev/null +++ b/framework/s6zer/CMakeLists.txt @@ -0,0 +1,23 @@ +cmake_minimum_required(VERSION 3.9 FATAL_ERROR) + +project(s6zer CXX) + +add_library(s6zer INTERFACE + #./src/s6zer/stream.hpp + #./src/s6zer/serialize.hpp +) + +add_library(MM::s6zer ALIAS s6zer) + +target_include_directories(s6zer INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}/src") + +target_compile_features(s6zer INTERFACE cxx_std_17) + +#target_link_libraries(s6zer + #INTERFACE +#) + +if (BUILD_TESTING) + add_subdirectory(test) +endif() + diff --git a/framework/s6zer/src/mm/s6zer/serialize.hpp b/framework/s6zer/src/mm/s6zer/serialize.hpp new file mode 100644 index 0000000..71b27d4 --- /dev/null +++ b/framework/s6zer/src/mm/s6zer/serialize.hpp @@ -0,0 +1,33 @@ +#pragma once + +#include "./stream.hpp" + +namespace MM::s6zer { + +// serialize macros + +// TODO: make use of ADL, like nlohmann::json does. + +/* + defines mm_serialize functions for you. + a "stream" object is in scope (StreamWriter/StreamReader), + as well as the object of Type called "data". +eg: +MM_DEFINE_SERIALIZE(Test1, + MM_S6ZER_BAIL(stream.serializeBits(data.seq, 16)) + MM_S6ZER_BAIL(stream.serializeBits(data.data1, 8)) +) +*/ +// TODO: refine, so we dont have to do MM_S6ZER_BAIL() everytime +#define MM_DEFINE_SERIALIZE(Type, ...) \ +inline bool mm_serialize(MM::s6zer::StreamWriter& stream, const Type& data) { \ + __VA_ARGS__ \ + return true; \ +} \ +inline bool mm_serialize(MM::s6zer::StreamReader& stream, Type& data) { \ + __VA_ARGS__ \ + return true; \ +} + +} // MM::s6zer + diff --git a/framework/s6zer/src/mm/s6zer/stream.hpp b/framework/s6zer/src/mm/s6zer/stream.hpp new file mode 100644 index 0000000..e110c9a --- /dev/null +++ b/framework/s6zer/src/mm/s6zer/stream.hpp @@ -0,0 +1,331 @@ +#pragma once + +#include // size_t +#include // uint8_t, etc + +#include + +#include + +// TODO: make asserts redefinable + +namespace MM::s6zer { + +// this is heavily inspired by Glenn Fiedler's (Gaffer On Games) serializers +// https://www.gafferongames.com/post/reading_and_writing_packets/ +// https://www.gafferongames.com/post/serialization_strategies/ + +// internal helpers +namespace detail { + // TODO: ugly, replace when c++20 + [[nodiscard]] constexpr size_t first_bit_set8(const uint8_t number) { + return + (number & 0b10000000) ? 8 : + (number & 0b01000000) ? 7 : + (number & 0b00100000) ? 6 : + (number & 0b00010000) ? 5 : + (number & 0b00001000) ? 4 : + (number & 0b00000100) ? 3 : + (number & 0b00000010) ? 2 : + (number & 0b00000001) ? 1 : + 0 + ; + } + + [[nodiscard]] constexpr size_t first_bit_set32(const uint32_t number) { + return + (number & 0xff000000) ? first_bit_set8((number >> 24) & 0xff) + 24 : + (number & 0x00ff0000) ? first_bit_set8((number >> 16) & 0xff) + 16 : + (number & 0x0000ff00) ? first_bit_set8((number >> 8) & 0xff) + 8 : + (number & 0x000000ff) ? first_bit_set8(number & 0xff) : + 0 + ; + } + + [[nodiscard]] constexpr uint32_t byte_swap(const uint32_t value) noexcept { + return + ((value & 0xff000000) >> 24) | + ((value & 0x00ff0000) >> 8) | + ((value & 0x0000ff00) << 8) | + ((value & 0x000000ff) << 24) + ; + } + + [[nodiscard]] constexpr uint16_t byte_swap(const uint16_t value) noexcept { + return + ((value & 0xff00) >> 8) | + ((value & 0x00ff) << 8) + ; + } + + [[nodiscard]] constexpr uint8_t byte_swap(const uint8_t value) noexcept { + // noop + return value; + } + + template + [[nodiscard]] constexpr const T& max(const T& a, const T& b) noexcept { + return (a < b) ? b : a; + } + +} // detail + +// TODO: maybe 64bit? +// TODO: is this detail? +[[nodiscard]] constexpr size_t bits_required(const uint32_t numbers) { + return detail::first_bit_set32(numbers); +} + +[[nodiscard]] constexpr uint32_t serialize_byte_order(const uint32_t value) { + // TODO: only works on little endian for now + if constexpr (true) { // native is little endian + return value; + } else { // native is big endian + return detail::byte_swap(value); + } +} + +// helper for fake exceptions +#ifndef MM_S6ZER_BAIL +#define MM_S6ZER_BAIL(...) { \ + if (! __VA_ARGS__) { \ + return false; \ + } \ +} +#endif + +struct StreamWriter { + StreamWriter(void) = delete; + StreamWriter(uint32_t* data, size_t size) : _data(data), _data_size(size) { + assert(size != 0); + assert(size % sizeof(uint32_t) == 0); + assert(data != nullptr); + } + + // do i still need them? + [[nodiscard]] static constexpr bool isWriting(void) noexcept { return true; } + [[nodiscard]] static constexpr bool isReading(void) noexcept { return false; } + + [[nodiscard]] bool flush(void) noexcept { + if (_scratch_bits != 0) { + // check if space in buffer + if (_data_size < (_word_index + 1) * sizeof(uint32_t)) { + return false; + } + + _data[_word_index] = serialize_byte_order(static_cast(_scratch & 0xffffffff)); + _scratch >>= 32; // new bits are allways unset, so we can just allways 32 + // we dont like negative + _scratch_bits = detail::max(static_cast(_scratch_bits) - 32, 0); + _word_index++; + } + + return true; + } + + template + [[nodiscard]] bool serializeBits(const T value, const size_t number_of_bits = sizeof(T)*8) noexcept { + static_assert(std::is_integral_v, "type needs to be an integer"); + static_assert(std::is_unsigned_v, "type needs to be unsigned"); + static_assert(sizeof(T) <= 4, "not yet defined for > 32bit"); + assert(number_of_bits <= sizeof(T)*8); + assert(number_of_bits > 0); + + // do scratching + _scratch |= static_cast(value) << _scratch_bits; + _scratch_bits += number_of_bits; + _bits_written += number_of_bits; + + if (_scratch_bits >= 32) { + return flush(); + } + + return true; + } + + [[nodiscard]] bool serializeBool(const bool value) noexcept { + return serializeBits(static_cast(value), 1); + } + + template + [[nodiscard]] bool serializeInt(const T value, const T min, const T max) noexcept { + static_assert(std::is_integral_v, "type needs to be an integer"); + static_assert(sizeof(T) <= 4, "not yet defined for > 32bit"); + assert(max >= min); + assert(value >= min); + assert(value <= max); + + const size_t bits = bits_required(max - min); + + return serializeBits(static_cast(value - min), bits); + } + + [[nodiscard]] bool serializeFloat(const float value) noexcept { + // TODO: dont use loop + for (size_t i = 0; i < sizeof(float); i++) { + MM_S6ZER_BAIL(serializeBits(reinterpret_cast(&value)[i], 8)); + } + + return true; + } + + [[nodiscard]] bool serializeDouble(const double value) noexcept { + // TODO: dont use loop + for (size_t i = 0; i < sizeof(double); i++) { + MM_S6ZER_BAIL(serializeBits(reinterpret_cast(&value)[i], 8)); + } + + return true; + } + + [[nodiscard]] bool serializeFloatCompressed(const float value, const float min, const float max, const float resolution) noexcept { + assert(max >= min); + assert(value >= min); + assert(value <= max); + + // TODO: handle those rounding errors + + const float numbers = (max - min) / resolution; + const size_t bits = bits_required(static_cast(numbers)); + + const uint32_t tmp_value = static_cast((value - min) / resolution); + + return serializeBits(tmp_value, bits); + } + + [[nodiscard]] size_t bytesWritten(void) noexcept { + // TODO: is this assert valid? + assert(_scratch_bits == 0); + + //return _bits_written/8 + ((_bits_written % 8) ? 1 : 0); + return (_bits_written+7) / 8; + } + + uint32_t* _data {nullptr}; + size_t _data_size {0}; + + uint64_t _scratch {0}; + size_t _scratch_bits {0}; + size_t _word_index {0}; + size_t _bits_written {0}; // includes bits still in scratch +}; + +struct StreamReader { + StreamReader(void) = delete; + // !! StreamReader assumes the data buffer has whole uint32_t, + // so at the end, even though data_size might be less then 4 bytes, + // here is actually a full, empty uint32 + // !! enable AddressSanitzier during development and testing + StreamReader(const uint32_t* data, size_t size) : _data(data), _data_size(size) { + assert(size != 0); + //assert(size % sizeof(uint32_t) == 0); + assert(data != nullptr); + } + + // do i still need them? + [[nodiscard]] static constexpr bool isWriting(void) noexcept { return false; } + [[nodiscard]] static constexpr bool isReading(void) noexcept { return true; } + + template + [[nodiscard]] bool serializeBits(T& value, const size_t number_of_bits = sizeof(T)*8) noexcept { + static_assert(std::is_integral_v, "type needs to be an integer"); + static_assert(std::is_unsigned_v, "type needs to be unsigned"); + static_assert(sizeof(T) <= 4, "not yet defined for > 32bit"); + assert(number_of_bits <= sizeof(T)*8); + assert(number_of_bits > 0); + + if (_scratch_bits < number_of_bits) { + if (_bits_read + number_of_bits > _data_size*8) { + // would read past end + return false; + } + + _scratch |= static_cast(serialize_byte_order(_data[_word_index])) << _scratch_bits; + _word_index++; + _scratch_bits += 32; + } + + value = _scratch & ((uint64_t(1) << number_of_bits) - 1); + + _scratch >>= number_of_bits; + _scratch_bits -= number_of_bits; + _bits_read += number_of_bits; + + return true; + } + + [[nodiscard]] bool serializeBool(bool& value) noexcept { + uint32_t tmp_value {0}; + MM_S6ZER_BAIL(serializeBits(tmp_value, 1)); + + // :) + value = tmp_value != 0; + + return true; + } + + template + [[nodiscard]] bool serializeInt(T& value, const T min, const T max) noexcept { + static_assert(std::is_integral_v, "type needs to be an integer"); + static_assert(sizeof(T) <= 4, "not yet defined for > 32bit"); + assert(max >= min); + + const size_t bits = bits_required(max - min); + + uint32_t tmp_val {0}; + MM_S6ZER_BAIL(serializeBits(tmp_val, bits)); + + value = static_cast(tmp_val) + min; + + return true; + } + + [[nodiscard]] bool serializeFloat(float& value) noexcept { + // TODO: dont use loop + for (size_t i = 0; i < sizeof(float); i++) { + MM_S6ZER_BAIL(serializeBits(reinterpret_cast(&value)[i], 8)); + } + + return true; + } + + [[nodiscard]] bool serializeDouble(double& value) noexcept { + // TODO: dont use loop + for (size_t i = 0; i < sizeof(double); i++) { + MM_S6ZER_BAIL(serializeBits(reinterpret_cast(&value)[i], 8)); + } + + return true; + } + + [[nodiscard]] bool serializeFloatCompressed(float& value, const float min, const float max, const float resolution) noexcept { + assert(max >= min); + + // TODO: use rounding, rn it snaps (floor) + + const float numbers = (max - min) / resolution; + const size_t bits = bits_required(static_cast(numbers)); + + uint32_t tmp_value {0}; + MM_S6ZER_BAIL(serializeBits(tmp_value, bits)); + + value = static_cast(tmp_value) * resolution + min; + + return true; + } + + [[nodiscard]] size_t bytesRead(void) noexcept { + return (_bits_read+7) / 8; + } + + const uint32_t* _data {nullptr}; + size_t _data_size {0}; + + uint64_t _scratch {0}; + size_t _scratch_bits {0}; + size_t _word_index {0}; + size_t _bits_read {0}; +}; + +} // MM::s6zer + diff --git a/framework/s6zer/test/CMakeLists.txt b/framework/s6zer/test/CMakeLists.txt new file mode 100644 index 0000000..120b3a7 --- /dev/null +++ b/framework/s6zer/test/CMakeLists.txt @@ -0,0 +1,18 @@ +add_executable(s6zer_test + test.cpp +) + +target_include_directories(s6zer_test PRIVATE ".") + +target_compile_features(s6zer_test PRIVATE cxx_std_17) + +target_link_libraries(s6zer_test + gtest_main + + s6zer + + random +) + +add_test(NAME s6zer_test COMMAND s6zer_test) + diff --git a/framework/s6zer/test/test.cpp b/framework/s6zer/test/test.cpp new file mode 100644 index 0000000..bcac55c --- /dev/null +++ b/framework/s6zer/test/test.cpp @@ -0,0 +1,553 @@ +#include + +#include +#include + +#include +#include + +#include + +namespace MM { + template + std::basic_ostream& operator<<(std::basic_ostream& out, const MM::ScalarRange2& range) { + return out << "{ min: " << static_cast(range.min()) << ", max: " << static_cast(range.max()) << " }"; + } +} // MM + +TEST(s6zer, bits_required_static) { + static_assert(MM::s6zer::bits_required(0)== 0); + static_assert(MM::s6zer::bits_required(1)== 1); + static_assert(MM::s6zer::bits_required(2)== 2); + static_assert(MM::s6zer::bits_required(3)== 2); + static_assert(MM::s6zer::bits_required(4)== 3); + static_assert(MM::s6zer::bits_required(32)== 6); + static_assert(MM::s6zer::bits_required(0xffffffff)== 32); + static_assert(MM::s6zer::bits_required(0xffffff00)== 32); + static_assert(MM::s6zer::bits_required(0xf0000a00)== 32); + static_assert(MM::s6zer::bits_required(0x0f000000)== 28); + static_assert(MM::s6zer::bits_required(0x0000f000)== 16); +} + +TEST(s6zer, byte_swap) { + static_assert(MM::s6zer::detail::byte_swap(static_cast(0x00)) == 0x00); + static_assert(MM::s6zer::detail::byte_swap(static_cast(0xff)) == 0xff); + static_assert(MM::s6zer::detail::byte_swap(static_cast(0x10)) == 0x10); + static_assert(MM::s6zer::detail::byte_swap(static_cast(0xfe)) == 0xfe); + + static_assert(MM::s6zer::detail::byte_swap(static_cast(0x0000)) == 0x0000); + static_assert(MM::s6zer::detail::byte_swap(static_cast(0xffff)) == 0xffff); + static_assert(MM::s6zer::detail::byte_swap(static_cast(0x00fe)) == 0xfe00); + static_assert(MM::s6zer::detail::byte_swap(static_cast(0xfefe)) == 0xfefe); + + static_assert(MM::s6zer::detail::byte_swap(static_cast(0x00000000)) == 0x00000000); + static_assert(MM::s6zer::detail::byte_swap(static_cast(0xffffffff)) == 0xffffffff); + static_assert(MM::s6zer::detail::byte_swap(static_cast(0xf0f00000)) == 0x0000f0f0); + static_assert(MM::s6zer::detail::byte_swap(static_cast(0xfe0000ef)) == 0xef0000fe); +} + +TEST(s6zer, stream_normalcase1) { + const uint32_t num1_orig {0b111}; + const uint32_t num1_orig_bits {3}; + const uint32_t num2_orig {0b1111111111}; + const uint32_t num2_orig_bits {10}; + const uint32_t num3_orig {0b111111111111111111111111}; + const uint32_t num3_orig_bits {24}; + + std::array buffer; + size_t buffer_size = buffer.size()*sizeof(uint32_t); + { + MM::s6zer::StreamWriter writer{buffer.data(), buffer_size}; + + bool r = false; + ASSERT_EQ(writer._scratch, 0x0); + ASSERT_EQ(writer._scratch_bits, 0); + ASSERT_EQ(writer._word_index, 0); + ASSERT_EQ(writer._bits_written, 0); + ASSERT_EQ(writer.bytesWritten(), 0); + + r = writer.serializeBits(num1_orig, num1_orig_bits); + ASSERT_TRUE(r); + ASSERT_EQ(writer._scratch, 0b0000000000000000000000000000000000000000000000000000000000000'111); + ASSERT_EQ(writer._scratch_bits, 3); + ASSERT_EQ(writer._word_index, 0); + ASSERT_EQ(writer._bits_written, num1_orig_bits); + + r = writer.serializeBits(num2_orig, num2_orig_bits); + ASSERT_TRUE(r); + ASSERT_EQ(writer._scratch, 0b000000000000000000000000000000000000000000000000000'111'1111111111); + ASSERT_EQ(writer._scratch_bits, 3+10); + ASSERT_EQ(writer._word_index, 0); + ASSERT_EQ(writer._bits_written, num1_orig_bits+num2_orig_bits); + + r = writer.serializeBits(num3_orig, num3_orig_bits); + ASSERT_TRUE(r); + ASSERT_EQ(writer._scratch, 0b00000000000000000000000000000000000000000000000000000000000'11111); + ASSERT_EQ(writer._scratch_bits, (3+10+24)-32); + ASSERT_EQ(writer._word_index, 1); + ASSERT_EQ(writer._bits_written, num1_orig_bits+num2_orig_bits+num3_orig_bits); + + r = writer.flush(); + ASSERT_TRUE(r); + ASSERT_EQ(writer._scratch, 0); + ASSERT_EQ(writer._scratch_bits, 0); + ASSERT_EQ(writer._word_index, 2); + ASSERT_EQ(writer._bits_written, num1_orig_bits+num2_orig_bits+num3_orig_bits); // flush does not change bits written + ASSERT_EQ(writer.bytesWritten(), 5); // 4.625 , so ceil + + buffer_size = writer.bytesWritten(); + } + + std::cout << "buffer_size: " << buffer_size << "\n"; + + ASSERT_EQ(buffer[0], 0xffffffff); + ASSERT_EQ(buffer[1], 0b000000000000000000000000000'11111); + + { + MM::s6zer::StreamReader reader{buffer.data(), buffer_size}; + + bool r = false; + ASSERT_EQ(reader._scratch, 0x0); + ASSERT_EQ(reader._scratch_bits, 0); + ASSERT_EQ(reader._word_index, 0); + + uint32_t num1 {0}; + r = reader.serializeBits(num1, num1_orig_bits); + ASSERT_TRUE(r); + ASSERT_EQ(num1, num1_orig); + ASSERT_EQ(reader._scratch, 0b00000000'00000000'00000000'00000000'000'11111111111111111111111111111); + ASSERT_EQ(reader._scratch_bits, 29); + ASSERT_EQ(reader._word_index, 1); // index refers to next dword + + uint32_t num2 {0}; + r = reader.serializeBits(num2, num2_orig_bits); + ASSERT_TRUE(r); + ASSERT_EQ(num2, num2_orig); + ASSERT_EQ(reader._scratch, 0b00000000'00000000'00000000'00000000'000'00000'00000'1111111111111111111); + ASSERT_EQ(reader._scratch_bits, 19); + ASSERT_EQ(reader._word_index, 1); // <=32, so should not yet have read next dword + + uint32_t num3 {0}; + r = reader.serializeBits(num3, num3_orig_bits); + ASSERT_TRUE(r); + ASSERT_EQ(num3, num3_orig); + ASSERT_EQ(reader._scratch, 0x0); // no data left + ASSERT_EQ(reader._scratch_bits, 27); + ASSERT_EQ(reader._word_index, 2); + + // error case + uint32_t num4 {0}; + r = reader.serializeBits(num4, 32); + ASSERT_FALSE(r); + } +} + +TEST(s6zer, stream_normalcase1_1) { + const uint32_t num1_orig {0b101}; + const uint32_t num1_orig_bits {3}; + const uint32_t num2_orig {0b1010101010}; + const uint32_t num2_orig_bits {10}; + const uint32_t num3_orig {0b101010101010101010101010}; + const uint32_t num3_orig_bits {24}; + + std::array buffer; + size_t buffer_size = buffer.size()*sizeof(uint32_t); + { + MM::s6zer::StreamWriter writer{buffer.data(), buffer_size}; + + bool r = false; + ASSERT_EQ(writer._scratch, 0x0); + ASSERT_EQ(writer._scratch_bits, 0); + ASSERT_EQ(writer._word_index, 0); + ASSERT_EQ(writer._bits_written, 0); + ASSERT_EQ(writer.bytesWritten(), 0); + + r = writer.serializeBits(num1_orig, num1_orig_bits); + ASSERT_TRUE(r); + ASSERT_EQ(writer._scratch, 0b0000000000000000000000000000000000000000000000000000000000000'101); + ASSERT_EQ(writer._scratch_bits, 3); + ASSERT_EQ(writer._word_index, 0); + ASSERT_EQ(writer._bits_written, num1_orig_bits); + + r = writer.serializeBits(num2_orig, num2_orig_bits); + ASSERT_TRUE(r); + ASSERT_EQ(writer._scratch, 0b000000000000000000000000000000000000000000000000000'1010101010'101); + ASSERT_EQ(writer._scratch_bits, 3+10); + ASSERT_EQ(writer._word_index, 0); + ASSERT_EQ(writer._bits_written, num1_orig_bits+num2_orig_bits); + + r = writer.serializeBits(num3_orig, num3_orig_bits); + ASSERT_TRUE(r); + ASSERT_EQ(writer._scratch, 0b00000000000000000000000000000000000000000000000000000000000'10101); // the high bits of the 24 + ASSERT_EQ(writer._scratch_bits, (3+10+24)-32); + ASSERT_EQ(writer._word_index, 1); + ASSERT_EQ(writer._bits_written, num1_orig_bits+num2_orig_bits+num3_orig_bits); + + r = writer.flush(); + ASSERT_TRUE(r); + ASSERT_EQ(writer._scratch, 0); + ASSERT_EQ(writer._scratch_bits, 0); + ASSERT_EQ(writer._word_index, 2); + ASSERT_EQ(writer._bits_written, num1_orig_bits+num2_orig_bits+num3_orig_bits); // flush does not change bits written + ASSERT_EQ(writer.bytesWritten(), 5); // 4.625 , so ceil + + buffer_size = writer.bytesWritten(); + } + + std::cout << "buffer_size: " << buffer_size << "\n"; + + ASSERT_EQ(buffer[0], 0b0101010101010101010'1010101010'101); + ASSERT_EQ(buffer[1], 0b000000000000000000000000000'10101); + + { + MM::s6zer::StreamReader reader{buffer.data(), buffer_size}; + + bool r = false; + ASSERT_EQ(reader._scratch, 0x0); + ASSERT_EQ(reader._scratch_bits, 0); + ASSERT_EQ(reader._word_index, 0); + + uint32_t num1 {0}; + r = reader.serializeBits(num1, num1_orig_bits); + ASSERT_TRUE(r); + ASSERT_EQ(num1, num1_orig); + ASSERT_EQ(reader._scratch, 0b00000000'00000000'00000000'00000000'000'0101010101010101010'1010101010); + ASSERT_EQ(reader._scratch_bits, 29); + ASSERT_EQ(reader._word_index, 1); // index refers to next dword + + uint32_t num2 {0}; + r = reader.serializeBits(num2, num2_orig_bits); + ASSERT_TRUE(r); + ASSERT_EQ(num2, num2_orig); + ASSERT_EQ(reader._scratch, 0b00000000'00000000'00000000'00000000'000'00000'00000'0101010101010101010); + ASSERT_EQ(reader._scratch_bits, 19); + ASSERT_EQ(reader._word_index, 1); // <=32, so should not yet have read next dword + + uint32_t num3 {0}; + r = reader.serializeBits(num3, num3_orig_bits); + ASSERT_TRUE(r); + ASSERT_EQ(num3, num3_orig); + ASSERT_EQ(reader._scratch, 0x0); // no data left + ASSERT_EQ(reader._scratch_bits, 27); + ASSERT_EQ(reader._word_index, 2); + + // error case + uint32_t num4 {0}; + r = reader.serializeBits(num4, 32); + ASSERT_FALSE(r); + } +} + +TEST(s6zer, stream_normalcase2) { + // we now take each number as its maximum, synthetic for testing + const uint32_t num1_orig {17}; + const uint32_t num1_orig_bits {MM::s6zer::bits_required(num1_orig)}; + const uint32_t num2_orig {1}; + const uint32_t num2_orig_bits {MM::s6zer::bits_required(num2_orig)}; + const uint32_t num3_orig {1298989}; + const uint32_t num3_orig_bits {MM::s6zer::bits_required(num3_orig)}; + + std::array buffer; + size_t buffer_size = buffer.size()*sizeof(uint32_t); + { + MM::s6zer::StreamWriter writer{buffer.data(), buffer_size}; + + // fewer asserts + bool r = false; + ASSERT_EQ(writer._scratch, 0x0); + ASSERT_EQ(writer._scratch_bits, 0); + ASSERT_EQ(writer._word_index, 0); + ASSERT_EQ(writer._bits_written, 0); + ASSERT_EQ(writer.bytesWritten(), 0); + + r = writer.serializeBits(num1_orig, num1_orig_bits); + ASSERT_TRUE(r); + ASSERT_EQ(writer._bits_written, num1_orig_bits); + + r = writer.serializeBits(num2_orig, num2_orig_bits); + ASSERT_TRUE(r); + ASSERT_EQ(writer._bits_written, num1_orig_bits+num2_orig_bits); + + r = writer.serializeBits(num3_orig, num3_orig_bits); + ASSERT_TRUE(r); + ASSERT_EQ(writer._bits_written, num1_orig_bits+num2_orig_bits+num3_orig_bits); + + r = writer.flush(); + ASSERT_TRUE(r); + ASSERT_EQ(writer._scratch, 0); + ASSERT_EQ(writer._scratch_bits, 0); + ASSERT_EQ(writer._bits_written, num1_orig_bits+num2_orig_bits+num3_orig_bits); // flush does not change bits written + + buffer_size = writer.bytesWritten(); + } + + std::cout << "buffer_size: " << buffer_size << "\n"; + + { + MM::s6zer::StreamReader reader{buffer.data(), buffer_size}; + + bool r = false; + ASSERT_EQ(reader._scratch, 0x0); + ASSERT_EQ(reader._scratch_bits, 0); + ASSERT_EQ(reader._word_index, 0); + ASSERT_EQ(reader._bits_read, 0); + + uint32_t num1 {0}; + r = reader.serializeBits(num1, num1_orig_bits); + ASSERT_TRUE(r); + ASSERT_EQ(num1, num1_orig); + + uint32_t num2 {0}; + r = reader.serializeBits(num2, num2_orig_bits); + ASSERT_TRUE(r); + ASSERT_EQ(num2, num2_orig); + + uint32_t num3 {0}; + r = reader.serializeBits(num3, num3_orig_bits); + ASSERT_TRUE(r); + ASSERT_EQ(num3, num3_orig); + + // error case + uint32_t num4 {0}; + r = reader.serializeBits(num4, 32); + ASSERT_FALSE(r); + } +} + +// emscripten cant do this +#ifndef __EMSCRIPTEN__ +bool serialize_int_death_fn(void) { + std::array dummy_buffer; + MM::s6zer::StreamReader reader{dummy_buffer.data(), dummy_buffer.size()*sizeof(uint32_t)}; + int32_t value{0}; + + MM_S6ZER_BAIL(reader.serializeInt(value, 20, -20)); // wrong order + + return true; +} + +TEST(s6zer, serialize_int_death) { + ASSERT_DEATH({ + [[maybe_unused]] bool ret = serialize_int_death_fn(); + }, "failed"); +} +#endif + +TEST(s6zer, reader_bits_bug1) { + const std::array buffer { + 0x27'5c'19'a1, + 0x00'00'3a'c3 + }; + const size_t buffer_size {6}; + + MM::s6zer::StreamReader reader{buffer.data(), buffer_size}; + + uint8_t value_0 {0}; + ASSERT_TRUE(reader.serializeBits(value_0)); + ASSERT_EQ(value_0, 0xa1); + + ASSERT_EQ(reader._scratch_bits, 24); + ASSERT_EQ(reader._scratch, 0x0000000000'27'5c'19); + + uint32_t value_1 {0}; + ASSERT_TRUE(reader.serializeBits(value_1)); + ASSERT_EQ(value_1, 0xc3'27'5c'19); + + ASSERT_EQ(reader._scratch_bits, 24); + ASSERT_EQ(reader._scratch, 0x00000000000000'3a); + + uint8_t value_2 {0}; + ASSERT_TRUE(reader.serializeBits(value_2)); + ASSERT_EQ(value_2, 0x3a); + + ASSERT_EQ(reader._scratch, 0x0000000000000000); +} + +struct TestStruct { + // integers bits + uint8_t u8 {0}; + uint16_t u16 {0}; + uint32_t u32 {0}; + //uint64_t u64 {0}; + + bool b1 {false}; + + // integers ranges + uint8_t r_u8 {0}; + constexpr static MM::ScalarRange2 r_u8_r{10, 60}; + int8_t r_i8 {0}; + constexpr static MM::ScalarRange2 r_i8_r{-10, 5}; + uint16_t r_u16 {0}; + constexpr static MM::ScalarRange2 r_u16_r{1, 1026}; + int16_t r_i16 {0}; + constexpr static MM::ScalarRange2 r_i16_r{-1, 1026}; + uint32_t r_u32 {0}; + constexpr static MM::ScalarRange2 r_u32_r{0, 12341234}; + int32_t r_i32 {0}; + constexpr static MM::ScalarRange2 r_i32_r{-12341234, 10}; + + // floats + float f32 {0.f}; + double f64 {0.}; + + // float compressed [0; 1] range + constexpr static float c0_f32_resolution = 0.001; + constexpr static MM::ScalarRange2 c0_f32_r{0.f, 1.f}; + float c0_f32_0 {0.f}; + float c0_f32_1 {0.f}; + float c0_f32_2 {0.f}; + float c0_f32_3 {0.f}; + + // float compressed [-1; 1] range + constexpr static float c1_f32_resolution = 0.05; + constexpr static MM::ScalarRange2 c1_f32_r{-1.f, 1.f}; + float c1_f32_0 {0.f}; + float c1_f32_1 {0.f}; + float c1_f32_2 {0.f}; + float c1_f32_3 {0.f}; + + // float compressed [-1000; 1000] range + constexpr static float c2_f32_resolution = 0.01; + constexpr static MM::ScalarRange2 c2_f32_r{-1000.f, 1000.f}; + float c2_f32_0 {0.f}; + float c2_f32_1 {0.f}; + float c2_f32_2 {0.f}; + float c2_f32_3 {0.f}; +}; + +MM_DEFINE_SERIALIZE(TestStruct, + MM_S6ZER_BAIL(stream.serializeBits(data.u8)) + MM_S6ZER_BAIL(stream.serializeBits(data.u16)) + MM_S6ZER_BAIL(stream.serializeBits(data.u32)) + + MM_S6ZER_BAIL(stream.serializeBool(data.b1)) + + MM_S6ZER_BAIL(stream.serializeInt(data.r_u8, data.r_u8_r.min(), data.r_u8_r.max())) + MM_S6ZER_BAIL(stream.serializeInt(data.r_i8, data.r_i8_r.min(), data.r_i8_r.max())) + MM_S6ZER_BAIL(stream.serializeInt(data.r_u16, data.r_u16_r.min(), data.r_u16_r.max())) + MM_S6ZER_BAIL(stream.serializeInt(data.r_i16, data.r_i16_r.min(), data.r_i16_r.max())) + MM_S6ZER_BAIL(stream.serializeInt(data.r_u32, data.r_u32_r.min(), data.r_u32_r.max())) + MM_S6ZER_BAIL(stream.serializeInt(data.r_i32, data.r_i32_r.min(), data.r_i32_r.max())) + + MM_S6ZER_BAIL(stream.serializeFloat(data.f32)) + MM_S6ZER_BAIL(stream.serializeDouble(data.f64)) + + MM_S6ZER_BAIL(stream.serializeFloatCompressed(data.c0_f32_0, data.c0_f32_r.min(), data.c0_f32_r.max(), data.c0_f32_resolution)) + MM_S6ZER_BAIL(stream.serializeFloatCompressed(data.c0_f32_1, data.c0_f32_r.min(), data.c0_f32_r.max(), data.c0_f32_resolution)) + MM_S6ZER_BAIL(stream.serializeFloatCompressed(data.c0_f32_2, data.c0_f32_r.min(), data.c0_f32_r.max(), data.c0_f32_resolution)) + MM_S6ZER_BAIL(stream.serializeFloatCompressed(data.c0_f32_3, data.c0_f32_r.min(), data.c0_f32_r.max(), data.c0_f32_resolution)) + + MM_S6ZER_BAIL(stream.serializeFloatCompressed(data.c1_f32_0, data.c1_f32_r.min(), data.c1_f32_r.max(), data.c1_f32_resolution)) + MM_S6ZER_BAIL(stream.serializeFloatCompressed(data.c1_f32_1, data.c1_f32_r.min(), data.c1_f32_r.max(), data.c1_f32_resolution)) + MM_S6ZER_BAIL(stream.serializeFloatCompressed(data.c1_f32_2, data.c1_f32_r.min(), data.c1_f32_r.max(), data.c1_f32_resolution)) + MM_S6ZER_BAIL(stream.serializeFloatCompressed(data.c1_f32_3, data.c1_f32_r.min(), data.c1_f32_r.max(), data.c1_f32_resolution)) + + MM_S6ZER_BAIL(stream.serializeFloatCompressed(data.c2_f32_0, data.c2_f32_r.min(), data.c2_f32_r.max(), data.c2_f32_resolution)) + MM_S6ZER_BAIL(stream.serializeFloatCompressed(data.c2_f32_1, data.c2_f32_r.min(), data.c2_f32_r.max(), data.c2_f32_resolution)) + MM_S6ZER_BAIL(stream.serializeFloatCompressed(data.c2_f32_2, data.c2_f32_r.min(), data.c2_f32_r.max(), data.c2_f32_resolution)) + MM_S6ZER_BAIL(stream.serializeFloatCompressed(data.c2_f32_3, data.c2_f32_r.min(), data.c2_f32_r.max(), data.c2_f32_resolution)) +) + +TEST(s6zer, stream_normalfull) { + std::array buffer; + size_t buffer_size = buffer.size()*sizeof(uint32_t); + + MM::Random::SRNG rng{1337, 0}; + + const TestStruct data_in{ + static_cast(rng()), + static_cast(rng()), + static_cast(rng()), + + rng.roll(0.5f), + + rng.range(TestStruct::r_u8_r), + rng.range(TestStruct::r_i8_r), + rng.range(TestStruct::r_u16_r), + rng.range(TestStruct::r_i16_r), + rng.range(TestStruct::r_u32_r), + rng.range(TestStruct::r_i32_r), + + rng.negOneToOne() * 10000000.f, + rng.negOneToOne() * 10000000., + + rng.range(TestStruct::c0_f32_r), + rng.range(TestStruct::c0_f32_r), + rng.range(TestStruct::c0_f32_r), + rng.range(TestStruct::c0_f32_r), + + rng.range(TestStruct::c1_f32_r), + rng.range(TestStruct::c1_f32_r), + rng.range(TestStruct::c1_f32_r), + rng.range(TestStruct::c1_f32_r), + + rng.range(TestStruct::c2_f32_r), + rng.range(TestStruct::c2_f32_r), + rng.range(TestStruct::c2_f32_r), + rng.range(TestStruct::c2_f32_r), + }; + + std::cout << "struct size: " << sizeof(TestStruct) << "\n"; + + { + MM::s6zer::StreamWriter writer{buffer.data(), buffer_size}; + + ASSERT_TRUE(mm_serialize(writer, data_in)); + + ASSERT_TRUE(writer.flush()); + buffer_size = writer.bytesWritten(); + } + + std::cout << "buffer_size: " << buffer_size << "\n"; + + TestStruct data_out{}; // all zero + + { + MM::s6zer::StreamReader reader{buffer.data(), buffer_size}; + + ASSERT_TRUE(mm_serialize(reader, data_out)); + + ASSERT_EQ(reader._scratch, 0x0000000000000000); + } + + std::cout << "buffer: "; + for (size_t i = 0; i < buffer_size; i++) { + std::cout << std::hex << static_cast(reinterpret_cast(buffer.data())[i]) << "'"; + } + std::cout << "\n"; + + //std::cout << "data_out: \n" << data_out; + + ASSERT_EQ(data_in.u8, data_out.u8); + ASSERT_EQ(data_in.u16, data_out.u16); + ASSERT_EQ(data_in.u32, data_out.u32); + + ASSERT_EQ(data_in.b1, data_out.b1); + + ASSERT_EQ(data_in.r_u8, data_out.r_u8) << "value range: " << TestStruct::r_u8_r; + ASSERT_EQ(data_in.r_i8, data_out.r_i8) << "value range: " << TestStruct::r_i8_r; + ASSERT_EQ(data_in.r_u16, data_out.r_u16) << "value range: " << TestStruct::r_u16_r; + ASSERT_EQ(data_in.r_i16, data_out.r_i16) << "value range: " << TestStruct::r_i16_r; + ASSERT_EQ(data_in.r_u32, data_out.r_u32) << "value range: " << TestStruct::r_u32_r; + ASSERT_EQ(data_in.r_i32, data_out.r_i32) << "value range: " << TestStruct::r_i32_r; + + // bit perfect copies, can have wrong results for special values <.< + ASSERT_EQ(data_in.f32, data_out.f32); + ASSERT_EQ(data_in.f64, data_out.f64); + + ASSERT_NEAR(data_in.c0_f32_0, data_out.c0_f32_0, TestStruct::c0_f32_resolution); + ASSERT_NEAR(data_in.c0_f32_1, data_out.c0_f32_1, TestStruct::c0_f32_resolution); + ASSERT_NEAR(data_in.c0_f32_2, data_out.c0_f32_2, TestStruct::c0_f32_resolution); + ASSERT_NEAR(data_in.c0_f32_3, data_out.c0_f32_3, TestStruct::c0_f32_resolution); + + ASSERT_NEAR(data_in.c1_f32_0, data_out.c1_f32_0, TestStruct::c1_f32_resolution); + ASSERT_NEAR(data_in.c1_f32_1, data_out.c1_f32_1, TestStruct::c1_f32_resolution); + ASSERT_NEAR(data_in.c1_f32_2, data_out.c1_f32_2, TestStruct::c1_f32_resolution); + ASSERT_NEAR(data_in.c1_f32_3, data_out.c1_f32_3, TestStruct::c1_f32_resolution); + + ASSERT_NEAR(data_in.c2_f32_0, data_out.c2_f32_0, TestStruct::c2_f32_resolution); + ASSERT_NEAR(data_in.c2_f32_1, data_out.c2_f32_1, TestStruct::c2_f32_resolution); + ASSERT_NEAR(data_in.c2_f32_2, data_out.c2_f32_2, TestStruct::c2_f32_resolution); + ASSERT_NEAR(data_in.c2_f32_3, data_out.c2_f32_3, TestStruct::c2_f32_resolution); +} +