add file2 zstd impl and test

2024-04-12 11:11:09 +02:00
parent f21dd1dbf0
commit 2b5b19cf31
7 changed files with 711 additions and 0 deletions
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@ -0,0 +1,14 @@
+cmake_minimum_required(VERSION 3.9...3.24 FATAL_ERROR)
+
+project(solanaceae)
+
+add_executable(solanaceae_file2_zstd_test
+	./test_file_zstd.cpp
+)
+
+target_link_libraries(solanaceae_file2_zstd_test PUBLIC
+	solanaceae_file2_zstd
+)
+
+add_test(NAME solanaceae_file2_zstd_test COMMAND solanaceae_file2_zstd_test)
+
--- a/test/test_file_zstd.cpp
+++ b/test/test_file_zstd.cpp
@ -0,0 +1,393 @@
+#include <solanaceae/util/span.hpp>
+#include <solanaceae/file/file2_mem.hpp>
+#include <solanaceae/file/file2_std.hpp>
+#include <solanaceae/file/file2_zstd.hpp>
+
+#include <filesystem>
+#include <iostream>
+#include <variant>
+#include <algorithm>
+#include <vector>
+#include <random>
+#include <cassert>
+
+const static std::string_view test_text1{"test1 1234 1234 :) 1234 5678 88888888\n"};
+const static ByteSpan data_test_text1{
+	reinterpret_cast<const uint8_t*>(test_text1.data()),
+	test_text1.size()
+};
+
+const static std::string_view test_text2{"test2 1234 1234 :) 1234 5678 88888888\n"};
+const static ByteSpan data_test_text2{
+	reinterpret_cast<const uint8_t*>(test_text2.data()),
+	test_text2.size()
+};
+
+const static std::string_view test_text3{
+	"00000000000000000000000000000000000000000000000000000 test 00000000000000000000000000000000000000\n"
+	"00000000000000000000000000000000000000000000000000000 test 00000000000000000000000000000000000000\n"
+	"00000000000000000000000000000000000000000000000000000 test 00000000000000000000000000000000000000\n"
+	"00000000000000000000000000000000000000000000000000000 test 00000000000000000000000000000000000000\n"
+	"00000000000000000000000000000000000000000000000000000 test 00000000000000000000000000000000000000\n"
+	"00000000000000000000000000000000000000000000000000000 test 00000000000000000000000000000000000000\n"
+};
+const static ByteSpan data_test_text3{
+	reinterpret_cast<const uint8_t*>(test_text3.data()),
+	test_text3.size()
+};
+
+int main(void) {
+	{ // first do a simple mem backed test
+		std::vector<uint8_t> buffer;
+		{ // write
+			File2MemW f_w_mem{buffer};
+			assert(f_w_mem.isGood());
+
+			File2ZSTDW f_w_zstd{f_w_mem};
+			assert(f_w_zstd.isGood());
+
+			bool res = f_w_zstd.write(data_test_text1);
+			assert(res);
+			assert(f_w_zstd.isGood());
+
+			// write another frame of the same data
+			res = f_w_zstd.write(data_test_text2);
+			assert(res);
+			assert(f_w_zstd.isGood());
+
+			// write larger frame
+			res = f_w_zstd.write(data_test_text3);
+			assert(res);
+			assert(f_w_zstd.isGood());
+		}
+
+		std::cout << "in mem size: " << buffer.size() << "\n";
+
+		{ // read
+			File2MemR f_r_mem{ByteSpan{buffer}};
+			assert(f_r_mem.isGood());
+
+			File2ZSTDR f_r_zstd{f_r_mem};
+			assert(f_r_zstd.isGood());
+
+			// reads return owning buffers
+
+			{ // readback data_test_text1
+				auto r_res_var = f_r_zstd.read(data_test_text1.size);
+
+				//assert(f_r_zstd.isGood());
+				//assert(f_r_file.isGood());
+				assert(std::holds_alternative<std::vector<uint8_t>>(r_res_var));
+				const auto& r_res_vec = std::get<std::vector<uint8_t>>(r_res_var);
+
+				//std::cout << "decomp: " << std::string_view{reinterpret_cast<const char*>(r_res_vec.data()), r_res_vec.size()};
+
+				assert(r_res_vec.size() == data_test_text1.size);
+				assert(std::equal(data_test_text1.cbegin(), data_test_text1.cend(), r_res_vec.cbegin()));
+			}
+
+			{ // readback data_test_text2
+				auto r_res_var = f_r_zstd.read(data_test_text2.size);
+
+				//assert(f_r_zstd.isGood());
+				//assert(f_r_file.isGood());
+				assert(std::holds_alternative<std::vector<uint8_t>>(r_res_var));
+				const auto& r_res_vec = std::get<std::vector<uint8_t>>(r_res_var);
+
+				//std::cout << "decomp: " << std::string_view{reinterpret_cast<const char*>(r_res_vec.data()), r_res_vec.size()};
+
+				assert(r_res_vec.size() == data_test_text2.size);
+				assert(std::equal(
+					data_test_text2.cbegin(),
+					data_test_text2.cend(),
+					r_res_vec.cbegin()
+				));
+			}
+
+			{ // readback data_test_text3
+				auto r_res_var = f_r_zstd.read(data_test_text3.size);
+
+				//assert(f_r_zstd.isGood());
+				//assert(f_r_file.isGood());
+				assert(std::holds_alternative<std::vector<uint8_t>>(r_res_var));
+				const auto& r_res_vec = std::get<std::vector<uint8_t>>(r_res_var);
+
+				//std::cout << "decomp: " << std::string_view{reinterpret_cast<const char*>(r_res_vec.data()), r_res_vec.size()};
+
+				assert(r_res_vec.size() == data_test_text3.size);
+				assert(std::equal(
+					data_test_text3.cbegin(),
+					data_test_text3.cend(),
+					r_res_vec.cbegin()
+				));
+			}
+
+			{ // assert eof somehow
+				// since its eof, reading a single byte should return a zero sized buffer
+				auto r_res_var = f_r_zstd.read(1);
+				if (std::holds_alternative<std::vector<uint8_t>>(r_res_var)) {
+					assert(std::get<std::vector<uint8_t>>(r_res_var).empty());
+				} else if (std::holds_alternative<ByteSpan>(r_res_var)) {
+					assert(std::get<ByteSpan>(r_res_var).empty());
+				} else {
+					assert(false);
+				}
+			}
+		}
+	}
+
+	const auto temp_dir = std::filesystem::temp_directory_path() / "file2_zstd_tests";
+
+	std::filesystem::create_directories(temp_dir); // making sure
+	assert(std::filesystem::exists(temp_dir));
+	std::cout << "test temp dir: " << temp_dir << "\n";
+
+	const auto test1_file_path = temp_dir / "testfile1.zstd";
+	{ // simple write test
+		File2WFile f_w_file{std::string_view{test1_file_path.u8string()}, true};
+		assert(f_w_file.isGood());
+
+		File2ZSTDW f_w_zstd{f_w_file};
+		assert(f_w_zstd.isGood());
+		assert(f_w_file.isGood());
+
+		//bool res = f_w_file.write(data_test_text1);
+		bool res = f_w_zstd.write(data_test_text1);
+		assert(res);
+		assert(f_w_zstd.isGood());
+		assert(f_w_file.isGood());
+
+		// write another frame of the same data
+		res = f_w_zstd.write(data_test_text2);
+		assert(res);
+		assert(f_w_zstd.isGood());
+		assert(f_w_file.isGood());
+
+		// write larger frame
+		res = f_w_zstd.write(data_test_text3);
+		assert(res);
+		assert(f_w_zstd.isGood());
+		assert(f_w_file.isGood());
+	}
+
+	// after flush
+	assert(std::filesystem::file_size(test1_file_path) != 0);
+
+	{ // simple read test (using write test created file)
+		File2RFile f_r_file{std::string_view{test1_file_path.u8string()}};
+		assert(f_r_file.isGood());
+
+		File2ZSTDR f_r_zstd{f_r_file};
+		assert(f_r_zstd.isGood());
+		assert(f_r_file.isGood());
+
+		// reads return owning buffers
+
+		{ // readback data_test_text1
+			auto r_res_var = f_r_zstd.read(data_test_text1.size);
+
+			//assert(f_r_zstd.isGood());
+			//assert(f_r_file.isGood());
+			assert(std::holds_alternative<std::vector<uint8_t>>(r_res_var));
+			const auto& r_res_vec = std::get<std::vector<uint8_t>>(r_res_var);
+
+			//std::cout << "decomp: " << std::string_view{reinterpret_cast<const char*>(r_res_vec.data()), r_res_vec.size()};
+
+			assert(r_res_vec.size() == data_test_text1.size);
+			assert(std::equal(data_test_text1.cbegin(), data_test_text1.cend(), r_res_vec.cbegin()));
+		}
+
+		{ // readback data_test_text2
+			auto r_res_var = f_r_zstd.read(data_test_text2.size);
+
+			//assert(f_r_zstd.isGood());
+			//assert(f_r_file.isGood());
+			assert(std::holds_alternative<std::vector<uint8_t>>(r_res_var));
+			const auto& r_res_vec = std::get<std::vector<uint8_t>>(r_res_var);
+
+			//std::cout << "decomp: " << std::string_view{reinterpret_cast<const char*>(r_res_vec.data()), r_res_vec.size()};
+
+			assert(r_res_vec.size() == data_test_text2.size);
+			assert(std::equal(
+				data_test_text2.cbegin(),
+				data_test_text2.cend(),
+				r_res_vec.cbegin()
+			));
+		}
+
+		{ // readback data_test_text3
+			auto r_res_var = f_r_zstd.read(data_test_text3.size);
+
+			//assert(f_r_zstd.isGood());
+			//assert(f_r_file.isGood());
+			assert(std::holds_alternative<std::vector<uint8_t>>(r_res_var));
+			const auto& r_res_vec = std::get<std::vector<uint8_t>>(r_res_var);
+
+			//std::cout << "decomp: " << std::string_view{reinterpret_cast<const char*>(r_res_vec.data()), r_res_vec.size()};
+
+			assert(r_res_vec.size() == data_test_text3.size);
+			assert(std::equal(
+				data_test_text3.cbegin(),
+				data_test_text3.cend(),
+				r_res_vec.cbegin()
+			));
+		}
+
+		{ // assert eof somehow
+			// since its eof, reading a single byte should return a zero sized buffer
+			auto r_res_var = f_r_zstd.read(1);
+			if (std::holds_alternative<std::vector<uint8_t>>(r_res_var)) {
+				assert(std::get<std::vector<uint8_t>>(r_res_var).empty());
+			} else if (std::holds_alternative<ByteSpan>(r_res_var)) {
+				assert(std::get<ByteSpan>(r_res_var).empty());
+			} else {
+				assert(false);
+			}
+		}
+	}
+
+	const auto test2_file_path = temp_dir / "testfile2.zstd";
+	{ // write and read a single frame with increasing size
+		for (size_t fslog = 1; fslog <= 25; fslog++) {
+			const size_t frame_size = 1<<fslog;
+			//std::cerr << "fs: " << frame_size << "\n";
+
+			{ // write
+				std::minstd_rand rng_data{11*1337};
+
+				File2WFile f_w_file{std::string_view{test2_file_path.u8string()}, true};
+				assert(f_w_file.isGood());
+
+				File2ZSTDW f_w_zstd{f_w_file};
+				assert(f_w_zstd.isGood());
+				assert(f_w_file.isGood());
+
+				std::vector<uint8_t> tmp_data(frame_size);
+				for (auto& e : tmp_data) {
+					e = uint8_t(rng_data() & 0xff); // cutoff bad but good enough
+				}
+				assert(tmp_data.size() == frame_size);
+
+				bool res = f_w_zstd.write(ByteSpan{tmp_data});
+				assert(res);
+				assert(f_w_zstd.isGood());
+				assert(f_w_file.isGood());
+			}
+
+			{ // read
+				std::minstd_rand rng_data{11*1337};
+
+				File2RFile f_r_file{std::string_view{test2_file_path.u8string()}};
+				assert(f_r_file.isGood());
+
+				File2ZSTDR f_r_zstd{f_r_file};
+				assert(f_r_zstd.isGood());
+				assert(f_r_file.isGood());
+
+				{ // read frame
+					auto r_res_var = f_r_zstd.read(frame_size);
+
+					assert(std::holds_alternative<std::vector<uint8_t>>(r_res_var));
+					const auto& r_res_vec = std::get<std::vector<uint8_t>>(r_res_var);
+					assert(r_res_vec.size() == frame_size);
+
+					// assert equal
+					for (auto& e : r_res_vec) {
+						assert(e == uint8_t(rng_data() & 0xff));
+					}
+				}
+
+				{ // eof test
+					auto r_res_var = f_r_zstd.read(1);
+					if (std::holds_alternative<std::vector<uint8_t>>(r_res_var)) {
+						assert(std::get<std::vector<uint8_t>>(r_res_var).empty());
+					} else if (std::holds_alternative<ByteSpan>(r_res_var)) {
+						assert(std::get<ByteSpan>(r_res_var).empty());
+					} else {
+						assert(false);
+					}
+				}
+			}
+
+			// since we spam file, we immediatly remove them
+			std::filesystem::remove(test2_file_path);
+		}
+	}
+
+	const auto test3_file_path = temp_dir / "testfile3.zstd";
+	{ // large file test write
+		File2WFile f_w_file{std::string_view{test3_file_path.u8string()}, true};
+		assert(f_w_file.isGood());
+
+		File2ZSTDW f_w_zstd{f_w_file};
+		assert(f_w_zstd.isGood());
+		assert(f_w_file.isGood());
+
+		std::minstd_rand rng{11*1337};
+		std::minstd_rand rng_data{11*1337}; // make investigating easier
+
+		size_t total_raw_size {0};
+		for (size_t i = 0; i < 2000; i++) {
+			const size_t frame_size = (rng() % ((2<<19) - 1)) + 1;
+
+			std::vector<uint8_t> tmp_data(frame_size);
+			for (auto& e : tmp_data) {
+				e = uint8_t(rng_data() & 0xff); // cutoff bad but good enough
+			}
+
+			bool res = f_w_zstd.write(ByteSpan{tmp_data});
+			assert(res);
+			assert(f_w_zstd.isGood());
+			assert(f_w_file.isGood());
+			total_raw_size += frame_size;
+		}
+		std::cout << "t3 total raw size: " << total_raw_size << "\n";
+	}
+
+	// after flush
+	std::cout << "t3 size on disk:   " << std::filesystem::file_size(test3_file_path) << "\n";
+
+	{ // large file test read
+		File2RFile f_r_file{std::string_view{test3_file_path.u8string()}};
+		assert(f_r_file.isGood());
+
+		File2ZSTDR f_r_zstd{f_r_file};
+		assert(f_r_zstd.isGood());
+		assert(f_r_file.isGood());
+
+		// using same rng state as write to compare
+		std::minstd_rand rng{11*1337};
+		std::minstd_rand rng_data{11*1337};
+
+		for (size_t i = 0; i < 2000; i++) {
+			const size_t frame_size = (rng() % ((2<<19) - 1)) + 1;
+			//std::cerr << "f: " << i << " fs: " << frame_size << "\n";
+
+			auto r_res_var = f_r_zstd.read(frame_size);
+
+			assert(std::holds_alternative<std::vector<uint8_t>>(r_res_var));
+			const auto& r_res_vec = std::get<std::vector<uint8_t>>(r_res_var);
+			assert(r_res_vec.size() == frame_size);
+
+			// assert equal
+			for (auto& e : r_res_vec) {
+				assert(e == uint8_t(rng_data() & 0xff));
+			}
+		}
+
+		{ // eof test
+			auto r_res_var = f_r_zstd.read(1);
+			if (std::holds_alternative<std::vector<uint8_t>>(r_res_var)) {
+				assert(std::get<std::vector<uint8_t>>(r_res_var).empty());
+			} else if (std::holds_alternative<ByteSpan>(r_res_var)) {
+				assert(std::get<ByteSpan>(r_res_var).empty());
+			} else {
+				assert(false);
+			}
+		}
+	}
+
+	// cleanup
+	std::filesystem::remove_all(temp_dir);
+}
+