From be24b3815bdf4d503a2db15ffaefc8cba7f2fce8 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sun, 26 May 2024 21:33:49 +0200 Subject: [PATCH] adopt file2 changes --- src/solanaceae/file/file2_zstd.cpp | 18 +-- src/solanaceae/file/file2_zstd.hpp | 4 +- .../backends/filesystem_storage.cpp | 23 +--- test/test_file_zstd.cpp | 112 +++++++----------- 4 files changed, 55 insertions(+), 102 deletions(-) diff --git a/src/solanaceae/file/file2_zstd.cpp b/src/solanaceae/file/file2_zstd.cpp index 77fa76f..8eefbca 100644 --- a/src/solanaceae/file/file2_zstd.cpp +++ b/src/solanaceae/file/file2_zstd.cpp @@ -46,8 +46,8 @@ bool File2ZSTDW::write(const ByteSpan data, int64_t pos) { return _real_file.write(ByteSpan{compressed_buffer.data(), ret}) && _real_file.isGood(); } -std::variant> File2ZSTDW::read(uint64_t, int64_t) { - return {}; +ByteSpanWithOwnership File2ZSTDW::read(uint64_t, int64_t) { + return ByteSpan{}; } // ######################################### decompression @@ -70,10 +70,10 @@ bool File2ZSTDR::write(const ByteSpan, int64_t) { return false; } -std::variant> File2ZSTDR::read(uint64_t size, int64_t pos) { +ByteSpanWithOwnership File2ZSTDR::read(uint64_t size, int64_t pos) { if (pos != -1) { // error, only support streaming (for now) - return {}; + return ByteSpan{}; } std::vector ret_data; @@ -92,7 +92,7 @@ std::variant> File2ZSTDR::read(uint64_t size, int if (_z_input.src == nullptr || _z_input.pos == _z_input.size) { const auto request_size = _in_buffer.size(); if (!feedInput(_real_file.read(request_size, -1))) { - return ret_data; + return ByteSpanWithOwnership{std::move(ret_data)}; } // if _z_input.size < _in_buffer.size() -> assume eof? @@ -108,7 +108,7 @@ std::variant> File2ZSTDR::read(uint64_t size, int if (ZSTD_isError(ret)) { // error <.< std::cerr << "---- error: decompression error\n"; - return ret_data; + return ByteSpanWithOwnership{std::move(ret_data)}; } // no new decomp data? @@ -145,7 +145,7 @@ std::variant> File2ZSTDR::read(uint64_t size, int } while (_z_input.pos < _z_input.size); } - return ret_data; + return ByteSpanWithOwnership{std::move(ret_data)}; } bool File2ZSTDR::feedInput(std::variant>&& read_buff) { @@ -154,7 +154,7 @@ bool File2ZSTDR::feedInput(std::variant>&& read_b const auto& span = std::get(read_buff); if (span.size > _in_buffer.size()) { // error, how did we read more than we asked for?? - return {}; + return false; } if (span.empty()) { @@ -172,7 +172,7 @@ bool File2ZSTDR::feedInput(std::variant>&& read_b auto& vec = std::get>(read_buff); if (vec.size() > _in_buffer.size()) { // error, how did we read more than we asked for?? - return {}; + return false; } // cpy diff --git a/src/solanaceae/file/file2_zstd.hpp b/src/solanaceae/file/file2_zstd.hpp index 785cfac..832bc77 100644 --- a/src/solanaceae/file/file2_zstd.hpp +++ b/src/solanaceae/file/file2_zstd.hpp @@ -18,7 +18,7 @@ struct File2ZSTDW : public File2I { // for simplicity and potential future seekability each write is its own frame bool write(const ByteSpan data, int64_t pos = -1) override; - std::variant> read(uint64_t size, int64_t pos = -1) override; + ByteSpanWithOwnership read(uint64_t size, int64_t pos = -1) override; }; // zstd decompression wrapper over another file @@ -40,7 +40,7 @@ struct File2ZSTDR : public File2I { bool isGood(void) override; bool write(const ByteSpan data, int64_t pos = -1) override; - std::variant> read(uint64_t size, int64_t pos = -1) override; + ByteSpanWithOwnership read(uint64_t size, int64_t pos = -1) override; private: bool feedInput(std::variant>&& read_buff); diff --git a/src/solanaceae/object_store/backends/filesystem_storage.cpp b/src/solanaceae/object_store/backends/filesystem_storage.cpp index 0218154..baf47ad 100644 --- a/src/solanaceae/object_store/backends/filesystem_storage.cpp +++ b/src/solanaceae/object_store/backends/filesystem_storage.cpp @@ -26,19 +26,6 @@ static const char* metaFileTypeSuffix(MetaFileType mft) { return ""; // .unk? } -// TODO: move to ... somewhere. (span? file2i?) -static ByteSpan spanFromRead(const std::variant>& data_var) { - if (std::holds_alternative>(data_var)) { - auto& vec = std::get>(data_var); - return {vec.data(), vec.size()}; - } else if (std::holds_alternative(data_var)) { - return std::get(data_var); - } else { - assert(false); - return {}; - } -} - namespace backend { @@ -347,8 +334,7 @@ bool FilesystemStorage::read(Object o, std::functionread(chunk_size); - ByteSpan data = spanFromRead(data_var); + auto data = data_file_stack.top()->read(chunk_size); if (data.empty()) { // error or probably eof @@ -567,11 +553,10 @@ size_t FilesystemStorage::scanPath(std::string_view path) { } // HACK: read fixed amout of data, but this way if we have neither enc nor comp we pass the span through - auto binary_read_value = binary_reader_stack.top()->read(10*1024*1024); // is 10MiB large enough for meta? - const auto binary_read_span = spanFromRead(binary_read_value); - assert(binary_read_span.size < 10*1024*1024); + auto binary_read = binary_reader_stack.top()->read(10*1024*1024); // is 10MiB large enough for meta? + assert(binary_read.size < 10*1024*1024); - j = nlohmann::json::from_msgpack(binary_read_span, true, false); + j = nlohmann::json::from_msgpack(binary_read, true, false); } else if (it.meta_ext == ".meta.json") { std::ifstream file(it.obj_path.generic_u8string() + it.meta_ext, std::ios::in | std::ios::binary); if (!file.is_open()) { diff --git a/test/test_file_zstd.cpp b/test/test_file_zstd.cpp index fa2389c..5191166 100644 --- a/test/test_file_zstd.cpp +++ b/test/test_file_zstd.cpp @@ -73,65 +73,56 @@ int main(void) { // reads return owning buffers { // readback data_test_text1 - auto r_res_var = f_r_zstd.read(data_test_text1.size); + auto r_res = f_r_zstd.read(data_test_text1.size); //assert(f_r_zstd.isGood()); //assert(f_r_file.isGood()); - assert(std::holds_alternative>(r_res_var)); - const auto& r_res_vec = std::get>(r_res_var); + assert(r_res.isOwning()); //std::cout << "decomp: " << std::string_view{reinterpret_cast(r_res_vec.data()), r_res_vec.size()}; - assert(r_res_vec.size() == data_test_text1.size); - assert(std::equal(data_test_text1.cbegin(), data_test_text1.cend(), r_res_vec.cbegin())); + assert(r_res.size == data_test_text1.size); + assert(std::equal(data_test_text1.cbegin(), data_test_text1.cend(), r_res.cbegin())); } { // readback data_test_text2 - auto r_res_var = f_r_zstd.read(data_test_text2.size); + auto r_res = f_r_zstd.read(data_test_text2.size); //assert(f_r_zstd.isGood()); //assert(f_r_file.isGood()); - assert(std::holds_alternative>(r_res_var)); - const auto& r_res_vec = std::get>(r_res_var); + assert(r_res.isOwning()); //std::cout << "decomp: " << std::string_view{reinterpret_cast(r_res_vec.data()), r_res_vec.size()}; - assert(r_res_vec.size() == data_test_text2.size); + assert(r_res.size == data_test_text2.size); assert(std::equal( data_test_text2.cbegin(), data_test_text2.cend(), - r_res_vec.cbegin() + r_res.cbegin() )); } { // readback data_test_text3 - auto r_res_var = f_r_zstd.read(data_test_text3.size); + auto r_res = f_r_zstd.read(data_test_text3.size); //assert(f_r_zstd.isGood()); //assert(f_r_file.isGood()); - assert(std::holds_alternative>(r_res_var)); - const auto& r_res_vec = std::get>(r_res_var); + assert(r_res.isOwning()); //std::cout << "decomp: " << std::string_view{reinterpret_cast(r_res_vec.data()), r_res_vec.size()}; - assert(r_res_vec.size() == data_test_text3.size); + assert(r_res.size == data_test_text3.size); assert(std::equal( data_test_text3.cbegin(), data_test_text3.cend(), - r_res_vec.cbegin() + r_res.cbegin() )); } { // assert eof somehow // since its eof, reading a single byte should return a zero sized buffer - auto r_res_var = f_r_zstd.read(1); - if (std::holds_alternative>(r_res_var)) { - assert(std::get>(r_res_var).empty()); - } else if (std::holds_alternative(r_res_var)) { - assert(std::get(r_res_var).empty()); - } else { - assert(false); - } + auto r_res = f_r_zstd.read(1); + assert(r_res.empty()); } } } @@ -184,65 +175,56 @@ int main(void) { // reads return owning buffers { // readback data_test_text1 - auto r_res_var = f_r_zstd.read(data_test_text1.size); + auto r_res = f_r_zstd.read(data_test_text1.size); //assert(f_r_zstd.isGood()); //assert(f_r_file.isGood()); - assert(std::holds_alternative>(r_res_var)); - const auto& r_res_vec = std::get>(r_res_var); + assert(r_res.isOwning()); //std::cout << "decomp: " << std::string_view{reinterpret_cast(r_res_vec.data()), r_res_vec.size()}; - assert(r_res_vec.size() == data_test_text1.size); - assert(std::equal(data_test_text1.cbegin(), data_test_text1.cend(), r_res_vec.cbegin())); + assert(r_res.size == data_test_text1.size); + assert(std::equal(data_test_text1.cbegin(), data_test_text1.cend(), r_res.cbegin())); } { // readback data_test_text2 - auto r_res_var = f_r_zstd.read(data_test_text2.size); + auto r_res = f_r_zstd.read(data_test_text2.size); //assert(f_r_zstd.isGood()); //assert(f_r_file.isGood()); - assert(std::holds_alternative>(r_res_var)); - const auto& r_res_vec = std::get>(r_res_var); + assert(r_res.isOwning()); //std::cout << "decomp: " << std::string_view{reinterpret_cast(r_res_vec.data()), r_res_vec.size()}; - assert(r_res_vec.size() == data_test_text2.size); + assert(r_res.size == data_test_text2.size); assert(std::equal( data_test_text2.cbegin(), data_test_text2.cend(), - r_res_vec.cbegin() + r_res.cbegin() )); } { // readback data_test_text3 - auto r_res_var = f_r_zstd.read(data_test_text3.size); + auto r_res = f_r_zstd.read(data_test_text3.size); //assert(f_r_zstd.isGood()); //assert(f_r_file.isGood()); - assert(std::holds_alternative>(r_res_var)); - const auto& r_res_vec = std::get>(r_res_var); + assert(r_res.isOwning()); //std::cout << "decomp: " << std::string_view{reinterpret_cast(r_res_vec.data()), r_res_vec.size()}; - assert(r_res_vec.size() == data_test_text3.size); + assert(r_res.size == data_test_text3.size); assert(std::equal( data_test_text3.cbegin(), data_test_text3.cend(), - r_res_vec.cbegin() + r_res.cbegin() )); } { // assert eof somehow // since its eof, reading a single byte should return a zero sized buffer - auto r_res_var = f_r_zstd.read(1); - if (std::holds_alternative>(r_res_var)) { - assert(std::get>(r_res_var).empty()); - } else if (std::holds_alternative(r_res_var)) { - assert(std::get(r_res_var).empty()); - } else { - assert(false); - } + auto r_res = f_r_zstd.read(1); + assert(r_res.empty()); } } @@ -285,27 +267,20 @@ int main(void) { assert(f_r_file.isGood()); { // read frame - auto r_res_var = f_r_zstd.read(frame_size); + auto r_res = f_r_zstd.read(frame_size); - assert(std::holds_alternative>(r_res_var)); - const auto& r_res_vec = std::get>(r_res_var); - assert(r_res_vec.size() == frame_size); + assert(r_res.isOwning()); + assert(r_res.size == frame_size); // assert equal - for (auto& e : r_res_vec) { + for (auto& e : r_res) { assert(e == uint8_t(rng_data() & 0xff)); } } { // eof test - auto r_res_var = f_r_zstd.read(1); - if (std::holds_alternative>(r_res_var)) { - assert(std::get>(r_res_var).empty()); - } else if (std::holds_alternative(r_res_var)) { - assert(std::get(r_res_var).empty()); - } else { - assert(false); - } + auto r_res = f_r_zstd.read(1); + assert(r_res.empty()); } } @@ -363,27 +338,20 @@ int main(void) { const size_t frame_size = (rng() % ((2<<19) - 1)) + 1; //std::cerr << "f: " << i << " fs: " << frame_size << "\n"; - auto r_res_var = f_r_zstd.read(frame_size); + auto r_res = f_r_zstd.read(frame_size); - assert(std::holds_alternative>(r_res_var)); - const auto& r_res_vec = std::get>(r_res_var); - assert(r_res_vec.size() == frame_size); + assert(r_res.isOwning()); + assert(r_res.size == frame_size); // assert equal - for (auto& e : r_res_vec) { + for (auto& e : r_res) { assert(e == uint8_t(rng_data() & 0xff)); } } { // eof test - auto r_res_var = f_r_zstd.read(1); - if (std::holds_alternative>(r_res_var)) { - assert(std::get>(r_res_var).empty()); - } else if (std::holds_alternative(r_res_var)) { - assert(std::get(r_res_var).empty()); - } else { - assert(false); - } + auto r_res = f_r_zstd.read(1); + assert(r_res.empty()); } }