Green-Sky/solanaceae_object_store
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

adopt file2 changes

Browse Source
This commit is contained in:
Green Sky 2024-05-26 21:33:49 +02:00
parent e26959c380
commit be24b3815b
No known key found for this signature in database
4 changed files with 55 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
src/solanaceae/file/file2_zstd.cpp
View File

@ -46,8 +46,8 @@ bool File2ZSTDW::write(const ByteSpan data, int64_t pos) {
return _real_file.write(ByteSpan{compressed_buffer.data(), ret}) && _real_file.isGood(); return _real_file.write(ByteSpan{compressed_buffer.data(), ret}) && _real_file.isGood();
} }
std::variant<ByteSpan, std::vector<uint8_t>> File2ZSTDW::read(uint64_t, int64_t) { ByteSpanWithOwnership File2ZSTDW::read(uint64_t, int64_t) {
return {}; return ByteSpan{};
} }
// ######################################### decompression // ######################################### decompression
@ -70,10 +70,10 @@ bool File2ZSTDR::write(const ByteSpan, int64_t) {
return false; return false;
} }
std::variant<ByteSpan, std::vector<uint8_t>> File2ZSTDR::read(uint64_t size, int64_t pos) { ByteSpanWithOwnership File2ZSTDR::read(uint64_t size, int64_t pos) {
if (pos != -1) { if (pos != -1) {
// error, only support streaming (for now) // error, only support streaming (for now)
return {}; return ByteSpan{};
} }
std::vector<uint8_t> ret_data; std::vector<uint8_t> ret_data;
@ -92,7 +92,7 @@ std::variant<ByteSpan, std::vector<uint8_t>> File2ZSTDR::read(uint64_t size, int
if (_z_input.src == nullptr || _z_input.pos == _z_input.size) { if (_z_input.src == nullptr || _z_input.pos == _z_input.size) {
const auto request_size = _in_buffer.size(); const auto request_size = _in_buffer.size();
if (!feedInput(_real_file.read(request_size, -1))) { if (!feedInput(_real_file.read(request_size, -1))) {
return ret_data; return ByteSpanWithOwnership{std::move(ret_data)};
} }
// if _z_input.size < _in_buffer.size() -> assume eof? // if _z_input.size < _in_buffer.size() -> assume eof?
@ -108,7 +108,7 @@ std::variant<ByteSpan, std::vector<uint8_t>> File2ZSTDR::read(uint64_t size, int
if (ZSTD_isError(ret)) { if (ZSTD_isError(ret)) {
// error <.< // error <.<
std::cerr << "---- error: decompression error\n"; std::cerr << "---- error: decompression error\n";
return ret_data; return ByteSpanWithOwnership{std::move(ret_data)};
} }
// no new decomp data? // no new decomp data?
@ -145,7 +145,7 @@ std::variant<ByteSpan, std::vector<uint8_t>> File2ZSTDR::read(uint64_t size, int
} while (_z_input.pos < _z_input.size); } while (_z_input.pos < _z_input.size);
} }
return ret_data; return ByteSpanWithOwnership{std::move(ret_data)};
} }
bool File2ZSTDR::feedInput(std::variant<ByteSpan, std::vector<uint8_t>>&& read_buff) { bool File2ZSTDR::feedInput(std::variant<ByteSpan, std::vector<uint8_t>>&& read_buff) {
@ -154,7 +154,7 @@ bool File2ZSTDR::feedInput(std::variant<ByteSpan, std::vector<uint8_t>>&& read_b
const auto& span = std::get<ByteSpan>(read_buff); const auto& span = std::get<ByteSpan>(read_buff);
if (span.size > _in_buffer.size()) { if (span.size > _in_buffer.size()) {
// error, how did we read more than we asked for?? // error, how did we read more than we asked for??
return {}; return false;
} }
if (span.empty()) { if (span.empty()) {
@ -172,7 +172,7 @@ bool File2ZSTDR::feedInput(std::variant<ByteSpan, std::vector<uint8_t>>&& read_b
auto& vec = std::get<std::vector<uint8_t>>(read_buff); auto& vec = std::get<std::vector<uint8_t>>(read_buff);
if (vec.size() > _in_buffer.size()) { if (vec.size() > _in_buffer.size()) {
// error, how did we read more than we asked for?? // error, how did we read more than we asked for??
return {}; return false;
} }
// cpy // cpy

4
src/solanaceae/file/file2_zstd.hpp
View File

@ -18,7 +18,7 @@ struct File2ZSTDW : public File2I {
// for simplicity and potential future seekability each write is its own frame // for simplicity and potential future seekability each write is its own frame
bool write(const ByteSpan data, int64_t pos = -1) override; bool write(const ByteSpan data, int64_t pos = -1) override;
std::variant<ByteSpan, std::vector<uint8_t>> read(uint64_t size, int64_t pos = -1) override; ByteSpanWithOwnership read(uint64_t size, int64_t pos = -1) override;
}; };
// zstd decompression wrapper over another file // zstd decompression wrapper over another file
@ -40,7 +40,7 @@ struct File2ZSTDR : public File2I {
bool isGood(void) override; bool isGood(void) override;
bool write(const ByteSpan data, int64_t pos = -1) override; bool write(const ByteSpan data, int64_t pos = -1) override;
std::variant<ByteSpan, std::vector<uint8_t>> read(uint64_t size, int64_t pos = -1) override; ByteSpanWithOwnership read(uint64_t size, int64_t pos = -1) override;
private: private:
bool feedInput(std::variant<ByteSpan, std::vector<uint8_t>>&& read_buff); bool feedInput(std::variant<ByteSpan, std::vector<uint8_t>>&& read_buff);

23
src/solanaceae/object_store/backends/filesystem_storage.cpp
View File

@ -26,19 +26,6 @@ static const char* metaFileTypeSuffix(MetaFileType mft) {
return ""; // .unk? return ""; // .unk?
} }
// TODO: move to ... somewhere. (span? file2i?)
static ByteSpan spanFromRead(const std::variant<ByteSpan, std::vector<uint8_t>>& data_var) {
if (std::holds_alternative<std::vector<uint8_t>>(data_var)) {
auto& vec = std::get<std::vector<uint8_t>>(data_var);
return {vec.data(), vec.size()};
} else if (std::holds_alternative<ByteSpan>(data_var)) {
return std::get<ByteSpan>(data_var);
} else {
assert(false);
return {};
}
}
namespace backend { namespace backend {
@ -347,8 +334,7 @@ bool FilesystemStorage::read(Object o, std::function<read_from_storage_put_data_
// TODO: make it read in a single chunk instead? // TODO: make it read in a single chunk instead?
static constexpr int64_t chunk_size {1024 * 1024}; // 1MiB should be good for read static constexpr int64_t chunk_size {1024 * 1024}; // 1MiB should be good for read
do { do {
auto data_var = data_file_stack.top()->read(chunk_size); auto data = data_file_stack.top()->read(chunk_size);
ByteSpan data = spanFromRead(data_var);
if (data.empty()) { if (data.empty()) {
// error or probably eof // error or probably eof
@ -567,11 +553,10 @@ size_t FilesystemStorage::scanPath(std::string_view path) {
} }
// HACK: read fixed amout of data, but this way if we have neither enc nor comp we pass the span through // HACK: read fixed amout of data, but this way if we have neither enc nor comp we pass the span through
auto binary_read_value = binary_reader_stack.top()->read(10*1024*1024); // is 10MiB large enough for meta? auto binary_read = binary_reader_stack.top()->read(10*1024*1024); // is 10MiB large enough for meta?
const auto binary_read_span = spanFromRead(binary_read_value); assert(binary_read.size < 10*1024*1024);
assert(binary_read_span.size < 10*1024*1024);
j = nlohmann::json::from_msgpack(binary_read_span, true, false); j = nlohmann::json::from_msgpack(binary_read, true, false);
} else if (it.meta_ext == ".meta.json") { } else if (it.meta_ext == ".meta.json") {
std::ifstream file(it.obj_path.generic_u8string() + it.meta_ext, std::ios::in | std::ios::binary); std::ifstream file(it.obj_path.generic_u8string() + it.meta_ext, std::ios::in | std::ios::binary);
if (!file.is_open()) { if (!file.is_open()) {

112
test/test_file_zstd.cpp
View File

@ -73,65 +73,56 @@ int main(void) {
// reads return owning buffers // reads return owning buffers
{ // readback data_test_text1 { // readback data_test_text1
auto r_res_var = f_r_zstd.read(data_test_text1.size); auto r_res = f_r_zstd.read(data_test_text1.size);
//assert(f_r_zstd.isGood()); //assert(f_r_zstd.isGood());
//assert(f_r_file.isGood()); //assert(f_r_file.isGood());
assert(std::holds_alternative<std::vector<uint8_t>>(r_res_var)); assert(r_res.isOwning());
const auto& r_res_vec = std::get<std::vector<uint8_t>>(r_res_var);
//std::cout << "decomp: " << std::string_view{reinterpret_cast<const char*>(r_res_vec.data()), r_res_vec.size()}; //std::cout << "decomp: " << std::string_view{reinterpret_cast<const char*>(r_res_vec.data()), r_res_vec.size()};
assert(r_res_vec.size() == data_test_text1.size); assert(r_res.size == data_test_text1.size);
assert(std::equal(data_test_text1.cbegin(), data_test_text1.cend(), r_res_vec.cbegin())); assert(std::equal(data_test_text1.cbegin(), data_test_text1.cend(), r_res.cbegin()));
} }
{ // readback data_test_text2 { // readback data_test_text2
auto r_res_var = f_r_zstd.read(data_test_text2.size); auto r_res = f_r_zstd.read(data_test_text2.size);
//assert(f_r_zstd.isGood()); //assert(f_r_zstd.isGood());
//assert(f_r_file.isGood()); //assert(f_r_file.isGood());
assert(std::holds_alternative<std::vector<uint8_t>>(r_res_var)); assert(r_res.isOwning());
const auto& r_res_vec = std::get<std::vector<uint8_t>>(r_res_var);
//std::cout << "decomp: " << std::string_view{reinterpret_cast<const char*>(r_res_vec.data()), r_res_vec.size()}; //std::cout << "decomp: " << std::string_view{reinterpret_cast<const char*>(r_res_vec.data()), r_res_vec.size()};
assert(r_res_vec.size() == data_test_text2.size); assert(r_res.size == data_test_text2.size);
assert(std::equal( assert(std::equal(
data_test_text2.cbegin(), data_test_text2.cbegin(),
data_test_text2.cend(), data_test_text2.cend(),
r_res_vec.cbegin() r_res.cbegin()
)); ));
} }
{ // readback data_test_text3 { // readback data_test_text3
auto r_res_var = f_r_zstd.read(data_test_text3.size); auto r_res = f_r_zstd.read(data_test_text3.size);
//assert(f_r_zstd.isGood()); //assert(f_r_zstd.isGood());
//assert(f_r_file.isGood()); //assert(f_r_file.isGood());
assert(std::holds_alternative<std::vector<uint8_t>>(r_res_var)); assert(r_res.isOwning());
const auto& r_res_vec = std::get<std::vector<uint8_t>>(r_res_var);
//std::cout << "decomp: " << std::string_view{reinterpret_cast<const char*>(r_res_vec.data()), r_res_vec.size()}; //std::cout << "decomp: " << std::string_view{reinterpret_cast<const char*>(r_res_vec.data()), r_res_vec.size()};
assert(r_res_vec.size() == data_test_text3.size); assert(r_res.size == data_test_text3.size);
assert(std::equal( assert(std::equal(
data_test_text3.cbegin(), data_test_text3.cbegin(),
data_test_text3.cend(), data_test_text3.cend(),
r_res_vec.cbegin() r_res.cbegin()
)); ));
} }
{ // assert eof somehow { // assert eof somehow
// since its eof, reading a single byte should return a zero sized buffer // since its eof, reading a single byte should return a zero sized buffer
auto r_res_var = f_r_zstd.read(1); auto r_res = f_r_zstd.read(1);
if (std::holds_alternative<std::vector<uint8_t>>(r_res_var)) { assert(r_res.empty());
assert(std::get<std::vector<uint8_t>>(r_res_var).empty());
} else if (std::holds_alternative<ByteSpan>(r_res_var)) {
assert(std::get<ByteSpan>(r_res_var).empty());
} else {
assert(false);
}
} }
} }
} }
@ -184,65 +175,56 @@ int main(void) {
// reads return owning buffers // reads return owning buffers
{ // readback data_test_text1 { // readback data_test_text1
auto r_res_var = f_r_zstd.read(data_test_text1.size); auto r_res = f_r_zstd.read(data_test_text1.size);
//assert(f_r_zstd.isGood()); //assert(f_r_zstd.isGood());
//assert(f_r_file.isGood()); //assert(f_r_file.isGood());
assert(std::holds_alternative<std::vector<uint8_t>>(r_res_var)); assert(r_res.isOwning());
const auto& r_res_vec = std::get<std::vector<uint8_t>>(r_res_var);
//std::cout << "decomp: " << std::string_view{reinterpret_cast<const char*>(r_res_vec.data()), r_res_vec.size()}; //std::cout << "decomp: " << std::string_view{reinterpret_cast<const char*>(r_res_vec.data()), r_res_vec.size()};
assert(r_res_vec.size() == data_test_text1.size); assert(r_res.size == data_test_text1.size);
assert(std::equal(data_test_text1.cbegin(), data_test_text1.cend(), r_res_vec.cbegin())); assert(std::equal(data_test_text1.cbegin(), data_test_text1.cend(), r_res.cbegin()));
} }
{ // readback data_test_text2 { // readback data_test_text2
auto r_res_var = f_r_zstd.read(data_test_text2.size); auto r_res = f_r_zstd.read(data_test_text2.size);
//assert(f_r_zstd.isGood()); //assert(f_r_zstd.isGood());
//assert(f_r_file.isGood()); //assert(f_r_file.isGood());
assert(std::holds_alternative<std::vector<uint8_t>>(r_res_var)); assert(r_res.isOwning());
const auto& r_res_vec = std::get<std::vector<uint8_t>>(r_res_var);
//std::cout << "decomp: " << std::string_view{reinterpret_cast<const char*>(r_res_vec.data()), r_res_vec.size()}; //std::cout << "decomp: " << std::string_view{reinterpret_cast<const char*>(r_res_vec.data()), r_res_vec.size()};
assert(r_res_vec.size() == data_test_text2.size); assert(r_res.size == data_test_text2.size);
assert(std::equal( assert(std::equal(
data_test_text2.cbegin(), data_test_text2.cbegin(),
data_test_text2.cend(), data_test_text2.cend(),
r_res_vec.cbegin() r_res.cbegin()
)); ));
} }
{ // readback data_test_text3 { // readback data_test_text3
auto r_res_var = f_r_zstd.read(data_test_text3.size); auto r_res = f_r_zstd.read(data_test_text3.size);
//assert(f_r_zstd.isGood()); //assert(f_r_zstd.isGood());
//assert(f_r_file.isGood()); //assert(f_r_file.isGood());
assert(std::holds_alternative<std::vector<uint8_t>>(r_res_var)); assert(r_res.isOwning());
const auto& r_res_vec = std::get<std::vector<uint8_t>>(r_res_var);
//std::cout << "decomp: " << std::string_view{reinterpret_cast<const char*>(r_res_vec.data()), r_res_vec.size()}; //std::cout << "decomp: " << std::string_view{reinterpret_cast<const char*>(r_res_vec.data()), r_res_vec.size()};
assert(r_res_vec.size() == data_test_text3.size); assert(r_res.size == data_test_text3.size);
assert(std::equal( assert(std::equal(
data_test_text3.cbegin(), data_test_text3.cbegin(),
data_test_text3.cend(), data_test_text3.cend(),
r_res_vec.cbegin() r_res.cbegin()
)); ));
} }
{ // assert eof somehow { // assert eof somehow
// since its eof, reading a single byte should return a zero sized buffer // since its eof, reading a single byte should return a zero sized buffer
auto r_res_var = f_r_zstd.read(1); auto r_res = f_r_zstd.read(1);
if (std::holds_alternative<std::vector<uint8_t>>(r_res_var)) { assert(r_res.empty());
assert(std::get<std::vector<uint8_t>>(r_res_var).empty());
} else if (std::holds_alternative<ByteSpan>(r_res_var)) {
assert(std::get<ByteSpan>(r_res_var).empty());
} else {
assert(false);
}
} }
} }
@ -285,27 +267,20 @@ int main(void) {
assert(f_r_file.isGood()); assert(f_r_file.isGood());
{ // read frame { // read frame
auto r_res_var = f_r_zstd.read(frame_size); auto r_res = f_r_zstd.read(frame_size);
assert(std::holds_alternative<std::vector<uint8_t>>(r_res_var)); assert(r_res.isOwning());
const auto& r_res_vec = std::get<std::vector<uint8_t>>(r_res_var); assert(r_res.size == frame_size);
assert(r_res_vec.size() == frame_size);
// assert equal // assert equal
for (auto& e : r_res_vec) { for (auto& e : r_res) {
assert(e == uint8_t(rng_data() & 0xff)); assert(e == uint8_t(rng_data() & 0xff));
} }
} }
{ // eof test { // eof test
auto r_res_var = f_r_zstd.read(1); auto r_res = f_r_zstd.read(1);
if (std::holds_alternative<std::vector<uint8_t>>(r_res_var)) { assert(r_res.empty());
assert(std::get<std::vector<uint8_t>>(r_res_var).empty());
} else if (std::holds_alternative<ByteSpan>(r_res_var)) {
assert(std::get<ByteSpan>(r_res_var).empty());
} else {
assert(false);
}
} }
} }
@ -363,27 +338,20 @@ int main(void) {
const size_t frame_size = (rng() % ((2<<19) - 1)) + 1; const size_t frame_size = (rng() % ((2<<19) - 1)) + 1;
//std::cerr << "f: " << i << " fs: " << frame_size << "\n"; //std::cerr << "f: " << i << " fs: " << frame_size << "\n";
auto r_res_var = f_r_zstd.read(frame_size); auto r_res = f_r_zstd.read(frame_size);
assert(std::holds_alternative<std::vector<uint8_t>>(r_res_var)); assert(r_res.isOwning());
const auto& r_res_vec = std::get<std::vector<uint8_t>>(r_res_var); assert(r_res.size == frame_size);
assert(r_res_vec.size() == frame_size);
// assert equal // assert equal
for (auto& e : r_res_vec) { for (auto& e : r_res) {
assert(e == uint8_t(rng_data() & 0xff)); assert(e == uint8_t(rng_data() & 0xff));
} }
} }
{ // eof test { // eof test
auto r_res_var = f_r_zstd.read(1); auto r_res = f_r_zstd.read(1);
if (std::holds_alternative<std::vector<uint8_t>>(r_res_var)) { assert(r_res.empty());
assert(std::get<std::vector<uint8_t>>(r_res_var).empty());
} else if (std::holds_alternative<ByteSpan>(r_res_var)) {
assert(std::get<ByteSpan>(r_res_var).empty());
} else {
assert(false);
}
} }
} }