Green-Sky/solanaceae_object_store
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

adopt file2 changes

Browse Source
This commit is contained in:
Green Sky 2024-05-26 21:33:49 +02:00
parent e26959c380
commit be24b3815b
No known key found for this signature in database
4 changed files with 55 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
src/solanaceae/file/file2_zstd.cpp
View File

@ -46,8 +46,8 @@ bool File2ZSTDW::write(const ByteSpan data, int64_t pos) {
return _real_file.write(ByteSpan{compressed_buffer.data(), ret}) && _real_file.isGood();
}
std::variant<ByteSpan, std::vector<uint8_t>> File2ZSTDW::read(uint64_t, int64_t) {
return {};
ByteSpanWithOwnership File2ZSTDW::read(uint64_t, int64_t) {
return ByteSpan{};
}
// ######################################### decompression
@ -70,10 +70,10 @@ bool File2ZSTDR::write(const ByteSpan, int64_t) {
return false;
}
std::variant<ByteSpan, std::vector<uint8_t>> File2ZSTDR::read(uint64_t size, int64_t pos) {
ByteSpanWithOwnership File2ZSTDR::read(uint64_t size, int64_t pos) {
if (pos != -1) {
// error, only support streaming (for now)
return {};
return ByteSpan{};
}
std::vector<uint8_t> ret_data;
@ -92,7 +92,7 @@ std::variant<ByteSpan, std::vector<uint8_t>> File2ZSTDR::read(uint64_t size, int
if (_z_input.src == nullptr || _z_input.pos == _z_input.size) {
const auto request_size = _in_buffer.size();
if (!feedInput(_real_file.read(request_size, -1))) {
return ret_data;
return ByteSpanWithOwnership{std::move(ret_data)};
}
// if _z_input.size < _in_buffer.size() -> assume eof?
@ -108,7 +108,7 @@ std::variant<ByteSpan, std::vector<uint8_t>> File2ZSTDR::read(uint64_t size, int
if (ZSTD_isError(ret)) {
// error <.<
std::cerr << "---- error: decompression error\n";
return ret_data;
return ByteSpanWithOwnership{std::move(ret_data)};
}
// no new decomp data?
@ -145,7 +145,7 @@ std::variant<ByteSpan, std::vector<uint8_t>> File2ZSTDR::read(uint64_t size, int
} while (_z_input.pos < _z_input.size);
}
return ret_data;
return ByteSpanWithOwnership{std::move(ret_data)};
}
bool File2ZSTDR::feedInput(std::variant<ByteSpan, std::vector<uint8_t>>&& read_buff) {
@ -154,7 +154,7 @@ bool File2ZSTDR::feedInput(std::variant<ByteSpan, std::vector<uint8_t>>&& read_b
const auto& span = std::get<ByteSpan>(read_buff);
if (span.size > _in_buffer.size()) {
// error, how did we read more than we asked for??
return {};
return false;
}
if (span.empty()) {
@ -172,7 +172,7 @@ bool File2ZSTDR::feedInput(std::variant<ByteSpan, std::vector<uint8_t>>&& read_b
auto& vec = std::get<std::vector<uint8_t>>(read_buff);
if (vec.size() > _in_buffer.size()) {
// error, how did we read more than we asked for??
return {};
return false;
}
// cpy

4
src/solanaceae/file/file2_zstd.hpp
View File

@ -18,7 +18,7 @@ struct File2ZSTDW : public File2I {
// for simplicity and potential future seekability each write is its own frame
bool write(const ByteSpan data, int64_t pos = -1) override;
std::variant<ByteSpan, std::vector<uint8_t>> read(uint64_t size, int64_t pos = -1) override;
ByteSpanWithOwnership read(uint64_t size, int64_t pos = -1) override;
};
// zstd decompression wrapper over another file
@ -40,7 +40,7 @@ struct File2ZSTDR : public File2I {
bool isGood(void) override;
bool write(const ByteSpan data, int64_t pos = -1) override;
std::variant<ByteSpan, std::vector<uint8_t>> read(uint64_t size, int64_t pos = -1) override;
ByteSpanWithOwnership read(uint64_t size, int64_t pos = -1) override;
private:
bool feedInput(std::variant<ByteSpan, std::vector<uint8_t>>&& read_buff);

23
src/solanaceae/object_store/backends/filesystem_storage.cpp
View File

@ -26,19 +26,6 @@ static const char* metaFileTypeSuffix(MetaFileType mft) {
return ""; // .unk?
}
// TODO: move to ... somewhere. (span? file2i?)
static ByteSpan spanFromRead(const std::variant<ByteSpan, std::vector<uint8_t>>& data_var) {
if (std::holds_alternative<std::vector<uint8_t>>(data_var)) {
auto& vec = std::get<std::vector<uint8_t>>(data_var);
return {vec.data(), vec.size()};
} else if (std::holds_alternative<ByteSpan>(data_var)) {
return std::get<ByteSpan>(data_var);
} else {
assert(false);
return {};
}
}
namespace backend {
@ -347,8 +334,7 @@ bool FilesystemStorage::read(Object o, std::function<read_from_storage_put_data_
// TODO: make it read in a single chunk instead?
static constexpr int64_t chunk_size {1024 * 1024}; // 1MiB should be good for read
do {
auto data_var = data_file_stack.top()->read(chunk_size);
ByteSpan data = spanFromRead(data_var);
auto data = data_file_stack.top()->read(chunk_size);
if (data.empty()) {
// error or probably eof
@ -567,11 +553,10 @@ size_t FilesystemStorage::scanPath(std::string_view path) {
}
// HACK: read fixed amout of data, but this way if we have neither enc nor comp we pass the span through
auto binary_read_value = binary_reader_stack.top()->read(10*1024*1024); // is 10MiB large enough for meta?
const auto binary_read_span = spanFromRead(binary_read_value);
assert(binary_read_span.size < 10*1024*1024);
auto binary_read = binary_reader_stack.top()->read(10*1024*1024); // is 10MiB large enough for meta?
assert(binary_read.size < 10*1024*1024);
j = nlohmann::json::from_msgpack(binary_read_span, true, false);
j = nlohmann::json::from_msgpack(binary_read, true, false);
} else if (it.meta_ext == ".meta.json") {
std::ifstream file(it.obj_path.generic_u8string() + it.meta_ext, std::ios::in | std::ios::binary);
if (!file.is_open()) {

112
test/test_file_zstd.cpp
View File

@ -73,65 +73,56 @@ int main(void) {
// reads return owning buffers
{ // readback data_test_text1
auto r_res_var = f_r_zstd.read(data_test_text1.size);
auto r_res = f_r_zstd.read(data_test_text1.size);
//assert(f_r_zstd.isGood());
//assert(f_r_file.isGood());
assert(std::holds_alternative<std::vector<uint8_t>>(r_res_var));
const auto& r_res_vec = std::get<std::vector<uint8_t>>(r_res_var);
assert(r_res.isOwning());
//std::cout << "decomp: " << std::string_view{reinterpret_cast<const char*>(r_res_vec.data()), r_res_vec.size()};
assert(r_res_vec.size() == data_test_text1.size);
assert(std::equal(data_test_text1.cbegin(), data_test_text1.cend(), r_res_vec.cbegin()));
assert(r_res.size == data_test_text1.size);
assert(std::equal(data_test_text1.cbegin(), data_test_text1.cend(), r_res.cbegin()));
}
{ // readback data_test_text2
auto r_res_var = f_r_zstd.read(data_test_text2.size);
auto r_res = f_r_zstd.read(data_test_text2.size);
//assert(f_r_zstd.isGood());
//assert(f_r_file.isGood());
assert(std::holds_alternative<std::vector<uint8_t>>(r_res_var));
const auto& r_res_vec = std::get<std::vector<uint8_t>>(r_res_var);
assert(r_res.isOwning());
//std::cout << "decomp: " << std::string_view{reinterpret_cast<const char*>(r_res_vec.data()), r_res_vec.size()};
assert(r_res_vec.size() == data_test_text2.size);
assert(r_res.size == data_test_text2.size);
assert(std::equal(
data_test_text2.cbegin(),
data_test_text2.cend(),
r_res_vec.cbegin()
r_res.cbegin()
));
}
{ // readback data_test_text3
auto r_res_var = f_r_zstd.read(data_test_text3.size);
auto r_res = f_r_zstd.read(data_test_text3.size);
//assert(f_r_zstd.isGood());
//assert(f_r_file.isGood());
assert(std::holds_alternative<std::vector<uint8_t>>(r_res_var));
const auto& r_res_vec = std::get<std::vector<uint8_t>>(r_res_var);
assert(r_res.isOwning());
//std::cout << "decomp: " << std::string_view{reinterpret_cast<const char*>(r_res_vec.data()), r_res_vec.size()};
assert(r_res_vec.size() == data_test_text3.size);
assert(r_res.size == data_test_text3.size);
assert(std::equal(
data_test_text3.cbegin(),
data_test_text3.cend(),
r_res_vec.cbegin()
r_res.cbegin()
));
}
{ // assert eof somehow
// since its eof, reading a single byte should return a zero sized buffer
auto r_res_var = f_r_zstd.read(1);
if (std::holds_alternative<std::vector<uint8_t>>(r_res_var)) {
assert(std::get<std::vector<uint8_t>>(r_res_var).empty());
} else if (std::holds_alternative<ByteSpan>(r_res_var)) {
assert(std::get<ByteSpan>(r_res_var).empty());
} else {
assert(false);
}
auto r_res = f_r_zstd.read(1);
assert(r_res.empty());
}
}
}
@ -184,65 +175,56 @@ int main(void) {
// reads return owning buffers
{ // readback data_test_text1
auto r_res_var = f_r_zstd.read(data_test_text1.size);
auto r_res = f_r_zstd.read(data_test_text1.size);
//assert(f_r_zstd.isGood());
//assert(f_r_file.isGood());
assert(std::holds_alternative<std::vector<uint8_t>>(r_res_var));
const auto& r_res_vec = std::get<std::vector<uint8_t>>(r_res_var);
assert(r_res.isOwning());
//std::cout << "decomp: " << std::string_view{reinterpret_cast<const char*>(r_res_vec.data()), r_res_vec.size()};
assert(r_res_vec.size() == data_test_text1.size);
assert(std::equal(data_test_text1.cbegin(), data_test_text1.cend(), r_res_vec.cbegin()));
assert(r_res.size == data_test_text1.size);
assert(std::equal(data_test_text1.cbegin(), data_test_text1.cend(), r_res.cbegin()));
}
{ // readback data_test_text2
auto r_res_var = f_r_zstd.read(data_test_text2.size);
auto r_res = f_r_zstd.read(data_test_text2.size);
//assert(f_r_zstd.isGood());
//assert(f_r_file.isGood());
assert(std::holds_alternative<std::vector<uint8_t>>(r_res_var));
const auto& r_res_vec = std::get<std::vector<uint8_t>>(r_res_var);
assert(r_res.isOwning());
//std::cout << "decomp: " << std::string_view{reinterpret_cast<const char*>(r_res_vec.data()), r_res_vec.size()};
assert(r_res_vec.size() == data_test_text2.size);
assert(r_res.size == data_test_text2.size);
assert(std::equal(
data_test_text2.cbegin(),
data_test_text2.cend(),
r_res_vec.cbegin()
r_res.cbegin()
));
}
{ // readback data_test_text3
auto r_res_var = f_r_zstd.read(data_test_text3.size);
auto r_res = f_r_zstd.read(data_test_text3.size);
//assert(f_r_zstd.isGood());
//assert(f_r_file.isGood());
assert(std::holds_alternative<std::vector<uint8_t>>(r_res_var));
const auto& r_res_vec = std::get<std::vector<uint8_t>>(r_res_var);
assert(r_res.isOwning());
//std::cout << "decomp: " << std::string_view{reinterpret_cast<const char*>(r_res_vec.data()), r_res_vec.size()};
assert(r_res_vec.size() == data_test_text3.size);
assert(r_res.size == data_test_text3.size);
assert(std::equal(
data_test_text3.cbegin(),
data_test_text3.cend(),
r_res_vec.cbegin()
r_res.cbegin()
));
}
{ // assert eof somehow
// since its eof, reading a single byte should return a zero sized buffer
auto r_res_var = f_r_zstd.read(1);
if (std::holds_alternative<std::vector<uint8_t>>(r_res_var)) {
assert(std::get<std::vector<uint8_t>>(r_res_var).empty());
} else if (std::holds_alternative<ByteSpan>(r_res_var)) {
assert(std::get<ByteSpan>(r_res_var).empty());
} else {
assert(false);
}
auto r_res = f_r_zstd.read(1);
assert(r_res.empty());
}
}
@ -285,27 +267,20 @@ int main(void) {
assert(f_r_file.isGood());
{ // read frame
auto r_res_var = f_r_zstd.read(frame_size);
auto r_res = f_r_zstd.read(frame_size);
assert(std::holds_alternative<std::vector<uint8_t>>(r_res_var));
const auto& r_res_vec = std::get<std::vector<uint8_t>>(r_res_var);
assert(r_res_vec.size() == frame_size);
assert(r_res.isOwning());
assert(r_res.size == frame_size);
// assert equal
for (auto& e : r_res_vec) {
for (auto& e : r_res) {
assert(e == uint8_t(rng_data() & 0xff));
}
}
{ // eof test
auto r_res_var = f_r_zstd.read(1);
if (std::holds_alternative<std::vector<uint8_t>>(r_res_var)) {
assert(std::get<std::vector<uint8_t>>(r_res_var).empty());
} else if (std::holds_alternative<ByteSpan>(r_res_var)) {
assert(std::get<ByteSpan>(r_res_var).empty());
} else {
assert(false);
}
auto r_res = f_r_zstd.read(1);
assert(r_res.empty());
}
}
@ -363,27 +338,20 @@ int main(void) {
const size_t frame_size = (rng() % ((2<<19) - 1)) + 1;
//std::cerr << "f: " << i << " fs: " << frame_size << "\n";
auto r_res_var = f_r_zstd.read(frame_size);
auto r_res = f_r_zstd.read(frame_size);
assert(std::holds_alternative<std::vector<uint8_t>>(r_res_var));
const auto& r_res_vec = std::get<std::vector<uint8_t>>(r_res_var);
assert(r_res_vec.size() == frame_size);
assert(r_res.isOwning());
assert(r_res.size == frame_size);
// assert equal
for (auto& e : r_res_vec) {
for (auto& e : r_res) {
assert(e == uint8_t(rng_data() & 0xff));
}
}
{ // eof test
auto r_res_var = f_r_zstd.read(1);
if (std::holds_alternative<std::vector<uint8_t>>(r_res_var)) {
assert(std::get<std::vector<uint8_t>>(r_res_var).empty());
} else if (std::holds_alternative<ByteSpan>(r_res_var)) {
assert(std::get<ByteSpan>(r_res_var).empty());
} else {
assert(false);
}
auto r_res = f_r_zstd.read(1);
assert(r_res.empty());
}
}