Green-Sky/solanaceae_object_store
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

adopt file2 changes

Browse Source
This commit is contained in:
Green Sky
2024-05-26 21:33:49 +02:00
parent e26959c380
commit be24b3815b
4 changed files with 55 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
src/solanaceae/file/file2_zstd.cpp
View File

@@ -46,8 +46,8 @@ bool File2ZSTDW::write(const ByteSpan data, int64_t pos) {
return _real_file.write(ByteSpan{compressed_buffer.data(), ret}) && _real_file.isGood();
}
std::variant<ByteSpan, std::vector<uint8_t>> File2ZSTDW::read(uint64_t, int64_t) {
return {};
ByteSpanWithOwnership File2ZSTDW::read(uint64_t, int64_t) {
return ByteSpan{};
}
// ######################################### decompression
@@ -70,10 +70,10 @@ bool File2ZSTDR::write(const ByteSpan, int64_t) {
return false;
}
std::variant<ByteSpan, std::vector<uint8_t>> File2ZSTDR::read(uint64_t size, int64_t pos) {
ByteSpanWithOwnership File2ZSTDR::read(uint64_t size, int64_t pos) {
if (pos != -1) {
// error, only support streaming (for now)
return {};
return ByteSpan{};
}
std::vector<uint8_t> ret_data;
@@ -92,7 +92,7 @@ std::variant<ByteSpan, std::vector<uint8_t>> File2ZSTDR::read(uint64_t size, int
if (_z_input.src == nullptr || _z_input.pos == _z_input.size) {
const auto request_size = _in_buffer.size();
if (!feedInput(_real_file.read(request_size, -1))) {
return ret_data;
return ByteSpanWithOwnership{std::move(ret_data)};
}
// if _z_input.size < _in_buffer.size() -> assume eof?
@@ -108,7 +108,7 @@ std::variant<ByteSpan, std::vector<uint8_t>> File2ZSTDR::read(uint64_t size, int
if (ZSTD_isError(ret)) {
// error <.<
std::cerr << "---- error: decompression error\n";
return ret_data;
return ByteSpanWithOwnership{std::move(ret_data)};
}
// no new decomp data?
@@ -145,7 +145,7 @@ std::variant<ByteSpan, std::vector<uint8_t>> File2ZSTDR::read(uint64_t size, int
} while (_z_input.pos < _z_input.size);
}
return ret_data;
return ByteSpanWithOwnership{std::move(ret_data)};
}
bool File2ZSTDR::feedInput(std::variant<ByteSpan, std::vector<uint8_t>>&& read_buff) {
@@ -154,7 +154,7 @@ bool File2ZSTDR::feedInput(std::variant<ByteSpan, std::vector<uint8_t>>&& read_b
const auto& span = std::get<ByteSpan>(read_buff);
if (span.size > _in_buffer.size()) {
// error, how did we read more than we asked for??
return {};
return false;
}
if (span.empty()) {
@@ -172,7 +172,7 @@ bool File2ZSTDR::feedInput(std::variant<ByteSpan, std::vector<uint8_t>>&& read_b
auto& vec = std::get<std::vector<uint8_t>>(read_buff);
if (vec.size() > _in_buffer.size()) {
// error, how did we read more than we asked for??
return {};
return false;
}
// cpy

4
src/solanaceae/file/file2_zstd.hpp
View File

@@ -18,7 +18,7 @@ struct File2ZSTDW : public File2I {
// for simplicity and potential future seekability each write is its own frame
bool write(const ByteSpan data, int64_t pos = -1) override;
std::variant<ByteSpan, std::vector<uint8_t>> read(uint64_t size, int64_t pos = -1) override;
ByteSpanWithOwnership read(uint64_t size, int64_t pos = -1) override;
};
// zstd decompression wrapper over another file
@@ -40,7 +40,7 @@ struct File2ZSTDR : public File2I {
bool isGood(void) override;
bool write(const ByteSpan data, int64_t pos = -1) override;
std::variant<ByteSpan, std::vector<uint8_t>> read(uint64_t size, int64_t pos = -1) override;
ByteSpanWithOwnership read(uint64_t size, int64_t pos = -1) override;
private:
bool feedInput(std::variant<ByteSpan, std::vector<uint8_t>>&& read_buff);

23
src/solanaceae/object_store/backends/filesystem_storage.cpp
View File

@@ -26,19 +26,6 @@ static const char* metaFileTypeSuffix(MetaFileType mft) {
return ""; // .unk?
}
// TODO: move to ... somewhere. (span? file2i?)
static ByteSpan spanFromRead(const std::variant<ByteSpan, std::vector<uint8_t>>& data_var) {
if (std::holds_alternative<std::vector<uint8_t>>(data_var)) {
auto& vec = std::get<std::vector<uint8_t>>(data_var);
return {vec.data(), vec.size()};
} else if (std::holds_alternative<ByteSpan>(data_var)) {
return std::get<ByteSpan>(data_var);
} else {
assert(false);
return {};
}
}
namespace backend {
@@ -347,8 +334,7 @@ bool FilesystemStorage::read(Object o, std::function<read_from_storage_put_data_
// TODO: make it read in a single chunk instead?
static constexpr int64_t chunk_size {1024 * 1024}; // 1MiB should be good for read
do {
auto data_var = data_file_stack.top()->read(chunk_size);
ByteSpan data = spanFromRead(data_var);
auto data = data_file_stack.top()->read(chunk_size);
if (data.empty()) {
// error or probably eof
@@ -567,11 +553,10 @@ size_t FilesystemStorage::scanPath(std::string_view path) {
}
// HACK: read fixed amout of data, but this way if we have neither enc nor comp we pass the span through
auto binary_read_value = binary_reader_stack.top()->read(10*1024*1024); // is 10MiB large enough for meta?
const auto binary_read_span = spanFromRead(binary_read_value);
assert(binary_read_span.size < 10*1024*1024);
auto binary_read = binary_reader_stack.top()->read(10*1024*1024); // is 10MiB large enough for meta?
assert(binary_read.size < 10*1024*1024);
j = nlohmann::json::from_msgpack(binary_read_span, true, false);
j = nlohmann::json::from_msgpack(binary_read, true, false);
} else if (it.meta_ext == ".meta.json") {
std::ifstream file(it.obj_path.generic_u8string() + it.meta_ext, std::ios::in | std::ios::binary);
if (!file.is_open()) {