Green-Sky/tomato
Green-Sky/tomato
1
0
Fork 1
Code Issues Pull Requests 1 Actions Packages Projects Releases 1 Wiki Activity

tested and works, cleaning up commented code

Browse Source
This commit is contained in:
Green Sky 2024-04-01 12:32:22 +02:00
parent 16d2238f35
commit 1b9363e7b5
No known key found for this signature in database
1 changed files with 2 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
src/fragment_store/fragment_store.cpp
View File

@ -430,14 +430,8 @@ bool FragmentStore::loadFromStorage(FragmentID fid, std::function<read_from_stor
std::stack<std::unique_ptr<File2I>> data_file_stack; std::stack<std::unique_ptr<File2I>> data_file_stack;
data_file_stack.push(std::make_unique<File2RFile>(std::string_view{frag_path})); data_file_stack.push(std::make_unique<File2RFile>(std::string_view{frag_path}));
//std::ifstream data_file{
//frag_path,
//std::ios::in | std::ios::binary // always binary, also for text
//};
if (!data_file_stack.top()->isGood()) { if (!data_file_stack.top()->isGood()) {
std::cerr << "FS error: fragment data file failed to open '" << frag_path << "'\n"; std::cerr << "FS error: fragment data file failed to open '" << frag_path << "'\n";
// error
return false; return false;
} }
@ -453,15 +447,14 @@ bool FragmentStore::loadFromStorage(FragmentID fid, std::function<read_from_stor
data_file_stack.push(std::make_unique<File2ZSTDR>(*data_file_stack.top().get())); data_file_stack.push(std::make_unique<File2ZSTDR>(*data_file_stack.top().get()));
if (!data_file_stack.top()->isGood()) { if (!data_file_stack.top()->isGood()) {
std::cerr << "FS error: fragment data file failed to add zstd decompression layer '" << frag_path << "'\n"; std::cerr << "FS error: fragment data file failed to add zstd decompression layer '" << frag_path << "'\n";
// error
return false; return false;
} }
} else { } else {
assert(data_comp == Compression::NONE); assert(data_comp == Compression::NONE);
} }
//if (data_comp == Compression::NONE) { // TODO: make it read in a single chunk instead?
static constexpr int64_t chunk_size {1024 * 1024}; static constexpr int64_t chunk_size {1024 * 1024}; // 1MiB should be good for read
do { do {
auto data_var = data_file_stack.top()->read(chunk_size); auto data_var = data_file_stack.top()->read(chunk_size);
ByteSpan data; ByteSpan data;
@ -486,35 +479,6 @@ bool FragmentStore::loadFromStorage(FragmentID fid, std::function<read_from_stor
break; break;
} }
} while (data_file_stack.top()->isGood()); } while (data_file_stack.top()->isGood());
//} else if (data_comp == Compression::ZSTD) {
//std::vector<uint8_t> in_buffer(ZSTD_DStreamInSize());
//std::vector<uint8_t> out_buffer(ZSTD_DStreamOutSize());
//std::unique_ptr<ZSTD_DCtx, decltype(&ZSTD_freeDCtx)> dctx{ZSTD_createDCtx(), &ZSTD_freeDCtx};
//uint64_t buffer_actual_size {0};
//do {
//data_file.read(reinterpret_cast<char*>(in_buffer.data()), in_buffer.size());
//buffer_actual_size = data_file.gcount();
//if (buffer_actual_size == 0) {
//break;
//}
//ZSTD_inBuffer input {in_buffer.data(), buffer_actual_size, 0 };
//do {
//ZSTD_outBuffer output = { out_buffer.data(), out_buffer.size(), 0 };
//size_t const ret = ZSTD_decompressStream(dctx.get(), &output , &input);
//if (ZSTD_isError(ret)) {
//// error <.<
//std::cerr << "FS error: decompression error\n";
//break;
//}
//data_cb(out_buffer.data(), output.pos);
//} while (input.pos < input.size);
//} while (buffer_actual_size == in_buffer.size() && !data_file.eof());
//} else {
//assert(false && "implement me");
//}
return true; return true;
} }