From 5767834f71eed9ff37f39d35582d2936a52b6383 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Tue, 2 Apr 2024 18:27:47 +0200 Subject: [PATCH] since its working, remove old commented code --- src/fragment_store/fragment_store.cpp | 73 ++++++--------------------- 1 file changed, 15 insertions(+), 58 deletions(-) diff --git a/src/fragment_store/fragment_store.cpp b/src/fragment_store/fragment_store.cpp index bc35aa2..31a1876 100644 --- a/src/fragment_store/fragment_store.cpp +++ b/src/fragment_store/fragment_store.cpp @@ -381,65 +381,22 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function buffer(chunk_size); - uint64_t buffer_actual_size {0}; - do { - buffer_actual_size = data_cb(buffer.data(), buffer.size()); - if (buffer_actual_size == 0) { - break; - } - if (buffer_actual_size > buffer.size()) { - // wtf - break; - } + // for zstd compression, chunk size is frame size. (no cross frame referencing) + static constexpr int64_t chunk_size{1024*1024}; // 1MiB should be enough + std::vector buffer(chunk_size); + uint64_t buffer_actual_size {0}; + do { + buffer_actual_size = data_cb(buffer.data(), buffer.size()); + if (buffer_actual_size == 0) { + break; + } + if (buffer_actual_size > buffer.size()) { + // wtf + break; + } - data_file_stack.top()->write({buffer.data(), buffer_actual_size}); - } while (buffer_actual_size == buffer.size()); - //} else if (data_comp == Compression::ZSTD) { - //std::vector buffer(ZSTD_CStreamInSize()); - //std::vector compressed_buffer(ZSTD_CStreamOutSize()); - //uint64_t buffer_actual_size {0}; - - //std::unique_ptr cctx{ZSTD_createCCtx(), &ZSTD_freeCCtx}; - //ZSTD_CCtx_setParameter(cctx.get(), ZSTD_c_compressionLevel, 0); // default (3) - //ZSTD_CCtx_setParameter(cctx.get(), ZSTD_c_checksumFlag, 1); // add extra checksums (to frames?) - //do { - //buffer_actual_size = data_cb(buffer.data(), buffer.size()); - ////if (buffer_actual_size == 0) { - ////break; - ////} - //if (buffer_actual_size > buffer.size()) { - //// wtf - //break; - //} - //bool const lastChunk = (buffer_actual_size < buffer.size()); - - //ZSTD_EndDirective const mode = lastChunk ? ZSTD_e_end : ZSTD_e_continue; - //ZSTD_inBuffer input = { buffer.data(), buffer_actual_size, 0 }; - - //while (input.pos < input.size) { - //ZSTD_outBuffer output = { compressed_buffer.data(), compressed_buffer.size(), 0 }; - - //size_t const remaining = ZSTD_compressStream2(cctx.get(), &output , &input, mode); - //if (ZSTD_isError(remaining)) { - //std::cerr << "FS error: compressing data failed\n"; - //break; - //} - - //data_file.write(reinterpret_cast(compressed_buffer.data()), output.pos); - - //if (remaining == 0) { - //break; - //} - //} - //// same as if lastChunk break; - //} while (buffer_actual_size == buffer.size()); - //} else { - //assert(false && "implement me"); - //} + data_file_stack.top()->write({buffer.data(), buffer_actual_size}); + } while (buffer_actual_size == buffer.size()); //meta_file.flush(); //meta_file.close();