From 379684196181b2edf811df057a9a679d22a1487b Mon Sep 17 00:00:00 2001 From: Green Sky Date: Wed, 10 Apr 2024 21:18:27 +0200 Subject: [PATCH] conversion improvements --- src/fragment_store/convert_frag_to_obj.cpp | 30 ++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/src/fragment_store/convert_frag_to_obj.cpp b/src/fragment_store/convert_frag_to_obj.cpp index 5b4cd4a7..0329212a 100644 --- a/src/fragment_store/convert_frag_to_obj.cpp +++ b/src/fragment_store/convert_frag_to_obj.cpp @@ -2,6 +2,8 @@ #include "./backends/filesystem_storage.hpp" #include "./meta_components.hpp" +#include + #include #include @@ -57,6 +59,31 @@ int main(int argc, const char** argv) { bool onEvent(const ObjectStore::Events::ObjectConstruct& e) override { assert(e.e.all_of()); assert(e.e.all_of()); + + // !! we read the obj first, so we can discard empty objects + // technically we could just copy the file, but meh + // read src and write dst data + std::vector tmp_buffer; + std::function cb = [&tmp_buffer](const ByteSpan buffer) { + tmp_buffer.insert(tmp_buffer.end(), buffer.cbegin(), buffer.cend()); + }; + if (!_fsb_src.read(e.e, cb)) { + std::cerr << "failed to read obj '" << bin2hex(e.e.get().v) << "'\n"; + return false; + } + + if (tmp_buffer.empty()) { + std::cerr << "discarded empty obj '" << bin2hex(e.e.get().v) << "'\n"; + return false; + } + { // try getting lucky and see if its an empty json + const auto j = nlohmann::json::parse(tmp_buffer, nullptr, false); + if (j.is_array() && j.empty()) { + std::cerr << "discarded empty json array obj '" << bin2hex(e.e.get().v) << "'\n"; + return false; + } + } + auto oh = _fsb_dst.newObject(e.e.get().type, ByteSpan{e.e.get().v}); if (!static_cast(oh)) { @@ -84,8 +111,7 @@ int main(int argc, const char** argv) { } } - // read src and write dst data - static_cast(_fsb_dst).write(oh, ByteSpan{}); + static_cast(_fsb_dst).write(oh, ByteSpan{tmp_buffer}); return false; }