From 57575330ddd829051b738d332294b25bc1f5f084 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Mon, 27 May 2024 11:20:37 +0200 Subject: [PATCH] port to file2, other minor improvements --- CMakeLists.txt | 1 + solanaceae/ngc_ft1/flow_only.cpp | 3 +- solanaceae/ngc_ft1_sha1/file_rw_mapped.hpp | 54 ++++++++++------ solanaceae/ngc_ft1_sha1/sha1_ngcft1.cpp | 72 +++++++++++----------- 4 files changed, 74 insertions(+), 56 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ddf7192..ccfb116 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -66,5 +66,6 @@ target_link_libraries(solanaceae_sha1_ngcft1 PUBLIC solanaceae_tox_contacts solanaceae_message3 solanaceae_object_store + solanaceae_file2 ) diff --git a/solanaceae/ngc_ft1/flow_only.cpp b/solanaceae/ngc_ft1/flow_only.cpp index 65b58a3..cba9848 100644 --- a/solanaceae/ngc_ft1/flow_only.cpp +++ b/solanaceae/ngc_ft1/flow_only.cpp @@ -6,7 +6,8 @@ #include float FlowOnly::getCurrentDelay(void) const { - return std::min(_rtt_ema, RTT_MAX); + // below 1ms is useless + return std::clamp(_rtt_ema, 0.001f, RTT_MAX); } void FlowOnly::addRTT(float new_delay) { diff --git a/solanaceae/ngc_ft1_sha1/file_rw_mapped.hpp b/solanaceae/ngc_ft1_sha1/file_rw_mapped.hpp index cfd7032..69b90a7 100644 --- a/solanaceae/ngc_ft1_sha1/file_rw_mapped.hpp +++ b/solanaceae/ngc_ft1_sha1/file_rw_mapped.hpp @@ -1,6 +1,6 @@ #pragma once -#include +#include #include "./mio.hpp" @@ -9,22 +9,27 @@ #include #include -struct FileRWMapped : public FileI { +struct File2RWMapped : public File2I { mio::ummap_sink _file_map; // TODO: add truncate support? - FileRWMapped(std::string_view file_path, uint64_t file_size) { - _file_size = file_size; + // TODO: rw always true? + File2RWMapped(std::string_view file_path, int64_t file_size = -1) : File2I(true, true) { std::filesystem::path native_file_path{file_path}; if (!std::filesystem::exists(native_file_path)) { std::ofstream(native_file_path) << '\0'; // force create the file } - std::filesystem::resize_file(native_file_path, file_size); // ensure size, usually sparse + + _file_size = std::filesystem::file_size(native_file_path); + if (file_size >= 0 && _file_size != file_size) { + _file_size = file_size; + std::filesystem::resize_file(native_file_path, file_size); // ensure size, usually sparse + } std::error_code err; // sink, is also read - _file_map.map(native_file_path.u8string(), 0, file_size, err); + _file_map.map(native_file_path.u8string(), 0, _file_size, err); if (err) { std::cerr << "FileRWMapped error: mapping file failed " << err << "\n"; @@ -32,29 +37,40 @@ struct FileRWMapped : public FileI { } } - virtual ~FileRWMapped(void) override {} + virtual ~File2RWMapped(void) override {} bool isGood(void) override { return _file_map.is_mapped(); } - std::vector read(uint64_t pos, uint64_t size) override { - if (pos+size > _file_size) { - //assert(false && "read past end"); - return {}; - } - - return {_file_map.data()+pos, _file_map.data()+(pos+size)}; - } - - bool write(uint64_t pos, const std::vector& data) override { - if (pos+data.size() > _file_size) { + bool write(const ByteSpan data, int64_t pos = -1) override { + // TODO: support streaming write + if (pos < 0) { return false; } - std::memcpy(_file_map.data()+pos, data.data(), data.size()); + if (data.empty()) { + return true; // false? + } + + // file size is fix for mmaped files + if (pos+data.size > _file_size) { + return false; + } + + std::memcpy(_file_map.data()+pos, data.ptr, data.size); return true; } + + ByteSpanWithOwnership read(uint64_t size, int64_t pos = -1) override { + if (pos+size > _file_size) { + //assert(false && "read past end"); + return ByteSpan{}; + } + + // return non-owning + return ByteSpan{_file_map.data()+pos, size}; + } }; diff --git a/solanaceae/ngc_ft1_sha1/sha1_ngcft1.cpp b/solanaceae/ngc_ft1_sha1/sha1_ngcft1.cpp index 6adad5c..82c5f49 100644 --- a/solanaceae/ngc_ft1_sha1/sha1_ngcft1.cpp +++ b/solanaceae/ngc_ft1_sha1/sha1_ngcft1.cpp @@ -7,8 +7,6 @@ #include #include -#include - #include "./ft1_sha1_info.hpp" #include "./hash_utils.hpp" @@ -568,7 +566,7 @@ bool SHA1_NGCFT1::onEvent(const Message::Events::MessageUpdated& e) { ce.emplace(std::vector{full_file_path}); const bool file_exists = std::filesystem::exists(full_file_path); - std::unique_ptr file_impl = std::make_unique(full_file_path, info.file_size); + std::unique_ptr file_impl = std::make_unique(full_file_path, info.file_size); if (!file_impl->isGood()) { std::cerr << "SHA1_NGCFT1 error: failed opening file '" << full_file_path << "'!\n"; @@ -589,22 +587,24 @@ bool SHA1_NGCFT1::onEvent(const Message::Events::MessageUpdated& e) { // iterate existing file for (size_t i = 0; i < info.chunks.size(); i++) { const uint64_t chunk_size = info.chunkSize(i); - auto existing_data = file_impl->read(i*uint64_t(info.chunk_size), chunk_size); - assert(existing_data.size() == chunk_size); + auto existing_data = file_impl->read(chunk_size, i*uint64_t(info.chunk_size)); - // TODO: avoid copy + assert(existing_data.size == chunk_size); + if (existing_data.size == chunk_size) { + const auto data_hash = SHA1Digest{hash_sha1(existing_data.ptr, existing_data.size)}; + const bool data_equal = data_hash == info.chunks.at(i); - const auto data_hash = SHA1Digest{hash_sha1(existing_data.data(), existing_data.size())}; - const bool data_equal = data_hash == info.chunks.at(i); + cc.have_chunk.push_back(data_equal); - cc.have_chunk.push_back(data_equal); - - if (data_equal) { - cc.have_count += 1; - bytes_received += chunk_size; - //std::cout << "existing i[" << info.chunks.at(i) << "] == d[" << data_hash << "]\n"; + if (data_equal) { + cc.have_count += 1; + bytes_received += chunk_size; + //std::cout << "existing i[" << info.chunks.at(i) << "] == d[" << data_hash << "]\n"; + } else { + //std::cout << "unk i[" << info.chunks.at(i) << "] != d[" << data_hash << "]\n"; + } } else { - //std::cout << "unk i[" << info.chunks.at(i) << "] != d[" << data_hash << "]\n"; + // error reading? } _chunks[info.chunks[i]] = ce; @@ -833,9 +833,7 @@ bool SHA1_NGCFT1::onEvent(const Events::NGCFT1_recv_data& e) { for (const auto chunk_index : std::get(tv).chunk_indices) { const auto offset_into_file = chunk_index* ce.get().chunk_size; - // TODO: avoid temporary copy - // TODO: check return - if (!file->write(offset_into_file + e.data_offset, {e.data, e.data + e.data_size})) { + if (!file->write({e.data, e.data_size}, offset_into_file + e.data_offset)) { std::cerr << "SHA1_NGCFT1 error: writing file failed o:" << offset_into_file + e.data_offset << "\n"; } } @@ -873,14 +871,17 @@ bool SHA1_NGCFT1::onEvent(const Events::NGCFT1_send_data& e) { auto& chunk_transfer = std::get(transfer.v); const auto& info = chunk_transfer.content.get(); // TODO: should we really use file? - const auto data = chunk_transfer.content.get()->read((chunk_transfer.chunk_index * uint64_t(info.chunk_size)) + e.data_offset, e.data_size); + const auto data = chunk_transfer.content.get()->read( + e.data_size, + (chunk_transfer.chunk_index * uint64_t(info.chunk_size)) + e.data_offset + ); // TODO: optimize - for (size_t i = 0; i < e.data_size && i < data.size(); i++) { + for (size_t i = 0; i < e.data_size && i < data.size; i++) { e.data[i] = data[i]; } - chunk_transfer.content.get_or_emplace().total += data.size(); + chunk_transfer.content.get_or_emplace().total += data.size; // TODO: add event to propergate to messages //_rmm.throwEventUpdate(transfer); // should we? @@ -961,10 +962,11 @@ bool SHA1_NGCFT1::onEvent(const Events::NGCFT1_recv_done& e) { const auto chunk_size = info.chunkSize(chunk_index); assert(offset_into_file+chunk_size <= info.file_size); - const auto chunk_data = ce.get()->read(offset_into_file, chunk_size); + const auto chunk_data = ce.get()->read(chunk_size, offset_into_file); + assert(!chunk_data.empty()); // check hash of chunk - auto got_hash = hash_sha1(chunk_data.data(), chunk_data.size()); + auto got_hash = hash_sha1(chunk_data.ptr, chunk_data.size); if (info.chunks.at(chunk_index) == got_hash) { std::cout << "SHA1_NGCFT1: got chunk [" << SHA1Digest{got_hash} << "]\n"; @@ -986,7 +988,7 @@ bool SHA1_NGCFT1::onEvent(const Events::NGCFT1_recv_done& e) { // HACK: remap file, to clear ram // TODO: error checking - ce.get() = std::make_unique( + ce.get() = std::make_unique( ce.get().file_list.front(), info.file_size ); @@ -994,7 +996,7 @@ bool SHA1_NGCFT1::onEvent(const Events::NGCFT1_recv_done& e) { // good chunk // TODO: have wasted + metadata - ce.get_or_emplace().total += chunk_data.size(); + ce.get_or_emplace().total += chunk_data.size; } } } else { @@ -1184,9 +1186,7 @@ bool SHA1_NGCFT1::sendFilePath(const Contact3 c, std::string_view file_name, std file_name_ = std::string(file_name), file_path_ = std::string(file_path) ]() mutable { - // TODO: rw? - // TODO: memory mapped would be king - auto file_impl = std::make_unique(file_path_); + auto file_impl = std::make_unique(file_path_, -1); if (!file_impl->isGood()) { { std::lock_guard l{self->_info_builder_queue_mutex}; @@ -1205,19 +1205,19 @@ bool SHA1_NGCFT1::sendFilePath(const Contact3 c, std::string_view file_name, std FT1InfoSHA1 sha1_info; // build info sha1_info.file_name = file_name_; - sha1_info.file_size = file_impl->_file_size; + sha1_info.file_size = file_impl->_file_size; // TODO: remove the reliance on implementation details { // build chunks // HACK: load file fully - // TODO: the speed is truly horrid - const auto file_data = file_impl->read(0, file_impl->_file_size); + // ... its only a hack if its not memory mapped, but reading in chunk_sized chunks is probably a good idea anyway + const auto file_data = file_impl->read(file_impl->_file_size, 0); size_t i = 0; - for (; i + sha1_info.chunk_size < file_data.size(); i += sha1_info.chunk_size) { - sha1_info.chunks.push_back(hash_sha1(file_data.data()+i, sha1_info.chunk_size)); + for (; i + sha1_info.chunk_size < file_data.size; i += sha1_info.chunk_size) { + sha1_info.chunks.push_back(hash_sha1(file_data.ptr+i, sha1_info.chunk_size)); } - if (i < file_data.size()) { - sha1_info.chunks.push_back(hash_sha1(file_data.data()+i, file_data.size()-i)); + if (i < file_data.size) { + sha1_info.chunks.push_back(hash_sha1(file_data.ptr+i, file_data.size-i)); } } @@ -1236,7 +1236,7 @@ bool SHA1_NGCFT1::sendFilePath(const Contact3 c, std::string_view file_name, std ]() mutable { // // back on iterate thread - auto file_impl = std::make_unique(file_path_); + auto file_impl = std::make_unique(file_path_, sha1_info.file_size); if (!file_impl->isGood()) { std::cerr << "SHA1_NGCFT1 error: failed opening file '" << file_path_ << "'!\n"; return;