Green-Sky/solanaceae_ngc_ft1
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

port to file2, other minor improvements

Browse Source
This commit is contained in:
Green Sky 2024-05-27 11:20:37 +02:00
parent eb2a19d8f3
commit 57575330dd
No known key found for this signature in database
4 changed files with 74 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CMakeLists.txt
View File

@ -66,5 +66,6 @@ target_link_libraries(solanaceae_sha1_ngcft1 PUBLIC
solanaceae_tox_contacts solanaceae_tox_contacts
solanaceae_message3 solanaceae_message3
solanaceae_object_store solanaceae_object_store
solanaceae_file2
) )

3
solanaceae/ngc_ft1/flow_only.cpp
View File

@ -6,7 +6,8 @@
#include <algorithm> #include <algorithm>
float FlowOnly::getCurrentDelay(void) const { float FlowOnly::getCurrentDelay(void) const {
return std::min(_rtt_ema, RTT_MAX); // below 1ms is useless
return std::clamp(_rtt_ema, 0.001f, RTT_MAX);
} }
void FlowOnly::addRTT(float new_delay) { void FlowOnly::addRTT(float new_delay) {

54
solanaceae/ngc_ft1_sha1/file_rw_mapped.hpp
View File

@ -1,6 +1,6 @@
#pragma once #pragma once
#include <solanaceae/message3/file.hpp> #include <solanaceae/file/file2.hpp>
#include "./mio.hpp" #include "./mio.hpp"
@ -9,22 +9,27 @@
#include <iostream> #include <iostream>
#include <cstring> #include <cstring>
struct FileRWMapped : public FileI { struct File2RWMapped : public File2I {
mio::ummap_sink _file_map; mio::ummap_sink _file_map;
// TODO: add truncate support? // TODO: add truncate support?
FileRWMapped(std::string_view file_path, uint64_t file_size) { // TODO: rw always true?
_file_size = file_size; File2RWMapped(std::string_view file_path, int64_t file_size = -1) : File2I(true, true) {
std::filesystem::path native_file_path{file_path}; std::filesystem::path native_file_path{file_path};
if (!std::filesystem::exists(native_file_path)) { if (!std::filesystem::exists(native_file_path)) {
std::ofstream(native_file_path) << '\0'; // force create the file std::ofstream(native_file_path) << '\0'; // force create the file
} }
std::filesystem::resize_file(native_file_path, file_size); // ensure size, usually sparse
_file_size = std::filesystem::file_size(native_file_path);
if (file_size >= 0 && _file_size != file_size) {
_file_size = file_size;
std::filesystem::resize_file(native_file_path, file_size); // ensure size, usually sparse
}
std::error_code err; std::error_code err;
// sink, is also read // sink, is also read
_file_map.map(native_file_path.u8string(), 0, file_size, err); _file_map.map(native_file_path.u8string(), 0, _file_size, err);
if (err) { if (err) {
std::cerr << "FileRWMapped error: mapping file failed " << err << "\n"; std::cerr << "FileRWMapped error: mapping file failed " << err << "\n";
@ -32,29 +37,40 @@ struct FileRWMapped : public FileI {
} }
} }
virtual ~FileRWMapped(void) override {} virtual ~File2RWMapped(void) override {}
bool isGood(void) override { bool isGood(void) override {
return _file_map.is_mapped(); return _file_map.is_mapped();
} }
std::vector<uint8_t> read(uint64_t pos, uint64_t size) override { bool write(const ByteSpan data, int64_t pos = -1) override {
if (pos+size > _file_size) { // TODO: support streaming write
//assert(false && "read past end"); if (pos < 0) {
return {};
}
return {_file_map.data()+pos, _file_map.data()+(pos+size)};
}
bool write(uint64_t pos, const std::vector<uint8_t>& data) override {
if (pos+data.size() > _file_size) {
return false; return false;
} }
std::memcpy(_file_map.data()+pos, data.data(), data.size()); if (data.empty()) {
return true; // false?
}
// file size is fix for mmaped files
if (pos+data.size > _file_size) {
return false;
}
std::memcpy(_file_map.data()+pos, data.ptr, data.size);
return true; return true;
} }
ByteSpanWithOwnership read(uint64_t size, int64_t pos = -1) override {
if (pos+size > _file_size) {
//assert(false && "read past end");
return ByteSpan{};
}
// return non-owning
return ByteSpan{_file_map.data()+pos, size};
}
}; };

72
solanaceae/ngc_ft1_sha1/sha1_ngcft1.cpp
View File

@ -7,8 +7,6 @@
#include <solanaceae/message3/components.hpp> #include <solanaceae/message3/components.hpp>
#include <solanaceae/tox_messages/components.hpp> #include <solanaceae/tox_messages/components.hpp>
#include <solanaceae/message3/file_r_file.hpp>
#include "./ft1_sha1_info.hpp" #include "./ft1_sha1_info.hpp"
#include "./hash_utils.hpp" #include "./hash_utils.hpp"
@ -568,7 +566,7 @@ bool SHA1_NGCFT1::onEvent(const Message::Events::MessageUpdated& e) {
ce.emplace<Message::Components::Transfer::FileInfoLocal>(std::vector{full_file_path}); ce.emplace<Message::Components::Transfer::FileInfoLocal>(std::vector{full_file_path});
const bool file_exists = std::filesystem::exists(full_file_path); const bool file_exists = std::filesystem::exists(full_file_path);
std::unique_ptr<FileRWMapped> file_impl = std::make_unique<FileRWMapped>(full_file_path, info.file_size); std::unique_ptr<File2I> file_impl = std::make_unique<File2RWMapped>(full_file_path, info.file_size);
if (!file_impl->isGood()) { if (!file_impl->isGood()) {
std::cerr << "SHA1_NGCFT1 error: failed opening file '" << full_file_path << "'!\n"; std::cerr << "SHA1_NGCFT1 error: failed opening file '" << full_file_path << "'!\n";
@ -589,22 +587,24 @@ bool SHA1_NGCFT1::onEvent(const Message::Events::MessageUpdated& e) {
// iterate existing file // iterate existing file
for (size_t i = 0; i < info.chunks.size(); i++) { for (size_t i = 0; i < info.chunks.size(); i++) {
const uint64_t chunk_size = info.chunkSize(i); const uint64_t chunk_size = info.chunkSize(i);
auto existing_data = file_impl->read(i*uint64_t(info.chunk_size), chunk_size); auto existing_data = file_impl->read(chunk_size, i*uint64_t(info.chunk_size));
assert(existing_data.size() == chunk_size);
// TODO: avoid copy assert(existing_data.size == chunk_size);
if (existing_data.size == chunk_size) {
const auto data_hash = SHA1Digest{hash_sha1(existing_data.ptr, existing_data.size)};
const bool data_equal = data_hash == info.chunks.at(i);
const auto data_hash = SHA1Digest{hash_sha1(existing_data.data(), existing_data.size())}; cc.have_chunk.push_back(data_equal);
const bool data_equal = data_hash == info.chunks.at(i);
cc.have_chunk.push_back(data_equal); if (data_equal) {
cc.have_count += 1;
if (data_equal) { bytes_received += chunk_size;
cc.have_count += 1; //std::cout << "existing i[" << info.chunks.at(i) << "] == d[" << data_hash << "]\n";
bytes_received += chunk_size; } else {
//std::cout << "existing i[" << info.chunks.at(i) << "] == d[" << data_hash << "]\n"; //std::cout << "unk i[" << info.chunks.at(i) << "] != d[" << data_hash << "]\n";
}
} else { } else {
//std::cout << "unk i[" << info.chunks.at(i) << "] != d[" << data_hash << "]\n"; // error reading?
} }
_chunks[info.chunks[i]] = ce; _chunks[info.chunks[i]] = ce;
@ -833,9 +833,7 @@ bool SHA1_NGCFT1::onEvent(const Events::NGCFT1_recv_data& e) {
for (const auto chunk_index : std::get<ReceivingTransfer::Chunk>(tv).chunk_indices) { for (const auto chunk_index : std::get<ReceivingTransfer::Chunk>(tv).chunk_indices) {
const auto offset_into_file = chunk_index* ce.get<Components::FT1InfoSHA1>().chunk_size; const auto offset_into_file = chunk_index* ce.get<Components::FT1InfoSHA1>().chunk_size;
// TODO: avoid temporary copy if (!file->write({e.data, e.data_size}, offset_into_file + e.data_offset)) {
// TODO: check return
if (!file->write(offset_into_file + e.data_offset, {e.data, e.data + e.data_size})) {
std::cerr << "SHA1_NGCFT1 error: writing file failed o:" << offset_into_file + e.data_offset << "\n"; std::cerr << "SHA1_NGCFT1 error: writing file failed o:" << offset_into_file + e.data_offset << "\n";
} }
} }
@ -873,14 +871,17 @@ bool SHA1_NGCFT1::onEvent(const Events::NGCFT1_send_data& e) {
auto& chunk_transfer = std::get<SendingTransfer::Chunk>(transfer.v); auto& chunk_transfer = std::get<SendingTransfer::Chunk>(transfer.v);
const auto& info = chunk_transfer.content.get<Components::FT1InfoSHA1>(); const auto& info = chunk_transfer.content.get<Components::FT1InfoSHA1>();
// TODO: should we really use file? // TODO: should we really use file?
const auto data = chunk_transfer.content.get<Message::Components::Transfer::File>()->read((chunk_transfer.chunk_index * uint64_t(info.chunk_size)) + e.data_offset, e.data_size); const auto data = chunk_transfer.content.get<Message::Components::Transfer::File>()->read(
e.data_size,
(chunk_transfer.chunk_index * uint64_t(info.chunk_size)) + e.data_offset
);
// TODO: optimize // TODO: optimize
for (size_t i = 0; i < e.data_size && i < data.size(); i++) { for (size_t i = 0; i < e.data_size && i < data.size; i++) {
e.data[i] = data[i]; e.data[i] = data[i];
} }
chunk_transfer.content.get_or_emplace<Message::Components::Transfer::BytesSent>().total += data.size(); chunk_transfer.content.get_or_emplace<Message::Components::Transfer::BytesSent>().total += data.size;
// TODO: add event to propergate to messages // TODO: add event to propergate to messages
//_rmm.throwEventUpdate(transfer); // should we? //_rmm.throwEventUpdate(transfer); // should we?
@ -961,10 +962,11 @@ bool SHA1_NGCFT1::onEvent(const Events::NGCFT1_recv_done& e) {
const auto chunk_size = info.chunkSize(chunk_index); const auto chunk_size = info.chunkSize(chunk_index);
assert(offset_into_file+chunk_size <= info.file_size); assert(offset_into_file+chunk_size <= info.file_size);
const auto chunk_data = ce.get<Message::Components::Transfer::File>()->read(offset_into_file, chunk_size); const auto chunk_data = ce.get<Message::Components::Transfer::File>()->read(chunk_size, offset_into_file);
assert(!chunk_data.empty());
// check hash of chunk // check hash of chunk
auto got_hash = hash_sha1(chunk_data.data(), chunk_data.size()); auto got_hash = hash_sha1(chunk_data.ptr, chunk_data.size);
if (info.chunks.at(chunk_index) == got_hash) { if (info.chunks.at(chunk_index) == got_hash) {
std::cout << "SHA1_NGCFT1: got chunk [" << SHA1Digest{got_hash} << "]\n"; std::cout << "SHA1_NGCFT1: got chunk [" << SHA1Digest{got_hash} << "]\n";
@ -986,7 +988,7 @@ bool SHA1_NGCFT1::onEvent(const Events::NGCFT1_recv_done& e) {
// HACK: remap file, to clear ram // HACK: remap file, to clear ram
// TODO: error checking // TODO: error checking
ce.get<Message::Components::Transfer::File>() = std::make_unique<FileRWMapped>( ce.get<Message::Components::Transfer::File>() = std::make_unique<File2RWMapped>(
ce.get<Message::Components::Transfer::FileInfoLocal>().file_list.front(), ce.get<Message::Components::Transfer::FileInfoLocal>().file_list.front(),
info.file_size info.file_size
); );
@ -994,7 +996,7 @@ bool SHA1_NGCFT1::onEvent(const Events::NGCFT1_recv_done& e) {
// good chunk // good chunk
// TODO: have wasted + metadata // TODO: have wasted + metadata
ce.get_or_emplace<Message::Components::Transfer::BytesReceived>().total += chunk_data.size(); ce.get_or_emplace<Message::Components::Transfer::BytesReceived>().total += chunk_data.size;
} }
} }
} else { } else {
@ -1184,9 +1186,7 @@ bool SHA1_NGCFT1::sendFilePath(const Contact3 c, std::string_view file_name, std
file_name_ = std::string(file_name), file_name_ = std::string(file_name),
file_path_ = std::string(file_path) file_path_ = std::string(file_path)
]() mutable { ]() mutable {
// TODO: rw? auto file_impl = std::make_unique<File2RWMapped>(file_path_, -1);
// TODO: memory mapped would be king
auto file_impl = std::make_unique<FileRFile>(file_path_);
if (!file_impl->isGood()) { if (!file_impl->isGood()) {
{ {
std::lock_guard l{self->_info_builder_queue_mutex}; std::lock_guard l{self->_info_builder_queue_mutex};
@ -1205,19 +1205,19 @@ bool SHA1_NGCFT1::sendFilePath(const Contact3 c, std::string_view file_name, std
FT1InfoSHA1 sha1_info; FT1InfoSHA1 sha1_info;
// build info // build info
sha1_info.file_name = file_name_; sha1_info.file_name = file_name_;
sha1_info.file_size = file_impl->_file_size; sha1_info.file_size = file_impl->_file_size; // TODO: remove the reliance on implementation details
{ // build chunks { // build chunks
// HACK: load file fully // HACK: load file fully
// TODO: the speed is truly horrid // ... its only a hack if its not memory mapped, but reading in chunk_sized chunks is probably a good idea anyway
const auto file_data = file_impl->read(0, file_impl->_file_size); const auto file_data = file_impl->read(file_impl->_file_size, 0);
size_t i = 0; size_t i = 0;
for (; i + sha1_info.chunk_size < file_data.size(); i += sha1_info.chunk_size) { for (; i + sha1_info.chunk_size < file_data.size; i += sha1_info.chunk_size) {
sha1_info.chunks.push_back(hash_sha1(file_data.data()+i, sha1_info.chunk_size)); sha1_info.chunks.push_back(hash_sha1(file_data.ptr+i, sha1_info.chunk_size));
} }
if (i < file_data.size()) { if (i < file_data.size) {
sha1_info.chunks.push_back(hash_sha1(file_data.data()+i, file_data.size()-i)); sha1_info.chunks.push_back(hash_sha1(file_data.ptr+i, file_data.size-i));
} }
} }
@ -1236,7 +1236,7 @@ bool SHA1_NGCFT1::sendFilePath(const Contact3 c, std::string_view file_name, std
]() mutable { // ]() mutable { //
// back on iterate thread // back on iterate thread
auto file_impl = std::make_unique<FileRFile>(file_path_); auto file_impl = std::make_unique<File2RWMapped>(file_path_, sha1_info.file_size);
if (!file_impl->isGood()) { if (!file_impl->isGood()) {
std::cerr << "SHA1_NGCFT1 error: failed opening file '" << file_path_ << "'!\n"; std::cerr << "SHA1_NGCFT1 error: failed opening file '" << file_path_ << "'!\n";
return; return;