From d888e91adad9147ab7aa326500602c8a5b266489 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Wed, 18 Jan 2023 00:47:08 +0100 Subject: [PATCH] add support for variable chunk size, double --- external/tox_ngc_ft1/tox_ngc_ft1 | 2 +- src/ft_sha1_info.cpp | 27 ++++++++++++++++++++++----- src/ft_sha1_info.hpp | 2 +- src/states/receive_start_sha1.cpp | 4 ++-- src/states/send_start_sha1.cpp | 5 +++-- src/states/sha1.cpp | 4 +++- src/tox_client.cpp | 1 + src/tox_client.hpp | 3 ++- 8 files changed, 35 insertions(+), 13 deletions(-) diff --git a/external/tox_ngc_ft1/tox_ngc_ft1 b/external/tox_ngc_ft1/tox_ngc_ft1 index 3675c2a..bbc96bd 160000 --- a/external/tox_ngc_ft1/tox_ngc_ft1 +++ b/external/tox_ngc_ft1/tox_ngc_ft1 @@ -1 +1 @@ -Subproject commit 3675c2a1a8dc01e3a892d4f487f98657a9ea0ce6 +Subproject commit bbc96bd776a29a6581545d60b5cb6e60e7453f1f diff --git a/src/ft_sha1_info.cpp b/src/ft_sha1_info.cpp index 3e93fc7..825f1a2 100644 --- a/src/ft_sha1_info.cpp +++ b/src/ft_sha1_info.cpp @@ -54,20 +54,28 @@ std::vector FTInfoSHA1::toBuffer(void) const { } assert(buffer.size() == 256+8); - // chunk size? + // chunk size + { // HACK: endianess + buffer.push_back((chunk_size>>(0*8)) & 0xff); + buffer.push_back((chunk_size>>(1*8)) & 0xff); + buffer.push_back((chunk_size>>(2*8)) & 0xff); + buffer.push_back((chunk_size>>(3*8)) & 0xff); + } + + assert(buffer.size() == 256+8+4); for (const auto& chunk : chunks) { for (size_t i = 0; i < chunk.data.size(); i++) { buffer.push_back(chunk.data[i]); } } - assert(buffer.size() == 256+8+20*chunks.size()); + assert(buffer.size() == 256+8+4+20*chunks.size()); return buffer; } void FTInfoSHA1::fromBuffer(const std::vector& buffer) { - assert(buffer.size() >= 256+8); + assert(buffer.size() >= 256+8+4); // TODO: optimize file_name.clear(); @@ -91,9 +99,17 @@ void FTInfoSHA1::fromBuffer(const std::vector& buffer) { file_size |= uint64_t(buffer[256+7]) << (7*8); } - assert((buffer.size()-(256+8)) % 20 == 0); + { // HACK: endianess + chunk_size = 0; + chunk_size |= uint32_t(buffer[256+8+0]) << (0*8); + chunk_size |= uint32_t(buffer[256+8+1]) << (1*8); + chunk_size |= uint32_t(buffer[256+8+2]) << (2*8); + chunk_size |= uint32_t(buffer[256+8+3]) << (3*8); + } - for (size_t offset = 256+8; offset < buffer.size();) { + assert((buffer.size()-(256+8+4)) % 20 == 0); + + for (size_t offset = 256+8+4; offset < buffer.size();) { assert(buffer.size() >= offset + 20); auto& chunk = chunks.emplace_back(); @@ -107,6 +123,7 @@ void FTInfoSHA1::fromBuffer(const std::vector& buffer) { std::ostream& operator<<(std::ostream& out, const FTInfoSHA1& v) { out << " file_name: " << v.file_name << "\n"; out << " file_size: " << v.file_size << "\n"; + out << " chunk_size: " << v.chunk_size << "\n"; out << " chunks.size(): " << v.chunks.size() << "\n"; return out; } diff --git a/src/ft_sha1_info.hpp b/src/ft_sha1_info.hpp index 2bb6f6b..4c5fbd7 100644 --- a/src/ft_sha1_info.hpp +++ b/src/ft_sha1_info.hpp @@ -43,7 +43,7 @@ namespace std { // inject struct FTInfoSHA1 { std::string file_name; uint64_t file_size {0}; - static constexpr size_t chunk_size {64*1024}; // 64KiB for now + uint32_t chunk_size {128*1024}; // 128KiB for now std::vector chunks; std::vector toBuffer(void) const; diff --git a/src/states/receive_start_sha1.cpp b/src/states/receive_start_sha1.cpp index e56b417..d0a27b4 100644 --- a/src/states/receive_start_sha1.cpp +++ b/src/states/receive_start_sha1.cpp @@ -94,8 +94,8 @@ std::unique_ptr ReceiveStartSHA1::nextState(void) { std::cout << "ReceiveStartSHA1 checking existing file\n"; size_t f_i {0}; size_t tmp_have_count {0}; - for (size_t c_i = 0; f_i + FTInfoSHA1::chunk_size < file_map.length(); f_i += FTInfoSHA1::chunk_size, c_i++) { - if (sha1_info.chunks[c_i] == hash_sha1(file_map.data()+f_i, FTInfoSHA1::chunk_size)) { + for (size_t c_i = 0; f_i + sha1_info.chunk_size < file_map.length(); f_i += sha1_info.chunk_size, c_i++) { + if (sha1_info.chunks[c_i] == hash_sha1(file_map.data()+f_i, sha1_info.chunk_size)) { have_chunk[c_i] = true; tmp_have_count++; } diff --git a/src/states/send_start_sha1.cpp b/src/states/send_start_sha1.cpp index dbcc0c8..b189779 100644 --- a/src/states/send_start_sha1.cpp +++ b/src/states/send_start_sha1.cpp @@ -29,11 +29,12 @@ SendStartSHA1::SendStartSHA1(ToxClient& tcl, const CommandLine& cl) : StateI(tcl // build info _sha1_info.file_name = std::filesystem::path(cl.send_path).filename().string(); _sha1_info.file_size = _file_map.length(); + //_sha1_info.chunk_size; { // build chunks size_t i = 0; - for (; i + FTInfoSHA1::chunk_size < _file_map.length(); i += FTInfoSHA1::chunk_size) { - _sha1_info.chunks.push_back(hash_sha1(_file_map.data()+i, FTInfoSHA1::chunk_size)); + for (; i + _sha1_info.chunk_size < _file_map.length(); i += _sha1_info.chunk_size) { + _sha1_info.chunks.push_back(hash_sha1(_file_map.data()+i, _sha1_info.chunk_size)); } if (i < _file_map.length()) { diff --git a/src/states/sha1.cpp b/src/states/sha1.cpp index b2fc323..7f8a998 100644 --- a/src/states/sha1.cpp +++ b/src/states/sha1.cpp @@ -188,7 +188,7 @@ bool SHA1::iterate(float delta) { // log _io_log_timer += delta; - static const float log_interval {15.f}; + static const float log_interval {10.f}; if (_io_log_timer >= log_interval) { _io_log_timer = 0.f; @@ -203,6 +203,8 @@ bool SHA1::iterate(float delta) { std::cout << "SHA1 speed down: " << down_kibs << "KiB/s up: " << up_kibs << "KiB/s\n"; std::cout << "SHA1 total down: " << _bytes_down / 1024 << "KiB up: " << _bytes_up / 1024 << "KiB\n"; + + std::cout << "SHA1 cwq:" << _chunk_want_queue.size() << " cwqr:" << _chunks_requested.size() << " trc:" << _transfers_receiving_chunk.size() << " tsc:" << _transfers_sending_chunk.size() << "\n"; } // TODO: unmap and remap the file every couple of minutes to keep ram usage down? diff --git a/src/tox_client.cpp b/src/tox_client.cpp index 6c0eb4b..75d97f6 100644 --- a/src/tox_client.cpp +++ b/src/tox_client.cpp @@ -274,6 +274,7 @@ void ToxClient::onToxGroupInvite(uint32_t friend_number, const uint8_t* invite_d void ToxClient::onToxGroupPeerJoin(uint32_t group_number, uint32_t peer_id) { std::cout << "TCL group peer join " << group_number << ":" << peer_id << "\n"; _groups[group_number].emplace(peer_id); + //_groups[group_number][peer_id] = tox_group_peer_get_connection_status(_tox, group_number, peer_id, nullptr); _tox_profile_dirty = true; } diff --git a/src/tox_client.hpp b/src/tox_client.hpp index 2d10845..9ffd20d 100644 --- a/src/tox_client.hpp +++ b/src/tox_client.hpp @@ -4,7 +4,6 @@ #include "./state.hpp" -#include #include #include #include @@ -12,6 +11,7 @@ #include #include #include +#include #include #include @@ -84,5 +84,6 @@ struct ToxClient { // key groupid, value set of peer ids std::map> _groups; + // std::map> _groups; };