From e1d8e9ed4c7f97898a5e883d0dae9039a58db20a Mon Sep 17 00:00:00 2001 From: Green Sky Date: Mon, 16 Jan 2023 00:00:42 +0100 Subject: [PATCH] send and receiver startup should be complete --- external/tox_ngc_ft1/tox_ngc_ft1 | 2 +- src/ft_sha1_info.cpp | 45 ++++++++++++++++++++++ src/ft_sha1_info.hpp | 3 ++ src/states/receive_start_sha1.cpp | 63 ++++++++++++++++++++++++++++--- src/states/send_start_sha1.cpp | 14 +++++-- src/states/send_start_sha1.hpp | 1 + src/states/sha1.cpp | 10 ++++- src/states/sha1.hpp | 4 +- src/tox_client.cpp | 8 +++- src/tox_client.hpp | 21 +++++++++++ 10 files changed, 155 insertions(+), 16 deletions(-) diff --git a/external/tox_ngc_ft1/tox_ngc_ft1 b/external/tox_ngc_ft1/tox_ngc_ft1 index 4c37f0c..04befb2 160000 --- a/external/tox_ngc_ft1/tox_ngc_ft1 +++ b/external/tox_ngc_ft1/tox_ngc_ft1 @@ -1 +1 @@ -Subproject commit 4c37f0c32bd22f55756c5adacf37956a9b7dac1e +Subproject commit 04befb21be0327996eb3597cb91d5bdd00066974 diff --git a/src/ft_sha1_info.cpp b/src/ft_sha1_info.cpp index d1db0c1..3e93fc7 100644 --- a/src/ft_sha1_info.cpp +++ b/src/ft_sha1_info.cpp @@ -66,3 +66,48 @@ std::vector FTInfoSHA1::toBuffer(void) const { return buffer; } +void FTInfoSHA1::fromBuffer(const std::vector& buffer) { + assert(buffer.size() >= 256+8); + + // TODO: optimize + file_name.clear(); + for (size_t i = 0; i < 256; i++) { + char next_char = static_cast(buffer[i]); + if (next_char == 0) { + break; + } + file_name.push_back(next_char); + } + + { // HACK: endianess + file_size = 0; + file_size |= uint64_t(buffer[256+0]) << (0*8); + file_size |= uint64_t(buffer[256+1]) << (1*8); + file_size |= uint64_t(buffer[256+2]) << (2*8); + file_size |= uint64_t(buffer[256+3]) << (3*8); + file_size |= uint64_t(buffer[256+4]) << (4*8); + file_size |= uint64_t(buffer[256+5]) << (5*8); + file_size |= uint64_t(buffer[256+6]) << (6*8); + file_size |= uint64_t(buffer[256+7]) << (7*8); + } + + assert((buffer.size()-(256+8)) % 20 == 0); + + for (size_t offset = 256+8; offset < buffer.size();) { + assert(buffer.size() >= offset + 20); + + auto& chunk = chunks.emplace_back(); + for (size_t i = 0; i < chunk.size(); i++, offset++) { + chunk.data[i] = buffer.at(offset); + } + // TODO: error/leftover checking + } +} + +std::ostream& operator<<(std::ostream& out, const FTInfoSHA1& v) { + out << " file_name: " << v.file_name << "\n"; + out << " file_size: " << v.file_size << "\n"; + out << " chunks.size(): " << v.chunks.size() << "\n"; + return out; +} + diff --git a/src/ft_sha1_info.hpp b/src/ft_sha1_info.hpp index a2672eb..18048d8 100644 --- a/src/ft_sha1_info.hpp +++ b/src/ft_sha1_info.hpp @@ -29,5 +29,8 @@ struct FTInfoSHA1 { std::vector chunks; std::vector toBuffer(void) const; + void fromBuffer(const std::vector& buffer); }; +std::ostream& operator<<(std::ostream& out, const FTInfoSHA1& v); + diff --git a/src/states/receive_start_sha1.cpp b/src/states/receive_start_sha1.cpp index 6b61d8a..41c3af1 100644 --- a/src/states/receive_start_sha1.cpp +++ b/src/states/receive_start_sha1.cpp @@ -2,16 +2,20 @@ #include "./sha1.hpp" +#include "../hash_utils.hpp" #include "../tox_utils.hpp" #include "../ft_sha1_info.hpp" #include "../tox_client.hpp" +#include +#include #include #include #include #include +#include #include namespace States { @@ -43,8 +47,18 @@ bool ReceiveStartSHA1::iterate(float delta) { } } else if (_time_since_last_request >= 15.f) { // blast ever 15sec _time_since_last_request = 0.f; - //_tcl.sendFT1RequestPrivate( - + // TODO: select random and try, not blas + // ... and we are blasing + _tcl.forEachGroup([this](const uint32_t group_number) { + _tcl.forEachGroupPeer(group_number, [this, group_number](uint32_t peer_number) { + _tcl.sendFT1RequestPrivate( + group_number, peer_number, + NGC_FT1_file_kind::HASH_SHA1_INFO, + _sha1_info_hash.data.data(), _sha1_info_hash.size() + ); + std::cout << "ReceiveStartSHA1 sendig info request to " << group_number << ":" << peer_number << "\n"; + }); + }); } // if not transfer, request from random peer (equal dist!!) @@ -53,16 +67,41 @@ bool ReceiveStartSHA1::iterate(float delta) { } std::unique_ptr ReceiveStartSHA1::nextState(void) { - std::cout << "ReceiveStartSHA1 switching state to SHA1\n"; - FTInfoSHA1 sha1_info; - // from buffer + sha1_info.fromBuffer(_sha1_info_data); + + std::cout << "ReceiveStartSHA1 info is: \n" << sha1_info; + + bool file_existed = std::filesystem::exists(sha1_info.file_name); + if (!file_existed) { + std::ofstream(sha1_info.file_name) << '\0'; // create the file + } + std::filesystem::resize_file(sha1_info.file_name, sha1_info.file_size); // open file for writing (pre allocate?) - mio::mmap_sink file_map; + std::error_code err; + mio::mmap_sink file_map = mio::make_mmap_sink(sha1_info.file_name, 0, sha1_info.file_size, err); std::vector have_chunk(sha1_info.chunks.size(), false); + // dont overwrite correct existing data + if (file_existed) { + std::cout << "ReceiveStartSHA1 checking existing file\n"; + size_t f_i = 0; + for (size_t c_i = 0; f_i + FTInfoSHA1::chunk_size < file_map.length(); f_i += FTInfoSHA1::chunk_size, c_i++) { + if (sha1_info.chunks[c_i] == hash_sha1(file_map.data()+f_i, FTInfoSHA1::chunk_size)) { + have_chunk[c_i] = true; + } + } + + if (f_i < file_map.length()) { + if (sha1_info.chunks.back() == hash_sha1(file_map.data()+f_i, file_map.length()-f_i)) { + have_chunk.back() = true; + } + } + } + + std::cout << "ReceiveStartSHA1 switching state to SHA1\n"; return std::make_unique( _tcl, std::move(file_map), @@ -112,7 +151,19 @@ void ReceiveStartSHA1::onFT1ReceiveDataSHA1Info(uint32_t group_number, uint32_t _sha1_info_data[data_offset+i] = data[i]; } + std::get(_transfer.value()) = 0.f; + + std::cout << "ReceiveStartSHA1 " << data_offset+data_size << "/" << _sha1_info_data.size() << " (" << float(data_offset+data_size) / _sha1_info_data.size() * 100.f << "%)\n"; + if (data_offset + data_size == _sha1_info_data.size()) { + // hash and verify + SHA1Digest test_hash = hash_sha1(_sha1_info_data.data(), _sha1_info_data.size()); + if (test_hash != _sha1_info_hash) { + std::cerr << "ReceiveStartSHA1 received info's hash does not match!, discarding\n"; + _transfer.reset(); + _sha1_info_data.clear(); + } + std::cout << "ReceiveStartSHA1 info tansfer finished " << group_number << ":" << peer_number << "." << transfer_id << "\n"; _done = true; } diff --git a/src/states/send_start_sha1.cpp b/src/states/send_start_sha1.cpp index cb17499..b492aa2 100644 --- a/src/states/send_start_sha1.cpp +++ b/src/states/send_start_sha1.cpp @@ -13,10 +13,11 @@ #include #include #include +#include namespace States { -SendStartSHA1::SendStartSHA1(ToxClient& tcl, const CommandLine& cl) : StateI(tcl) { +SendStartSHA1::SendStartSHA1(ToxClient& tcl, const CommandLine& cl) : StateI(tcl), _file_path(cl.send_path) { std::cout << "SendStartSHA1 start building sha1_info\n"; std::error_code err; _file_map = mio::make_mmap_source(cl.send_path, 0, mio::map_entire_file, err); @@ -26,7 +27,7 @@ SendStartSHA1::SendStartSHA1(ToxClient& tcl, const CommandLine& cl) : StateI(tcl assert(!_file_map.empty()); // build info - _sha1_info.file_name = "testfile.bin"; + _sha1_info.file_name = std::filesystem::path(cl.send_path).filename(); _sha1_info.file_size = _file_map.length(); { // build chunks @@ -45,7 +46,7 @@ SendStartSHA1::SendStartSHA1(ToxClient& tcl, const CommandLine& cl) : StateI(tcl _file_map = mio::make_mmap_source(cl.send_path, 0, mio::map_entire_file, err); } - std::cout << "SendStartSHA1 chunks: " << _sha1_info.chunks.size() << "\n"; + std::cout << "SendStartSHA1 info is: \n" << _sha1_info; _sha1_info_data = _sha1_info.toBuffer(); @@ -63,10 +64,15 @@ bool SendStartSHA1::iterate(float) { std::unique_ptr SendStartSHA1::nextState(void) { std::cout << "SendStartSHA1 switching state to SHA1\n"; std::vector have_chunk(_sha1_info.chunks.size(), true); + + _file_map.unmap(); + std::error_code err; + auto new_file_map = mio::make_mmap_sink(_file_path, 0, mio::map_entire_file, err); + // we are done setting up return std::make_unique( _tcl, - std::move(_file_map), + std::move(new_file_map), std::move(_sha1_info), std::move(_sha1_info_data), std::move(_sha1_info_hash), diff --git a/src/states/send_start_sha1.hpp b/src/states/send_start_sha1.hpp index 18e5370..4789ed1 100644 --- a/src/states/send_start_sha1.hpp +++ b/src/states/send_start_sha1.hpp @@ -33,6 +33,7 @@ struct SendStartSHA1 final : public StateI { void onFT1SendDataSHA1Chunk(uint32_t group_number, uint32_t peer_number, uint8_t transfer_id, size_t data_offset, uint8_t* data, size_t data_size) override; private: + std::string _file_path; mio::mmap_source _file_map; FTInfoSHA1 _sha1_info; std::vector _sha1_info_data; diff --git a/src/states/sha1.cpp b/src/states/sha1.cpp index 3af79bf..152ae19 100644 --- a/src/states/sha1.cpp +++ b/src/states/sha1.cpp @@ -8,7 +8,7 @@ namespace States { SHA1::SHA1( ToxClient& tcl, - mio::mmap_source&& file_map, + mio::mmap_sink&& file_map, const FTInfoSHA1&& sha1_info, const std::vector&& sha1_info_data, //const std::vector&& sha1_info_hash, @@ -128,6 +128,14 @@ void SHA1::onFT1SendDataSHA1Info(uint32_t group_number, uint32_t peer_number, ui data[i] = _sha1_info_data.at(data_offset+i); } + // TODO: sub optimal + for (auto it = _transfers_requested_info.begin(); it != _transfers_requested_info.end(); it++) { + if (std::get<0>(*it) == group_number && std::get<1>(*it) == peer_number && std::get<2>(*it) == transfer_id) { + std::get(*it) = 0.f; + break; + } + } + // if last data block if (data_offset + data_size == _sha1_info_data.size()) { // this transfer is "done" (ft still could have to retransfer) diff --git a/src/states/sha1.hpp b/src/states/sha1.hpp index 0ff4fb6..5dc5695 100644 --- a/src/states/sha1.hpp +++ b/src/states/sha1.hpp @@ -17,7 +17,7 @@ struct SHA1 final : public StateI { public: // general interface SHA1( ToxClient& tcl, - mio::mmap_source&& file_map, + mio::mmap_sink&& file_map, const FTInfoSHA1&& sha1_info, const std::vector&& sha1_info_data, //const std::vector&& sha1_info_hash, @@ -48,7 +48,7 @@ struct SHA1 final : public StateI { void queueUpRequestInfo(uint32_t group_number, uint32_t peer_number); private: - mio::mmap_source _file_map; + mio::mmap_sink _file_map; // writable if not all const FTInfoSHA1 _sha1_info; const std::vector _sha1_info_data; const SHA1Digest _sha1_info_hash; diff --git a/src/tox_client.cpp b/src/tox_client.cpp index 2e43757..7894fc2 100644 --- a/src/tox_client.cpp +++ b/src/tox_client.cpp @@ -233,22 +233,26 @@ void ToxClient::onToxGroupCustomPrivatePacket(uint32_t group_number, uint32_t pe void ToxClient::onToxGroupInvite(uint32_t friend_number, const uint8_t* invite_data, size_t invite_length, std::string_view group_name) { std::cout << "TCL accepting group invite (" << group_name << ")\n"; - tox_group_invite_accept(_tox, friend_number, invite_data, invite_length, reinterpret_cast(_self_name.data()), _self_name.size(), nullptr, 0, nullptr); + uint32_t new_group_number = tox_group_invite_accept(_tox, friend_number, invite_data, invite_length, reinterpret_cast(_self_name.data()), _self_name.size(), nullptr, 0, nullptr); + _groups[new_group_number] = {}; _tox_profile_dirty = true; } void ToxClient::onToxGroupPeerJoin(uint32_t group_number, uint32_t peer_id) { std::cout << "TCL group peer join " << group_number << ":" << peer_id << "\n"; + _groups[group_number].emplace(peer_id); _tox_profile_dirty = true; } void ToxClient::onToxGroupPeerExit(uint32_t group_number, uint32_t peer_id, Tox_Group_Exit_Type exit_type, std::string_view name, std::string_view part_message) { - std::cout << "TCL group peer esit " << group_number << ":" << peer_id << "\n"; + std::cout << "TCL group peer exit " << group_number << ":" << peer_id << "\n"; + _groups[group_number].erase(peer_id); _tox_profile_dirty = true; } void ToxClient::onToxGroupSelfJoin(uint32_t group_number) { std::cout << "TCL group self join " << group_number << "\n"; + // ??? _tox_profile_dirty = true; } diff --git a/src/tox_client.hpp b/src/tox_client.hpp index 94527f0..a3dd2a6 100644 --- a/src/tox_client.hpp +++ b/src/tox_client.hpp @@ -11,6 +11,8 @@ #include #include +#include +#include // fwd namespace States { @@ -29,6 +31,22 @@ struct ToxClient { std::string getOwnAddress(void) const; + template + void forEachGroup(FN&& fn) const { + for (const auto& it : _groups) { + fn(it.first); + } + } + + template + void forEachGroupPeer(uint32_t group_number, FN&& fn) const { + if (_groups.count(group_number)) { + for (const uint32_t peer_number : _groups.at(group_number)) { + fn(peer_number); + } + } + } + public: // tox callbacks void onToxSelfConnectionStatus(TOX_CONNECTION connection_status); void onToxFriendRequest(const uint8_t* public_key, std::string_view message); @@ -60,5 +78,8 @@ struct ToxClient { bool _tox_profile_dirty {false}; // set in callbacks std::unique_ptr _state; + + // key groupid, value set of peer ids + std::map> _groups; };