#include "./sha1.hpp" #include "../tox_client.hpp" #include "../hash_utils.hpp" #include #include namespace States { SHA1::SHA1( ToxClient& tcl, mio::mmap_sink&& file_map, const FTInfoSHA1&& sha1_info, const std::vector&& sha1_info_data, //const std::vector&& sha1_info_hash, const SHA1Digest&& sha1_info_hash, std::vector&& have_chunk ) : StateI(tcl), _file_map(std::move(file_map)), _sha1_info(std::move(sha1_info)), _sha1_info_data(std::move(sha1_info_data)), _sha1_info_hash(std::move(sha1_info_hash)), _have_chunk(std::move(have_chunk)) { assert(_have_chunk.size() == _sha1_info.chunks.size()); _have_all = true; _have_count = 0; for (size_t i = 0; i < _have_chunk.size(); i++) { if (_have_chunk[i]) { _have_count++; } else { _have_all = false; _chunk_want_queue.push_back(i); } } // if not sequential, shuffle _chunk_want_queue // build lookup table for (size_t i = 0; i < _sha1_info.chunks.size(); i++) { _chunk_hash_to_index[_sha1_info.chunks[i]] = i; } } bool SHA1::iterate(float delta) { { // timer and timeouts // info for (auto it = _transfers_requested_info.begin(); it != _transfers_requested_info.end();) { float& time_since_remove_activity = std::get(*it); time_since_remove_activity += delta; // if we have not heard for 10sec, timeout if (time_since_remove_activity >= 10.f) { std::cerr << "SHA1 info tansfer timed out " << std::get<0>(*it) << ":" << std::get<1>(*it) << "." << int(std::get<2>(*it)) << "\n"; it = _transfers_requested_info.erase(it); } else { it++; } } // chunk sending for (auto it = _transfers_sending_chunk.begin(); it != _transfers_sending_chunk.end();) { float& time_since_remove_activity = std::get(*it); time_since_remove_activity += delta; // if we have not heard for 10sec, timeout if (time_since_remove_activity >= 10.f) { std::cerr << "SHA1 sending chunk tansfer timed out " << std::get<0>(*it) << ":" << std::get<1>(*it) << "." << int(std::get<2>(*it)) << "\n"; it = _transfers_sending_chunk.erase(it); } else { it++; } } // chunk receiving for (auto it = _transfers_receiving_chunk.begin(); it != _transfers_receiving_chunk.end();) { float& time_since_remove_activity = std::get(*it); time_since_remove_activity += delta; // if we have not heard for 10sec, timeout if (time_since_remove_activity >= 10.f) { std::cerr << "SHA1 receiving chunk tansfer timed out " << std::get<0>(*it) << ":" << std::get<1>(*it) << "." << int(std::get<2>(*it)) << "\n"; _chunk_want_queue.push_front(std::get<4>(*it)); // put it back it = _transfers_receiving_chunk.erase(it); } else { it++; } } // sent requests for (auto it = _chunks_requested.begin(); it != _chunks_requested.end();) { it->second += delta; // if we have not heard for 15sec, timeout if (it->second >= 15.f) { _chunk_want_queue.push_front(it->first); // put it back it = _chunks_requested.erase(it); } else { it++; } } } // if we have not reached the total cap for transfers if (_transfers_requested_info.size() + _transfers_sending_chunk.size() < _max_concurrent_out) { // for each peer? transfer cap per peer? // first check requests for info if (!_queue_requested_info.empty()) { // send init to _queue_requested_info const auto [group_number, peer_number] = _queue_requested_info.front(); uint8_t transfer_id {0}; _tcl.sendFT1InitPrivate( group_number, peer_number, NGC_FT1_file_kind::HASH_SHA1_INFO, _sha1_info_hash.data.data(), _sha1_info_hash.size(), // id (info hash) _sha1_info_data.size(), // "file_size" transfer_id ); _transfers_requested_info.push_back({ group_number, peer_number, transfer_id, 0.f }); _queue_requested_info.pop_front(); } else if (!_queue_requested_chunk.empty()) { // then check for chunk requests const auto [group_number, peer_number, chunk_hash] = _queue_requested_chunk.front(); size_t chunk_index = chunkIndex(chunk_hash).value(); size_t chunk_file_size = chunkSize(chunk_index); uint8_t transfer_id {0}; _tcl.sendFT1InitPrivate( group_number, peer_number, NGC_FT1_file_kind::HASH_SHA1_CHUNK, chunk_hash.data.data(), chunk_hash.size(), // id (chunk hash) chunk_file_size, transfer_id ); _transfers_sending_chunk.push_back({ group_number, peer_number, transfer_id, 0.f, chunk_index }); _queue_requested_chunk.pop_front(); } } if (!_have_all && !_chunk_want_queue.empty() && _chunks_requested.size() + _transfers_receiving_chunk.size() < _max_concurrent_in) { // send out request, no burst tho std::vector> target_peers; _tcl.forEachGroup([&target_peers, this](uint32_t group_number) { _tcl.forEachGroupPeer(group_number, [&target_peers, group_number](uint32_t peer_number) { target_peers.push_back({group_number, peer_number}); }); }); if (!target_peers.empty()) { //if (_distrib.max() != target_peers.size()) { //std::uniform_int_distribution new_dist{0, target_peers.size()-1}; //_distrib.param(new_dist.param()); //} //size_t target_index = _distrib(_rng); size_t target_index = _rng()%target_peers.size(); auto [group_number, peer_number] = target_peers.at(target_index); size_t chunk_index = _chunk_want_queue.front(); _chunks_requested[chunk_index] = 0.f; _chunk_want_queue.pop_front(); _tcl.sendFT1RequestPrivate(group_number, peer_number, NGC_FT1_file_kind::HASH_SHA1_CHUNK, _sha1_info.chunks[chunk_index].data.data(), 20); } } // TODO: unmap and remap the file every couple of minutes to keep ram usage down? // TODO: when to stop? return false; } std::unique_ptr SHA1::nextState(void) { return nullptr; } // sha1_info void SHA1::onFT1ReceiveRequestSHA1Info(uint32_t group_number, uint32_t peer_number, const uint8_t* file_id, size_t file_id_size) { // start tf (init) for sha1_info if (file_id_size != _sha1_info_hash.size()) { std::cerr << "SHA1 got request for sha1_info of wrong size!!\n"; return; } SHA1Digest requested_hash(file_id, file_id_size); if (requested_hash != _sha1_info_hash) { std::cout << "SHA1 ignoring different info request " << requested_hash << "\n"; return; } // same hash, should respond // prio higher then chunks? // add to requested queue queueUpRequestInfo(group_number, peer_number); } bool SHA1::onFT1ReceiveInitSHA1Info(uint32_t, uint32_t, const uint8_t*, size_t, const uint8_t, const size_t) { // no, in this state we have init return false; } void SHA1::onFT1ReceiveDataSHA1Info(uint32_t, uint32_t, uint8_t, size_t, const uint8_t*, size_t) { // no, in this state we have init assert(false && "ft should have said dropped this for us!"); } void SHA1::onFT1SendDataSHA1Info(uint32_t group_number, uint32_t peer_number, uint8_t transfer_id, size_t data_offset, uint8_t* data, size_t data_size) { // should all be fine (void)group_number; (void)peer_number; (void)transfer_id; for (size_t i = 0; i < data_size; i++) { data[i] = _sha1_info_data.at(data_offset+i); } // TODO: sub optimal for (auto it = _transfers_requested_info.begin(); it != _transfers_requested_info.end(); it++) { if (std::get<0>(*it) == group_number && std::get<1>(*it) == peer_number && std::get<2>(*it) == transfer_id) { std::get(*it) = 0.f; break; } } // if last data block if (data_offset + data_size == _sha1_info_data.size()) { // this transfer is "done" (ft still could have to retransfer) for (auto it = _transfers_requested_info.cbegin(); it != _transfers_requested_info.cend(); it++) { if (std::get<0>(*it) == group_number && std::get<1>(*it) == peer_number && std::get<2>(*it) == transfer_id) { std::cout << "SHA1 info tansfer finished " << std::get<0>(*it) << ":" << std::get<1>(*it) << "." << std::get<2>(*it) << "\n"; _transfers_requested_info.erase(it); break; } } } } // sha1_chunk void SHA1::onFT1ReceiveRequestSHA1Chunk(uint32_t group_number, uint32_t peer_number, const uint8_t* file_id, size_t file_id_size) { if (file_id_size != 20) { std::cerr << "SHA1 got request for sha1_chunk of wrong size!!\n"; return; } SHA1Digest requested_hash(file_id, file_id_size); if (!haveChunk(requested_hash)) { // we dont have, ignore return; } queueUpRequestChunk(group_number, peer_number, requested_hash); } bool SHA1::onFT1ReceiveInitSHA1Chunk(uint32_t group_number, uint32_t peer_number, const uint8_t* file_id, size_t file_id_size, const uint8_t transfer_id, const size_t file_size) { if (_transfers_receiving_chunk.size() >= _max_concurrent_in) { // reject, max tf in return false; } if (file_id_size != 20) { std::cerr << "SHA1 got request for sha1_chunk of wrong size!!\n"; return false; } SHA1Digest incomming_hash(file_id, file_id_size); if (haveChunk(incomming_hash)) { std::cout << "SHA1 ignoring init for chunk we already have " << incomming_hash << "\n"; return false; } auto chunk_i_opt = chunkIndex(incomming_hash); if (!chunk_i_opt.has_value()) { std::cout << "SHA1 ignoring init for unrelated chunk " << incomming_hash << "\n"; return false; } size_t chunk_index = chunk_i_opt.value(); // check transfers for (const auto& it : _transfers_receiving_chunk) { if (std::get<4>(it) == chunk_index) { // already in transition return false; } } _transfers_receiving_chunk.push_back( std::make_tuple( group_number, peer_number, transfer_id, 0.f, chunk_index ) ); // remove form requests _chunks_requested.erase(chunk_index); return true; } void SHA1::onFT1ReceiveDataSHA1Chunk(uint32_t group_number, uint32_t peer_number, uint8_t transfer_id, size_t data_offset, const uint8_t* data, size_t data_size) { // check transfers for (auto it = _transfers_receiving_chunk.begin(); it != _transfers_receiving_chunk.end(); it++) { if (std::get<0>(*it) == group_number && std::get<1>(*it) == peer_number && std::get<2>(*it) == transfer_id) { std::get(*it) = 0.f; // time const size_t chunk_index = std::get<4>(*it); size_t file_offset = chunk_index * _sha1_info.chunk_size; // TODO: optimize for (size_t i = 0; i < data_size; i++) { _file_map[file_offset+data_offset+i] = data[i]; } size_t chunk_file_size = chunkSize(chunk_index); // if last data block if (data_offset + data_size == chunk_file_size) { // hash and verify SHA1Digest test_hash = hash_sha1(_file_map.data()+file_offset, chunk_file_size); if (test_hash != _sha1_info.chunks[chunk_index]) { std::cerr << "SHA1 received chunks's hash does not match!, discarding\n"; _transfers_receiving_chunk.erase(it); _chunk_want_queue.push_front(chunk_index); // put back in queue break; } _have_chunk[chunk_index] = true; _have_count++; _have_all = _have_count == _sha1_info.chunks.size(); std::cout << "SHA1 chunk received " << std::get<0>(*it) << ":" << std::get<1>(*it) << "." << int(std::get<2>(*it)) << " " << chunk_index << " (" << 100.f * float(_have_count) / _sha1_info.chunks.size() << "%)\n"; _transfers_receiving_chunk.erase(it); } break; } } } void SHA1::onFT1SendDataSHA1Chunk(uint32_t group_number, uint32_t peer_number, uint8_t transfer_id, size_t data_offset, uint8_t* data, size_t data_size) { // TODO: sub optimal for (auto it = _transfers_sending_chunk.begin(); it != _transfers_sending_chunk.end(); it++) { if (std::get<0>(*it) == group_number && std::get<1>(*it) == peer_number && std::get<2>(*it) == transfer_id) { std::get(*it) = 0.f; // time const size_t chunk_index = std::get<4>(*it); size_t file_offset = chunk_index * _sha1_info.chunk_size; // TODO: optimize for (size_t i = 0; i < data_size; i++) { data[i] = _file_map[file_offset+data_offset+i]; } // if last data block if (data_offset + data_size == chunkSize(chunk_index)) { std::cout << "SHA1 chunk sent " << std::get<0>(*it) << ":" << std::get<1>(*it) << "." << int(std::get<2>(*it)) << " " << chunk_index << "\n"; _transfers_sending_chunk.erase(it); } break; } } } void SHA1::queueUpRequestInfo(uint32_t group_number, uint32_t peer_number) { // check ongoing transfers for dup for (const auto& it : _transfers_requested_info) { // if already in queue if (std::get<0>(it) == group_number && std::get<1>(it) == peer_number) { return; } } for (auto& [i_g, i_p] : _queue_requested_info) { // if already in queue if (i_g == group_number && i_p == peer_number) { return; } } // not in queue yet _queue_requested_info.push_back(std::make_pair(group_number, peer_number)); } void SHA1::queueUpRequestChunk(uint32_t group_number, uint32_t peer_number, const SHA1Digest& hash) { // TODO: transfers for (auto& [i_g, i_p, i_h] : _queue_requested_chunk) { // if already in queue if (i_g == group_number && i_p == peer_number && i_h == hash) { return; } } // not in queue yet _queue_requested_chunk.push_back(std::make_tuple(group_number, peer_number, hash)); } std::optional SHA1::chunkIndex(const SHA1Digest& hash) const { const auto it = _chunk_hash_to_index.find(hash); if (it != _chunk_hash_to_index.cend()) { return it->second; } else { return std::nullopt; } } size_t SHA1::chunkSize(size_t chunk_index) const { if (chunk_index+1 == _sha1_info.chunks.size()) { // last chunk return _sha1_info.file_size - chunk_index * _sha1_info.chunk_size; } else { return _sha1_info.chunk_size; } } bool SHA1::haveChunk(const SHA1Digest& hash) const { if (_have_all) { return true; } if (auto i_opt = chunkIndex(hash); i_opt.has_value()) { return _have_chunk[i_opt.value()]; } // not part of this file return false; } } // States