From b9a7c75d200aab3375d34b5c69c6d1aa31d889ed Mon Sep 17 00:00:00 2001 From: Green Sky Date: Thu, 5 Dec 2024 23:06:44 +0100 Subject: [PATCH] hs: progess on the log file generator, fully written but untested ft: timeout changes and wording fixes --- CMakeLists.txt | 1 + external/CMakeLists.txt | 10 ++ solanaceae/ngc_ft1/ngcft1_file_kind.hpp | 2 +- .../ngc_ft1_sha1/receiving_transfers.cpp | 4 +- solanaceae/ngc_ft1_sha1/sha1_ngcft1.cpp | 2 +- solanaceae/ngc_hs2/ngc_hs2_send.cpp | 149 +++++++++++++++++- solanaceae/ngc_hs2/ngc_hs2_send.hpp | 8 +- solanaceae/ngc_hs2/spec_ngc_hs2.md | 2 +- 8 files changed, 168 insertions(+), 10 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 87f5bb1..9683e07 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -56,6 +56,7 @@ target_link_libraries(solanaceae_ngchs2 PUBLIC solanaceae_tox_contacts solanaceae_message3 solanaceae_object_store + nlohmann_json::nlohmann_json ) ######################################## diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt index 01a690c..18d1d56 100644 --- a/external/CMakeLists.txt +++ b/external/CMakeLists.txt @@ -2,3 +2,13 @@ cmake_minimum_required(VERSION 3.9 FATAL_ERROR) add_subdirectory(./sha1) +# we are running a custom msgpack serialization for hs2 +if (NOT TARGET nlohmann_json::nlohmann_json) + FetchContent_Declare(json + URL https://github.com/nlohmann/json/releases/download/v3.11.3/json.tar.xz + URL_HASH SHA256=d6c65aca6b1ed68e7a182f4757257b107ae403032760ed6ef121c9d55e81757d + EXCLUDE_FROM_ALL + ) + FetchContent_MakeAvailable(json) +endif() + diff --git a/solanaceae/ngc_ft1/ngcft1_file_kind.hpp b/solanaceae/ngc_ft1/ngcft1_file_kind.hpp index cc92488..ec90a52 100644 --- a/solanaceae/ngc_ft1/ngcft1_file_kind.hpp +++ b/solanaceae/ngc_ft1/ngcft1_file_kind.hpp @@ -16,7 +16,7 @@ enum class NGCFT1_file_kind : uint32_t { // id: TOX_FILE_ID_LENGTH (32) bytes // this is basically and id and probably not a hash, like the tox friend api // this id can be unique between 2 peers - ID = 8u, + ID = 8u, // TODO: this is actually DATA and 0 // id: hash of the info, like a torrent infohash (using the same hash as the data) // TODO: determain internal format diff --git a/solanaceae/ngc_ft1_sha1/receiving_transfers.cpp b/solanaceae/ngc_ft1_sha1/receiving_transfers.cpp index f7b6f61..9154181 100644 --- a/solanaceae/ngc_ft1_sha1/receiving_transfers.cpp +++ b/solanaceae/ngc_ft1_sha1/receiving_transfers.cpp @@ -7,8 +7,8 @@ void ReceivingTransfers::tick(float delta) { for (auto it = peer_it->second.begin(); it != peer_it->second.end();) { it->second.time_since_activity += delta; - // if we have not heard for 20sec, timeout - if (it->second.time_since_activity >= 20.f) { + // if we have not heard for 60sec, timeout + if (it->second.time_since_activity >= 60.f) { std::cerr << "SHA1_NGCFT1 warning: receiving tansfer timed out " << "." << int(it->first) << "\n"; // TODO: if info, requeue? or just keep the timer comp? - no, timer comp will continue ticking, even if loading //it->second.v diff --git a/solanaceae/ngc_ft1_sha1/sha1_ngcft1.cpp b/solanaceae/ngc_ft1_sha1/sha1_ngcft1.cpp index d51d3fc..3e3a513 100644 --- a/solanaceae/ngc_ft1_sha1/sha1_ngcft1.cpp +++ b/solanaceae/ngc_ft1_sha1/sha1_ngcft1.cpp @@ -868,7 +868,7 @@ bool SHA1_NGCFT1::onEvent(const Events::NGCFT1_recv_init& e) { bool SHA1_NGCFT1::onEvent(const Events::NGCFT1_recv_data& e) { if (!_receiving_transfers.containsPeerTransfer(e.group_number, e.peer_number, e.transfer_id)) { - std::cerr << "SHA1_NGCFT1 waring: unknown transfer " << e.transfer_id << " from " << e.group_number << ":" << e.peer_number << "\n"; + std::cerr << "SHA1_NGCFT1 warning: unknown transfer " << (int)e.transfer_id << " from " << e.group_number << ":" << e.peer_number << "\n"; return false; } diff --git a/solanaceae/ngc_hs2/ngc_hs2_send.cpp b/solanaceae/ngc_hs2/ngc_hs2_send.cpp index dbf8e96..2b02df3 100644 --- a/solanaceae/ngc_hs2/ngc_hs2_send.cpp +++ b/solanaceae/ngc_hs2/ngc_hs2_send.cpp @@ -5,6 +5,19 @@ #include #include +#include +#include +#include + +//#include +// TODO: this is kinda bad, needs improvement +// use tox fileid/filekind instead ! +#include +#include + +#include + +#include #include @@ -40,10 +53,7 @@ NGCHS2Send::NGCHS2Send( { _nftep_sr .subscribe(NGCFT1_Event::recv_request) - //.subscribe(NGCFT1_Event::recv_init) // we only send init - //.subscribe(NGCFT1_Event::recv_data) // we only send data .subscribe(NGCFT1_Event::send_data) - //.subscribe(NGCFT1_Event::recv_done) .subscribe(NGCFT1_Event::send_done) ; } @@ -189,6 +199,139 @@ void NGCHS2Send::handleSingleMessage(Contact3Handle c, const Events::NGCFT1_recv } #endif +std::vector NGCHS2Send::buildHSFileRange(Contact3Handle c, uint64_t ts_start, uint64_t ts_end) { + const Message3Registry* reg_ptr = static_cast(_rmm).get(c); + if (reg_ptr == nullptr) { + return {}; + } + const Message3Registry& msg_reg = *reg_ptr; + + + if (msg_reg.storage() == nullptr) { + // nothing to do here + return {}; + } + + std::cout << "!!!! starting msg ts search, ts_start:" << ts_start << " ts_end:" << ts_end << "\n"; + + auto ts_view = msg_reg.view(); + + // we iterate "forward", so from newest to oldest + + // start is the newest ts + auto ts_start_it = ts_view.end(); // start invalid + // end is the oldest ts + // + { // binary search for first value not newer than start ts + // -> first value smaller than start ts + auto res = std::lower_bound( + ts_view.begin(), ts_view.end(), + ts_start, + [&ts_view](const auto& a, const auto& b) { + const auto& [a_comp] = ts_view.get(a); + return a_comp.ts > b; // > bc ts is sorted high to low? + } + ); + + if (res != ts_view.end()) { + const auto& [ts_comp] = ts_view.get(*res); + std::cout << "!!!! first value not newer than start ts is " << ts_comp.ts << "\n"; + ts_start_it = res; + } else { + std::cout << "!!!! no first value not newer than start ts\n"; + } + } + + // we only search for the start point, because we walk to the end anyway + + auto j_array = nlohmann::json::array_t{}; + + // hmm + // maybe use other view or something? + for (auto it = ts_start_it; it != ts_view.end(); it++) { + const auto e = *it; + const auto& [ts_comp] = ts_view.get(e); + + if (ts_comp.ts > ts_start) { + std::cerr << "!!!! msg ent in view too new\n"; + continue; + } else if (ts_comp.ts < ts_end) { + // too old, we hit the end of the range + break; + } + + if (!msg_reg.all_of< + Message::Components::ContactFrom, + Message::Components::ContactTo, + Message::Components::ToxGroupMessageID + >(e)) { + continue; // ?? + } + if (!msg_reg.any_of(e)) { + continue; // skip + } + + + const auto& [c_from_c, c_to_c] = msg_reg.get(e); + + if (c_to_c.c != c) { + // message was not public + continue; + } + + if (!_cr.valid(c_from_c.c)) { + continue; // ??? + } + + Contact3Handle c_from{_cr, c_from_c.c}; + + if (!c_from.all_of()) { + continue; // ??? + } + + if (_only_send_self_observed && msg_reg.all_of(e) && c.all_of()) { + if (!msg_reg.get(e).ts.count(c.get().self)) { + continue; // did not observe ourselfs, skip + } + } + + auto j_entry = nlohmann::json::object_t{}; + + j_entry["ts"] = ts_comp.ts/100; // millisec -> decisec + { + const auto& ppk_ref = c_from.get().peer_key.data; + j_entry["ppk"] = nlohmann::json::binary_t{std::vector{ppk_ref.cbegin(), ppk_ref.cend()}}; + } + j_entry["mid"] = msg_reg.get(e).id; + + if (msg_reg.all_of(e)) { + if (msg_reg.all_of(e)) { + j_entry["msgtype"] = "action"; // TODO: textaction? + } else { + j_entry["msgtype"] = "text"; + } + j_entry["text"] = msg_reg.get(e).text; + } else if (msg_reg.any_of(e)) { + const auto& o = msg_reg.get(e).o; + if (!o) { + continue; + } + + // HACK: use tox fild_id and file_kind instead!! + if (o.all_of()) { + j_entry["fkind"] = NGCFT1_file_kind::HASH_SHA1_INFO; + j_entry["fid"] = nlohmann::json::binary_t{o.get().hash}; + } else { + continue; // unknown file type + } + } + + j_array.push_back(j_entry); + } + + return nlohmann::json::to_msgpack(j_array); +} + bool NGCHS2Send::onEvent(const Message::Events::MessageConstruct&) { return false; } diff --git a/solanaceae/ngc_hs2/ngc_hs2_send.hpp b/solanaceae/ngc_hs2/ngc_hs2_send.hpp index a1657fc..40775d3 100644 --- a/solanaceae/ngc_hs2/ngc_hs2_send.hpp +++ b/solanaceae/ngc_hs2/ngc_hs2_send.hpp @@ -51,10 +51,10 @@ class NGCHS2Send : public RegistryMessageModelEventI, public NGCFT1EventI { float _iterate_heat {0.f}; constexpr static float _iterate_cooldown {1.22f}; // sec - // open/running info requests (by c) + // open/running range requests (by c) // comp on peer c - // open/running info responses (by c) + // open/running range responses (by c) // comp on peer c // limit to 2 uploads per peer simultaniously @@ -79,6 +79,10 @@ class NGCHS2Send : public RegistryMessageModelEventI, public NGCFT1EventI { void handleTimeRange(Contact3Handle c, const Events::NGCFT1_recv_request&); + // msg reg contact + // time ranges + std::vector buildHSFileRange(Contact3Handle c, uint64_t ts_start, uint64_t ts_end); + protected: bool onEvent(const Message::Events::MessageConstruct&) override; bool onEvent(const Message::Events::MessageUpdated&) override; diff --git a/solanaceae/ngc_hs2/spec_ngc_hs2.md b/solanaceae/ngc_hs2/spec_ngc_hs2.md index 8071144..79e0ed9 100644 --- a/solanaceae/ngc_hs2/spec_ngc_hs2.md +++ b/solanaceae/ngc_hs2/spec_ngc_hs2.md @@ -92,7 +92,7 @@ Msgpack array of messages. - ppk | 32bytes - mid | 16bit - msgtype | enum (string or number?) - - if text/textaction | + - if text/action | - text | string | maybe byte array instead? - if file | - fkind | 32bit enum | is this right?