From 930c82903115bbea1212c997bf42f353b02f6898 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Mon, 9 Dec 2024 22:58:36 +0100 Subject: [PATCH] rizzler working, more fixes everywhere there still are some crashes that needs workarounds --- CMakeLists.txt | 2 + plugins/plugin_ngchs2.cpp | 4 +- .../ngc_ft1_sha1/re_announce_systems.cpp | 3 +- solanaceae/ngc_ft1_sha1/sha1_ngcft1.cpp | 79 +++- solanaceae/ngc_ft1_sha1/sha1_ngcft1.hpp | 4 + solanaceae/ngc_hs2/ngc_hs2_rizzler.cpp | 433 ++++++++++++++++-- solanaceae/ngc_hs2/ngc_hs2_rizzler.hpp | 8 +- solanaceae/ngc_hs2/ngc_hs2_sigma.cpp | 74 +-- solanaceae/ngc_hs2/ngc_hs2_sigma.hpp | 24 - solanaceae/ngc_hs2/serl.hpp | 32 ++ 10 files changed, 559 insertions(+), 104 deletions(-) create mode 100644 solanaceae/ngc_hs2/serl.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index d77dd15..61a8a4f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -134,6 +134,8 @@ endif() ######################################## add_library(solanaceae_ngchs2 + ./solanaceae/ngc_hs2/serl.hpp + ./solanaceae/ngc_hs2/ts_find_start.hpp ./solanaceae/ngc_hs2/ngc_hs2_sigma.hpp diff --git a/plugins/plugin_ngchs2.cpp b/plugins/plugin_ngchs2.cpp index 1646b23..ce93919 100644 --- a/plugins/plugin_ngchs2.cpp +++ b/plugins/plugin_ngchs2.cpp @@ -3,6 +3,7 @@ #include #include +#include // this hurts #include #include @@ -43,11 +44,12 @@ SOLANA_PLUGIN_EXPORT uint32_t solana_plugin_start(struct SolanaAPI* solana_api) auto* rmm = PLUG_RESOLVE_INSTANCE(RegistryMessageModelI); auto* tcm = PLUG_RESOLVE_INSTANCE(ToxContactModel2); auto* ngcft1 = PLUG_RESOLVE_INSTANCE(NGCFT1); + auto* sha1_ngcft1 = PLUG_RESOLVE_INSTANCE(SHA1_NGCFT1); // static store, could be anywhere tho // construct with fetched dependencies g_ngchs2s = std::make_unique(*cr, *rmm, *tcm, *ngcft1); - g_ngchs2r = std::make_unique(*cr, *rmm, *tcm, *ngcft1, *tox_event_provider_i); + g_ngchs2r = std::make_unique(*cr, *rmm, *tcm, *ngcft1, *tox_event_provider_i, *sha1_ngcft1); // register types PLUG_PROVIDE_INSTANCE(NGCHS2Sigma, plugin_name, g_ngchs2s.get()); diff --git a/solanaceae/ngc_ft1_sha1/re_announce_systems.cpp b/solanaceae/ngc_ft1_sha1/re_announce_systems.cpp index e853ad0..32d1a0c 100644 --- a/solanaceae/ngc_ft1_sha1/re_announce_systems.cpp +++ b/solanaceae/ngc_ft1_sha1/re_announce_systems.cpp @@ -28,7 +28,8 @@ void re_announce( // if not downloading or info incomplete -> remove if (!o.all_of()) { to_remove.push_back(ov); - assert(false && "transfer in broken state"); + // TODO: triggers with hs, figure out why + //assert(false && "transfer in broken state"); return; } diff --git a/solanaceae/ngc_ft1_sha1/sha1_ngcft1.cpp b/solanaceae/ngc_ft1_sha1/sha1_ngcft1.cpp index 94d4aa4..a82eccd 100644 --- a/solanaceae/ngc_ft1_sha1/sha1_ngcft1.cpp +++ b/solanaceae/ngc_ft1_sha1/sha1_ngcft1.cpp @@ -552,6 +552,75 @@ void SHA1_NGCFT1::onSendFileHashFinished(ObjectHandle o, Message3Registry* reg_p updateMessages(o); // nop // TODO: remove } +void SHA1_NGCFT1::constructFileMessageInPlace(Message3Handle msg, NGCFT1_file_kind file_kind, ByteSpan file_id) { + if (file_kind != NGCFT1_file_kind::HASH_SHA1_INFO) { + return; + } + + // check if content exists + const std::vector sha1_info_hash{file_id.cbegin(), file_id.cend()}; + ObjectHandle o; + if (_info_to_content.count(sha1_info_hash)) { + o = _info_to_content.at(sha1_info_hash); + std::cout << "SHA1_NGCFT1: new message has existing content\n"; + } else { + // TODO: backend + o = _mfb.newObject(ByteSpan{sha1_info_hash}); + _info_to_content[sha1_info_hash] = o; + o.emplace(sha1_info_hash); + std::cout << "SHA1_NGCFT1: new message has new content\n"; + } + o.get_or_emplace().messages.push_back(msg); + msg.emplace_or_replace(o); + + // TODO: remove this assumption, this gets very unrelieable with hs + if (const auto* from_c_comp = msg.try_get(); from_c_comp != nullptr && _cr.valid(from_c_comp->c)) { + Contact3Handle c{_cr, from_c_comp->c}; + // HACK: assume the message sender is participating. usually a safe bet. + if (addParticipation(c, o)) { + // something happend, update chunk picker + assert(static_cast(c)); + c.emplace_or_replace(); + } + + // HACK: assume the message sender has all + o.get_or_emplace().others[c] = {true, {}}; + + // TODO: check if public + // since public + if (c.all_of()) { + // TODO: if this is a dummy contact, should it have parent? + o.get_or_emplace().targets.emplace(c.get().parent); + } + } + + // TODO: queue info dl + // TODO: queue info/check if we already have info + if (!o.all_of() && !o.all_of()) { + bool in_info_want {false}; + for (const auto it : _queue_content_want_info) { + if (it == o) { + in_info_want = true; + break; + } + } + if (!in_info_want) { + // TODO: check if already receiving + _queue_content_want_info.push_back(o); + } + } else if (o.all_of()){ + // remove from info want + o.remove(); + + auto it = std::find(_queue_content_want_info.cbegin(), _queue_content_want_info.cend(), o); + if (it != _queue_content_want_info.cend()) { + _queue_content_want_info.erase(it); + } + } + + _os.throwEventUpdate(o); +} + bool SHA1_NGCFT1::onEvent(const ObjectStore::Events::ObjectUpdate& e) { if (!e.e.all_of()) { return false; @@ -868,7 +937,8 @@ bool SHA1_NGCFT1::onEvent(const Events::NGCFT1_recv_init& e) { bool SHA1_NGCFT1::onEvent(const Events::NGCFT1_recv_data& e) { if (!_receiving_transfers.containsPeerTransfer(e.group_number, e.peer_number, e.transfer_id)) { - std::cerr << "SHA1_NGCFT1 warning: unknown transfer " << (int)e.transfer_id << " from " << e.group_number << ":" << e.peer_number << "\n"; + // not ours + //std::cerr << "SHA1_NGCFT1 warning: unknown transfer " << (int)e.transfer_id << " from " << e.group_number << ":" << e.peer_number << "\n"; return false; } @@ -920,7 +990,8 @@ bool SHA1_NGCFT1::onEvent(const Events::NGCFT1_recv_data& e) { bool SHA1_NGCFT1::onEvent(const Events::NGCFT1_send_data& e) { if (!_sending_transfers.containsPeerTransfer(e.group_number, e.peer_number, e.transfer_id)) { - std::cerr << "SHA1_NGCFT1 error: ngcft1 requested data for unknown transfer\n"; + // not ours + //std::cerr << "SHA1_NGCFT1 error: ngcft1 requested data for unknown transfer\n"; return false; } @@ -1228,6 +1299,9 @@ bool SHA1_NGCFT1::onEvent(const Events::NGCFT1_recv_message& e) { rb.try_emplace(self_c, ts); } + constructFileMessageInPlace({reg, new_msg_e}, e.file_kind, {e.file_id, e.file_id_size}); + +#if 0 // check if content exists const auto sha1_info_hash = std::vector{e.file_id, e.file_id+e.file_id_size}; ObjectHandle o; @@ -1282,6 +1356,7 @@ bool SHA1_NGCFT1::onEvent(const Events::NGCFT1_recv_message& e) { o.get_or_emplace().targets.emplace(c.get().parent); _os.throwEventUpdate(o); +#endif _rmm.throwEventConstruct(reg, new_msg_e); diff --git a/solanaceae/ngc_ft1_sha1/sha1_ngcft1.hpp b/solanaceae/ngc_ft1_sha1/sha1_ngcft1.hpp index badbae9..45cea7d 100644 --- a/solanaceae/ngc_ft1_sha1/sha1_ngcft1.hpp +++ b/solanaceae/ngc_ft1_sha1/sha1_ngcft1.hpp @@ -107,6 +107,10 @@ class SHA1_NGCFT1 : public ToxEventI, public RegistryMessageModelEventI, public void onSendFileHashFinished(ObjectHandle o, Message3Registry* reg_ptr, Contact3 c, uint64_t ts); + // construct the file part in a partially constructed message + // TODO: maybe return file object? + void constructFileMessageInPlace(Message3Handle msg, NGCFT1_file_kind file_kind, ByteSpan file_id); + protected: // rmm events (actions) bool sendFilePath(const Contact3 c, std::string_view file_name, std::string_view file_path) override; diff --git a/solanaceae/ngc_hs2/ngc_hs2_rizzler.cpp b/solanaceae/ngc_hs2/ngc_hs2_rizzler.cpp index ce9d3cf..0a7c763 100644 --- a/solanaceae/ngc_hs2/ngc_hs2_rizzler.cpp +++ b/solanaceae/ngc_hs2/ngc_hs2_rizzler.cpp @@ -1,28 +1,95 @@ #include "./ngc_hs2_rizzler.hpp" -#include -#include +#include #include +#include +#include +#include +#include +#include #include #include +#include + +#include + +#include "./serl.hpp" + +#include +#include +#include + #include +// TODO: move to own file +namespace Components { + struct RequestedChatLogs { + struct Entry { + uint64_t ts_start; + uint64_t ts_end; + //std::vector fid; // ? + }; + std::deque list; + bool contains(uint64_t ts_start, uint64_t ts_end); + void addRequest(uint64_t ts_start, uint64_t ts_end); + }; + + struct RunningChatLogs { + struct Entry { + uint64_t ts_start; + uint64_t ts_end; + std::vector data; + float last_activity {0.f}; + }; + // list of transfers + entt::dense_map list; + }; + + bool RequestedChatLogs::contains(uint64_t ts_start, uint64_t ts_end) { + auto it = std::find_if(list.cbegin(), list.cend(), [ts_start, ts_end](const auto& value) { + return value.ts_start == ts_start && value.ts_end == ts_end; + }); + return it != list.cend(); + } + + void RequestedChatLogs::addRequest(uint64_t ts_start, uint64_t ts_end) { + if (contains(ts_start, ts_end)) { + return; // pre existing + } + list.push_back(Entry{ts_start, ts_end}); + } + +} // Components + +// TODO: move to contact reg? +static Contact3 findContactByID(Contact3Registry& cr, const std::vector& id) { + // TODO: id lookup table, this is very inefficent + for (const auto& [c_it, id_it] : cr.view().each()) { + if (id == id_it.data) { + return c_it; + } + } + + return entt::null; +} + NGCHS2Rizzler::NGCHS2Rizzler( Contact3Registry& cr, RegistryMessageModelI& rmm, ToxContactModel2& tcm, NGCFT1& nft, - ToxEventProviderI& tep + ToxEventProviderI& tep, + SHA1_NGCFT1& sha1_nft ) : _cr(cr), _rmm(rmm), _tcm(tcm), _nft(nft), _nftep_sr(_nft.newSubRef(this)), - _tep_sr(tep.newSubRef(this)) - + _tep_sr(tep.newSubRef(this)), + _sha1_nft(sha1_nft) { _nftep_sr .subscribe(NGCFT1_Event::recv_init) @@ -46,18 +113,23 @@ float NGCHS2Rizzler::iterate(float delta) { continue; } - if (!_cr.all_of(it->first)) { + const Contact3Handle c {_cr, it->first}; + + if (!c.all_of()) { // peer nolonger online it = _request_queue.erase(it); continue; } - const auto [group_number, peer_number] = _cr.get(it->first); + const auto [group_number, peer_number] = c.get(); // now in sec const uint64_t ts_now = Message::getTimeMS()/1000; - if (sendRequest(group_number, peer_number, ts_now, ts_now-(60*60*48))) { + const uint64_t ts_start = ts_now; + const uint64_t ts_end = ts_now-(60*60*48); + + if (sendRequest(group_number, peer_number, ts_start, ts_end)) { // TODO: requeue // TODO: segment // TODO: dont request already received ranges @@ -73,6 +145,8 @@ float NGCHS2Rizzler::iterate(float delta) { //std::cout << "ZOX #### requeued request in " << it->second.delay << "s\n"; + auto& rcl = c.get_or_emplace(); + rcl.addRequest(ts_start, ts_end); } else { // on failure, assume disconnected } @@ -84,32 +158,6 @@ float NGCHS2Rizzler::iterate(float delta) { return 1000.f; } -template -static uint64_t deserlSimpleType(ByteSpan bytes) { - if (bytes.size < sizeof(Type)) { - throw int(1); - } - - Type value{}; - - for (size_t i = 0; i < sizeof(Type); i++) { - value |= Type(bytes[i]) << (i*8); - } - - return value; -} - -static uint64_t deserlTS(ByteSpan ts_bytes) { - return deserlSimpleType(ts_bytes); -} - -template -static void serlSimpleType(std::vector& bytes, const Type& value) { - for (size_t i = 0; i < sizeof(Type); i++) { - bytes.push_back(uint8_t(value >> (i*8) & 0xff)); - } -} - bool NGCHS2Rizzler::sendRequest( uint32_t group_number, uint32_t peer_number, uint64_t ts_start, uint64_t ts_end @@ -132,17 +180,324 @@ bool NGCHS2Rizzler::sendRequest( ); } +void NGCHS2Rizzler::handleMsgPack(Contact3Handle sync_by_c, const std::vector& data) { + assert(sync_by_c); + + auto* reg_ptr = _rmm.get(sync_by_c); + if (reg_ptr == nullptr) { + std::cerr << "NGCHS2Rizzler error: group without msg reg\n"; + return; + } + + Message3Registry& reg = *reg_ptr; + + uint64_t now_ts = Message::getTimeMS(); + + std::cout << "NGCHS2Rizzler: start parsing msgpack chatlog from " << entt::to_integral(sync_by_c.entity()) << "\n"; + try { + const auto j = nlohmann::json::from_msgpack(data); + if (!j.is_array()) { + std::cerr << "NGCHS2Rizzler error: chatlog not array\n"; + return; + } + + std::cout << "NGCHS2Rizzler: chatlog has " << j.size() << " entries\n"; + + for (const auto j_entry : j) { + try { + // deci seconds + uint64_t ts = j_entry.at("ts"); + // TODO: check against ts range + + ts *= 100; // convert to ms + + const auto& j_ppk = j_entry.at("ppk"); + + uint32_t mid = j_entry.at("mid"); + + const std::string& type = j_entry.at("msgtype"); + + if ( + !(j_entry.count("text")) && + !(j_entry.count("fkind") && j_entry.count("fid")) + ) { + std::cerr << "NGCHS2Rizzler error: msg neither contains text nor file fields\n"; + continue; + } + + if ( + type != "text" && type != "action" && + type != "file" + ) { + std::cerr << "NGCHS2Rizzler error: unknown entry '" << j_entry.dump() << "'\n"; + continue; + } + + Contact3 from_c{entt::null}; + { // from_c + std::vector id; + if (j_ppk.is_binary()) { + id = j_ppk.get_binary(); + } else { + j_ppk.at("bytes").get_to(id); + } + + from_c = findContactByID(_cr, id); + if (!_cr.valid(from_c)) { + // create sparse contact with id only + from_c = _cr.create(); + _cr.emplace_or_replace(from_c, id); + } + } + + // TODO: from_c perm check + // hard to do without numbers + + Message3Handle new_real_msg{reg, reg.create()}; + + new_real_msg.emplace(from_c); + new_real_msg.emplace(sync_by_c.get().parent); + + new_real_msg.emplace(mid); + + if (type == "text" || type == "action") { + bool is_action = type == "action"; + const std::string& text = j_entry.at("text"); + + new_real_msg.emplace(text); + + if (is_action) { + new_real_msg.emplace(); + } +#if 0 + std::cout + << "msg ts:" << ts + //<< " ppk:" << j_ppk + << " mid:" << mid + << " type:" << type + << " text:" << text + << "\n" + ; +#endif + } else if (type == "file") { + uint32_t fkind = j_entry.at("fkind"); + + const auto& j_fid = j_entry.at("fid"); + + std::vector fid; + if (j_fid.is_binary()) { + fid = j_fid.get_binary(); + } else { + j_fid.at("bytes").get_to(fid); + } + + if (fkind == (uint32_t)NGCFT1_file_kind::HASH_SHA1_INFO) { + _sha1_nft.constructFileMessageInPlace( + new_real_msg, + NGCFT1_file_kind::HASH_SHA1_INFO, + ByteSpan{fid} + ); + } else { + std::cerr << "NGCHS2Rizzler error: unknown file kind " << fkind << "\n"; + } + +#if 0 + std::cout + << "msg ts:" << ts + //<< " ppk:" << j_ppk + << " mid:" << mid + << " type:" << type + << " fkind:" << fkind + << " fid:" << j_fid + << "\n" + ; +#endif + } + + // now check against pre existing + // TODO: dont do this afterwards + Message3Handle dup_msg{}; + { // check preexisting + // get comparator from contact + const Contact3Handle reg_c {_cr, reg.ctx().get()}; + if (reg_c.all_of()) { + auto& comp = reg_c.get().comp; + // walking EVERY existing message OOF + // this needs optimizing + for (const Message3 other_msg : reg.view()) { + if (other_msg == new_real_msg) { + continue; // skip self + } + + if (comp({reg, other_msg}, new_real_msg)) { + // dup + dup_msg = {reg, other_msg}; + break; + } + } + } // else, default heuristic?? + } + + Message3Handle new_msg = new_real_msg; + + if (dup_msg) { + reg.destroy(new_msg); + new_msg = dup_msg; + } + + { // by whom + auto& synced_by = new_msg.get_or_emplace().ts; + // dont overwrite + synced_by.try_emplace(sync_by_c, now_ts); + } + + { // now we also know they got the message + auto& list = new_msg.get_or_emplace().ts; + // dont overwrite + list.try_emplace(sync_by_c, now_ts); + } + + if (new_msg == dup_msg) { + // TODO: maybe update a timestamp? + _rmm.throwEventUpdate(reg, new_msg); + } else { + // pure new msg + + new_msg.emplace(now_ts); + new_msg.emplace(ts); + new_msg.emplace(ts); // reactive? + + new_msg.emplace(); + _rmm.throwEventConstruct(reg, new_msg); + } + } catch (...) { + std::cerr << "NGCHS2Rizzler error: parsing entry '" << j_entry.dump() << "'\n"; + } + } + } catch (...) { + std::cerr << "NGCHS2Rizzler error: failed parsing data as msgpack\n"; + } +} + bool NGCHS2Rizzler::onEvent(const Events::NGCFT1_recv_init& e) { + if (e.file_kind != NGCFT1_file_kind::HS2_RANGE_TIME_MSGPACK) { + return false; // not for us + } + std::cout << "NGCHS2Rizzler: recv_init " << e.group_number << ":" << e.peer_number << "." << (int)e.transfer_id << "\n"; - return false; + + auto c = _tcm.getContactGroupPeer(e.group_number, e.peer_number); + if (!c) { + return false; // huh? + } + + if (!c.all_of()) { + return false; + } + + // parse start end + // TODO: extract + ByteSpan fid{e.file_id, e.file_id_size}; + // TODO: better size check + if (fid.size != sizeof(uint64_t)+sizeof(uint64_t)) { + std::cerr << "NGCHS2S error: range not lange enough\n"; + return true; + } + + // seconds + uint64_t ts_start{0}; + uint64_t ts_end{0}; + + // parse + try { + ByteSpan ts_start_bytes{fid.ptr, sizeof(uint64_t)}; + ts_start = deserlTS(ts_start_bytes); + + ByteSpan ts_end_bytes{ts_start_bytes.ptr+ts_start_bytes.size, sizeof(uint64_t)}; + ts_end = deserlTS(ts_end_bytes); + } catch (...) { + std::cerr << "NGCHS2R error: failed to parse range\n"; + return true; + } + + if (ts_end >= ts_start) { + std::cerr << "NGCHS2R error: end not < start\n"; + return true; + } + + auto& reqcl = c.get(); + + if (!reqcl.contains(ts_start, ts_end)) { + // warn? + return true; + } + + auto& rnncl = c.get_or_emplace(); + + auto& transfer = rnncl.list[e.transfer_id]; + transfer.data.reserve(e.file_size); // danger? + transfer.last_activity = 0.f; + transfer.ts_start = ts_start; + transfer.ts_end = ts_end; + + e.accept = true; + + return true; } -bool NGCHS2Rizzler::onEvent(const Events::NGCFT1_recv_data&) { - return false; +bool NGCHS2Rizzler::onEvent(const Events::NGCFT1_recv_data& e) { + auto c = _tcm.getContactGroupPeer(e.group_number, e.peer_number); + if (!c) { + return false; + } + + if (!c.all_of()) { + return false; // not ours + } + + auto& rnncl = c.get(); + if (!rnncl.list.count(e.transfer_id)) { + return false; // not ours + } + + std::cout << "NGCHS2Rizzler: recv_data " << e.group_number << ":" << e.peer_number << "." << (int)e.transfer_id << " " << e.data_size << "@" << e.data_offset << "\n"; + + auto& transfer = rnncl.list.at(e.transfer_id); + transfer.data.resize(e.data_offset+e.data_size); + std::memcpy(&transfer.data[e.data_offset], e.data, e.data_size); + + transfer.last_activity = 0.f; + + return true; } -bool NGCHS2Rizzler::onEvent(const Events::NGCFT1_recv_done&) { - return false; +bool NGCHS2Rizzler::onEvent(const Events::NGCFT1_recv_done& e) { + auto c = _tcm.getContactGroupPeer(e.group_number, e.peer_number); + // TODO: fix disconnect + if (!c) { + return false; + } + + if (!c.all_of()) { + return false; // not ours + } + + auto& rnncl = c.get(); + if (!rnncl.list.count(e.transfer_id)) { + return false; // not ours + } + + std::cout << "NGCHS2Rizzler: recv_done " << e.group_number << ":" << e.peer_number << "." << (int)e.transfer_id << "\n"; + { + auto& transfer = rnncl.list.at(e.transfer_id); + + // use data + // TODO: move out of packet handler + handleMsgPack(c, transfer.data); + } + rnncl.list.erase(e.transfer_id); + + return true; } bool NGCHS2Rizzler::onToxEvent(const Tox_Event_Group_Peer_Join* e) { diff --git a/solanaceae/ngc_hs2/ngc_hs2_rizzler.hpp b/solanaceae/ngc_hs2/ngc_hs2_rizzler.hpp index 253e531..349809d 100644 --- a/solanaceae/ngc_hs2/ngc_hs2_rizzler.hpp +++ b/solanaceae/ngc_hs2/ngc_hs2_rizzler.hpp @@ -4,11 +4,13 @@ #include #include +#include // fwd class ToxContactModel2; class RegistryMessageModelI; + class NGCHS2Rizzler : public ToxEventI, public NGCFT1EventI { Contact3Registry& _cr; RegistryMessageModelI& _rmm; @@ -16,6 +18,7 @@ class NGCHS2Rizzler : public ToxEventI, public NGCFT1EventI { NGCFT1& _nft; NGCFT1EventProviderI::SubscriptionReference _nftep_sr; ToxEventProviderI::SubscriptionReference _tep_sr; + SHA1_NGCFT1& _sha1_nft; // 5s-6s const float _delay_before_first_request_min {5.f}; @@ -39,7 +42,8 @@ class NGCHS2Rizzler : public ToxEventI, public NGCFT1EventI { RegistryMessageModelI& rmm, ToxContactModel2& tcm, NGCFT1& nft, - ToxEventProviderI& tep + ToxEventProviderI& tep, + SHA1_NGCFT1& sha1_nft ); ~NGCHS2Rizzler(void); @@ -52,6 +56,8 @@ class NGCHS2Rizzler : public ToxEventI, public NGCFT1EventI { uint64_t ts_start, uint64_t ts_end ); + void handleMsgPack(Contact3Handle c, const std::vector& data); + protected: bool onEvent(const Events::NGCFT1_recv_init&) override; bool onEvent(const Events::NGCFT1_recv_data&) override; diff --git a/solanaceae/ngc_hs2/ngc_hs2_sigma.cpp b/solanaceae/ngc_hs2/ngc_hs2_sigma.cpp index 2b161ec..fe86eff 100644 --- a/solanaceae/ngc_hs2/ngc_hs2_sigma.cpp +++ b/solanaceae/ngc_hs2/ngc_hs2_sigma.cpp @@ -17,28 +17,52 @@ #include +#include "./serl.hpp" + #include "./ts_find_start.hpp" #include // https://www.youtube.com/watch?v=AdAqsgga3qo +// TODO: move to own file namespace Components { -void IncommingTimeRangeRequestQueue::queueRequest(const TimeRangeRequest& new_request, const ByteSpan fid) { - // TODO: do more than exact dedupe - for (const auto& [time_range, _] : _queue) { - if (time_range.ts_start == new_request.ts_start && time_range.ts_end == new_request.ts_end) { - return; // already enqueued - // TODO: what about fid? - } - } + struct IncommingTimeRangeRequestQueue { + struct Entry { + TimeRangeRequest ir; + std::vector fid; + }; + std::deque _queue; - _queue.emplace_back(Entry{ - new_request, - std::vector{fid.cbegin(), fid.cend()} - }); -} + // we should remove/notadd queued requests + // that are subsets of same or larger ranges + void queueRequest(const TimeRangeRequest& new_request, const ByteSpan fid); + }; + + struct IncommingTimeRangeRequestRunning { + struct Entry { + TimeRangeRequest ir; + std::vector data; // transfer data in memory + float last_activity {0.f}; + }; + entt::dense_map _list; + }; + + void IncommingTimeRangeRequestQueue::queueRequest(const TimeRangeRequest& new_request, const ByteSpan fid) { + // TODO: do more than exact dedupe + for (const auto& [time_range, _] : _queue) { + if (time_range.ts_start == new_request.ts_start && time_range.ts_end == new_request.ts_end) { + return; // already enqueued + // TODO: what about fid? + } + } + + _queue.emplace_back(Entry{ + new_request, + std::vector{fid.cbegin(), fid.cend()} + }); + } } // Components @@ -167,29 +191,6 @@ float NGCHS2Sigma::iterate(float delta) { return 1000.f; } -template -static uint64_t deserlSimpleType(ByteSpan bytes) { - if (bytes.size < sizeof(Type)) { - throw int(1); - } - - Type value{}; - - for (size_t i = 0; i < sizeof(Type); i++) { - value |= Type(bytes[i]) << (i*8); - } - - return value; -} - -//static uint32_t deserlMID(ByteSpan mid_bytes) { -// return deserlSimpleType(mid_bytes); -//} - -static uint64_t deserlTS(ByteSpan ts_bytes) { - return deserlSimpleType(ts_bytes); -} - void NGCHS2Sigma::handleTimeRange(Contact3Handle c, const Events::NGCFT1_recv_request& e) { ByteSpan fid{e.file_id, e.file_id_size}; // TODO: better size check @@ -333,6 +334,7 @@ std::vector NGCHS2Sigma::buildChatLogFileRange(Contact3Handle c, uint64 // HACK: use tox fild_id and file_kind instead!! if (o.all_of()) { + j_entry["msgtype"] = "file"; j_entry["fkind"] = NGCFT1_file_kind::HASH_SHA1_INFO; j_entry["fid"] = nlohmann::json::binary_t{o.get().hash}; } else { diff --git a/solanaceae/ngc_hs2/ngc_hs2_sigma.hpp b/solanaceae/ngc_hs2/ngc_hs2_sigma.hpp index de721b1..6747afb 100644 --- a/solanaceae/ngc_hs2/ngc_hs2_sigma.hpp +++ b/solanaceae/ngc_hs2/ngc_hs2_sigma.hpp @@ -23,30 +23,6 @@ struct TimeRangeRequest { uint64_t ts_end{0}; }; -// TODO: move to own file -namespace Components { - struct IncommingTimeRangeRequestQueue { - struct Entry { - TimeRangeRequest ir; - std::vector fid; - }; - std::deque _queue; - - // we should remove/notadd queued requests - // that are subsets of same or larger ranges - void queueRequest(const TimeRangeRequest& new_request, const ByteSpan fid); - }; - - struct IncommingTimeRangeRequestRunning { - struct Entry { - TimeRangeRequest ir; - std::vector data; // transfer data in memory - float last_activity {0.f}; - }; - entt::dense_map _list; - }; -} // Components - class NGCHS2Sigma : public RegistryMessageModelEventI, public NGCFT1EventI { Contact3Registry& _cr; RegistryMessageModelI& _rmm; diff --git a/solanaceae/ngc_hs2/serl.hpp b/solanaceae/ngc_hs2/serl.hpp new file mode 100644 index 0000000..a49b1f3 --- /dev/null +++ b/solanaceae/ngc_hs2/serl.hpp @@ -0,0 +1,32 @@ +#pragma once + +#include + +#include + +template +static uint64_t deserlSimpleType(ByteSpan bytes) { + if (bytes.size < sizeof(Type)) { + throw int(1); + } + + Type value{}; + + for (size_t i = 0; i < sizeof(Type); i++) { + value |= Type(bytes[i]) << (i*8); + } + + return value; +} + +static uint64_t deserlTS(ByteSpan ts_bytes) { + return deserlSimpleType(ts_bytes); +} + +template +static void serlSimpleType(std::vector& bytes, const Type& value) { + for (size_t i = 0; i < sizeof(Type); i++) { + bytes.push_back(uint8_t(value >> (i*8) & 0xff)); + } +} +