hs: progess on the log file generator, fully written but untested

ft: timeout changes and wording fixes

CMakeLists.txt

@@ -56,6 +56,7 @@ target_link_libraries(solanaceae_ngchs2 PUBLIC
 	solanaceae_tox_contacts
 	solanaceae_message3
 	solanaceae_object_store
+	nlohmann_json::nlohmann_json
 )
 
 ########################################

external/CMakeLists.txt

@@ -2,3 +2,13 @@ cmake_minimum_required(VERSION 3.9 FATAL_ERROR)
 
 add_subdirectory(./sha1)
 
+# we are running a custom msgpack serialization for hs2
+if (NOT TARGET nlohmann_json::nlohmann_json)
+	FetchContent_Declare(json
+		URL https://github.com/nlohmann/json/releases/download/v3.11.3/json.tar.xz
+		URL_HASH SHA256=d6c65aca6b1ed68e7a182f4757257b107ae403032760ed6ef121c9d55e81757d
+		EXCLUDE_FROM_ALL
+	)
+	FetchContent_MakeAvailable(json)
+endif()
+

solanaceae/ngc_ft1/ngcft1_file_kind.hpp

@@ -16,7 +16,7 @@ enum class NGCFT1_file_kind : uint32_t {
 	// id: TOX_FILE_ID_LENGTH (32) bytes
 	// this is basically and id and probably not a hash, like the tox friend api
 	// this id can be unique between 2 peers
-	ID = 8u,
+	ID = 8u, // TODO: this is actually DATA and 0
 
 	// id: hash of the info, like a torrent infohash (using the same hash as the data)
 	// TODO: determain internal format

solanaceae/ngc_ft1_sha1/receiving_transfers.cpp

@@ -7,8 +7,8 @@ void ReceivingTransfers::tick(float delta) {
 		for (auto it = peer_it->second.begin(); it != peer_it->second.end();) {
 			it->second.time_since_activity += delta;
 
-			// if we have not heard for 20sec, timeout
+			// if we have not heard for 60sec, timeout
-			if (it->second.time_since_activity >= 20.f) {
+			if (it->second.time_since_activity >= 60.f) {
 				std::cerr << "SHA1_NGCFT1 warning: receiving tansfer timed out " << "." << int(it->first) << "\n";
 				// TODO: if info, requeue? or just keep the timer comp? - no, timer comp will continue ticking, even if loading
 				//it->second.v

solanaceae/ngc_ft1_sha1/sha1_ngcft1.cpp

@@ -868,7 +868,7 @@ bool SHA1_NGCFT1::onEvent(const Events::NGCFT1_recv_init& e) {
 
 bool SHA1_NGCFT1::onEvent(const Events::NGCFT1_recv_data& e) {
 	if (!_receiving_transfers.containsPeerTransfer(e.group_number, e.peer_number, e.transfer_id)) {
-		std::cerr << "SHA1_NGCFT1 waring: unknown transfer " << e.transfer_id << " from " << e.group_number << ":" << e.peer_number << "\n";
+		std::cerr << "SHA1_NGCFT1 warning: unknown transfer " << (int)e.transfer_id << " from " << e.group_number << ":" << e.peer_number << "\n";
 		return false;
 	}
 

solanaceae/ngc_hs2/ngc_hs2_send.cpp

@@ -5,6 +5,19 @@
 #include <solanaceae/tox_contacts/tox_contact_model2.hpp>
 
 #include <solanaceae/contact/components.hpp>
+#include <solanaceae/tox_contacts/components.hpp>
+#include <solanaceae/message3/components.hpp>
+#include <solanaceae/tox_messages/msg_components.hpp>
+
+//#include <solanaceae/tox_messages/obj_components.hpp>
+// TODO: this is kinda bad, needs improvement
+// use tox fileid/filekind instead !
+#include <solanaceae/ngc_ft1/ngcft1_file_kind.hpp>
+#include <solanaceae/ngc_ft1_sha1/components.hpp>
+
+#include <nlohmann/json.hpp>
+
+#include <algorithm>
 
 #include <iostream>
 
@@ -40,10 +53,7 @@ NGCHS2Send::NGCHS2Send(
 {
 	_nftep_sr
 		.subscribe(NGCFT1_Event::recv_request)
-		//.subscribe(NGCFT1_Event::recv_init) // we only send init
-		//.subscribe(NGCFT1_Event::recv_data) // we only send data
 		.subscribe(NGCFT1_Event::send_data)
-		//.subscribe(NGCFT1_Event::recv_done)
 		.subscribe(NGCFT1_Event::send_done)
 	;
 }
@@ -189,6 +199,139 @@ void NGCHS2Send::handleSingleMessage(Contact3Handle c, const Events::NGCFT1_recv
 }
 #endif
 
+std::vector<uint8_t> NGCHS2Send::buildHSFileRange(Contact3Handle c, uint64_t ts_start, uint64_t ts_end) {
+	const Message3Registry* reg_ptr = static_cast<const RegistryMessageModelI&>(_rmm).get(c);
+	if (reg_ptr == nullptr) {
+		return {};
+	}
+	const Message3Registry& msg_reg = *reg_ptr;
+
+
+	if (msg_reg.storage<Message::Components::Timestamp>() == nullptr) {
+		// nothing to do here
+		return {};
+	}
+
+	std::cout << "!!!! starting msg ts search, ts_start:" << ts_start << " ts_end:" << ts_end << "\n";
+
+	auto ts_view = msg_reg.view<Message::Components::Timestamp>();
+
+	// we iterate "forward", so from newest to oldest
+
+	// start is the newest ts
+	auto ts_start_it = ts_view.end(); // start invalid
+	// end is the oldest ts
+	//
+	{ // binary search for first value not newer than start ts
+		// -> first value smaller than start ts
+		auto res = std::lower_bound(
+			ts_view.begin(), ts_view.end(),
+			ts_start,
+			[&ts_view](const auto& a, const auto& b) {
+				const auto& [a_comp] = ts_view.get(a);
+				return a_comp.ts > b; // > bc ts is sorted high to low?
+			}
+		);
+
+		if (res != ts_view.end()) {
+			const auto& [ts_comp] = ts_view.get(*res);
+			std::cout << "!!!! first value not newer than start ts is " << ts_comp.ts << "\n";
+			ts_start_it = res;
+		} else {
+			std::cout << "!!!! no first value not newer than start ts\n";
+		}
+	}
+
+	// we only search for the start point, because we walk to the end anyway
+
+	auto j_array = nlohmann::json::array_t{};
+
+	// hmm
+	// maybe use other view or something?
+	for (auto it = ts_start_it; it != ts_view.end(); it++) {
+		const auto e = *it;
+		const auto& [ts_comp] = ts_view.get(e);
+
+		if (ts_comp.ts > ts_start) {
+			std::cerr << "!!!! msg ent in view too new\n";
+			continue;
+		} else if (ts_comp.ts < ts_end) {
+			// too old, we hit the end of the range
+			break;
+		}
+
+		if (!msg_reg.all_of<
+			Message::Components::ContactFrom,
+			Message::Components::ContactTo,
+			Message::Components::ToxGroupMessageID
+		>(e)) {
+			continue; // ??
+		}
+		if (!msg_reg.any_of<Message::Components::MessageText, Message::Components::MessageFileObject>(e)) {
+			continue; // skip
+		}
+
+
+		const auto& [c_from_c, c_to_c] = msg_reg.get<Message::Components::ContactFrom, Message::Components::ContactTo>(e);
+
+		if (c_to_c.c != c) {
+			// message was not public
+			continue;
+		}
+
+		if (!_cr.valid(c_from_c.c)) {
+			continue; // ???
+		}
+
+		Contact3Handle c_from{_cr, c_from_c.c};
+
+		if (!c_from.all_of<Contact::Components::ToxGroupPeerPersistent>()) {
+			continue; // ???
+		}
+
+		if (_only_send_self_observed && msg_reg.all_of<Message::Components::SyncedBy>(e) && c.all_of<Contact::Components::Self>()) {
+			if (!msg_reg.get<Message::Components::SyncedBy>(e).ts.count(c.get<Contact::Components::Self>().self)) {
+				continue; // did not observe ourselfs, skip
+			}
+		}
+
+		auto j_entry = nlohmann::json::object_t{};
+
+		j_entry["ts"] = ts_comp.ts/100; // millisec -> decisec
+		{
+			const auto& ppk_ref = c_from.get<Contact::Components::ToxGroupPeerPersistent>().peer_key.data;
+			j_entry["ppk"] = nlohmann::json::binary_t{std::vector<uint8_t>{ppk_ref.cbegin(), ppk_ref.cend()}};
+		}
+		j_entry["mid"] = msg_reg.get<Message::Components::ToxGroupMessageID>(e).id;
+
+		if (msg_reg.all_of<Message::Components::MessageText>(e)) {
+			if (msg_reg.all_of<Message::Components::TagMessageIsAction>(e)) {
+				j_entry["msgtype"] = "action"; // TODO: textaction?
+			} else {
+				j_entry["msgtype"] = "text";
+			}
+			j_entry["text"] = msg_reg.get<Message::Components::MessageText>(e).text;
+		} else if (msg_reg.any_of<Message::Components::MessageFileObject>(e)) {
+			const auto& o = msg_reg.get<Message::Components::MessageFileObject>(e).o;
+			if (!o) {
+				continue;
+			}
+
+			// HACK: use tox fild_id and file_kind instead!!
+			if (o.all_of<Components::FT1InfoSHA1Hash>()) {
+				j_entry["fkind"] = NGCFT1_file_kind::HASH_SHA1_INFO;
+				j_entry["fid"] = nlohmann::json::binary_t{o.get<Components::FT1InfoSHA1Hash>().hash};
+			} else {
+				continue; // unknown file type
+			}
+		}
+
+		j_array.push_back(j_entry);
+	}
+
+	return nlohmann::json::to_msgpack(j_array);
+}
+
 bool NGCHS2Send::onEvent(const Message::Events::MessageConstruct&) {
 	return false;
 }

solanaceae/ngc_hs2/ngc_hs2_send.hpp

@@ -51,10 +51,10 @@ class NGCHS2Send : public RegistryMessageModelEventI, public NGCFT1EventI {
 	float _iterate_heat {0.f};
 	constexpr static float _iterate_cooldown {1.22f}; // sec
 
-	// open/running info requests (by c)
+	// open/running range requests (by c)
 	// comp on peer c
 
-	// open/running info responses (by c)
+	// open/running range responses (by c)
 	// comp on peer c
 
 	// limit to 2 uploads per peer simultaniously
@@ -79,6 +79,10 @@ class NGCHS2Send : public RegistryMessageModelEventI, public NGCFT1EventI {
 
 		void handleTimeRange(Contact3Handle c, const Events::NGCFT1_recv_request&);
 
+		// msg reg contact
+		// time ranges
+		std::vector<uint8_t> buildHSFileRange(Contact3Handle c, uint64_t ts_start, uint64_t ts_end);
+
 	protected:
 		bool onEvent(const Message::Events::MessageConstruct&) override;
 		bool onEvent(const Message::Events::MessageUpdated&) override;

solanaceae/ngc_hs2/spec_ngc_hs2.md

@@ -92,7 +92,7 @@ Msgpack array of messages.
   - ppk                  | 32bytes
   - mid                  | 16bit
   - msgtype              | enum (string or number?)
-  - if text/textaction   |
+  - if text/action       |
     - text               | string            | maybe byte array instead?
   - if file              |
     - fkind              | 32bit enum        | is this right?