accounted

especially preventing a stall on some packetloss scenarios

CMakeLists.txt

@@ -43,6 +43,21 @@ target_link_libraries(solanaceae_ngcft1 PUBLIC
 
 ########################################
 
+add_library(solanaceae_ngchs2
+	./solanaceae/ngc_hs2/ngc_hs2.hpp
+	./solanaceae/ngc_hs2/ngc_hs2.cpp
+)
+target_include_directories(solanaceae_ngchs2 PUBLIC .)
+target_compile_features(solanaceae_ngchs2 PUBLIC cxx_std_17)
+target_link_libraries(solanaceae_ngchs2 PUBLIC
+	solanaceae_ngcft1
+	solanaceae_tox_contacts
+	solanaceae_message3
+	solanaceae_object_store
+)
+
+########################################
+
 add_library(solanaceae_sha1_ngcft1
 	# hacky deps
 	./solanaceae/ngc_ft1_sha1/mio.hpp

solanaceae/ngc_ext/ngcext.cpp

@@ -3,9 +3,11 @@
 #include <iostream>
 #include <cassert>
 
-NGCEXTEventProvider::NGCEXTEventProvider(ToxI& t, ToxEventProviderI& tep) : _t(t), _tep(tep) {
-	_tep.subscribe(this, Tox_Event_Type::TOX_EVENT_GROUP_CUSTOM_PACKET);
-	_tep.subscribe(this, Tox_Event_Type::TOX_EVENT_GROUP_CUSTOM_PRIVATE_PACKET);
+NGCEXTEventProvider::NGCEXTEventProvider(ToxI& t, ToxEventProviderI& tep) : _t(t), _tep(tep), _tep_sr(_tep.newSubRef(this)) {
+	_tep_sr
+		.subscribe(Tox_Event_Type::TOX_EVENT_GROUP_CUSTOM_PACKET)
+		.subscribe(Tox_Event_Type::TOX_EVENT_GROUP_CUSTOM_PRIVATE_PACKET)
+	;
 }
 
 #define _DATA_HAVE(x, error) if ((data_size - curser) < (x)) { error; }
@@ -78,7 +80,7 @@ bool NGCEXTEventProvider::parse_ft1_init(
 	e.file_size = 0u;
 	_DATA_HAVE(sizeof(e.file_size), std::cerr << "NGCEXT: packet too small, missing file_size\n"; return false)
 	for (size_t i = 0; i < sizeof(e.file_size); i++, curser++) {
-		e.file_size |= size_t(data[curser]) << (i*8);
+		e.file_size |= uint64_t(data[curser]) << (i*8);
 	}
 
 	// - 1 byte (temporary_file_tf_id)
@@ -121,6 +123,83 @@ bool NGCEXTEventProvider::parse_ft1_init_ack(
 	);
 }
 
+bool NGCEXTEventProvider::parse_ft1_init_ack_v2(
+	uint32_t group_number, uint32_t peer_number,
+	const uint8_t* data, size_t data_size,
+	bool _private
+) {
+	if (!_private) {
+		std::cerr << "NGCEXT: ft1_init_ack_v2 cant be public\n";
+		return false;
+	}
+
+	Events::NGCEXT_ft1_init_ack e;
+	e.group_number = group_number;
+	e.peer_number = peer_number;
+	size_t curser = 0;
+
+	// - 1 byte (temporary_file_tf_id)
+	_DATA_HAVE(sizeof(e.transfer_id), std::cerr << "NGCEXT: packet too small, missing transfer_id\n"; return false)
+	e.transfer_id = data[curser++];
+
+	// - 2 byte (max_lossy_data_size)
+	if ((data_size - curser) >= sizeof(e.max_lossy_data_size)) {
+		e.max_lossy_data_size = 0;
+		for (size_t i = 0; i < sizeof(e.max_lossy_data_size); i++, curser++) {
+			e.max_lossy_data_size |= uint16_t(data[curser]) << (i*8);
+		}
+	} else {
+		e.max_lossy_data_size = 500-4; // default
+	}
+
+	return dispatch(
+		NGCEXT_Event::FT1_INIT_ACK,
+		e
+	);
+}
+
+bool NGCEXTEventProvider::parse_ft1_init_ack_v3(
+	uint32_t group_number, uint32_t peer_number,
+	const uint8_t* data, size_t data_size,
+	bool _private
+) {
+	if (!_private) {
+		std::cerr << "NGCEXT: ft1_init_ack_v3 cant be public\n";
+		return false;
+	}
+
+	Events::NGCEXT_ft1_init_ack e;
+	e.group_number = group_number;
+	e.peer_number = peer_number;
+	size_t curser = 0;
+
+	// - 1 byte (temporary_file_tf_id)
+	_DATA_HAVE(sizeof(e.transfer_id), std::cerr << "NGCEXT: packet too small, missing transfer_id\n"; return false)
+	e.transfer_id = data[curser++];
+
+	// - 2 byte (max_lossy_data_size)
+	if ((data_size - curser) >= sizeof(e.max_lossy_data_size)) {
+		e.max_lossy_data_size = 0;
+		for (size_t i = 0; i < sizeof(e.max_lossy_data_size); i++, curser++) {
+			e.max_lossy_data_size |= uint16_t(data[curser]) << (i*8);
+		}
+	} else {
+		e.max_lossy_data_size = 500-4; // default
+	}
+
+	// - 1 byte (feature_flags)
+	if ((data_size - curser) >= sizeof(e.feature_flags)) {
+		e.feature_flags = data[curser++];
+	} else {
+		e.feature_flags = 0x00; // default
+	}
+
+	return dispatch(
+		NGCEXT_Event::FT1_INIT_ACK,
+		e
+	);
+}
+
 bool NGCEXTEventProvider::parse_ft1_data(
 	uint32_t group_number, uint32_t peer_number,
 	const uint8_t* data, size_t data_size,
@@ -227,41 +306,6 @@ bool NGCEXTEventProvider::parse_ft1_message(
 	);
 }
 
-bool NGCEXTEventProvider::parse_ft1_init_ack_v2(
-	uint32_t group_number, uint32_t peer_number,
-	const uint8_t* data, size_t data_size,
-	bool _private
-) {
-	if (!_private) {
-		std::cerr << "NGCEXT: ft1_init_ack_v2 cant be public\n";
-		return false;
-	}
-
-	Events::NGCEXT_ft1_init_ack e;
-	e.group_number = group_number;
-	e.peer_number = peer_number;
-	size_t curser = 0;
-
-	// - 1 byte (temporary_file_tf_id)
-	_DATA_HAVE(sizeof(e.transfer_id), std::cerr << "NGCEXT: packet too small, missing transfer_id\n"; return false)
-	e.transfer_id = data[curser++];
-
-	// - 2 byte (max_lossy_data_size)
-	if ((data_size - curser) >= sizeof(e.max_lossy_data_size)) {
-		e.max_lossy_data_size = 0;
-		for (size_t i = 0; i < sizeof(e.max_lossy_data_size); i++, curser++) {
-			e.max_lossy_data_size |= uint16_t(data[curser]) << (i*8);
-		}
-	} else {
-		e.max_lossy_data_size = 500-4; // default
-	}
-
-	return dispatch(
-		NGCEXT_Event::FT1_INIT_ACK,
-		e
-	);
-}
-
 bool NGCEXTEventProvider::parse_ft1_have(
 	uint32_t group_number, uint32_t peer_number,
 	const uint8_t* data, size_t data_size,
@@ -399,6 +443,52 @@ bool NGCEXTEventProvider::parse_ft1_have_all(
 	);
 }
 
+bool NGCEXTEventProvider::parse_ft1_init2(
+	uint32_t group_number, uint32_t peer_number,
+	const uint8_t* data, size_t data_size,
+	bool _private
+) {
+	if (!_private) {
+		std::cerr << "NGCEXT: ft1_init2 cant be public\n";
+		return false;
+	}
+
+	Events::NGCEXT_ft1_init2 e;
+	e.group_number = group_number;
+	e.peer_number = peer_number;
+	size_t curser = 0;
+
+	// - 4 byte (file_kind)
+	e.file_kind = 0u;
+	_DATA_HAVE(sizeof(e.file_kind), std::cerr << "NGCEXT: packet too small, missing file_kind\n"; return false)
+	for (size_t i = 0; i < sizeof(e.file_kind); i++, curser++) {
+		e.file_kind |= uint32_t(data[curser]) << (i*8);
+	}
+
+	// - 8 bytes (data size)
+	e.file_size = 0u;
+	_DATA_HAVE(sizeof(e.file_size), std::cerr << "NGCEXT: packet too small, missing file_size\n"; return false)
+	for (size_t i = 0; i < sizeof(e.file_size); i++, curser++) {
+		e.file_size |= uint64_t(data[curser]) << (i*8);
+	}
+
+	// - 1 byte (temporary_file_tf_id)
+	_DATA_HAVE(sizeof(e.transfer_id), std::cerr << "NGCEXT: packet too small, missing transfer_id\n"; return false)
+	e.transfer_id = data[curser++];
+
+	// - 1 byte feature flags
+	_DATA_HAVE(sizeof(e.feature_flags), std::cerr << "NGCEXT: packet too small, missing feature_flags\n"; return false)
+	e.feature_flags = data[curser++];
+
+	// - X bytes (file_kind dependent id, differnt sizes)
+	e.file_id = {data+curser, data+curser+(data_size-curser)};
+
+	return dispatch(
+		NGCEXT_Event::FT1_INIT2,
+		e
+	);
+}
+
 bool NGCEXTEventProvider::parse_pc1_announce(
 	uint32_t group_number, uint32_t peer_number,
 	const uint8_t* data, size_t data_size,
@@ -443,7 +533,8 @@ bool NGCEXTEventProvider::handlePacket(
 			return parse_ft1_init(group_number, peer_number, data+1, data_size-1, _private);
 		case NGCEXT_Event::FT1_INIT_ACK:
 			//return parse_ft1_init_ack(group_number, peer_number, data+1, data_size-1, _private);
-			return parse_ft1_init_ack_v2(group_number, peer_number, data+1, data_size-1, _private);
+			//return parse_ft1_init_ack_v2(group_number, peer_number, data+1, data_size-1, _private);
+			return parse_ft1_init_ack_v3(group_number, peer_number, data+1, data_size-1, _private);
 		case NGCEXT_Event::FT1_DATA:
 			return parse_ft1_data(group_number, peer_number, data+1, data_size-1, _private);
 		case NGCEXT_Event::FT1_DATA_ACK:
@@ -456,6 +547,8 @@ bool NGCEXTEventProvider::handlePacket(
 			return parse_ft1_bitset(group_number, peer_number, data+1, data_size-1, _private);
 		case NGCEXT_Event::FT1_HAVE_ALL:
 			return parse_ft1_have_all(group_number, peer_number, data+1, data_size-1, _private);
+		case NGCEXT_Event::FT1_INIT2:
+			return parse_ft1_init2(group_number, peer_number, data+1, data_size-1, _private);
 		case NGCEXT_Event::PC1_ANNOUNCE:
 			return parse_pc1_announce(group_number, peer_number, data+1, data_size-1, _private);
 		default:
@@ -699,6 +792,39 @@ bool NGCEXTEventProvider::send_ft1_have_all(
 	return _t.toxGroupSendCustomPrivatePacket(group_number, peer_number, true, pkg) == TOX_ERR_GROUP_SEND_CUSTOM_PRIVATE_PACKET_OK;
 }
 
+bool NGCEXTEventProvider::send_ft1_init2(
+	uint32_t group_number, uint32_t peer_number,
+	uint32_t file_kind,
+	uint64_t file_size,
+	uint8_t transfer_id,
+	uint8_t feature_flags,
+	const uint8_t* file_id, size_t file_id_size
+) {
+	// - 1 byte packet id
+	// - 4 byte (file_kind)
+	// - 8 bytes (data size)
+	// - 1 byte (temporary_file_tf_id, for this peer only, technically just a prefix to distinguish between simultainious fts)
+	// - 1 byte (feature_flags)
+	// - X bytes (file_kind dependent id, differnt sizes)
+
+	std::vector<uint8_t> pkg;
+	pkg.push_back(static_cast<uint8_t>(NGCEXT_Event::FT1_INIT2));
+	for (size_t i = 0; i < sizeof(file_kind); i++) {
+		pkg.push_back((file_kind>>(i*8)) & 0xff);
+	}
+	for (size_t i = 0; i < sizeof(file_size); i++) {
+		pkg.push_back((file_size>>(i*8)) & 0xff);
+	}
+	pkg.push_back(transfer_id);
+	pkg.push_back(feature_flags);
+	for (size_t i = 0; i < file_id_size; i++) {
+		pkg.push_back(file_id[i]);
+	}
+
+	// lossless
+	return _t.toxGroupSendCustomPrivatePacket(group_number, peer_number, true, pkg) == TOX_ERR_GROUP_SEND_CUSTOM_PRIVATE_PACKET_OK;
+}
+
 static std::vector<uint8_t> build_pc1_announce(const uint8_t* id_data, size_t id_size) {
 	// - 1 byte packet id
 	// - X bytes (id, differnt sizes)

solanaceae/ngc_ext/ngcext.hpp

@@ -30,6 +30,7 @@ namespace Events {
 		uint32_t peer_number;
 
 		// respond to a request with 0 or more message ids, sorted by newest first
+
 		// - peer_key bytes (the msg_ids are from)
 		ToxKey peer_key;
 
@@ -47,6 +48,7 @@ namespace Events {
 		uint32_t peer_number;
 
 		// request the other side to initiate a FT
+
 		// - 4 byte (file_kind)
 		uint32_t file_kind;
 
@@ -54,11 +56,13 @@ namespace Events {
 		std::vector<uint8_t> file_id;
 	};
 
+	// DEPRECATED: use FT1_INIT2 instead
 	struct NGCEXT_ft1_init {
 		uint32_t group_number;
 		uint32_t peer_number;
 
 		// tell the other side you want to start a FT
+
 		// - 4 byte (file_kind)
 		uint32_t file_kind;
 
@@ -81,6 +85,11 @@ namespace Events {
 
 		// - 2 byte (self_max_lossy_data_size)
 		uint16_t max_lossy_data_size;
+
+		// - 1 byte feature flags
+		//   - 0x01 advertised zstd compression
+		//   - 0x02
+		uint8_t feature_flags;
 	};
 
 	struct NGCEXT_ft1_data {
@@ -88,6 +97,7 @@ namespace Events {
 		uint32_t peer_number;
 
 		// data fragment
+
 		// - 1 byte (temporary_file_tf_id)
 		uint8_t transfer_id;
 
@@ -172,6 +182,30 @@ namespace Events {
 		std::vector<uint8_t> file_id;
 	};
 
+	struct NGCEXT_ft1_init2 {
+		uint32_t group_number;
+		uint32_t peer_number;
+
+		// tell the other side you want to start a FT
+
+		// - 4 byte (file_kind)
+		uint32_t file_kind;
+
+		// - 8 bytes (data size)
+		uint64_t file_size;
+
+		// - 1 byte (temporary_file_tf_id, for this peer only, technically just a prefix to distinguish between simultainious fts)
+		uint8_t transfer_id;
+
+		// - 1 byte feature flags
+		//   - 0x01 advertise zstd compression
+		//   - 0x02
+		uint8_t feature_flags;
+
+		// - X bytes (file_kind dependent id, differnt sizes)
+		std::vector<uint8_t> file_id;
+	};
+
 	struct NGCEXT_pc1_announce {
 		uint32_t group_number;
 		uint32_t peer_number;
@@ -206,6 +240,7 @@ enum class NGCEXT_Event : uint8_t {
 
 	// tell the other side you want to start a FT
 	// TODO: might use id layer instead. with it, it would look similar to friends_ft
+	// DEPRECATED: use FT1_INIT2 instead
 	// - 4 byte (file_kind)
 	// - 8 bytes (data size, can be 0 if unknown, BUT files have to be atleast 1 byte)
 	// - 1 byte (temporary_file_tf_id, for this peer only, technically just a prefix to distinguish between simultainious fts)
@@ -215,7 +250,8 @@ enum class NGCEXT_Event : uint8_t {
 	// acknowlage init (like an accept)
 	// like tox ft control continue
 	// - 1 byte (transfer_id)
-	// - 2 byte (self_max_lossy_data_size) (optional since v2)
+	// - 2 byte (self_max_lossy_data_size) (optimal since v2)
+	// - 1 byte feature flags (optimal since v3, requires prev)
 	FT1_INIT_ACK,
 
 	// TODO: init deny, speed up non acceptance
@@ -273,8 +309,18 @@ enum class NGCEXT_Event : uint8_t {
 	// - X bytes (file_kind dependent id, differnt sizes)
 	FT1_HAVE_ALL,
 
+	// tell the other side you want to start a FT
+	// update: added feature flags (compression)
+	// - 4 byte (file_kind)
+	// - 8 bytes (data size, can be 0 if unknown, BUT files have to be atleast 1 byte)
+	// - 1 byte (temporary_file_tf_id, for this peer only, technically just a prefix to distinguish between simultainious fts)
+	// - 1 byte feature flags
+	// - X bytes (file_kind dependent id, differnt sizes)
+	FT1_INIT2,
+
 	// TODO: FT1_IDONTHAVE, tell a peer you no longer have said chunk
 	// TODO: FT1_REJECT, tell a peer you wont fulfil the request
+	// TODO: FT1_CANCEL, tell a peer you stop the transfer
 
 	// tell another peer that you are participating in X
 	// you can reply with PC1_ANNOUNCE, to let the other side know, you too are participating in X
@@ -283,6 +329,9 @@ enum class NGCEXT_Event : uint8_t {
 	// - x bytes (id, different sizes)
 	PC1_ANNOUNCE = 0x80 | 32u,
 
+	// uses sub splitting
+	P2PRNG = 0x80 | 38u,
+
 	MAX
 };
 
@@ -299,6 +348,7 @@ struct NGCEXTEventI {
 	virtual bool onEvent(const Events::NGCEXT_ft1_have&) { return false; }
 	virtual bool onEvent(const Events::NGCEXT_ft1_bitset&) { return false; }
 	virtual bool onEvent(const Events::NGCEXT_ft1_have_all&) { return false; }
+	virtual bool onEvent(const Events::NGCEXT_ft1_init2&) { return false; }
 	virtual bool onEvent(const Events::NGCEXT_pc1_announce&) { return false; }
 };
 
@@ -307,6 +357,7 @@ using NGCEXTEventProviderI = EventProviderI<NGCEXTEventI>;
 class NGCEXTEventProvider : public ToxEventI, public NGCEXTEventProviderI {
 	ToxI& _t;
 	ToxEventProviderI& _tep;
+	ToxEventProviderI::SubscriptionReference _tep_sr;
 
 	public:
 		NGCEXTEventProvider(ToxI& t, ToxEventProviderI& tep);
@@ -342,6 +393,18 @@ class NGCEXTEventProvider : public ToxEventI, public NGCEXTEventProviderI {
 			bool _private
 		);
 
+		bool parse_ft1_init_ack_v2(
+			uint32_t group_number, uint32_t peer_number,
+			const uint8_t* data, size_t data_size,
+			bool _private
+		);
+
+		bool parse_ft1_init_ack_v3(
+			uint32_t group_number, uint32_t peer_number,
+			const uint8_t* data, size_t data_size,
+			bool _private
+		);
+
 		bool parse_ft1_data(
 			uint32_t group_number, uint32_t peer_number,
 			const uint8_t* data, size_t data_size,
@@ -360,12 +423,6 @@ class NGCEXTEventProvider : public ToxEventI, public NGCEXTEventProviderI {
 			bool _private
 		);
 
-		bool parse_ft1_init_ack_v2(
-			uint32_t group_number, uint32_t peer_number,
-			const uint8_t* data, size_t data_size,
-			bool _private
-		);
-
 		bool parse_ft1_have(
 			uint32_t group_number, uint32_t peer_number,
 			const uint8_t* data, size_t data_size,
@@ -384,6 +441,12 @@ class NGCEXTEventProvider : public ToxEventI, public NGCEXTEventProviderI {
 			bool _private
 		);
 
+		bool parse_ft1_init2(
+			uint32_t group_number, uint32_t peer_number,
+			const uint8_t* data, size_t data_size,
+			bool _private
+		);
+
 		bool parse_pc1_announce(
 			uint32_t group_number, uint32_t peer_number,
 			const uint8_t* data, size_t data_size,
@@ -460,6 +523,15 @@ class NGCEXTEventProvider : public ToxEventI, public NGCEXTEventProviderI {
 			const uint8_t* file_id, size_t file_id_size
 		);
 
+		bool send_ft1_init2(
+			uint32_t group_number, uint32_t peer_number,
+			uint32_t file_kind,
+			uint64_t file_size,
+			uint8_t transfer_id,
+			uint8_t feature_flags,
+			const uint8_t* file_id, size_t file_id_size
+		);
+
 		bool send_pc1_announce(
 			uint32_t group_number, uint32_t peer_number,
 			const uint8_t* id_data, size_t id_size

solanaceae/ngc_ft1/cca.hpp

@@ -61,6 +61,10 @@ struct CCAI {
 		// returns -1 if not implemented, can return 0
 		virtual int64_t inFlightBytes(void) const { return -1; }
 
+		// returns -1 if not implemented, can return 0
+		// excluded timed out packets (not those currently resent)
+		virtual int64_t inFlightBytesAccounted(void) const { return -1; }
+
 	public: // callbacks
 		// data size is without overhead
 		virtual void onSent(SeqIDType seq, size_t data_size) = 0;

solanaceae/ngc_ft1/cubic.cpp

@@ -7,7 +7,9 @@ void CUBIC::updateReductionTimer(float time_delta) {
 	const auto now {getTimeNow()};
 
 	// only keep updating while the cca interaction is not too long ago
-	if (now - _time_point_last_update <= getCurrentDelay()*4.f) {
+	// or simply when there are packets in flight
+	// (you need space to resend timedout, which still use up pipe space)
+	if (!_in_flight.empty() || now - _time_point_last_update <= getCurrentDelay()*4.f) {
 		_time_since_reduction += time_delta;
 	}
 }
@@ -86,12 +88,15 @@ int64_t CUBIC::canSend(float time_delta) {
 	updateReductionTimer(time_delta);
 
 	if (fspace_pkgs == 0u) {
+		std::cerr << "CUBIC: flow said 0\n";
 		return 0u;
 	}
 
 	const auto window = getCWnD();
-	int64_t cspace_bytes = window - _in_flight_bytes;
+	//int64_t cspace_bytes = window - _in_flight_bytes;
+	int64_t cspace_bytes = window - _in_flight_bytes_accounted;
 	if (cspace_bytes < MAXIMUM_SEGMENT_DATA_SIZE) {
+		//std::cerr << "CUBIC: cspace < seg size\n";
 		return 0u;
 	}
 

solanaceae/ngc_ft1/flow_only.cpp

@@ -29,6 +29,25 @@ void FlowOnly::updateWindow(void) {
 	_fwnd = std::max(_fwnd, 2.f * MAXIMUM_SEGMENT_DATA_SIZE);
 }
 
+void FlowOnly::updateAccounted(void) {
+	int64_t size_timedout {0};
+
+	{ // can be expensive
+		// code see getTimeouts()
+		// after 3 rtt delay, we trigger timeout
+		const auto now_adjusted = getTimeNow() - getCurrentDelay()*3.f;
+
+		for (const auto& [seq, time_stamp, size, _] : _in_flight) {
+			if (now_adjusted > time_stamp) {
+				//list.push_back(seq);
+				size_timedout += size;
+			}
+		}
+	}
+
+	_in_flight_bytes_accounted = _in_flight_bytes - size_timedout;
+}
+
 void FlowOnly::updateCongestion(void) {
 	updateWindow();
 	const auto tmp_window = getWindow();
@@ -65,12 +84,15 @@ float FlowOnly::getWindow(void) const {
 int64_t FlowOnly::canSend(float time_delta) {
 	if (_in_flight.empty()) {
 		assert(_in_flight_bytes == 0);
-		return MAXIMUM_SEGMENT_DATA_SIZE;
+		// TODO: should we really exit early here??
+		return 2*MAXIMUM_SEGMENT_DATA_SIZE;
 	}
 
 	updateWindow();
+	updateAccounted();
 
-	int64_t fspace = _fwnd - _in_flight_bytes;
+	//int64_t fspace = _fwnd - _in_flight_bytes;
+	int64_t fspace = _fwnd - _in_flight_bytes_accounted;
 	if (fspace < MAXIMUM_SEGMENT_DATA_SIZE) {
 		return 0u;
 	}
@@ -106,6 +128,10 @@ int64_t FlowOnly::inFlightBytes(void) const {
 	return _in_flight_bytes;
 }
 
+int64_t FlowOnly::inFlightBytesAccounted(void) const {
+	return _in_flight_bytes_accounted;
+}
+
 void FlowOnly::onSent(SeqIDType seq, size_t data_size) {
 	if constexpr (true) {
 		size_t sum {0u};

solanaceae/ngc_ft1/flow_only.hpp

@@ -32,6 +32,7 @@ struct FlowOnly : public CCAI {
 		};
 		std::vector<FlyingBunch> _in_flight;
 		int64_t _in_flight_bytes {0};
+		int64_t _in_flight_bytes_accounted {0};
 
 		int32_t _consecutive_events {0};
 
@@ -58,6 +59,8 @@ struct FlowOnly : public CCAI {
 
 		void updateWindow(void);
 
+		void updateAccounted(void);
+
 		virtual void onCongestion(void) {};
 
 		// internal logic, calls the onCongestion() event
@@ -77,6 +80,7 @@ struct FlowOnly : public CCAI {
 
 		int64_t inFlightCount(void) const override;
 		int64_t inFlightBytes(void) const override;
+		int64_t inFlightBytesAccounted(void) const override;
 
 	public: // callbacks
 		// data size is without overhead

solanaceae/ngc_ft1/ngcft1.cpp

@@ -40,22 +40,24 @@ void NGCFT1::updateSendTransfer(float time_delta, uint32_t group_number, uint32_
 				} else {
 					// timed out, resend
 					std::cerr << "NGCFT1 warning: ft init timed out, resending\n";
-					//sendPKG_FT1_INIT(group_number, peer_number, tf.file_kind, tf.file_size, idx, tf.file_id.data(), tf.file_id.size());
 					_neep.send_ft1_init(group_number, peer_number, tf.file_kind, tf.file_size, idx, tf.file_id.data(), tf.file_id.size());
 					tf.inits_sent++;
 					tf.time_since_activity = 0.f;
 				}
 			}
-			//break;
-			return;
+			break;
 		case State::FINISHING: // we still have unacked packets
 			tf.ssb.for_each(time_delta, [&](uint16_t id, const std::vector<uint8_t>& data, float& time_since_activity) {
-				if (can_packet_size >= data.size() && timeouts_set.count({idx, id})) {
-					_neep.send_ft1_data(group_number, peer_number, idx, id, data.data(), data.size());
-					peer.cca->onLoss({idx, id}, false);
-					time_since_activity = 0.f;
-					timeouts_set.erase({idx, id});
-					can_packet_size -= data.size();
+				if (timeouts_set.count({idx, id})) {
+					if (can_packet_size >= data.size()) {
+						_neep.send_ft1_data(group_number, peer_number, idx, id, data.data(), data.size());
+						peer.cca->onLoss({idx, id}, false);
+						time_since_activity = 0.f;
+						timeouts_set.erase({idx, id});
+						can_packet_size -= data.size();
+					} else {
+						std::cerr << "NGCFT1 warning: no space to resend timedout\n";
+					}
 				}
 			});
 			if (tf.time_since_activity >= sending_give_up_after) {
@@ -122,7 +124,7 @@ void NGCFT1::updateSendTransfer(float time_delta, uint32_t group_number, uint32_
 					size_t chunk_size = std::min<size_t>({
 						peer.cca->MAXIMUM_SEGMENT_DATA_SIZE,
 						static_cast<size_t>(can_packet_size),
-						tf.file_size - tf.file_size_current
+						static_cast<size_t>(tf.file_size - tf.file_size_current),
 					});
 					if (chunk_size == 0) {
 						tf.state = State::FINISHING;
@@ -139,7 +141,7 @@ void NGCFT1::updateSendTransfer(float time_delta, uint32_t group_number, uint32_
 							group_number, peer_number,
 							static_cast<uint8_t>(idx),
 							tf.file_size_current,
-							new_data.data(), new_data.size(),
+							new_data.data(), static_cast<uint32_t>(new_data.size()),
 						}
 					);
 
@@ -198,7 +200,6 @@ void NGCFT1::iteratePeer(float time_delta, uint32_t group_number, uint32_t peer_
 		}
 	}
 
-	//for (auto& transfer_opt : peer.recv_transfers) {
 	for (size_t idx = 0; idx < peer.recv_transfers.size(); idx++) {
 		if (!peer.recv_transfers.at(idx).has_value()) {
 			continue;
@@ -210,13 +211,13 @@ void NGCFT1::iteratePeer(float time_delta, uint32_t group_number, uint32_t peer_
 		if (transfer.state == Group::Peer::RecvTransfer::State::FINISHING) {
 			transfer.finishing_timer -= time_delta;
 			if (transfer.finishing_timer <= 0.f) {
-				dispatch(
-					NGCFT1_Event::recv_done,
-					Events::NGCFT1_recv_done{
-						group_number, peer_number,
-						uint8_t(idx)
-					}
-				);
+				//dispatch(
+				//    NGCFT1_Event::recv_done,
+				//    Events::NGCFT1_recv_done{
+				//        group_number, peer_number,
+				//        uint8_t(idx)
+				//    }
+				//);
 
 				peer.recv_transfers.at(idx).reset();
 			}
@@ -251,16 +252,18 @@ NGCFT1::NGCFT1(
 	ToxI& t,
 	ToxEventProviderI& tep,
 	NGCEXTEventProvider& neep
-) : _t(t), _tep(tep), _neep(neep)
+) : _t(t), _tep(tep), _tep_sr(_tep.newSubRef(this)), _neep(neep), _neep_sr(_neep.newSubRef(this))
 {
-	_neep.subscribe(this, NGCEXT_Event::FT1_REQUEST);
-	_neep.subscribe(this, NGCEXT_Event::FT1_INIT);
-	_neep.subscribe(this, NGCEXT_Event::FT1_INIT_ACK);
-	_neep.subscribe(this, NGCEXT_Event::FT1_DATA);
-	_neep.subscribe(this, NGCEXT_Event::FT1_DATA_ACK);
-	_neep.subscribe(this, NGCEXT_Event::FT1_MESSAGE);
+	_neep_sr
+		.subscribe(NGCEXT_Event::FT1_REQUEST)
+		.subscribe(NGCEXT_Event::FT1_INIT)
+		.subscribe(NGCEXT_Event::FT1_INIT_ACK)
+		.subscribe(NGCEXT_Event::FT1_DATA)
+		.subscribe(NGCEXT_Event::FT1_DATA_ACK)
+		.subscribe(NGCEXT_Event::FT1_MESSAGE)
+	;
 
-	_tep.subscribe(this, Tox_Event_Type::TOX_EVENT_GROUP_PEER_EXIT);
+	_tep_sr.subscribe(Tox_Event_Type::TOX_EVENT_GROUP_PEER_EXIT);
 }
 
 float NGCFT1::iterate(float time_delta) {
@@ -306,7 +309,7 @@ float NGCFT1::iterate(float time_delta) {
 void NGCFT1::NGC_FT1_send_request_private(
 	uint32_t group_number, uint32_t peer_number,
 	uint32_t file_kind,
-	const uint8_t* file_id, size_t file_id_size
+	const uint8_t* file_id, uint32_t file_id_size
 ) {
 	// TODO: error check
 	_neep.send_ft1_request(group_number, peer_number, file_kind, file_id, file_id_size);
@@ -315,9 +318,10 @@ void NGCFT1::NGC_FT1_send_request_private(
 bool NGCFT1::NGC_FT1_send_init_private(
 	uint32_t group_number, uint32_t peer_number,
 	uint32_t file_kind,
-	const uint8_t* file_id, size_t file_id_size,
-	size_t file_size,
-	uint8_t* transfer_id
+	const uint8_t* file_id, uint32_t file_id_size,
+	uint64_t file_size,
+	uint8_t* transfer_id,
+	bool can_compress
 ) {
 	if (std::get<0>(_t.toxGroupPeerGetConnectionStatus(group_number, peer_number)).value_or(TOX_CONNECTION_NONE) == TOX_CONNECTION_NONE) {
 		std::cerr << "NGCFT1 error: cant init ft, peer offline\n";
@@ -347,6 +351,8 @@ bool NGCFT1::NGC_FT1_send_init_private(
 			std::cerr << "NGCFT1 error: cant init ft, no free transfer slot\n";
 			return false;
 		}
+
+		idx = i;
 	}
 
 	// TODO: check return value
@@ -374,7 +380,7 @@ bool NGCFT1::NGC_FT1_send_message_public(
 	uint32_t group_number,
 	uint32_t& message_id,
 	uint32_t file_kind,
-	const uint8_t* file_id, size_t file_id_size
+	const uint8_t* file_id, uint32_t file_id_size
 ) {
 	// create msg_id
 	message_id = randombytes_random();
@@ -441,7 +447,7 @@ bool NGCFT1::onEvent(const Events::NGCEXT_ft1_request& e) {
 		Events::NGCFT1_recv_request{
 			e.group_number, e.peer_number,
 			static_cast<NGCFT1_file_kind>(e.file_kind),
-			e.file_id.data(), e.file_id.size()
+			e.file_id.data(), static_cast<uint32_t>(e.file_id.size())
 		}
 	);
 }
@@ -450,14 +456,14 @@ bool NGCFT1::onEvent(const Events::NGCEXT_ft1_init& e) {
 //#if !NDEBUG
 	std::cout << "NGCFT1: got FT1_INIT fk:" << e.file_kind << " fs:" << e.file_size << " tid:" << int(e.transfer_id) << " [" << bin2hex(e.file_id) << "]\n";
 //#endif
-
+#if 0
 	bool accept = false;
 	dispatch(
 		NGCFT1_Event::recv_init,
 		Events::NGCFT1_recv_init{
 			e.group_number, e.peer_number,
 			static_cast<NGCFT1_file_kind>(e.file_kind),
-			e.file_id.data(), e.file_id.size(),
+			e.file_id.data(), static_cast<uint32_t>(e.file_id.size()),
 			e.transfer_id,
 			e.file_size,
 			accept
@@ -486,8 +492,19 @@ bool NGCFT1::onEvent(const Events::NGCEXT_ft1_init& e) {
 		0u,
 		{} // rsb
 	};
-
 	return true;
+#else
+	// HACK: simply forward to init2 hanlder
+	return onEvent(Events::NGCEXT_ft1_init2{
+		e.group_number,
+		e.peer_number,
+		e.file_kind,
+		e.file_size,
+		e.transfer_id,
+		0x00, // non set
+		e.file_id, // sadly a copy, wont matter in the future
+	});
+#endif
 }
 
 bool NGCFT1::onEvent(const Events::NGCEXT_ft1_init_ack& e) {
@@ -516,6 +533,11 @@ bool NGCFT1::onEvent(const Events::NGCEXT_ft1_init_ack& e) {
 		return true;
 	}
 
+	if (e.max_lossy_data_size < 16) {
+		std::cerr << "NGCFT1 error: init_ack max_lossy_data_size is less than 16 bytes\n";
+		return true;
+	}
+
 	// negotiated packet_data_size
 	const auto negotiated_packet_data_size = std::min<uint32_t>(e.max_lossy_data_size, _t.toxGroupMaxCustomLossyPacketLength()-4);
 	// TODO: reset cca with new pkg size
@@ -532,6 +554,8 @@ bool NGCFT1::onEvent(const Events::NGCEXT_ft1_init_ack& e) {
 		//peer.cca = std::make_unique<LEDBAT>(peer.max_packet_data_size);
 		//peer.cca = std::make_unique<FlowOnly>(peer.max_packet_data_size);
 		//peer.cca->max_byterate_allowed = 1.f *1024*1024;
+	} else {
+		std::cerr << "NGCFT1: reusing cca. rtt:" << peer.cca->getCurrentDelay() << " w:" << peer.cca->getWindow() << " ifc:" << peer.cca->inFlightCount() << "\n";
 	}
 
 	// iterate will now call NGC_FT1_send_data_cb
@@ -543,7 +567,7 @@ bool NGCFT1::onEvent(const Events::NGCEXT_ft1_init_ack& e) {
 
 bool NGCFT1::onEvent(const Events::NGCEXT_ft1_data& e) {
 #if !NDEBUG
-	//std::cout << "NGCFT1: got FT1_DATA\n";
+	//std::cout << "NGCFT1: got FT1_DATA " << e.sequence_id << "\n";
 #endif
 
 	if (e.data.empty()) {
@@ -578,7 +602,7 @@ bool NGCFT1::onEvent(const Events::NGCEXT_ft1_data& e) {
 				e.group_number, e.peer_number,
 				e.transfer_id,
 				transfer.file_size_current,
-				data.data(), data.size()
+				data.data(), static_cast<uint32_t>(data.size())
 			}
 		);
 
@@ -601,7 +625,15 @@ bool NGCFT1::onEvent(const Events::NGCEXT_ft1_data& e) {
 
 		// TODO: keep around for remote timeout + delay + offset, so we can be sure all acks where received
 		// or implement a dedicated finished that needs to be acked
-		transfer.finishing_timer = 0.5f; // TODO: we are receiving, we dont know delay
+		transfer.finishing_timer = 0.75f; // TODO: we are receiving, we dont know delay
+
+		dispatch(
+			NGCFT1_Event::recv_done,
+			Events::NGCFT1_recv_done{
+				e.group_number, e.peer_number,
+				e.transfer_id
+			}
+		);
 	}
 
 	return true;
@@ -620,7 +652,8 @@ bool NGCFT1::onEvent(const Events::NGCEXT_ft1_data_ack& e) {
 	Group::Peer& peer = groups[e.group_number].peers[e.peer_number];
 	if (!peer.send_transfers[e.transfer_id].has_value()) {
 		// we delete directly, packets might still be in flight (in practice they are when ce)
-		//std::cerr << "NGCFT1 warning: data_ack for unknown transfer\n";
+		// update: we no longer delete directly, but its kinda hacky
+		std::cerr << "NGCFT1 warning: data_ack for unknown transfer\n";
 		return true;
 	}
 
@@ -672,11 +705,55 @@ bool NGCFT1::onEvent(const Events::NGCEXT_ft1_message& e) {
 			e.group_number, e.peer_number,
 			e.message_id,
 			static_cast<NGCFT1_file_kind>(e.file_kind),
-			e.file_id.data(), e.file_id.size()
+			e.file_id.data(), static_cast<uint32_t>(e.file_id.size())
 		}
 	);
 }
 
+bool NGCFT1::onEvent(const Events::NGCEXT_ft1_init2& e) {
+//#if !NDEBUG
+	std::cout << "NGCFT1: got FT1_INIT2 fk:" << e.file_kind << " fs:" << e.file_size << " tid:" << int(e.transfer_id) << " ff:" << int(e.feature_flags) << " [" << bin2hex(e.file_id) << "]\n";
+//#endif
+
+	bool accept = false;
+	dispatch(
+		NGCFT1_Event::recv_init,
+		Events::NGCFT1_recv_init{
+			e.group_number, e.peer_number,
+			static_cast<NGCFT1_file_kind>(e.file_kind),
+			e.file_id.data(), static_cast<uint32_t>(e.file_id.size()),
+			e.transfer_id,
+			e.file_size,
+			accept
+		}
+	);
+
+	if (!accept) {
+		std::cout << "NGCFT1: rejected init2\n";
+		return true; // return true?
+	}
+
+	_neep.send_ft1_init_ack(e.group_number, e.peer_number, e.transfer_id);
+
+	std::cout << "NGCFT1: accepted init2\n";
+
+	auto& peer = groups[e.group_number].peers[e.peer_number];
+	if (peer.recv_transfers[e.transfer_id].has_value()) {
+		std::cerr << "NGCFT1 warning: overwriting existing recv_transfer " << int(e.transfer_id) << ", other peer started new transfer on preexising\n";
+	}
+
+	peer.recv_transfers[e.transfer_id] = Group::Peer::RecvTransfer{
+		e.file_kind,
+		e.file_id,
+		Group::Peer::RecvTransfer::State::INITED,
+		e.file_size,
+		0u,
+		{} // rsb
+	};
+
+	return true;
+}
+
 bool NGCFT1::onToxEvent(const Tox_Event_Group_Peer_Exit* e) {
 	const auto group_number = tox_event_group_peer_exit_get_group_number(e);
 	const auto peer_number = tox_event_group_peer_exit_get_peer_id(e);

solanaceae/ngc_ft1/ngcft1.hpp

@@ -29,7 +29,7 @@ namespace Events {
 		NGCFT1_file_kind file_kind;
 
 		const uint8_t* file_id;
-		size_t file_id_size;
+		uint32_t file_id_size;
 	};
 
 	struct NGCFT1_recv_init {
@@ -39,10 +39,10 @@ namespace Events {
 		NGCFT1_file_kind file_kind;
 
 		const uint8_t* file_id;
-		size_t file_id_size;
+		uint32_t file_id_size;
 
 		const uint8_t transfer_id;
-		const size_t file_size;
+		const uint64_t file_size;
 
 		// return true to accept, false to deny
 		bool& accept;
@@ -54,9 +54,9 @@ namespace Events {
 
 		uint8_t transfer_id;
 
-		size_t data_offset;
+		uint64_t data_offset;
 		const uint8_t* data;
-		size_t data_size;
+		uint32_t data_size;
 	};
 
 	// request to fill data_size bytes into data
@@ -66,9 +66,9 @@ namespace Events {
 
 		uint8_t transfer_id;
 
-		size_t data_offset;
+		uint64_t data_offset;
 		uint8_t* data;
-		size_t data_size;
+		uint32_t data_size;
 	};
 
 	struct NGCFT1_recv_done {
@@ -96,7 +96,7 @@ namespace Events {
 		NGCFT1_file_kind file_kind;
 
 		const uint8_t* file_id;
-		size_t file_id_size;
+		uint32_t file_id_size;
 	};
 
 } // Events
@@ -132,7 +132,9 @@ using NGCFT1EventProviderI = EventProviderI<NGCFT1EventI>;
 class NGCFT1 : public ToxEventI, public NGCEXTEventI, public NGCFT1EventProviderI {
 	ToxI& _t;
 	ToxEventProviderI& _tep;
+	ToxEventProviderI::SubscriptionReference _tep_sr;
 	NGCEXTEventProvider& _neep; // not the interface?
+	NGCEXTEventProvider::SubscriptionReference _neep_sr;
 
 	std::default_random_engine _rng{std::random_device{}()};
 
@@ -141,7 +143,7 @@ class NGCFT1 : public ToxEventI, public NGCEXTEventI, public NGCFT1EventProvider
 	// TODO: config
 	size_t acks_per_packet {3u}; // 3
 	float init_retry_timeout_after {4.f};
-	float sending_give_up_after {15.f}; // 30sec (per active transfer)
+	float sending_give_up_after {10.f}; // sec (per active transfer)
 
 	struct Group {
 		struct Peer {
@@ -159,8 +161,8 @@ class NGCFT1 : public ToxEventI, public NGCEXTEventI, public NGCFT1EventProvider
 					FINISHING, // got all the data, but we wait for 2*delay, since its likely there is data still arriving
 				} state;
 
-				size_t file_size {0};
-				size_t file_size_current {0};
+				uint64_t file_size {0};
+				uint64_t file_size_current {0};
 
 				// if state FINISHING and it reaches 0, delete
 				float finishing_timer {0.f};
@@ -188,8 +190,8 @@ class NGCFT1 : public ToxEventI, public NGCEXTEventI, public NGCFT1EventProvider
 				size_t inits_sent {1}; // is sent when creating
 
 				float time_since_activity {0.f};
-				size_t file_size {0};
-				size_t file_size_current {0};
+				uint64_t file_size {0};
+				uint64_t file_size_current {0};
 
 				// sequence array
 				// list of sent but not acked seq_ids
@@ -224,16 +226,17 @@ class NGCFT1 : public ToxEventI, public NGCEXTEventI, public NGCFT1EventProvider
 		void NGC_FT1_send_request_private(
 			uint32_t group_number, uint32_t peer_number,
 			uint32_t file_kind,
-			const uint8_t* file_id, size_t file_id_size
+			const uint8_t* file_id, uint32_t file_id_size
 		);
 
 		// public does not make sense here
 		bool NGC_FT1_send_init_private(
 			uint32_t group_number, uint32_t peer_number,
 			uint32_t file_kind,
-			const uint8_t* file_id, size_t file_id_size,
-			size_t file_size,
-			uint8_t* transfer_id
+			const uint8_t* file_id, uint32_t file_id_size,
+			uint64_t file_size,
+			uint8_t* transfer_id,
+			bool can_compress = false // set this if you know the data is compressable (eg text)
 		);
 
 		// sends the message and fills in message_id
@@ -241,7 +244,7 @@ class NGCFT1 : public ToxEventI, public NGCEXTEventI, public NGCFT1EventProvider
 			uint32_t group_number,
 			uint32_t& message_id,
 			uint32_t file_kind,
-			const uint8_t* file_id, size_t file_id_size
+			const uint8_t* file_id, uint32_t file_id_size
 		);
 
 	public: // cca stuff
@@ -268,6 +271,7 @@ class NGCFT1 : public ToxEventI, public NGCEXTEventI, public NGCFT1EventProvider
 		bool onEvent(const Events::NGCEXT_ft1_data&) override;
 		bool onEvent(const Events::NGCEXT_ft1_data_ack&) override;
 		bool onEvent(const Events::NGCEXT_ft1_message&) override;
+		bool onEvent(const Events::NGCEXT_ft1_init2&) override;
 
 	protected:
 		bool onToxEvent(const Tox_Event_Group_Peer_Exit* e) override;

solanaceae/ngc_ft1/ngcft1_file_kind.hpp

@@ -72,5 +72,23 @@ enum class NGCFT1_file_kind : uint32_t {
 	// id: sha256
 	// always of size 16KiB, except if last piece in file
 	TORRENT_V2_PIECE,
+
+	// https://gist.github.com/Green-Sky/440cd9817a7114786850eb4c62dc57c3
+	// id: ts start, ts end
+	// content:
+	//  - ts start (do we need this? when this is part of the id?)
+	//  - ts end (same)
+	//  - list size
+	//    - ppk
+	//    - mid
+	//    - ts
+	HS2_INFO_RANGE_TIME = 0x00000f00,
+	// TODO: half open ranges
+	// TODO: id based
+	// TODO: ppk based?
+
+	// id: ppk, mid, ts
+	HS2_SINGLE_MESSAGE,
+	// TODO: message pack
 };
 

solanaceae/ngc_ft1_sha1/backends/sha1_mapped_filesystem.cpp

@@ -86,6 +86,15 @@ void SHA1MappedFilesystem::newFromFile(std::string_view file_name, std::string_v
 		// build info
 		sha1_info.file_name = file_name_;
 		sha1_info.file_size = file_impl->_file_size; // TODO: remove the reliance on implementation details
+		sha1_info.chunk_size = chunkSizeFromFileSize(sha1_info.file_size);
+		{
+			// TOOD: remove
+			const uint32_t cs_low {32*1024};
+			const uint32_t cs_high {4*1024*1024};
+
+			assert(sha1_info.chunk_size >= cs_low);
+			assert(sha1_info.chunk_size <= cs_high);
+		}
 
 		{ // build chunks
 			// HACK: load file fully

solanaceae/ngc_ft1_sha1/components.hpp

@@ -108,7 +108,7 @@ namespace Components {
 		struct Peer {
 			struct Entry {
 				float time_point {0.f};
-				size_t bytes {0u};
+				uint64_t bytes {0u};
 				bool accounted {false};
 			};
 			std::deque<Entry> recently_sent;

solanaceae/ngc_ft1_sha1/ft1_sha1_info.cpp

@@ -1,5 +1,8 @@
 #include "./ft1_sha1_info.hpp"
 
+// next power of two
+#include <entt/core/memory.hpp>
+
 #include <sodium.h>
 
 SHA1Digest::SHA1Digest(const std::vector<uint8_t>& v) {
@@ -28,6 +31,27 @@ std::ostream& operator<<(std::ostream& out, const SHA1Digest& v) {
 	return out;
 }
 
+uint32_t chunkSizeFromFileSize(uint64_t file_size) {
+	const uint64_t fs_low {UINT64_C(512)*1024};
+	const uint64_t fs_high {UINT64_C(2)*1024*1024*1024};
+
+	const uint32_t cs_low {32*1024};
+	const uint32_t cs_high {4*1024*1024};
+
+	if (file_size <= fs_low) { // 512kib
+		return cs_low; // 32kib
+	} else if (file_size >= fs_high) { // 2gib
+		return cs_high; // 4mib
+	}
+
+	double t = file_size - fs_low;
+	t /= fs_high;
+
+	double x = (1 - t) * cs_low + t * cs_high;
+
+	return entt::next_power_of_two(uint64_t(x));
+}
+
 size_t FT1InfoSHA1::chunkSize(size_t chunk_index) const {
 	if (chunk_index+1 == chunks.size()) {
 		// last chunk

solanaceae/ngc_ft1_sha1/ft1_sha1_info.hpp

@@ -25,21 +25,23 @@ std::ostream& operator<<(std::ostream& out, const SHA1Digest& v);
 
 namespace std { // inject
 	template<> struct hash<SHA1Digest> {
-		std::size_t operator()(const SHA1Digest& h) const noexcept {
+		std::uint64_t operator()(const SHA1Digest& h) const noexcept {
 			return
-				size_t(h.data[0]) << (0*8) |
-				size_t(h.data[1]) << (1*8) |
-				size_t(h.data[2]) << (2*8) |
-				size_t(h.data[3]) << (3*8) |
-				size_t(h.data[4]) << (4*8) |
-				size_t(h.data[5]) << (5*8) |
-				size_t(h.data[6]) << (6*8) |
-				size_t(h.data[7]) << (7*8)
+				std::uint64_t(h.data[0]) << (0*8) |
+				std::uint64_t(h.data[1]) << (1*8) |
+				std::uint64_t(h.data[2]) << (2*8) |
+				std::uint64_t(h.data[3]) << (3*8) |
+				std::uint64_t(h.data[4]) << (4*8) |
+				std::uint64_t(h.data[5]) << (5*8) |
+				std::uint64_t(h.data[6]) << (6*8) |
+				std::uint64_t(h.data[7]) << (7*8)
 			;
 		}
 	};
 } // std
 
+uint32_t chunkSizeFromFileSize(uint64_t file_size);
+
 struct FT1InfoSHA1 {
 	std::string file_name;
 	uint64_t file_size {0};

solanaceae/ngc_ft1_sha1/sha1_ngcft1.cpp

@@ -194,43 +194,56 @@ File2I* SHA1_NGCFT1::objGetFile2Read(ObjectHandle o) {
 SHA1_NGCFT1::SHA1_NGCFT1(
 	ObjectStore2& os,
 	Contact3Registry& cr,
-	RegistryMessageModel& rmm,
+	RegistryMessageModelI& rmm,
 	NGCFT1& nft,
 	ToxContactModel2& tcm,
 	ToxEventProviderI& tep,
 	NGCEXTEventProvider& neep
 ) :
 	_os(os),
+	_os_sr(_os.newSubRef(this)),
 	_cr(cr),
 	_rmm(rmm),
+	_rmm_sr(_rmm.newSubRef(this)),
 	_nft(nft),
+	_nft_sr(_nft.newSubRef(this)),
 	_tcm(tcm),
 	_tep(tep),
+	_tep_sr(_tep.newSubRef(this)),
 	_neep(neep),
+	_neep_sr(_neep.newSubRef(this)),
 	_mfb(os)
 {
+	_os_sr
 	// TODO: also create and destroy
-	//_os.subscribe(this, ObjectStore_Event::object_construct);
-	_os.subscribe(this, ObjectStore_Event::object_update);
-	//_os.subscribe(this, ObjectStore_Event::object_destroy);
+	//	.subscribe(ObjectStore_Event::object_construct)
+		.subscribe(ObjectStore_Event::object_update)
+	//	.subscribe(ObjectStore_Event::object_destroy)
+	;
 
-	_nft.subscribe(this, NGCFT1_Event::recv_request);
-	_nft.subscribe(this, NGCFT1_Event::recv_init);
-	_nft.subscribe(this, NGCFT1_Event::recv_data);
-	_nft.subscribe(this, NGCFT1_Event::send_data);
-	_nft.subscribe(this, NGCFT1_Event::recv_done);
-	_nft.subscribe(this, NGCFT1_Event::send_done);
-	_nft.subscribe(this, NGCFT1_Event::recv_message);
+	_nft_sr
+		.subscribe(NGCFT1_Event::recv_request)
+		.subscribe(NGCFT1_Event::recv_init)
+		.subscribe(NGCFT1_Event::recv_data)
+		.subscribe(NGCFT1_Event::send_data)
+		.subscribe(NGCFT1_Event::recv_done)
+		.subscribe(NGCFT1_Event::send_done)
+		.subscribe(NGCFT1_Event::recv_message)
+	;
 
-	_rmm.subscribe(this, RegistryMessageModel_Event::send_file_path);
+	_rmm_sr.subscribe(RegistryMessageModel_Event::send_file_path);
 
-	_tep.subscribe(this, Tox_Event_Type::TOX_EVENT_GROUP_PEER_JOIN);
-	_tep.subscribe(this, Tox_Event_Type::TOX_EVENT_GROUP_PEER_EXIT);
+	_tep_sr
+		.subscribe(Tox_Event_Type::TOX_EVENT_GROUP_PEER_JOIN)
+		.subscribe(Tox_Event_Type::TOX_EVENT_GROUP_PEER_EXIT)
+	;
 
-	_neep.subscribe(this, NGCEXT_Event::FT1_HAVE);
-	_neep.subscribe(this, NGCEXT_Event::FT1_BITSET);
-	_neep.subscribe(this, NGCEXT_Event::FT1_HAVE_ALL);
-	_neep.subscribe(this, NGCEXT_Event::PC1_ANNOUNCE);
+	_neep_sr
+		.subscribe(NGCEXT_Event::FT1_HAVE)
+		.subscribe(NGCEXT_Event::FT1_BITSET)
+		.subscribe(NGCEXT_Event::FT1_HAVE_ALL)
+		.subscribe(NGCEXT_Event::PC1_ANNOUNCE)
+	;
 }
 
 float SHA1_NGCFT1::iterate(float delta) {
@@ -687,21 +700,21 @@ bool SHA1_NGCFT1::onEvent(const Events::NGCFT1_recv_request& e) {
 		// TODO: queue instead
 		//queueUpRequestInfo(e.group_number, e.peer_number, info_hash);
 		uint8_t transfer_id {0};
-		_nft.NGC_FT1_send_init_private(
+		if (_nft.NGC_FT1_send_init_private(
 			e.group_number, e.peer_number,
 			static_cast<uint32_t>(e.file_kind),
 			e.file_id, e.file_id_size,
 			o.get<Components::FT1InfoSHA1Data>().data.size(),
 			&transfer_id
-		);
-
-		_sending_transfers.emplaceInfo(
-			e.group_number, e.peer_number,
-			transfer_id,
-			SendingTransfers::Entry::Info{
-				o.get<Components::FT1InfoSHA1Data>().data
-			}
-		);
+		)) {
+			_sending_transfers.emplaceInfo(
+				e.group_number, e.peer_number,
+				transfer_id,
+				SendingTransfers::Entry::Info{
+					o.get<Components::FT1InfoSHA1Data>().data
+				}
+			);
+		}
 
 		const auto c = _tcm.getContactGroupPeer(e.group_number, e.peer_number);
 		_tox_peer_to_contact[combine_ids(e.group_number, e.peer_number)] = c; // workaround
@@ -855,6 +868,7 @@ bool SHA1_NGCFT1::onEvent(const Events::NGCFT1_recv_init& e) {
 
 bool SHA1_NGCFT1::onEvent(const Events::NGCFT1_recv_data& e) {
 	if (!_receiving_transfers.containsPeerTransfer(e.group_number, e.peer_number, e.transfer_id)) {
+		std::cerr << "SHA1_NGCFT1 waring: unknown transfer " << e.transfer_id << " from " << e.group_number << ":" << e.peer_number << "\n";
 		return false;
 	}
 
@@ -862,6 +876,7 @@ bool SHA1_NGCFT1::onEvent(const Events::NGCFT1_recv_data& e) {
 
 	transfer.time_since_activity = 0.f;
 	if (transfer.isInfo()) {
+		std::cout << "SHA1_NGCFT1: got info data " << e.data_size << "@" << e.data_offset << " from " << e.group_number << ":" << e.peer_number << "\n";
 		auto& info_data = transfer.getInfo().info_data;
 		for (size_t i = 0; i < e.data_size && i + e.data_offset < info_data.size(); i++) {
 			info_data[i+e.data_offset] = e.data[i];
@@ -875,6 +890,7 @@ bool SHA1_NGCFT1::onEvent(const Events::NGCFT1_recv_data& e) {
 
 			auto* file2 = objGetFile2Write(o);
 			if (file2 == nullptr) {
+				std::cerr << "SHA1_NGCFT1 error: writing file failed, no file object\n";
 				return false; // early out
 			}
 			if (!file2->write({e.data, e.data_size}, offset_into_file + e.data_offset)) {
@@ -904,6 +920,7 @@ bool SHA1_NGCFT1::onEvent(const Events::NGCFT1_recv_data& e) {
 
 bool SHA1_NGCFT1::onEvent(const Events::NGCFT1_send_data& e) {
 	if (!_sending_transfers.containsPeerTransfer(e.group_number, e.peer_number, e.transfer_id)) {
+		std::cerr << "SHA1_NGCFT1 error: ngcft1 requested data for unknown transfer\n";
 		return false;
 	}
 
@@ -1102,8 +1119,11 @@ bool SHA1_NGCFT1::onEvent(const Events::NGCFT1_recv_done& e) {
 
 			// something happend, update chunk picker
 			auto c = _tcm.getContactGroupPeer(e.group_number, e.peer_number);
-			assert(static_cast<bool>(c));
-			c.emplace_or_replace<ChunkPickerUpdateTag>();
+			//assert(static_cast<bool>(c));
+			// happened, went offline but chunk was still done o.o
+			if (static_cast<bool>(c)) {
+				c.emplace_or_replace<ChunkPickerUpdateTag>();
+			}
 		} else {
 			// bad chunk
 			std::cout << "SHA1_NGCFT1: got BAD chunk from " << e.group_number << ":" << e.peer_number << " [" << info.chunks.at(chunk_index) << "] ; instead got [" << SHA1Digest{got_hash} << "]\n";

solanaceae/ngc_ft1_sha1/sha1_ngcft1.hpp

@@ -22,13 +22,18 @@
 
 class SHA1_NGCFT1 : public ToxEventI, public RegistryMessageModelEventI, public ObjectStoreEventI, public NGCFT1EventI, public NGCEXTEventI {
 	ObjectStore2& _os;
+	ObjectStore2::SubscriptionReference _os_sr;
 	// TODO: backend abstraction
 	Contact3Registry& _cr;
-	RegistryMessageModel& _rmm;
+	RegistryMessageModelI& _rmm;
+	RegistryMessageModelI::SubscriptionReference _rmm_sr;
 	NGCFT1& _nft;
+	NGCFT1::SubscriptionReference _nft_sr;
 	ToxContactModel2& _tcm;
 	ToxEventProviderI& _tep;
+	ToxEventProviderI::SubscriptionReference _tep_sr;
 	NGCEXTEventProvider& _neep;
+	NGCEXTEventProvider::SubscriptionReference _neep_sr;
 
 	Backends::SHA1MappedFilesystem _mfb;
 
@@ -88,7 +93,7 @@ class SHA1_NGCFT1 : public ToxEventI, public RegistryMessageModelEventI, public
 		SHA1_NGCFT1(
 			ObjectStore2& os,
 			Contact3Registry& cr,
-			RegistryMessageModel& rmm,
+			RegistryMessageModelI& rmm,
 			NGCFT1& nft,
 			ToxContactModel2& tcm,
 			ToxEventProviderI& tep,

solanaceae/ngc_hs2/ngc_hs2.cpp

@@ -0,0 +1,91 @@
+#include "./ngc_hs2.hpp"
+
+#include <solanaceae/tox_contacts/tox_contact_model2.hpp>
+
+NGCHS2::NGCHS2(
+	ToxContactModel2& tcm,
+	ToxEventProviderI& tep,
+	NGCFT1& nft
+) :
+	_tcm(tcm),
+	_tep_sr(tep.newSubRef(this)),
+	_nft(nft),
+	_nftep_sr(_nft.newSubRef(this))
+{
+	_tep_sr
+		.subscribe(TOX_EVENT_GROUP_PEER_JOIN)
+		.subscribe(TOX_EVENT_GROUP_PEER_EXIT)
+	;
+
+	_nftep_sr
+		.subscribe(NGCFT1_Event::recv_init)
+		.subscribe(NGCFT1_Event::recv_request)
+		.subscribe(NGCFT1_Event::recv_init)
+		.subscribe(NGCFT1_Event::recv_data)
+		.subscribe(NGCFT1_Event::send_data)
+		.subscribe(NGCFT1_Event::recv_done)
+		.subscribe(NGCFT1_Event::send_done)
+	;
+}
+
+NGCHS2::~NGCHS2(void) {
+}
+
+float NGCHS2::iterate(float delta) {
+	return 1000.f;
+}
+
+bool NGCHS2::onEvent(const Events::NGCFT1_recv_request& e) {
+	if (
+		e.file_kind != NGCFT1_file_kind::HS2_INFO_RANGE_TIME &&
+		e.file_kind != NGCFT1_file_kind::HS2_SINGLE_MESSAGE
+	) {
+		return false; // not for us
+	}
+
+	return false;
+}
+
+bool NGCHS2::onEvent(const Events::NGCFT1_recv_init& e) {
+	if (
+		e.file_kind != NGCFT1_file_kind::HS2_INFO_RANGE_TIME &&
+		e.file_kind != NGCFT1_file_kind::HS2_SINGLE_MESSAGE
+	) {
+		return false; // not for us
+	}
+
+	return false;
+}
+
+bool NGCHS2::onEvent(const Events::NGCFT1_recv_data&) {
+	return false;
+}
+
+bool NGCHS2::onEvent(const Events::NGCFT1_send_data&) {
+	return false;
+}
+
+bool NGCHS2::onEvent(const Events::NGCFT1_recv_done&) {
+	return false;
+}
+
+bool NGCHS2::onEvent(const Events::NGCFT1_send_done&) {
+	return false;
+}
+
+bool NGCHS2::onToxEvent(const Tox_Event_Group_Peer_Join* e) {
+	const auto group_number = tox_event_group_peer_join_get_group_number(e);
+	const auto peer_number = tox_event_group_peer_join_get_peer_id(e);
+
+	const auto c = _tcm.getContactGroupPeer(group_number, peer_number);
+	assert(c);
+
+	// add to check list with inital cooldown
+
+	return false;
+}
+
+bool NGCHS2::onToxEvent(const Tox_Event_Group_Peer_Exit* e) {
+	return false;
+}
+

solanaceae/ngc_hs2/ngc_hs2.hpp

@@ -0,0 +1,44 @@
+#pragma once
+
+//#include <solanaceae/contact/contact_model3.hpp>
+#include <solanaceae/toxcore/tox_event_interface.hpp>
+
+//#include <solanaceae/message3/registry_message_model.hpp>
+
+#include <solanaceae/ngc_ft1/ngcft1.hpp>
+
+// fwd
+class ToxContactModel2;
+
+class NGCHS2 : public ToxEventI, public NGCFT1EventI {
+	ToxContactModel2& _tcm;
+	//Contact3Registry& _cr;
+	//RegistryMessageModelI& _rmm;
+	ToxEventProviderI::SubscriptionReference _tep_sr;
+	NGCFT1& _nft;
+	NGCFT1EventProviderI::SubscriptionReference _nftep_sr;
+
+	public:
+		NGCHS2(
+			ToxContactModel2& tcm,
+			ToxEventProviderI& tep,
+			NGCFT1& nf
+		);
+
+		~NGCHS2(void);
+
+		float iterate(float delta);
+
+	protected:
+		bool onEvent(const Events::NGCFT1_recv_request&) override;
+		bool onEvent(const Events::NGCFT1_recv_init&) override;
+		bool onEvent(const Events::NGCFT1_recv_data&) override;
+		bool onEvent(const Events::NGCFT1_send_data&) override;
+		bool onEvent(const Events::NGCFT1_recv_done&) override;
+		bool onEvent(const Events::NGCFT1_send_done&) override;
+
+	protected:
+		bool onToxEvent(const Tox_Event_Group_Peer_Join* e) override;
+		bool onToxEvent(const Tox_Event_Group_Peer_Exit* e) override;
+};
+