detect and reuse existing data

src/CMakeLists.txt

@@ -30,6 +30,25 @@ target_link_libraries(ngcft1 PUBLIC
 
 ########################################
 
+add_library(sha1_ngcft1 STATIC
+	./hash_utils.hpp
+	./hash_utils.cpp
+
+	./ft1_sha1_info.hpp
+	./ft1_sha1_info.cpp
+
+	./sha1_ngcft1.hpp
+	./sha1_ngcft1.cpp
+)
+target_link_libraries(sha1_ngcft1 PUBLIC
+	ngcft1
+	sha1::sha1
+	solanaceae_tox_contacts
+	solanaceae_message3
+)
+
+########################################
+
 add_library(plugin_ngcft1 SHARED
 	./plugin_ngcft1.cpp
 )
@@ -38,6 +57,7 @@ target_link_libraries(plugin_ngcft1 PUBLIC
 	solanaceae_plugin
 	ngcext
 	ngcft1
+	sha1_ngcft1
 )
 
 ########################################

src/ft1_sha1_info.cpp

@@ -0,0 +1,139 @@
+#include "./ft1_sha1_info.hpp"
+
+#include <sodium.h>
+
+SHA1Digest::SHA1Digest(const std::vector<uint8_t>& v) {
+	assert(v.size() == data.size());
+	for (size_t i = 0; i < data.size(); i++) {
+		data[i] = v[i];
+	}
+}
+
+SHA1Digest::SHA1Digest(const uint8_t* d, size_t s) {
+	assert(s == data.size());
+	for (size_t i = 0; i < data.size(); i++) {
+		data[i] = d[i];
+	}
+}
+
+std::ostream& operator<<(std::ostream& out, const SHA1Digest& v) {
+	std::string str{};
+	str.resize(v.size()*2, '?');
+
+	// HECK, std is 1 larger than size returns ('\0')
+	sodium_bin2hex(str.data(), str.size()+1, v.data.data(), v.data.size());
+
+	out << str;
+
+	return out;
+}
+
+size_t FT1InfoSHA1::chunkSize(size_t chunk_index) const {
+	if (chunk_index+1 == chunks.size()) {
+		// last chunk
+		return file_size - chunk_index * chunk_size;
+	} else {
+		return chunk_size;
+	}
+}
+
+std::vector<uint8_t> FT1InfoSHA1::toBuffer(void) const {
+	std::vector<uint8_t> buffer;
+
+	assert(!file_name.empty());
+	// TODO: optimize
+	for (size_t i = 0; i < 256; i++) {
+		if (i < file_name.size()) {
+			buffer.push_back(file_name.at(i));
+		} else {
+			buffer.push_back(0);
+		}
+	}
+	assert(buffer.size() == 256);
+
+	{ // HACK: endianess
+		buffer.push_back((file_size>>(0*8)) & 0xff);
+		buffer.push_back((file_size>>(1*8)) & 0xff);
+		buffer.push_back((file_size>>(2*8)) & 0xff);
+		buffer.push_back((file_size>>(3*8)) & 0xff);
+		buffer.push_back((file_size>>(4*8)) & 0xff);
+		buffer.push_back((file_size>>(5*8)) & 0xff);
+		buffer.push_back((file_size>>(6*8)) & 0xff);
+		buffer.push_back((file_size>>(7*8)) & 0xff);
+	}
+	assert(buffer.size() == 256+8);
+
+	// chunk size
+	{ // HACK: endianess
+		buffer.push_back((chunk_size>>(0*8)) & 0xff);
+		buffer.push_back((chunk_size>>(1*8)) & 0xff);
+		buffer.push_back((chunk_size>>(2*8)) & 0xff);
+		buffer.push_back((chunk_size>>(3*8)) & 0xff);
+	}
+
+	assert(buffer.size() == 256+8+4);
+
+	for (const auto& chunk : chunks) {
+		for (size_t i = 0; i < chunk.data.size(); i++) {
+			buffer.push_back(chunk.data[i]);
+		}
+	}
+	assert(buffer.size() == 256+8+4+20*chunks.size());
+
+	return buffer;
+}
+
+void FT1InfoSHA1::fromBuffer(const std::vector<uint8_t>& buffer) {
+	assert(buffer.size() >= 256+8+4);
+
+	// TODO: optimize
+	file_name.clear();
+	for (size_t i = 0; i < 256; i++) {
+		char next_char = static_cast<char>(buffer[i]);
+		if (next_char == 0) {
+			break;
+		}
+		file_name.push_back(next_char);
+	}
+
+	{ // HACK: endianess
+		file_size = 0;
+		file_size |= uint64_t(buffer[256+0]) << (0*8);
+		file_size |= uint64_t(buffer[256+1]) << (1*8);
+		file_size |= uint64_t(buffer[256+2]) << (2*8);
+		file_size |= uint64_t(buffer[256+3]) << (3*8);
+		file_size |= uint64_t(buffer[256+4]) << (4*8);
+		file_size |= uint64_t(buffer[256+5]) << (5*8);
+		file_size |= uint64_t(buffer[256+6]) << (6*8);
+		file_size |= uint64_t(buffer[256+7]) << (7*8);
+	}
+
+	{ // HACK: endianess
+		chunk_size = 0;
+		chunk_size |= uint32_t(buffer[256+8+0]) << (0*8);
+		chunk_size |= uint32_t(buffer[256+8+1]) << (1*8);
+		chunk_size |= uint32_t(buffer[256+8+2]) << (2*8);
+		chunk_size |= uint32_t(buffer[256+8+3]) << (3*8);
+	}
+
+	assert((buffer.size()-(256+8+4)) % 20 == 0);
+
+	for (size_t offset = 256+8+4; offset < buffer.size();) {
+		assert(buffer.size() >= offset + 20);
+
+		auto& chunk = chunks.emplace_back();
+		for (size_t i = 0; i < chunk.size(); i++, offset++) {
+			chunk.data[i] = buffer.at(offset);
+		}
+		// TODO: error/leftover checking
+	}
+}
+
+std::ostream& operator<<(std::ostream& out, const FT1InfoSHA1& v) {
+	out << "  file_name: " << v.file_name << "\n";
+	out << "  file_size: " << v.file_size << "\n";
+	out << "  chunk_size: " << v.chunk_size << "\n";
+	out << "  chunks.size(): " << v.chunks.size() << "\n";
+	return out;
+}
+

src/ft1_sha1_info.hpp

@@ -0,0 +1,55 @@
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <array>
+#include <ostream>
+#include <vector>
+#include <cassert>
+#include <string>
+
+struct SHA1Digest {
+	std::array<uint8_t, 20> data;
+
+	SHA1Digest(void) = default;
+	SHA1Digest(const std::vector<uint8_t>& v);
+	SHA1Digest(const uint8_t* d, size_t s);
+
+	bool operator==(const SHA1Digest& other) const { return data == other.data; }
+	bool operator!=(const SHA1Digest& other) const { return data != other.data; }
+
+	size_t size(void) const { return data.size(); }
+};
+
+std::ostream& operator<<(std::ostream& out, const SHA1Digest& v);
+
+namespace std { // inject
+	template<> struct hash<SHA1Digest> {
+		std::size_t operator()(const SHA1Digest& h) const noexcept {
+			return
+				size_t(h.data[0]) << (0*8) |
+				size_t(h.data[1]) << (1*8) |
+				size_t(h.data[2]) << (2*8) |
+				size_t(h.data[3]) << (3*8) |
+				size_t(h.data[4]) << (4*8) |
+				size_t(h.data[5]) << (5*8) |
+				size_t(h.data[6]) << (6*8) |
+				size_t(h.data[7]) << (7*8)
+			;
+		}
+	};
+} // std
+
+struct FT1InfoSHA1 {
+	std::string file_name;
+	uint64_t file_size {0};
+	uint32_t chunk_size {128*1024}; // 128KiB for now
+	std::vector<SHA1Digest> chunks;
+
+	size_t chunkSize(size_t chunk_index) const;
+
+	std::vector<uint8_t> toBuffer(void) const;
+	void fromBuffer(const std::vector<uint8_t>& buffer);
+};
+std::ostream& operator<<(std::ostream& out, const FT1InfoSHA1& v);
+

src/hash_utils.cpp

@@ -0,0 +1,26 @@
+#include "./hash_utils.hpp"
+
+#include <sha1.h>
+
+// returns the 20bytes sha1 hash
+std::vector<uint8_t> hash_sha1(const uint8_t* data, size_t size) {
+	SHA1_CTX ctx;
+	SHA1Init(&ctx);
+
+	{ // lib only takes uint32_t sizes, so chunk it
+		constexpr size_t hash_block_size {0xffffffff};
+		size_t i = 0;
+		for (; i + hash_block_size < size; i += hash_block_size) {
+			SHA1Update(&ctx, reinterpret_cast<const uint8_t*>(data) + i, hash_block_size);
+		}
+
+		if (i < size) {
+			SHA1Update(&ctx, reinterpret_cast<const uint8_t*>(data) + i, size - i);
+		}
+	}
+
+	std::vector<uint8_t> sha1_hash(20);
+	SHA1Final(sha1_hash.data(), &ctx);
+	return sha1_hash;
+}
+

src/hash_utils.hpp

@@ -0,0 +1,10 @@
+#pragma once
+
+#include <cstdint>
+#include <vector>
+
+// returns the 20bytes sha1 hash
+std::vector<uint8_t> hash_sha1(const uint8_t* data, size_t size);
+
+inline std::vector<uint8_t> hash_sha1(const char* data, size_t size) { return hash_sha1(reinterpret_cast<const uint8_t*>(data), size); }
+

src/ngcext.cpp

@@ -186,6 +186,44 @@ bool NGCEXTEventProvider::parse_ft1_data_ack(
 	);
 }
 
+bool NGCEXTEventProvider::parse_ft1_message(
+	uint32_t group_number, uint32_t peer_number,
+	const uint8_t* data, size_t data_size,
+	bool _private
+) {
+	if (_private) {
+		std::cerr << "NGCEXT: ft1_message cant be private (yet)\n";
+		return false;
+	}
+
+	Events::NGCEXT_ft1_message e;
+	e.group_number = group_number;
+	e.peer_number = peer_number;
+	size_t curser = 0;
+
+	// - 4 byte (message_id)
+	e.message_id = 0u;
+	_DATA_HAVE(sizeof(e.message_id), std::cerr << "NGCEXT: packet too small, missing message_id\n"; return false)
+	for (size_t i = 0; i < sizeof(e.message_id); i++, curser++) {
+		e.message_id |= uint32_t(data[curser]) << (i*8);
+	}
+
+	// - 4 byte (file_kind)
+	e.file_kind = 0u;
+	_DATA_HAVE(sizeof(e.file_kind), std::cerr << "NGCEXT: packet too small, missing file_kind\n"; return false)
+	for (size_t i = 0; i < sizeof(e.file_kind); i++, curser++) {
+		e.file_kind |= uint32_t(data[curser]) << (i*8);
+	}
+
+	// - X bytes (file_kind dependent id, differnt sizes)
+	e.file_id = {data+curser, data+curser+(data_size-curser)};
+
+	return dispatch(
+		NGCEXT_Event::FT1_MESSAGE,
+		e
+	);
+}
+
 bool NGCEXTEventProvider::handlePacket(
 	const uint32_t group_number,
 	const uint32_t peer_number,
@@ -214,6 +252,8 @@ bool NGCEXTEventProvider::handlePacket(
 			return parse_ft1_data(group_number, peer_number, data+1, data_size-1, _private);
 		case NGCEXT_Event::FT1_DATA_ACK:
 			return parse_ft1_data_ack(group_number, peer_number, data+1, data_size-1, _private);
+		case NGCEXT_Event::FT1_MESSAGE:
+			return parse_ft1_message(group_number, peer_number, data+1, data_size-1, _private);
 		default:
 			return false;
 	}

src/ngcext.hpp

@@ -113,6 +113,21 @@ namespace Events {
 		std::vector<uint16_t> sequence_ids;
 	};
 
+	struct NGCEXT_ft1_message {
+		uint32_t group_number;
+		uint32_t peer_number;
+
+		// - 4 byte (message_id)
+		uint32_t message_id;
+
+		// request the other side to initiate a FT
+		// - 4 byte (file_kind)
+		uint32_t file_kind;
+
+		// - X bytes (file_kind dependent id, differnt sizes)
+		std::vector<uint8_t> file_id;
+	};
+
 } // Events
 
 enum class NGCEXT_Event : uint8_t {
@@ -131,7 +146,7 @@ enum class NGCEXT_Event : uint8_t {
 	HS1_RESPONSE_LAST_IDS,
 
 	// request the other side to initiate a FT
-	// - 1 byte (file_kind)
+	// - 4 byte (file_kind)
 	// - X bytes (file_kind dependent id, differnt sizes)
 	FT1_REQUEST = 0x80 | 8u,
 
@@ -139,7 +154,7 @@ enum class NGCEXT_Event : uint8_t {
 
 	// tell the other side you want to start a FT
 	// TODO: might use id layer instead. with it, it would look similar to friends_ft
-	// - 1 byte (file_kind)
+	// - 4 byte (file_kind)
 	// - 8 bytes (data size, can be 0 if unknown, BUT files have to be atleast 1 byte)
 	// - 1 byte (temporary_file_tf_id, for this peer only, technically just a prefix to distinguish between simultainious fts)
 	// - X bytes (file_kind dependent id, differnt sizes)
@@ -168,6 +183,14 @@ enum class NGCEXT_Event : uint8_t {
 	// - ]
 	FT1_DATA_ACK,
 
+	// send file as message
+	// basically the opposite of request
+	// contains file_kind and file_id (and timestamp?)
+	// - 4 byte (message_id)
+	// - 4 byte (file_kind)
+	// - X bytes (file_kind dependent id, differnt sizes)
+	FT1_MESSAGE,
+
 	MAX
 };
 
@@ -180,6 +203,7 @@ struct NGCEXTEventI {
 	virtual bool onEvent(const Events::NGCEXT_ft1_init_ack&) { return false; }
 	virtual bool onEvent(const Events::NGCEXT_ft1_data&) { return false; }
 	virtual bool onEvent(const Events::NGCEXT_ft1_data_ack&) { return false; }
+	virtual bool onEvent(const Events::NGCEXT_ft1_message&) { return false; }
 };
 
 using NGCEXTEventProviderI = EventProviderI<NGCEXTEventI>;
@@ -233,6 +257,12 @@ class NGCEXTEventProvider : public ToxEventI, public NGCEXTEventProviderI {
 			bool _private
 		);
 
+		bool parse_ft1_message(
+			uint32_t group_number, uint32_t peer_number,
+			const uint8_t* data, size_t data_size,
+			bool _private
+		);
+
 		bool handlePacket(
 			const uint32_t group_number,
 			const uint32_t peer_number,

src/ngcft1.cpp

@@ -2,6 +2,8 @@
 
 #include <solanaceae/toxcore/utils.hpp>
 
+#include <sodium.h>
+
 #include <iostream>
 #include <set>
 #include <algorithm>
@@ -118,6 +120,29 @@ bool NGCFT1::sendPKG_FT1_DATA_ACK(
 	return _t.toxGroupSendCustomPrivatePacket(group_number, peer_number, false, pkg) == TOX_ERR_GROUP_SEND_CUSTOM_PRIVATE_PACKET_OK;
 }
 
+bool NGCFT1::sendPKG_FT1_MESSAGE(
+	uint32_t group_number,
+	uint32_t message_id,
+	uint32_t file_kind,
+	const uint8_t* file_id, size_t file_id_size
+) {
+	std::vector<uint8_t> pkg;
+	pkg.push_back(static_cast<uint8_t>(NGCEXT_Event::FT1_MESSAGE));
+
+	for (size_t i = 0; i < sizeof(message_id); i++) {
+		pkg.push_back((message_id>>(i*8)) & 0xff);
+	}
+	for (size_t i = 0; i < sizeof(file_kind); i++) {
+		pkg.push_back((file_kind>>(i*8)) & 0xff);
+	}
+	for (size_t i = 0; i < file_id_size; i++) {
+		pkg.push_back(file_id[i]);
+	}
+
+	// lossless
+	return _t.toxGroupSendCustomPacket(group_number, true, pkg) == TOX_ERR_GROUP_SEND_CUSTOM_PACKET_OK;
+}
+
 void NGCFT1::updateSendTransfer(float time_delta, uint32_t group_number, uint32_t peer_number, Group::Peer& peer, size_t idx, std::set<LEDBAT::SeqIDType>& timeouts_set) {
 	auto& tf_opt = peer.send_transfers.at(idx);
 	assert(tf_opt.has_value());
@@ -132,6 +157,13 @@ void NGCFT1::updateSendTransfer(float time_delta, uint32_t group_number, uint32_
 				if (tf.inits_sent >= 3) {
 					// delete, timed out 3 times
 					std::cerr << "NGCFT1 warning: ft init timed out, deleting\n";
+					dispatch(
+						NGCFT1_Event::send_done,
+						Events::NGCFT1_send_done{
+							group_number, peer_number,
+							static_cast<uint8_t>(idx),
+						}
+					);
 					tf_opt.reset();
 				} else {
 					// timed out, resend
@@ -150,7 +182,7 @@ void NGCFT1::updateSendTransfer(float time_delta, uint32_t group_number, uint32_
 					if (timeouts_set.count({idx, id})) {
 						// TODO: can fail
 						sendPKG_FT1_DATA(group_number, peer_number, idx, id, data.data(), data.size());
-						peer.cca.onLoss({idx, id}, false);
+						peer.cca->onLoss({idx, id}, false);
 						time_since_activity = 0.f;
 						timeouts_set.erase({idx, id});
 					}
@@ -158,12 +190,18 @@ void NGCFT1::updateSendTransfer(float time_delta, uint32_t group_number, uint32_
 
 				if (tf.time_since_activity >= sending_give_up_after) {
 					// no ack after 30sec, close ft
-					// TODO: notify app
 					std::cerr << "NGCFT1 warning: sending ft in progress timed out, deleting\n";
+					dispatch(
+						NGCFT1_Event::send_done,
+						Events::NGCFT1_send_done{
+							group_number, peer_number,
+							static_cast<uint8_t>(idx),
+						}
+					);
 
 					// clean up cca
 					tf.ssb.for_each(time_delta, [&](uint16_t id, const std::vector<uint8_t>& data, float& time_since_activity) {
-						peer.cca.onLoss({idx, id}, true);
+						peer.cca->onLoss({idx, id}, true);
 						timeouts_set.erase({idx, id});
 					});
 
@@ -174,7 +212,7 @@ void NGCFT1::updateSendTransfer(float time_delta, uint32_t group_number, uint32_
 
 				// if chunks in flight < window size (2)
 				//while (tf.ssb.size() < ngc_ft1_ctx->options.packet_window_size) {
-				int64_t can_packet_size {static_cast<int64_t>(peer.cca.canSend())};
+				int64_t can_packet_size {static_cast<int64_t>(peer.cca->canSend())};
 				//if (can_packet_size) {
 					//std::cerr << "FT: can_packet_size: " << can_packet_size;
 				//}
@@ -188,7 +226,7 @@ void NGCFT1::updateSendTransfer(float time_delta, uint32_t group_number, uint32_
 					size_t chunk_size = std::min<size_t>({
 						//496u,
 						//996u,
-						peer.cca.MAXIMUM_SEGMENT_DATA_SIZE,
+						peer.cca->MAXIMUM_SEGMENT_DATA_SIZE,
 						static_cast<size_t>(can_packet_size),
 						tf.file_size - tf.file_size_current
 					});
@@ -221,7 +259,7 @@ void NGCFT1::updateSendTransfer(float time_delta, uint32_t group_number, uint32_
 
 					uint16_t seq_id = tf.ssb.add(std::move(new_data));
 					sendPKG_FT1_DATA(group_number, peer_number, idx, seq_id, tf.ssb.entries.at(seq_id).data.data(), tf.ssb.entries.at(seq_id).data.size());
-					peer.cca.onSent({idx, seq_id}, chunk_size);
+					peer.cca->onSent({idx, seq_id}, chunk_size);
 
 #if defined(EXTRA_LOGGING) && EXTRA_LOGGING == 1
 					fprintf(stderr, "FT: sent data size: %ld (seq %d)\n", chunk_size, seq_id);
@@ -242,7 +280,7 @@ void NGCFT1::updateSendTransfer(float time_delta, uint32_t group_number, uint32_
 				//if (time_since_activity >= ngc_ft1_ctx->options.sending_resend_without_ack_after) {
 				if (timeouts_set.count({idx, id})) {
 					sendPKG_FT1_DATA(group_number, peer_number, idx, id, data.data(), data.size());
-					peer.cca.onLoss({idx, id}, false);
+					peer.cca->onLoss({idx, id}, false);
 					time_since_activity = 0.f;
 					timeouts_set.erase({idx, id});
 				}
@@ -254,7 +292,7 @@ void NGCFT1::updateSendTransfer(float time_delta, uint32_t group_number, uint32_
 
 				// clean up cca
 				tf.ssb.for_each(time_delta, [&](uint16_t id, const std::vector<uint8_t>& data, float& time_since_activity) {
-					peer.cca.onLoss({idx, id}, true);
+					peer.cca->onLoss({idx, id}, true);
 					timeouts_set.erase({idx, id});
 				});
 
@@ -270,7 +308,7 @@ void NGCFT1::updateSendTransfer(float time_delta, uint32_t group_number, uint32_
 }
 
 void NGCFT1::iteratePeer(float time_delta, uint32_t group_number, uint32_t peer_number, Group::Peer& peer) {
-	auto timeouts = peer.cca.getTimeouts();
+	auto timeouts = peer.cca->getTimeouts();
 	std::set<LEDBAT::SeqIDType> timeouts_set{timeouts.cbegin(), timeouts.cend()};
 
 	for (size_t idx = 0; idx < peer.send_transfers.size(); idx++) {
@@ -293,6 +331,9 @@ NGCFT1::NGCFT1(
 	_neep.subscribe(this, NGCEXT_Event::FT1_INIT_ACK);
 	_neep.subscribe(this, NGCEXT_Event::FT1_DATA);
 	_neep.subscribe(this, NGCEXT_Event::FT1_DATA_ACK);
+	_neep.subscribe(this, NGCEXT_Event::FT1_MESSAGE);
+
+	_tep.subscribe(this, Tox_Event::TOX_EVENT_GROUP_PEER_EXIT);
 }
 
 void NGCFT1::iterate(float time_delta) {
@@ -370,8 +411,23 @@ bool NGCFT1::NGC_FT1_send_init_private(
 	return true;
 }
 
+bool NGCFT1::NGC_FT1_send_message_public(
+	uint32_t group_number,
+	uint32_t& message_id,
+	uint32_t file_kind,
+	const uint8_t* file_id, size_t file_id_size
+) {
+	// create msg_id
+	message_id = randombytes_random();
+
+	// TODO: check return value
+	return sendPKG_FT1_MESSAGE(group_number, message_id, file_kind, file_id, file_id_size);
+}
+
 bool NGCFT1::onEvent(const Events::NGCEXT_ft1_request& e) {
+//#if !NDEBUG
 	std::cout << "NGCFT1: FT1_REQUEST fk:" << e.file_kind << " [" << bin2hex(e.file_id) << "]\n";
+//#endif
 
 	// .... just rethrow??
 	// TODO: dont
@@ -386,7 +442,9 @@ bool NGCFT1::onEvent(const Events::NGCEXT_ft1_request& e) {
 }
 
 bool NGCFT1::onEvent(const Events::NGCEXT_ft1_init& e) {
-	std::cout << "NGCFT1: FT1_INIT fk:" << e.file_kind << " fs:" << e.file_size << " tid:" << e.transfer_id << " [" << bin2hex(e.file_id) << "]\n";
+//#if !NDEBUG
+	std::cout << "NGCFT1: FT1_INIT fk:" << e.file_kind << " fs:" << e.file_size << " tid:" << int(e.transfer_id) << " [" << bin2hex(e.file_id) << "]\n";
+//#endif
 
 	bool accept = false;
 	dispatch(
@@ -412,7 +470,7 @@ bool NGCFT1::onEvent(const Events::NGCEXT_ft1_init& e) {
 
 	auto& peer = groups[e.group_number].peers[e.peer_number];
 	if (peer.recv_transfers[e.transfer_id].has_value()) {
-		std::cerr << "NGCFT1 warning: overwriting existing recv_transfer " << e.transfer_id << "\n";
+		std::cerr << "NGCFT1 warning: overwriting existing recv_transfer " << int(e.transfer_id) << "\n";
 	}
 
 	peer.recv_transfers[e.transfer_id] = Group::Peer::RecvTransfer{
@@ -428,7 +486,9 @@ bool NGCFT1::onEvent(const Events::NGCEXT_ft1_init& e) {
 }
 
 bool NGCFT1::onEvent(const Events::NGCEXT_ft1_init_ack& e) {
+//#if !NDEBUG
 	std::cout << "NGCFT1: FT1_INIT_ACK\n";
+//#endif
 
 	// we now should start sending data
 
@@ -459,7 +519,9 @@ bool NGCFT1::onEvent(const Events::NGCEXT_ft1_init_ack& e) {
 }
 
 bool NGCFT1::onEvent(const Events::NGCEXT_ft1_data& e) {
+#if !NDEBUG
 	std::cout << "NGCFT1: FT1_DATA\n";
+#endif
 
 	if (e.data.empty()) {
 		std::cerr << "NGCFT1 error: data of size 0!\n";
@@ -508,11 +570,26 @@ bool NGCFT1::onEvent(const Events::NGCEXT_ft1_data& e) {
 		sendPKG_FT1_DATA_ACK(e.group_number, e.peer_number, e.transfer_id, ack_seq_ids.data(), ack_seq_ids.size());
 	}
 
+
+	if (transfer.file_size_current == transfer.file_size) {
+		// TODO: set all data received, and clean up
+		//transfer.state = Group::Peer::RecvTransfer::State::RECV;
+		dispatch(
+			NGCFT1_Event::recv_done,
+			Events::NGCFT1_recv_done{
+				e.group_number, e.peer_number,
+				e.transfer_id
+			}
+		);
+	}
+
 	return true;
 }
 
 bool NGCFT1::onEvent(const Events::NGCEXT_ft1_data_ack& e) {
+#if !NDEBUG
 	std::cout << "NGCFT1: FT1_DATA_ACK\n";
+#endif
 
 	if (!groups.count(e.group_number)) {
 		std::cerr << "NGCFT1 warning: data_ack for unknown group\n";
@@ -546,14 +623,97 @@ bool NGCFT1::onEvent(const Events::NGCEXT_ft1_data_ack& e) {
 		seqs.push_back({e.transfer_id, it});
 		transfer.ssb.erase(it);
 	}
-	peer.cca.onAck(seqs);
+	peer.cca->onAck(seqs);
 
 	// delete if all packets acked
 	if (transfer.file_size == transfer.file_size_current && transfer.ssb.size() == 0) {
-		std::cout << "NGCFT1: " << e.transfer_id << " done\n";
+		std::cout << "NGCFT1: " << int(e.transfer_id) << " done\n";
+		dispatch(
+			NGCFT1_Event::send_done,
+			Events::NGCFT1_send_done{
+				e.group_number, e.peer_number,
+				e.transfer_id,
+			}
+		);
 		peer.send_transfers[e.transfer_id].reset();
 	}
 
 	return true;
 }
 
+bool NGCFT1::onEvent(const Events::NGCEXT_ft1_message& e) {
+	std::cout << "NGCFT1: FT1_MESSAGE mid:" << e.message_id << " fk:" << e.file_kind << " [" << bin2hex(e.file_id) << "]\n";
+
+	// .... just rethrow??
+	// TODO: dont
+	return dispatch(
+		NGCFT1_Event::recv_message,
+		Events::NGCFT1_recv_message{
+			e.group_number, e.peer_number,
+			e.message_id,
+			static_cast<NGCFT1_file_kind>(e.file_kind),
+			e.file_id.data(), e.file_id.size()
+		}
+	);
+}
+
+bool NGCFT1::onToxEvent(const Tox_Event_Group_Peer_Exit* e) {
+	const auto group_number = tox_event_group_peer_exit_get_group_number(e);
+	const auto peer_number = tox_event_group_peer_exit_get_peer_id(e);
+
+	// peer disconnected, end all transfers
+
+	if (!groups.count(group_number)) {
+		return false;
+	}
+
+	auto& group = groups.at(group_number);
+
+	if (!group.peers.count(peer_number)) {
+		return false;
+	}
+
+	auto& peer = group.peers.at(peer_number);
+
+	for (size_t i = 0; i < peer.send_transfers.size(); i++) {
+		auto& it_opt = peer.send_transfers.at(i);
+		if (!it_opt.has_value()) {
+			continue;
+		}
+
+		std::cout << "NGCFT1: sending " << int(i) << " canceled bc peer offline\n";
+		dispatch(
+			NGCFT1_Event::send_done,
+			Events::NGCFT1_send_done{
+				group_number, peer_number,
+				static_cast<uint8_t>(i),
+			}
+		);
+
+		it_opt.reset();
+	}
+
+	for (size_t i = 0; i < peer.recv_transfers.size(); i++) {
+		auto& it_opt = peer.recv_transfers.at(i);
+		if (!it_opt.has_value()) {
+			continue;
+		}
+
+		std::cout << "NGCFT1: receiving " << int(i) << " canceled bc peer offline\n";
+		dispatch(
+			NGCFT1_Event::recv_done,
+			Events::NGCFT1_recv_done{
+				group_number, peer_number,
+				static_cast<uint8_t>(i),
+			}
+		);
+
+		it_opt.reset();
+	}
+
+	// reset cca
+	peer.cca = std::make_unique<LEDBAT>(500-4); // TODO: replace with tox_group_max_custom_lossy_packet_length()-4
+
+	return false;
+}
+

src/ngcft1.hpp

@@ -15,6 +15,7 @@
 
 #include <map>
 #include <set>
+#include <memory>
 
 namespace Events {
 
@@ -67,14 +68,48 @@ namespace Events {
 		size_t data_size;
 	};
 
+	struct NGCFT1_recv_done {
+		uint32_t group_number;
+		uint32_t peer_number;
+
+		uint8_t transfer_id;
+		// TODO: reason
+	};
+
+	struct NGCFT1_send_done {
+		uint32_t group_number;
+		uint32_t peer_number;
+
+		uint8_t transfer_id;
+		// TODO: reason
+	};
+
+	struct NGCFT1_recv_message {
+		uint32_t group_number;
+		uint32_t peer_number;
+
+		uint32_t message_id;
+
+		NGCFT1_file_kind file_kind;
+
+		const uint8_t* file_id;
+		size_t file_id_size;
+	};
+
 } // Events
 
 enum class NGCFT1_Event : uint8_t {
 	recv_request,
 	recv_init,
+
 	recv_data,
 	send_data,
 
+	recv_done,
+	send_done,
+
+	recv_message,
+
 	MAX
 };
 
@@ -84,6 +119,9 @@ struct NGCFT1EventI {
 	virtual bool onEvent(const Events::NGCFT1_recv_init&) { return false; }
 	virtual bool onEvent(const Events::NGCFT1_recv_data&) { return false; }
 	virtual bool onEvent(const Events::NGCFT1_send_data&) { return false; } // const?
+	virtual bool onEvent(const Events::NGCFT1_recv_done&) { return false; }
+	virtual bool onEvent(const Events::NGCFT1_send_done&) { return false; }
+	virtual bool onEvent(const Events::NGCFT1_recv_message&) { return false; }
 };
 
 using NGCFT1EventProviderI = EventProviderI<NGCFT1EventI>;
@@ -101,7 +139,7 @@ class NGCFT1 : public ToxEventI, public NGCEXTEventI, public NGCFT1EventProvider
 
 	struct Group {
 		struct Peer {
-			LEDBAT cca{500-4}; // TODO: replace with tox_group_max_custom_lossy_packet_length()-4
+			std::unique_ptr<LEDBAT> cca = std::make_unique<LEDBAT>(500-4); // TODO: replace with tox_group_max_custom_lossy_packet_length()-4
 
 			struct RecvTransfer {
 				uint32_t file_kind;
@@ -159,6 +197,7 @@ class NGCFT1 : public ToxEventI, public NGCEXTEventI, public NGCFT1EventProvider
 		bool sendPKG_FT1_INIT_ACK(uint32_t group_number, uint32_t peer_number, uint8_t transfer_id);
 		bool sendPKG_FT1_DATA(uint32_t group_number, uint32_t peer_number, uint8_t transfer_id, uint16_t sequence_id, const uint8_t* data, size_t data_size);
 		bool sendPKG_FT1_DATA_ACK(uint32_t group_number, uint32_t peer_number, uint8_t transfer_id, const uint16_t* seq_ids, size_t seq_ids_size);
+		bool sendPKG_FT1_MESSAGE(uint32_t group_number, uint32_t message_id, uint32_t file_kind, const uint8_t* file_id, size_t file_id_size);
 
 		void updateSendTransfer(float time_delta, uint32_t group_number, uint32_t peer_number, Group::Peer& peer, size_t idx, std::set<LEDBAT::SeqIDType>& timeouts_set);
 		void iteratePeer(float time_delta, uint32_t group_number, uint32_t peer_number, Group::Peer& peer);
@@ -189,14 +228,24 @@ class NGCFT1 : public ToxEventI, public NGCEXTEventI, public NGCFT1EventProvider
 			uint8_t* transfer_id
 		);
 
+		// sends the message and fills in message_id
+		bool NGC_FT1_send_message_public(
+			uint32_t group_number,
+			uint32_t& message_id,
+			uint32_t file_kind,
+			const uint8_t* file_id, size_t file_id_size
+		);
+
 	protected:
 		bool onEvent(const Events::NGCEXT_ft1_request&) override;
 		bool onEvent(const Events::NGCEXT_ft1_init&) override;
 		bool onEvent(const Events::NGCEXT_ft1_init_ack&) override;
 		bool onEvent(const Events::NGCEXT_ft1_data&) override;
 		bool onEvent(const Events::NGCEXT_ft1_data_ack&) override;
+		bool onEvent(const Events::NGCEXT_ft1_message&) override;
 
 	protected:
+		bool onToxEvent(const Tox_Event_Group_Peer_Exit* e) override;
 		//bool onToxEvent(const Tox_Event_Group_Custom_Packet* e) override;
 		//bool onToxEvent(const Tox_Event_Group_Custom_Private_Packet* e) override;
 };

src/plugin_ngcft1.cpp

@@ -2,19 +2,18 @@
 
 #include "./ngcext.hpp"
 #include "./ngcft1.hpp"
+#include "./sha1_ngcft1.hpp"
 
 #include <memory>
 #include <iostream>
 
-// fwd
-//class RegMessageModel;
-
 #define RESOLVE_INSTANCE(x) static_cast<x*>(solana_api->resolveInstance(#x))
 #define PROVIDE_INSTANCE(x, p, v) solana_api->provideInstance(#x, p, static_cast<x*>(v))
 
 static std::unique_ptr<NGCEXTEventProvider> g_ngcextep = nullptr;
 // TODO: make sep plug
 static std::unique_ptr<NGCFT1> g_ngcft1 = nullptr;
+static std::unique_ptr<SHA1_NGCFT1> g_sha1_ngcft1 = nullptr;
 
 extern "C" {
 
@@ -35,10 +34,16 @@ SOLANA_PLUGIN_EXPORT uint32_t solana_plugin_start(struct SolanaAPI* solana_api)
 
 	ToxI* tox_i = nullptr;
 	ToxEventProviderI* tox_event_provider_i = nullptr;
+	Contact3Registry* cr = nullptr;
+	RegistryMessageModel* rmm = nullptr;
+	ToxContactModel2* tcm = nullptr;
 
 	{ // make sure required types are loaded
 		tox_i = RESOLVE_INSTANCE(ToxI);
 		tox_event_provider_i = RESOLVE_INSTANCE(ToxEventProviderI);
+		cr = RESOLVE_INSTANCE(Contact3Registry);
+		rmm = RESOLVE_INSTANCE(RegistryMessageModel);
+		tcm = RESOLVE_INSTANCE(ToxContactModel2);
 
 		if (tox_i == nullptr) {
 			std::cerr << "PLUGIN NGCEXT missing ToxI\n";
@@ -49,12 +54,28 @@ SOLANA_PLUGIN_EXPORT uint32_t solana_plugin_start(struct SolanaAPI* solana_api)
 			std::cerr << "PLUGIN NGCEXT missing ToxEventProviderI\n";
 			return 2;
 		}
+
+		if (cr == nullptr) {
+			std::cerr << "PLUGIN NGCEXT missing Contact3Registry\n";
+			return 2;
+		}
+
+		if (rmm == nullptr) {
+			std::cerr << "PLUGIN NGCEXT missing RegistryMessageModel\n";
+			return 2;
+		}
+
+		if (tcm == nullptr) {
+			std::cerr << "PLUGIN NGCEXT missing ToxContactModel2\n";
+			return 2;
+		}
 	}
 
 	// static store, could be anywhere tho
 	// construct with fetched dependencies
 	g_ngcextep = std::make_unique<NGCEXTEventProvider>(*tox_event_provider_i);
 	g_ngcft1 = std::make_unique<NGCFT1>(*tox_i, *tox_event_provider_i, *g_ngcextep.get());
+	g_sha1_ngcft1 = std::make_unique<SHA1_NGCFT1>(*cr, *rmm, *g_ngcft1.get(), *tcm);
 
 	// register types
 	PROVIDE_INSTANCE(NGCEXTEventProviderI, "NGCEXT", g_ngcextep.get());
@@ -62,12 +83,15 @@ SOLANA_PLUGIN_EXPORT uint32_t solana_plugin_start(struct SolanaAPI* solana_api)
 	PROVIDE_INSTANCE(NGCFT1EventProviderI, "NGCEXT", g_ngcft1.get());
 	PROVIDE_INSTANCE(NGCFT1, "NGCEXT", g_ngcft1.get());
 
+	PROVIDE_INSTANCE(SHA1_NGCFT1, "NGCEXT", g_sha1_ngcft1.get());
+
 	return 0;
 }
 
 SOLANA_PLUGIN_EXPORT void solana_plugin_stop(void) {
 	std::cout << "PLUGIN NGCEXT STOP()\n";
 
+	g_sha1_ngcft1.reset();
 	g_ngcft1.reset();
 	g_ngcextep.reset();
 }
@@ -76,6 +100,7 @@ SOLANA_PLUGIN_EXPORT void solana_plugin_tick(float delta) {
 	//std::cout << "PLUGIN NGCEXT TICK()\n";
 
 	g_ngcft1->iterate(delta);
+	g_sha1_ngcft1->iterate(delta);
 }
 
 } // extern C

src/sha1_ngcft1.hpp

@@ -0,0 +1,134 @@
+#pragma once
+
+// solanaceae port of sha1 fts for NGCFT1
+
+#include <solanaceae/contact/contact_model3.hpp>
+#include <solanaceae/message3/registry_message_model.hpp>
+#include <solanaceae/tox_contacts/tox_contact_model2.hpp>
+
+#include "./ngcft1.hpp"
+
+#include "./ft1_sha1_info.hpp"
+
+#include <entt/entity/registry.hpp>
+#include <entt/entity/handle.hpp>
+#include <entt/container/dense_map.hpp>
+
+#include <variant>
+#include <random>
+
+enum class Content : uint32_t {};
+using ContentRegistry = entt::basic_registry<Content>;
+using ContentHandle = entt::basic_handle<ContentRegistry>;
+
+class SHA1_NGCFT1 : public RegistryMessageModelEventI, public NGCFT1EventI {
+	Contact3Registry& _cr;
+	RegistryMessageModel& _rmm;
+	NGCFT1& _nft;
+	ToxContactModel2& _tcm;
+
+	std::minstd_rand _rng {1337*11};
+
+	// registry per group?
+	ContentRegistry _contentr;
+
+	// limit this to each group?
+	entt::dense_map<SHA1Digest, ContentHandle> _info_to_content;
+
+	// sha1 chunk index
+	// TODO: optimize lookup
+	// TODO: multiple contents. hashes might be unique, but data is not
+	entt::dense_map<SHA1Digest, ContentHandle> _chunks;
+
+	// group_number, peer_number, content, chunk_hash, timer
+	std::deque<std::tuple<uint32_t, uint32_t, ContentHandle, SHA1Digest, float>> _queue_requested_chunk;
+	//void queueUpRequestInfo(uint32_t group_number, uint32_t peer_number, const SHA1Digest& hash);
+	void queueUpRequestChunk(uint32_t group_number, uint32_t peer_number, ContentHandle content, const SHA1Digest& hash);
+
+	struct SendingTransfer {
+		struct Info {
+			// copy of info data
+			// too large?
+			std::vector<uint8_t> info_data;
+		};
+
+		struct Chunk {
+			ContentHandle content;
+			size_t chunk_index; // <.< remove offset_into_file
+			//uint64_t offset_into_file;
+			// or data?
+			// if memmapped, this would be just a pointer
+		};
+
+		std::variant<Info, Chunk> v;
+
+		float time_since_activity {0.f};
+	};
+	// key is groupid + peerid
+	entt::dense_map<uint64_t, entt::dense_map<uint8_t, SendingTransfer>> _sending_transfers;
+
+	struct ReceivingTransfer {
+		struct Info {
+			ContentHandle content;
+			// copy of info data
+			// too large?
+			std::vector<uint8_t> info_data;
+		};
+
+		struct Chunk {
+			ContentHandle content;
+			std::vector<size_t> chunk_indices;
+			// or data?
+			// if memmapped, this would be just a pointer
+		};
+
+		std::variant<Info, Chunk> v;
+
+		float time_since_activity {0.f};
+	};
+	// key is groupid + peerid
+	entt::dense_map<uint64_t, entt::dense_map<uint8_t, ReceivingTransfer>> _receiving_transfers;
+
+	// makes request rotate around open content
+	std::deque<ContentHandle> _queue_content_want_info;
+	std::deque<ContentHandle> _queue_content_want_chunk;
+
+	static uint64_t combineIds(const uint32_t group_number, const uint32_t peer_number);
+
+	void updateMessages(ContentHandle ce);
+
+	std::optional<std::pair<uint32_t, uint32_t>> selectPeerForRequest(ContentHandle ce);
+
+	public: // TODO: config
+		bool _udp_only {false};
+
+		size_t _max_concurrent_in {4};
+		size_t _max_concurrent_out {6};
+		// TODO: probably also includes running transfers rn (meh)
+		size_t _max_pending_requests {16}; // per content
+
+	public:
+		SHA1_NGCFT1(
+			Contact3Registry& cr,
+			RegistryMessageModel& rmm,
+			NGCFT1& nft,
+			ToxContactModel2& tcm
+		);
+
+		void iterate(float delta);
+
+	protected: // rmm events (actions)
+		bool onEvent(const Message::Events::MessageUpdated&) override;
+
+	protected: // events
+		bool onEvent(const Events::NGCFT1_recv_request&) override;
+		bool onEvent(const Events::NGCFT1_recv_init&) override;
+		bool onEvent(const Events::NGCFT1_recv_data&) override;
+		bool onEvent(const Events::NGCFT1_send_data&) override; // const?
+		bool onEvent(const Events::NGCFT1_recv_done&) override;
+		bool onEvent(const Events::NGCFT1_send_done&) override;
+		bool onEvent(const Events::NGCFT1_recv_message&) override;
+
+		bool sendFilePath(const Contact3 c, std::string_view file_name, std::string_view file_path) override;
+};
+