add support for variable chunk size, double

external/tox_ngc_ft1/tox_ngc_ft1

@@ -1 +1 @@
-Subproject commit 3675c2a1a8dc01e3a892d4f487f98657a9ea0ce6
+Subproject commit bbc96bd776a29a6581545d60b5cb6e60e7453f1f

src/ft_sha1_info.cpp

@@ -54,20 +54,28 @@ std::vector<uint8_t> FTInfoSHA1::toBuffer(void) const {
 	}
 	assert(buffer.size() == 256+8);
 
-	// chunk size?
+	// chunk size
+	{ // HACK: endianess
+		buffer.push_back((chunk_size>>(0*8)) & 0xff);
+		buffer.push_back((chunk_size>>(1*8)) & 0xff);
+		buffer.push_back((chunk_size>>(2*8)) & 0xff);
+		buffer.push_back((chunk_size>>(3*8)) & 0xff);
+	}
+
+	assert(buffer.size() == 256+8+4);
 
 	for (const auto& chunk : chunks) {
 		for (size_t i = 0; i < chunk.data.size(); i++) {
 			buffer.push_back(chunk.data[i]);
 		}
 	}
-	assert(buffer.size() == 256+8+20*chunks.size());
+	assert(buffer.size() == 256+8+4+20*chunks.size());
 
 	return buffer;
 }
 
 void FTInfoSHA1::fromBuffer(const std::vector<uint8_t>& buffer) {
-	assert(buffer.size() >= 256+8);
+	assert(buffer.size() >= 256+8+4);
 
 	// TODO: optimize
 	file_name.clear();
@@ -91,9 +99,17 @@ void FTInfoSHA1::fromBuffer(const std::vector<uint8_t>& buffer) {
 		file_size |= uint64_t(buffer[256+7]) << (7*8);
 	}
 
-	assert((buffer.size()-(256+8)) % 20 == 0);
+	{ // HACK: endianess
+		chunk_size = 0;
+		chunk_size |= uint32_t(buffer[256+8+0]) << (0*8);
+		chunk_size |= uint32_t(buffer[256+8+1]) << (1*8);
+		chunk_size |= uint32_t(buffer[256+8+2]) << (2*8);
+		chunk_size |= uint32_t(buffer[256+8+3]) << (3*8);
+	}
 
-	for (size_t offset = 256+8; offset < buffer.size();) {
+	assert((buffer.size()-(256+8+4)) % 20 == 0);
+
+	for (size_t offset = 256+8+4; offset < buffer.size();) {
 		assert(buffer.size() >= offset + 20);
 
 		auto& chunk = chunks.emplace_back();
@@ -107,6 +123,7 @@ void FTInfoSHA1::fromBuffer(const std::vector<uint8_t>& buffer) {
 std::ostream& operator<<(std::ostream& out, const FTInfoSHA1& v) {
 	out << "  file_name: " << v.file_name << "\n";
 	out << "  file_size: " << v.file_size << "\n";
+	out << "  chunk_size: " << v.chunk_size << "\n";
 	out << "  chunks.size(): " << v.chunks.size() << "\n";
 	return out;
 }

src/ft_sha1_info.hpp

@@ -43,7 +43,7 @@ namespace std { // inject
 struct FTInfoSHA1 {
 	std::string file_name;
 	uint64_t file_size {0};
-	static constexpr size_t chunk_size {64*1024}; // 64KiB for now
+	uint32_t chunk_size {128*1024}; // 128KiB for now
 	std::vector<SHA1Digest> chunks;
 
 	std::vector<uint8_t> toBuffer(void) const;

src/states/receive_start_sha1.cpp

@@ -94,8 +94,8 @@ std::unique_ptr<StateI> ReceiveStartSHA1::nextState(void) {
 		std::cout << "ReceiveStartSHA1 checking existing file\n";
 		size_t f_i {0};
 		size_t tmp_have_count {0};
-		for (size_t c_i = 0; f_i + FTInfoSHA1::chunk_size < file_map.length(); f_i += FTInfoSHA1::chunk_size, c_i++) {
-			if (sha1_info.chunks[c_i] == hash_sha1(file_map.data()+f_i, FTInfoSHA1::chunk_size)) {
+		for (size_t c_i = 0; f_i + sha1_info.chunk_size < file_map.length(); f_i += sha1_info.chunk_size, c_i++) {
+			if (sha1_info.chunks[c_i] == hash_sha1(file_map.data()+f_i, sha1_info.chunk_size)) {
 				have_chunk[c_i] = true;
 				tmp_have_count++;
 			}

src/states/send_start_sha1.cpp

@@ -29,11 +29,12 @@ SendStartSHA1::SendStartSHA1(ToxClient& tcl, const CommandLine& cl) : StateI(tcl
 	// build info
 	_sha1_info.file_name = std::filesystem::path(cl.send_path).filename().string();
 	_sha1_info.file_size = _file_map.length();
+	//_sha1_info.chunk_size;
 
 	{ // build chunks
 		size_t i = 0;
-		for (; i + FTInfoSHA1::chunk_size < _file_map.length(); i += FTInfoSHA1::chunk_size) {
-			_sha1_info.chunks.push_back(hash_sha1(_file_map.data()+i, FTInfoSHA1::chunk_size));
+		for (; i + _sha1_info.chunk_size < _file_map.length(); i += _sha1_info.chunk_size) {
+			_sha1_info.chunks.push_back(hash_sha1(_file_map.data()+i, _sha1_info.chunk_size));
 		}
 
 		if (i < _file_map.length()) {

src/states/sha1.cpp

@@ -188,7 +188,7 @@ bool SHA1::iterate(float delta) {
 
 	// log
 	_io_log_timer += delta;
-	static const float log_interval {15.f};
+	static const float log_interval {10.f};
 	if (_io_log_timer >= log_interval) {
 		_io_log_timer = 0.f;
 
@@ -203,6 +203,8 @@ bool SHA1::iterate(float delta) {
 
 		std::cout << "SHA1 speed down: " << down_kibs << "KiB/s up: " << up_kibs << "KiB/s\n";
 		std::cout << "SHA1 total down: " << _bytes_down / 1024 << "KiB   up: " << _bytes_up / 1024 << "KiB\n";
+
+		std::cout << "SHA1 cwq:" << _chunk_want_queue.size() << " cwqr:" << _chunks_requested.size() << " trc:" << _transfers_receiving_chunk.size() << " tsc:" << _transfers_sending_chunk.size() << "\n";
 	}
 
 	// TODO: unmap and remap the file every couple of minutes to keep ram usage down?

src/tox_client.cpp

@@ -274,6 +274,7 @@ void ToxClient::onToxGroupInvite(uint32_t friend_number, const uint8_t* invite_d
 void ToxClient::onToxGroupPeerJoin(uint32_t group_number, uint32_t peer_id) {
 	std::cout << "TCL group peer join " << group_number << ":" << peer_id << "\n";
 	_groups[group_number].emplace(peer_id);
+	//_groups[group_number][peer_id] = tox_group_peer_get_connection_status(_tox, group_number, peer_id, nullptr);
 	_tox_profile_dirty = true;
 }
 

src/tox_client.hpp

@@ -4,7 +4,6 @@
 
 #include "./state.hpp"
 
-#include <memory>
 #include <tox/tox.h>
 #include <ngc_ext.h>
 #include <ngc_ft1.h>
@@ -12,6 +11,7 @@
 #include <string>
 #include <string_view>
 #include <map>
+#include <memory>
 #include <set>
 #include <vector>
 
@@ -84,5 +84,6 @@ struct ToxClient {
 
 		// key groupid, value set of peer ids
 		std::map<uint32_t, std::set<uint32_t>> _groups;
+	//	std::map<uint32_t, std::map<uint32_t, Tox_Connection>> _groups;
 };