after 2 weeks of porting over the ngc_ft1 code to solanaceae and rewriting the highlevel logic

(29 commits predate this)

solanaceae/ngc_ft1_sha1/ft1_sha1_info.cpp

@@ -0,0 +1,139 @@
+#include "./ft1_sha1_info.hpp"
+
+#include <sodium.h>
+
+SHA1Digest::SHA1Digest(const std::vector<uint8_t>& v) {
+	assert(v.size() == data.size());
+	for (size_t i = 0; i < data.size(); i++) {
+		data[i] = v[i];
+	}
+}
+
+SHA1Digest::SHA1Digest(const uint8_t* d, size_t s) {
+	assert(s == data.size());
+	for (size_t i = 0; i < data.size(); i++) {
+		data[i] = d[i];
+	}
+}
+
+std::ostream& operator<<(std::ostream& out, const SHA1Digest& v) {
+	std::string str{};
+	str.resize(v.size()*2, '?');
+
+	// HECK, std is 1 larger than size returns ('\0')
+	sodium_bin2hex(str.data(), str.size()+1, v.data.data(), v.data.size());
+
+	out << str;
+
+	return out;
+}
+
+size_t FT1InfoSHA1::chunkSize(size_t chunk_index) const {
+	if (chunk_index+1 == chunks.size()) {
+		// last chunk
+		return file_size - chunk_index * chunk_size;
+	} else {
+		return chunk_size;
+	}
+}
+
+std::vector<uint8_t> FT1InfoSHA1::toBuffer(void) const {
+	std::vector<uint8_t> buffer;
+
+	assert(!file_name.empty());
+	// TODO: optimize
+	for (size_t i = 0; i < 256; i++) {
+		if (i < file_name.size()) {
+			buffer.push_back(file_name.at(i));
+		} else {
+			buffer.push_back(0);
+		}
+	}
+	assert(buffer.size() == 256);
+
+	{ // HACK: endianess
+		buffer.push_back((file_size>>(0*8)) & 0xff);
+		buffer.push_back((file_size>>(1*8)) & 0xff);
+		buffer.push_back((file_size>>(2*8)) & 0xff);
+		buffer.push_back((file_size>>(3*8)) & 0xff);
+		buffer.push_back((file_size>>(4*8)) & 0xff);
+		buffer.push_back((file_size>>(5*8)) & 0xff);
+		buffer.push_back((file_size>>(6*8)) & 0xff);
+		buffer.push_back((file_size>>(7*8)) & 0xff);
+	}
+	assert(buffer.size() == 256+8);
+
+	// chunk size
+	{ // HACK: endianess
+		buffer.push_back((chunk_size>>(0*8)) & 0xff);
+		buffer.push_back((chunk_size>>(1*8)) & 0xff);
+		buffer.push_back((chunk_size>>(2*8)) & 0xff);
+		buffer.push_back((chunk_size>>(3*8)) & 0xff);
+	}
+
+	assert(buffer.size() == 256+8+4);
+
+	for (const auto& chunk : chunks) {
+		for (size_t i = 0; i < chunk.data.size(); i++) {
+			buffer.push_back(chunk.data[i]);
+		}
+	}
+	assert(buffer.size() == 256+8+4+20*chunks.size());
+
+	return buffer;
+}
+
+void FT1InfoSHA1::fromBuffer(const std::vector<uint8_t>& buffer) {
+	assert(buffer.size() >= 256+8+4);
+
+	// TODO: optimize
+	file_name.clear();
+	for (size_t i = 0; i < 256; i++) {
+		char next_char = static_cast<char>(buffer[i]);
+		if (next_char == 0) {
+			break;
+		}
+		file_name.push_back(next_char);
+	}
+
+	{ // HACK: endianess
+		file_size = 0;
+		file_size |= uint64_t(buffer[256+0]) << (0*8);
+		file_size |= uint64_t(buffer[256+1]) << (1*8);
+		file_size |= uint64_t(buffer[256+2]) << (2*8);
+		file_size |= uint64_t(buffer[256+3]) << (3*8);
+		file_size |= uint64_t(buffer[256+4]) << (4*8);
+		file_size |= uint64_t(buffer[256+5]) << (5*8);
+		file_size |= uint64_t(buffer[256+6]) << (6*8);
+		file_size |= uint64_t(buffer[256+7]) << (7*8);
+	}
+
+	{ // HACK: endianess
+		chunk_size = 0;
+		chunk_size |= uint32_t(buffer[256+8+0]) << (0*8);
+		chunk_size |= uint32_t(buffer[256+8+1]) << (1*8);
+		chunk_size |= uint32_t(buffer[256+8+2]) << (2*8);
+		chunk_size |= uint32_t(buffer[256+8+3]) << (3*8);
+	}
+
+	assert((buffer.size()-(256+8+4)) % 20 == 0);
+
+	for (size_t offset = 256+8+4; offset < buffer.size();) {
+		assert(buffer.size() >= offset + 20);
+
+		auto& chunk = chunks.emplace_back();
+		for (size_t i = 0; i < chunk.size(); i++, offset++) {
+			chunk.data[i] = buffer.at(offset);
+		}
+		// TODO: error/leftover checking
+	}
+}
+
+std::ostream& operator<<(std::ostream& out, const FT1InfoSHA1& v) {
+	out << "  file_name: " << v.file_name << "\n";
+	out << "  file_size: " << v.file_size << "\n";
+	out << "  chunk_size: " << v.chunk_size << "\n";
+	out << "  chunks.size(): " << v.chunks.size() << "\n";
+	return out;
+}
+

solanaceae/ngc_ft1_sha1/ft1_sha1_info.hpp

@@ -0,0 +1,55 @@
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <array>
+#include <ostream>
+#include <vector>
+#include <cassert>
+#include <string>
+
+struct SHA1Digest {
+	std::array<uint8_t, 20> data;
+
+	SHA1Digest(void) = default;
+	SHA1Digest(const std::vector<uint8_t>& v);
+	SHA1Digest(const uint8_t* d, size_t s);
+
+	bool operator==(const SHA1Digest& other) const { return data == other.data; }
+	bool operator!=(const SHA1Digest& other) const { return data != other.data; }
+
+	size_t size(void) const { return data.size(); }
+};
+
+std::ostream& operator<<(std::ostream& out, const SHA1Digest& v);
+
+namespace std { // inject
+	template<> struct hash<SHA1Digest> {
+		std::size_t operator()(const SHA1Digest& h) const noexcept {
+			return
+				size_t(h.data[0]) << (0*8) |
+				size_t(h.data[1]) << (1*8) |
+				size_t(h.data[2]) << (2*8) |
+				size_t(h.data[3]) << (3*8) |
+				size_t(h.data[4]) << (4*8) |
+				size_t(h.data[5]) << (5*8) |
+				size_t(h.data[6]) << (6*8) |
+				size_t(h.data[7]) << (7*8)
+			;
+		}
+	};
+} // std
+
+struct FT1InfoSHA1 {
+	std::string file_name;
+	uint64_t file_size {0};
+	uint32_t chunk_size {128*1024}; // 128KiB for now
+	std::vector<SHA1Digest> chunks;
+
+	size_t chunkSize(size_t chunk_index) const;
+
+	std::vector<uint8_t> toBuffer(void) const;
+	void fromBuffer(const std::vector<uint8_t>& buffer);
+};
+std::ostream& operator<<(std::ostream& out, const FT1InfoSHA1& v);
+

solanaceae/ngc_ft1_sha1/hash_utils.cpp

@@ -0,0 +1,26 @@
+#include "./hash_utils.hpp"
+
+#include <sha1.h>
+
+// returns the 20bytes sha1 hash
+std::vector<uint8_t> hash_sha1(const uint8_t* data, size_t size) {
+	SHA1_CTX ctx;
+	SHA1Init(&ctx);
+
+	{ // lib only takes uint32_t sizes, so chunk it
+		constexpr size_t hash_block_size {0xffffffff};
+		size_t i = 0;
+		for (; i + hash_block_size < size; i += hash_block_size) {
+			SHA1Update(&ctx, reinterpret_cast<const uint8_t*>(data) + i, hash_block_size);
+		}
+
+		if (i < size) {
+			SHA1Update(&ctx, reinterpret_cast<const uint8_t*>(data) + i, size - i);
+		}
+	}
+
+	std::vector<uint8_t> sha1_hash(20);
+	SHA1Final(sha1_hash.data(), &ctx);
+	return sha1_hash;
+}
+

solanaceae/ngc_ft1_sha1/hash_utils.hpp

@@ -0,0 +1,10 @@
+#pragma once
+
+#include <cstdint>
+#include <vector>
+
+// returns the 20bytes sha1 hash
+std::vector<uint8_t> hash_sha1(const uint8_t* data, size_t size);
+
+inline std::vector<uint8_t> hash_sha1(const char* data, size_t size) { return hash_sha1(reinterpret_cast<const uint8_t*>(data), size); }
+

solanaceae/ngc_ft1_sha1/sha1_ngcft1.hpp

@@ -0,0 +1,134 @@
+#pragma once
+
+// solanaceae port of sha1 fts for NGCFT1
+
+#include <solanaceae/contact/contact_model3.hpp>
+#include <solanaceae/message3/registry_message_model.hpp>
+#include <solanaceae/tox_contacts/tox_contact_model2.hpp>
+
+#include "./ngcft1.hpp"
+
+#include "./ft1_sha1_info.hpp"
+
+#include <entt/entity/registry.hpp>
+#include <entt/entity/handle.hpp>
+#include <entt/container/dense_map.hpp>
+
+#include <variant>
+#include <random>
+
+enum class Content : uint32_t {};
+using ContentRegistry = entt::basic_registry<Content>;
+using ContentHandle = entt::basic_handle<ContentRegistry>;
+
+class SHA1_NGCFT1 : public RegistryMessageModelEventI, public NGCFT1EventI {
+	Contact3Registry& _cr;
+	RegistryMessageModel& _rmm;
+	NGCFT1& _nft;
+	ToxContactModel2& _tcm;
+
+	std::minstd_rand _rng {1337*11};
+
+	// registry per group?
+	ContentRegistry _contentr;
+
+	// limit this to each group?
+	entt::dense_map<SHA1Digest, ContentHandle> _info_to_content;
+
+	// sha1 chunk index
+	// TODO: optimize lookup
+	// TODO: multiple contents. hashes might be unique, but data is not
+	entt::dense_map<SHA1Digest, ContentHandle> _chunks;
+
+	// group_number, peer_number, content, chunk_hash, timer
+	std::deque<std::tuple<uint32_t, uint32_t, ContentHandle, SHA1Digest, float>> _queue_requested_chunk;
+	//void queueUpRequestInfo(uint32_t group_number, uint32_t peer_number, const SHA1Digest& hash);
+	void queueUpRequestChunk(uint32_t group_number, uint32_t peer_number, ContentHandle content, const SHA1Digest& hash);
+
+	struct SendingTransfer {
+		struct Info {
+			// copy of info data
+			// too large?
+			std::vector<uint8_t> info_data;
+		};
+
+		struct Chunk {
+			ContentHandle content;
+			size_t chunk_index; // <.< remove offset_into_file
+			//uint64_t offset_into_file;
+			// or data?
+			// if memmapped, this would be just a pointer
+		};
+
+		std::variant<Info, Chunk> v;
+
+		float time_since_activity {0.f};
+	};
+	// key is groupid + peerid
+	entt::dense_map<uint64_t, entt::dense_map<uint8_t, SendingTransfer>> _sending_transfers;
+
+	struct ReceivingTransfer {
+		struct Info {
+			ContentHandle content;
+			// copy of info data
+			// too large?
+			std::vector<uint8_t> info_data;
+		};
+
+		struct Chunk {
+			ContentHandle content;
+			std::vector<size_t> chunk_indices;
+			// or data?
+			// if memmapped, this would be just a pointer
+		};
+
+		std::variant<Info, Chunk> v;
+
+		float time_since_activity {0.f};
+	};
+	// key is groupid + peerid
+	entt::dense_map<uint64_t, entt::dense_map<uint8_t, ReceivingTransfer>> _receiving_transfers;
+
+	// makes request rotate around open content
+	std::deque<ContentHandle> _queue_content_want_info;
+	std::deque<ContentHandle> _queue_content_want_chunk;
+
+	static uint64_t combineIds(const uint32_t group_number, const uint32_t peer_number);
+
+	void updateMessages(ContentHandle ce);
+
+	std::optional<std::pair<uint32_t, uint32_t>> selectPeerForRequest(ContentHandle ce);
+
+	public: // TODO: config
+		bool _udp_only {false};
+
+		size_t _max_concurrent_in {4};
+		size_t _max_concurrent_out {6};
+		// TODO: probably also includes running transfers rn (meh)
+		size_t _max_pending_requests {16}; // per content
+
+	public:
+		SHA1_NGCFT1(
+			Contact3Registry& cr,
+			RegistryMessageModel& rmm,
+			NGCFT1& nft,
+			ToxContactModel2& tcm
+		);
+
+		void iterate(float delta);
+
+	protected: // rmm events (actions)
+		bool onEvent(const Message::Events::MessageUpdated&) override;
+
+	protected: // events
+		bool onEvent(const Events::NGCFT1_recv_request&) override;
+		bool onEvent(const Events::NGCFT1_recv_init&) override;
+		bool onEvent(const Events::NGCFT1_recv_data&) override;
+		bool onEvent(const Events::NGCFT1_send_data&) override; // const?
+		bool onEvent(const Events::NGCFT1_recv_done&) override;
+		bool onEvent(const Events::NGCFT1_send_done&) override;
+		bool onEvent(const Events::NGCFT1_recv_message&) override;
+
+		bool sendFilePath(const Contact3 c, std::string_view file_name, std::string_view file_path) override;
+};
+