move self have_chunk to bitset

solanaceae/ngc_ft1_sha1/bitset.hpp

@@ -11,13 +11,16 @@
 struct BitSet {
 	std::vector<uint8_t> _bytes;
 
-	BitSet(void) = delete;
+	BitSet(void) = default;
 	BitSet(const BitSet&) = default;
 	BitSet(BitSet&&) = default;
 	BitSet(size_t size) {
 		_bytes.resize((size+7)/8);
 	}
 
+	BitSet& operator=(const BitSet&) = default;
+	BitSet& operator=(BitSet&&) = default;
+
 	bool operator[](size_t pos) const {
 		assert(pos < size_bits());
 		if (pos >= size_bits()) {

solanaceae/ngc_ft1_sha1/components.hpp

@@ -8,6 +8,7 @@
 
 #include "./ft1_sha1_info.hpp"
 #include "./hash_utils.hpp"
+#include "./bitset.hpp"
 
 
 #include <vector>
@@ -31,7 +32,11 @@ namespace Components {
 	};
 
 	struct FT1ChunkSHA1Cache {
-		std::vector<bool> have_chunk;
+		//std::vector<bool> have_chunk;
+		// have_chunk is the size of info.chunks.size(), or empty if have_all
+		// keep in mind bitset rounds up to 8s
+		BitSet have_chunk{0};
+
 		bool have_all {false};
 		size_t have_count {0};
 		entt::dense_map<SHA1Digest, std::vector<size_t>> chunk_hash_to_index;

solanaceae/ngc_ft1_sha1/sha1_ngcft1.cpp

@@ -424,49 +424,54 @@ void SHA1_NGCFT1::iterate(float delta) {
 					auto& cc = ce.get<Components::FT1ChunkSHA1Cache>();
 					const auto& info = ce.get<Components::FT1InfoSHA1>();
 
-					// naive, choose first chunk we dont have (double requests!!)
+					if (cc.have_all) {
-					for (size_t chunk_idx = 0; chunk_idx < cc.have_chunk.size(); chunk_idx++) {
+						_queue_content_want_chunk.pop_front();
-						if (cc.have_chunk[chunk_idx]) {
+					} else {
-							continue;
+						// naive, choose first chunk we dont have (double requests!!)
-						}
+						// TODO: piece picker, choose what other have (invert selectPeerForRequest)
-
+						for (size_t chunk_idx = 0; chunk_idx < info.chunks.size() /* cc.total_ */; chunk_idx++) {
-						// check by hash
+							if (cc.have_chunk[chunk_idx]) {
-						if (cc.haveChunk(info.chunks.at(chunk_idx))) {
+								continue;
-							// TODO: fix this, a completed chunk should fill all the indecies it occupies
-							cc.have_chunk[chunk_idx] = true;
-							cc.have_count += 1;
-							if (cc.have_count == info.chunks.size()) {
-								cc.have_all = true;
-								cc.have_chunk.clear();
-								break;
 							}
-							continue;
+
+							// check by hash
+							if (cc.haveChunk(info.chunks.at(chunk_idx))) {
+								// TODO: fix this, a completed chunk should fill all the indecies it occupies
+								cc.have_chunk.set(chunk_idx);
+								cc.have_count += 1;
+								if (cc.have_count == info.chunks.size()) {
+									cc.have_all = true;
+									cc.have_chunk = BitSet(0); // conserve space
+									break;
+								}
+								continue;
+							}
+
+							if (requested_chunks.count(chunk_idx)) {
+								// already requested
+								continue;
+							}
+
+							// request chunk_idx
+							_nft.NGC_FT1_send_request_private(
+								group_number, peer_number,
+								static_cast<uint32_t>(NGCFT1_file_kind::HASH_SHA1_CHUNK),
+								info.chunks.at(chunk_idx).data.data(), info.chunks.at(chunk_idx).size()
+							);
+							requested_chunks[chunk_idx] = 0.f;
+							std::cout << "SHA1_NGCFT1: requesting chunk [" << info.chunks.at(chunk_idx) << "] from " << group_number << ":" << peer_number << "\n";
+
+							break;
 						}
 
-						if (requested_chunks.count(chunk_idx)) {
+						// ...
-							// already requested
+
-							continue;
+						// TODO: properly determine
+						if (!cc.have_all) {
+							_queue_content_want_chunk.push_back(ce);
 						}
-
+						_queue_content_want_chunk.pop_front();
-						// request chunk_idx
-						_nft.NGC_FT1_send_request_private(
-							group_number, peer_number,
-							static_cast<uint32_t>(NGCFT1_file_kind::HASH_SHA1_CHUNK),
-							info.chunks.at(chunk_idx).data.data(), info.chunks.at(chunk_idx).size()
-						);
-						requested_chunks[chunk_idx] = 0.f;
-						std::cout << "SHA1_NGCFT1: requesting chunk [" << info.chunks.at(chunk_idx) << "] from " << group_number << ":" << peer_number << "\n";
-
-						break;
 					}
-
-					// ...
-
-					// TODO: properly determine
-					if (!cc.have_all) {
-						_queue_content_want_chunk.push_back(ce);
-					}
-					_queue_content_want_chunk.pop_front();
 				}
 			}
 		}
@@ -518,6 +523,7 @@ bool SHA1_NGCFT1::onEvent(const Message::Events::MessageUpdated& e) {
 	{ // next, create chuck cache and check for existing data
 		auto& cc = ce.emplace<Components::FT1ChunkSHA1Cache>();
 		auto& bytes_received = ce.get_or_emplace<Message::Components::Transfer::BytesReceived>().total;
+		cc.have_chunk = BitSet(info.chunks.size());
 		cc.have_all = false;
 		cc.have_count = 0;
 
@@ -534,9 +540,8 @@ bool SHA1_NGCFT1::onEvent(const Message::Events::MessageUpdated& e) {
 					const auto data_hash = SHA1Digest{hash_sha1(existing_data.ptr, existing_data.size)};
 					const bool data_equal = data_hash == info.chunks.at(i);
 
-					cc.have_chunk.push_back(data_equal);
-
 					if (data_equal) {
+						cc.have_chunk.set(i);
 						cc.have_count += 1;
 						bytes_received += chunk_size;
 						//std::cout << "existing i[" << info.chunks.at(i) << "] == d[" << data_hash << "]\n";
@@ -558,7 +563,6 @@ bool SHA1_NGCFT1::onEvent(const Message::Events::MessageUpdated& e) {
 			}
 		} else {
 			for (size_t i = 0; i < info.chunks.size(); i++) {
-				cc.have_chunk.push_back(false);
 				_chunks[info.chunks[i]] = ce;
 				cc.chunk_hash_to_index[info.chunks[i]].push_back(i);
 			}
@@ -940,17 +944,17 @@ bool SHA1_NGCFT1::onEvent(const Events::NGCFT1_recv_done& e) {
 
 			if (!cc.have_all) {
 				for (const auto inner_chunk_index : std::get<ReceivingTransfer::Chunk>(tv).chunk_indices) {
-					if (!cc.have_all && !cc.have_chunk.at(inner_chunk_index)) {
+					if (!cc.have_all && !cc.have_chunk[inner_chunk_index]) {
-						cc.have_chunk.at(inner_chunk_index) = true;
+						cc.have_chunk.set(inner_chunk_index);
 						cc.have_count += 1;
 						if (cc.have_count == info.chunks.size()) {
 							// debug check
-							for ([[maybe_unused]] const bool it : cc.have_chunk) {
+							for ([[maybe_unused]] size_t i = 0; i < info.chunks.size(); i++) {
-								assert(it);
+								assert(cc.have_chunk[i]);
 							}
 
 							cc.have_all = true;
-							cc.have_chunk.clear(); // not wasting memory
+							cc.have_chunk = BitSet(0); // not wasting memory
 							std::cout << "SHA1_NGCFT1: got all chunks for \n" << info << "\n";
 
 							// HACK: remap file, to clear ram