move self have_chunk to bitset

This commit is contained in:
Green Sky 2024-06-25 12:08:17 +02:00
parent c8619561ec
commit e9e38db1d5
No known key found for this signature in database
3 changed files with 60 additions and 48 deletions

View File

@ -11,13 +11,16 @@
struct BitSet { struct BitSet {
std::vector<uint8_t> _bytes; std::vector<uint8_t> _bytes;
BitSet(void) = delete; BitSet(void) = default;
BitSet(const BitSet&) = default; BitSet(const BitSet&) = default;
BitSet(BitSet&&) = default; BitSet(BitSet&&) = default;
BitSet(size_t size) { BitSet(size_t size) {
_bytes.resize((size+7)/8); _bytes.resize((size+7)/8);
} }
BitSet& operator=(const BitSet&) = default;
BitSet& operator=(BitSet&&) = default;
bool operator[](size_t pos) const { bool operator[](size_t pos) const {
assert(pos < size_bits()); assert(pos < size_bits());
if (pos >= size_bits()) { if (pos >= size_bits()) {

View File

@ -8,6 +8,7 @@
#include "./ft1_sha1_info.hpp" #include "./ft1_sha1_info.hpp"
#include "./hash_utils.hpp" #include "./hash_utils.hpp"
#include "./bitset.hpp"
#include <vector> #include <vector>
@ -31,7 +32,11 @@ namespace Components {
}; };
struct FT1ChunkSHA1Cache { struct FT1ChunkSHA1Cache {
std::vector<bool> have_chunk; //std::vector<bool> have_chunk;
// have_chunk is the size of info.chunks.size(), or empty if have_all
// keep in mind bitset rounds up to 8s
BitSet have_chunk{0};
bool have_all {false}; bool have_all {false};
size_t have_count {0}; size_t have_count {0};
entt::dense_map<SHA1Digest, std::vector<size_t>> chunk_hash_to_index; entt::dense_map<SHA1Digest, std::vector<size_t>> chunk_hash_to_index;

View File

@ -424,49 +424,54 @@ void SHA1_NGCFT1::iterate(float delta) {
auto& cc = ce.get<Components::FT1ChunkSHA1Cache>(); auto& cc = ce.get<Components::FT1ChunkSHA1Cache>();
const auto& info = ce.get<Components::FT1InfoSHA1>(); const auto& info = ce.get<Components::FT1InfoSHA1>();
// naive, choose first chunk we dont have (double requests!!) if (cc.have_all) {
for (size_t chunk_idx = 0; chunk_idx < cc.have_chunk.size(); chunk_idx++) { _queue_content_want_chunk.pop_front();
if (cc.have_chunk[chunk_idx]) { } else {
continue; // naive, choose first chunk we dont have (double requests!!)
} // TODO: piece picker, choose what other have (invert selectPeerForRequest)
for (size_t chunk_idx = 0; chunk_idx < info.chunks.size() /* cc.total_ */; chunk_idx++) {
// check by hash if (cc.have_chunk[chunk_idx]) {
if (cc.haveChunk(info.chunks.at(chunk_idx))) { continue;
// TODO: fix this, a completed chunk should fill all the indecies it occupies
cc.have_chunk[chunk_idx] = true;
cc.have_count += 1;
if (cc.have_count == info.chunks.size()) {
cc.have_all = true;
cc.have_chunk.clear();
break;
} }
continue;
// check by hash
if (cc.haveChunk(info.chunks.at(chunk_idx))) {
// TODO: fix this, a completed chunk should fill all the indecies it occupies
cc.have_chunk.set(chunk_idx);
cc.have_count += 1;
if (cc.have_count == info.chunks.size()) {
cc.have_all = true;
cc.have_chunk = BitSet(0); // conserve space
break;
}
continue;
}
if (requested_chunks.count(chunk_idx)) {
// already requested
continue;
}
// request chunk_idx
_nft.NGC_FT1_send_request_private(
group_number, peer_number,
static_cast<uint32_t>(NGCFT1_file_kind::HASH_SHA1_CHUNK),
info.chunks.at(chunk_idx).data.data(), info.chunks.at(chunk_idx).size()
);
requested_chunks[chunk_idx] = 0.f;
std::cout << "SHA1_NGCFT1: requesting chunk [" << info.chunks.at(chunk_idx) << "] from " << group_number << ":" << peer_number << "\n";
break;
} }
if (requested_chunks.count(chunk_idx)) { // ...
// already requested
continue; // TODO: properly determine
if (!cc.have_all) {
_queue_content_want_chunk.push_back(ce);
} }
_queue_content_want_chunk.pop_front();
// request chunk_idx
_nft.NGC_FT1_send_request_private(
group_number, peer_number,
static_cast<uint32_t>(NGCFT1_file_kind::HASH_SHA1_CHUNK),
info.chunks.at(chunk_idx).data.data(), info.chunks.at(chunk_idx).size()
);
requested_chunks[chunk_idx] = 0.f;
std::cout << "SHA1_NGCFT1: requesting chunk [" << info.chunks.at(chunk_idx) << "] from " << group_number << ":" << peer_number << "\n";
break;
} }
// ...
// TODO: properly determine
if (!cc.have_all) {
_queue_content_want_chunk.push_back(ce);
}
_queue_content_want_chunk.pop_front();
} }
} }
} }
@ -518,6 +523,7 @@ bool SHA1_NGCFT1::onEvent(const Message::Events::MessageUpdated& e) {
{ // next, create chuck cache and check for existing data { // next, create chuck cache and check for existing data
auto& cc = ce.emplace<Components::FT1ChunkSHA1Cache>(); auto& cc = ce.emplace<Components::FT1ChunkSHA1Cache>();
auto& bytes_received = ce.get_or_emplace<Message::Components::Transfer::BytesReceived>().total; auto& bytes_received = ce.get_or_emplace<Message::Components::Transfer::BytesReceived>().total;
cc.have_chunk = BitSet(info.chunks.size());
cc.have_all = false; cc.have_all = false;
cc.have_count = 0; cc.have_count = 0;
@ -534,9 +540,8 @@ bool SHA1_NGCFT1::onEvent(const Message::Events::MessageUpdated& e) {
const auto data_hash = SHA1Digest{hash_sha1(existing_data.ptr, existing_data.size)}; const auto data_hash = SHA1Digest{hash_sha1(existing_data.ptr, existing_data.size)};
const bool data_equal = data_hash == info.chunks.at(i); const bool data_equal = data_hash == info.chunks.at(i);
cc.have_chunk.push_back(data_equal);
if (data_equal) { if (data_equal) {
cc.have_chunk.set(i);
cc.have_count += 1; cc.have_count += 1;
bytes_received += chunk_size; bytes_received += chunk_size;
//std::cout << "existing i[" << info.chunks.at(i) << "] == d[" << data_hash << "]\n"; //std::cout << "existing i[" << info.chunks.at(i) << "] == d[" << data_hash << "]\n";
@ -558,7 +563,6 @@ bool SHA1_NGCFT1::onEvent(const Message::Events::MessageUpdated& e) {
} }
} else { } else {
for (size_t i = 0; i < info.chunks.size(); i++) { for (size_t i = 0; i < info.chunks.size(); i++) {
cc.have_chunk.push_back(false);
_chunks[info.chunks[i]] = ce; _chunks[info.chunks[i]] = ce;
cc.chunk_hash_to_index[info.chunks[i]].push_back(i); cc.chunk_hash_to_index[info.chunks[i]].push_back(i);
} }
@ -940,17 +944,17 @@ bool SHA1_NGCFT1::onEvent(const Events::NGCFT1_recv_done& e) {
if (!cc.have_all) { if (!cc.have_all) {
for (const auto inner_chunk_index : std::get<ReceivingTransfer::Chunk>(tv).chunk_indices) { for (const auto inner_chunk_index : std::get<ReceivingTransfer::Chunk>(tv).chunk_indices) {
if (!cc.have_all && !cc.have_chunk.at(inner_chunk_index)) { if (!cc.have_all && !cc.have_chunk[inner_chunk_index]) {
cc.have_chunk.at(inner_chunk_index) = true; cc.have_chunk.set(inner_chunk_index);
cc.have_count += 1; cc.have_count += 1;
if (cc.have_count == info.chunks.size()) { if (cc.have_count == info.chunks.size()) {
// debug check // debug check
for ([[maybe_unused]] const bool it : cc.have_chunk) { for ([[maybe_unused]] size_t i = 0; i < info.chunks.size(); i++) {
assert(it); assert(cc.have_chunk[i]);
} }
cc.have_all = true; cc.have_all = true;
cc.have_chunk.clear(); // not wasting memory cc.have_chunk = BitSet(0); // not wasting memory
std::cout << "SHA1_NGCFT1: got all chunks for \n" << info << "\n"; std::cout << "SHA1_NGCFT1: got all chunks for \n" << info << "\n";
// HACK: remap file, to clear ram // HACK: remap file, to clear ram