Green-Sky/solanaceae_ngc_ft1
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

hack replace content with improper use of objectstore

Browse Source
This commit is contained in:
Green Sky 2024-04-29 11:55:11 +02:00
parent dfcb5dee97
commit eb2a19d8f3
No known key found for this signature in database
3 changed files with 35 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CMakeLists.txt
View File

@ -65,5 +65,6 @@ target_link_libraries(solanaceae_sha1_ngcft1 PUBLIC
sha1::sha1 sha1::sha1
solanaceae_tox_contacts solanaceae_tox_contacts
solanaceae_message3 solanaceae_message3
solanaceae_object_store
) )

33
solanaceae/ngc_ft1_sha1/sha1_ngcft1.cpp
View File

@ -26,11 +26,11 @@
namespace Message::Components { namespace Message::Components {
using Content = ContentHandle; using Content = ObjectHandle;
} // Message::Components } // Message::Components
// TODO: rename to content components // TODO: rename to object components
namespace Components { namespace Components {
struct Messages { struct Messages {
@ -112,7 +112,7 @@ static size_t chunkSize(const FT1InfoSHA1& sha1_info, size_t chunk_index) {
} }
} }
void SHA1_NGCFT1::queueUpRequestChunk(uint32_t group_number, uint32_t peer_number, ContentHandle content, const SHA1Digest& hash) { void SHA1_NGCFT1::queueUpRequestChunk(uint32_t group_number, uint32_t peer_number, ObjectHandle content, const SHA1Digest& hash) {
for (auto& [i_g, i_p, i_m, i_h, i_t] : _queue_requested_chunk) { for (auto& [i_g, i_p, i_m, i_h, i_t] : _queue_requested_chunk) {
// if already in queue // if already in queue
if (i_g == group_number && i_p == peer_number && i_h == hash) { if (i_g == group_number && i_p == peer_number && i_h == hash) {
@ -156,7 +156,7 @@ uint64_t SHA1_NGCFT1::combineIds(const uint32_t group_number, const uint32_t pee
return (uint64_t(group_number) << 32) | peer_number; return (uint64_t(group_number) << 32) | peer_number;
} }
void SHA1_NGCFT1::updateMessages(ContentHandle ce) { void SHA1_NGCFT1::updateMessages(ObjectHandle ce) {
assert(ce.all_of<Components::Messages>()); assert(ce.all_of<Components::Messages>());
for (auto msg : ce.get<Components::Messages>().messages) { for (auto msg : ce.get<Components::Messages>().messages) {
@ -185,7 +185,7 @@ void SHA1_NGCFT1::updateMessages(ContentHandle ce) {
} }
} }
std::optional<std::pair<uint32_t, uint32_t>> SHA1_NGCFT1::selectPeerForRequest(ContentHandle ce) { std::optional<std::pair<uint32_t, uint32_t>> SHA1_NGCFT1::selectPeerForRequest(ObjectHandle ce) {
// get a list of peers we can request this file from // get a list of peers we can request this file from
// TODO: randomly request from non SuspectedParticipants // TODO: randomly request from non SuspectedParticipants
std::vector<std::pair<uint32_t, uint32_t>> tox_peers; std::vector<std::pair<uint32_t, uint32_t>> tox_peers;
@ -248,11 +248,13 @@ std::optional<std::pair<uint32_t, uint32_t>> SHA1_NGCFT1::selectPeerForRequest(C
} }
SHA1_NGCFT1::SHA1_NGCFT1( SHA1_NGCFT1::SHA1_NGCFT1(
ObjectStore2& os,
Contact3Registry& cr, Contact3Registry& cr,
RegistryMessageModel& rmm, RegistryMessageModel& rmm,
NGCFT1& nft, NGCFT1& nft,
ToxContactModel2& tcm ToxContactModel2& tcm
) : ) :
_os(os),
_cr(cr), _cr(cr),
_rmm(rmm), _rmm(rmm),
_nft(nft), _nft(nft),
@ -350,8 +352,8 @@ void SHA1_NGCFT1::iterate(float delta) {
} }
{ // requested info timers { // requested info timers
std::vector<Content> timed_out; std::vector<Object> timed_out;
_contentr.view<Components::ReRequestInfoTimer>().each([delta, &timed_out](Content e, Components::ReRequestInfoTimer& rrit) { _os.registry().view<Components::ReRequestInfoTimer>().each([delta, &timed_out](Object e, Components::ReRequestInfoTimer& rrit) {
rrit.timer += delta; rrit.timer += delta;
// 15sec, TODO: config // 15sec, TODO: config
@ -361,12 +363,13 @@ void SHA1_NGCFT1::iterate(float delta) {
}); });
for (const auto e : timed_out) { for (const auto e : timed_out) {
// TODO: avoid dups // TODO: avoid dups
_queue_content_want_info.push_back({_contentr, e}); _queue_content_want_info.push_back(_os.objectHandle(e));
_contentr.remove<Components::ReRequestInfoTimer>(e); _os.registry().remove<Components::ReRequestInfoTimer>(e);
// TODO: throw update?
} }
} }
{ // requested chunk timers { // requested chunk timers
_contentr.view<Components::FT1ChunkSHA1Requested>().each([delta](Components::FT1ChunkSHA1Requested& ftchunk_requested) { _os.registry().view<Components::FT1ChunkSHA1Requested>().each([delta](Components::FT1ChunkSHA1Requested& ftchunk_requested) {
for (auto it = ftchunk_requested.chunks.begin(); it != ftchunk_requested.chunks.end();) { for (auto it = ftchunk_requested.chunks.begin(); it != ftchunk_requested.chunks.end();) {
it->second += delta; it->second += delta;
@ -1087,12 +1090,13 @@ bool SHA1_NGCFT1::onEvent(const Events::NGCFT1_recv_message& e) {
// check if content exists // check if content exists
const auto sha1_info_hash = std::vector<uint8_t>{e.file_id, e.file_id+e.file_id_size}; const auto sha1_info_hash = std::vector<uint8_t>{e.file_id, e.file_id+e.file_id_size};
ContentHandle ce; ObjectHandle ce;
if (_info_to_content.count(sha1_info_hash)) { if (_info_to_content.count(sha1_info_hash)) {
ce = _info_to_content.at(sha1_info_hash); ce = _info_to_content.at(sha1_info_hash);
std::cout << "SHA1_NGCFT1: new message has existing content\n"; std::cout << "SHA1_NGCFT1: new message has existing content\n";
} else { } else {
ce = {_contentr, _contentr.create()}; // TODO: backend
ce = {_os.registry(), _os.registry().create()};
_info_to_content[sha1_info_hash] = ce; _info_to_content[sha1_info_hash] = ce;
std::cout << "SHA1_NGCFT1: new message has new content\n"; std::cout << "SHA1_NGCFT1: new message has new content\n";
@ -1249,7 +1253,7 @@ bool SHA1_NGCFT1::sendFilePath(const Contact3 c, std::string_view file_name, std
std::cout << "SHA1_NGCFT1 sha1_info_hash: " << bin2hex(sha1_info_hash) << "\n"; std::cout << "SHA1_NGCFT1 sha1_info_hash: " << bin2hex(sha1_info_hash) << "\n";
// check if content exists // check if content exists
ContentHandle ce; ObjectHandle ce;
if (self->_info_to_content.count(sha1_info_hash)) { if (self->_info_to_content.count(sha1_info_hash)) {
ce = self->_info_to_content.at(sha1_info_hash); ce = self->_info_to_content.at(sha1_info_hash);
@ -1318,7 +1322,8 @@ bool SHA1_NGCFT1::sendFilePath(const Contact3 c, std::string_view file_name, std
it = self->_queue_content_want_chunk.erase(it); it = self->_queue_content_want_chunk.erase(it);
} }
} else { } else {
ce = {self->_contentr, self->_contentr.create()}; // TODO: backend
ce = {self->_os.registry(), self->_os.registry().create()};
self->_info_to_content[sha1_info_hash] = ce; self->_info_to_content[sha1_info_hash] = ce;
ce.emplace<Components::FT1InfoSHA1>(sha1_info); ce.emplace<Components::FT1InfoSHA1>(sha1_info);

33
solanaceae/ngc_ft1_sha1/sha1_ngcft1.hpp
View File

@ -2,6 +2,7 @@
// solanaceae port of sha1 fts for NGCFT1 // solanaceae port of sha1 fts for NGCFT1
#include <solanaceae/object_store/object_store.hpp>
#include <solanaceae/contact/contact_model3.hpp> #include <solanaceae/contact/contact_model3.hpp>
#include <solanaceae/message3/registry_message_model.hpp> #include <solanaceae/message3/registry_message_model.hpp>
#include <solanaceae/tox_contacts/tox_contact_model2.hpp> #include <solanaceae/tox_contacts/tox_contact_model2.hpp>
@ -20,11 +21,9 @@
#include <mutex> #include <mutex>
#include <list> #include <list>
enum class Content : uint32_t {};
using ContentRegistry = entt::basic_registry<Content>;
using ContentHandle = entt::basic_handle<ContentRegistry>;
class SHA1_NGCFT1 : public RegistryMessageModelEventI, public NGCFT1EventI { class SHA1_NGCFT1 : public RegistryMessageModelEventI, public NGCFT1EventI {
ObjectStore2& _os;
// TODO: backend abstraction
Contact3Registry& _cr; Contact3Registry& _cr;
RegistryMessageModel& _rmm; RegistryMessageModel& _rmm;
NGCFT1& _nft; NGCFT1& _nft;
@ -32,21 +31,18 @@ class SHA1_NGCFT1 : public RegistryMessageModelEventI, public NGCFT1EventI {
std::minstd_rand _rng {1337*11}; std::minstd_rand _rng {1337*11};
// registry per group?
ContentRegistry _contentr;
// limit this to each group? // limit this to each group?
entt::dense_map<SHA1Digest, ContentHandle> _info_to_content; entt::dense_map<SHA1Digest, ObjectHandle> _info_to_content;
// sha1 chunk index // sha1 chunk index
// TODO: optimize lookup // TODO: optimize lookup
// TODO: multiple contents. hashes might be unique, but data is not // TODO: multiple contents. hashes might be unique, but data is not
entt::dense_map<SHA1Digest, ContentHandle> _chunks; entt::dense_map<SHA1Digest, ObjectHandle> _chunks;
// group_number, peer_number, content, chunk_hash, timer // group_number, peer_number, content, chunk_hash, timer
std::deque<std::tuple<uint32_t, uint32_t, ContentHandle, SHA1Digest, float>> _queue_requested_chunk; std::deque<std::tuple<uint32_t, uint32_t, ObjectHandle, SHA1Digest, float>> _queue_requested_chunk;
//void queueUpRequestInfo(uint32_t group_number, uint32_t peer_number, const SHA1Digest& hash); //void queueUpRequestInfo(uint32_t group_number, uint32_t peer_number, const SHA1Digest& hash);
void queueUpRequestChunk(uint32_t group_number, uint32_t peer_number, ContentHandle content, const SHA1Digest& hash); void queueUpRequestChunk(uint32_t group_number, uint32_t peer_number, ObjectHandle content, const SHA1Digest& hash);
struct SendingTransfer { struct SendingTransfer {
struct Info { struct Info {
@ -56,7 +52,7 @@ class SHA1_NGCFT1 : public RegistryMessageModelEventI, public NGCFT1EventI {
}; };
struct Chunk { struct Chunk {
ContentHandle content; ObjectHandle content;
size_t chunk_index; // <.< remove offset_into_file size_t chunk_index; // <.< remove offset_into_file
//uint64_t offset_into_file; //uint64_t offset_into_file;
// or data? // or data?
@ -72,14 +68,14 @@ class SHA1_NGCFT1 : public RegistryMessageModelEventI, public NGCFT1EventI {
struct ReceivingTransfer { struct ReceivingTransfer {
struct Info { struct Info {
ContentHandle content; ObjectHandle content;
// copy of info data // copy of info data
// too large? // too large?
std::vector<uint8_t> info_data; std::vector<uint8_t> info_data;
}; };
struct Chunk { struct Chunk {
ContentHandle content; ObjectHandle content;
std::vector<size_t> chunk_indices; std::vector<size_t> chunk_indices;
// or data? // or data?
// if memmapped, this would be just a pointer // if memmapped, this would be just a pointer
@ -93,8 +89,8 @@ class SHA1_NGCFT1 : public RegistryMessageModelEventI, public NGCFT1EventI {
entt::dense_map<uint64_t, entt::dense_map<uint8_t, ReceivingTransfer>> _receiving_transfers; entt::dense_map<uint64_t, entt::dense_map<uint8_t, ReceivingTransfer>> _receiving_transfers;
// makes request rotate around open content // makes request rotate around open content
std::deque<ContentHandle> _queue_content_want_info; std::deque<ObjectHandle> _queue_content_want_info;
std::deque<ContentHandle> _queue_content_want_chunk; std::deque<ObjectHandle> _queue_content_want_chunk;
std::atomic_bool _info_builder_dirty {false}; std::atomic_bool _info_builder_dirty {false};
std::mutex _info_builder_queue_mutex; std::mutex _info_builder_queue_mutex;
@ -108,9 +104,9 @@ class SHA1_NGCFT1 : public RegistryMessageModelEventI, public NGCFT1EventI {
static uint64_t combineIds(const uint32_t group_number, const uint32_t peer_number); static uint64_t combineIds(const uint32_t group_number, const uint32_t peer_number);
void updateMessages(ContentHandle ce); void updateMessages(ObjectHandle ce);
std::optional<std::pair<uint32_t, uint32_t>> selectPeerForRequest(ContentHandle ce); std::optional<std::pair<uint32_t, uint32_t>> selectPeerForRequest(ObjectHandle ce);
public: // TODO: config public: // TODO: config
bool _udp_only {false}; bool _udp_only {false};
@ -122,6 +118,7 @@ class SHA1_NGCFT1 : public RegistryMessageModelEventI, public NGCFT1EventI {
public: public:
SHA1_NGCFT1( SHA1_NGCFT1(
ObjectStore2& os,
Contact3Registry& cr, Contact3Registry& cr,
RegistryMessageModel& rmm, RegistryMessageModel& rmm,
NGCFT1& nft, NGCFT1& nft,