Compare commits
No commits in common. "54ace9d0b2e98d3b3ff70bd9bc7821b1d150dfdc" and "f73084477152b201bd20c785b1fa0390dd8a8ae2" have entirely different histories.
54ace9d0b2
...
f730844771
@ -50,9 +50,6 @@ add_library(solanaceae_sha1_ngcft1
|
||||
./solanaceae/ngc_ft1_sha1/file_constructor.hpp
|
||||
./solanaceae/ngc_ft1_sha1/file_constructor.cpp
|
||||
|
||||
./solanaceae/ngc_ft1_sha1/backends/sha1_mapped_filesystem.hpp
|
||||
./solanaceae/ngc_ft1_sha1/backends/sha1_mapped_filesystem.cpp
|
||||
|
||||
./solanaceae/ngc_ft1_sha1/hash_utils.hpp
|
||||
./solanaceae/ngc_ft1_sha1/hash_utils.cpp
|
||||
|
||||
|
||||
@ -1,253 +0,0 @@
|
||||
#include "./sha1_mapped_filesystem.hpp"
|
||||
|
||||
#include <solanaceae/object_store/meta_components.hpp>
|
||||
|
||||
#include "../file_constructor.hpp"
|
||||
#include "../ft1_sha1_info.hpp"
|
||||
#include "../hash_utils.hpp"
|
||||
#include "../components.hpp"
|
||||
|
||||
#include <solanaceae/util/utils.hpp>
|
||||
|
||||
#include <atomic>
|
||||
#include <mutex>
|
||||
#include <list>
|
||||
#include <thread>
|
||||
|
||||
#include <iostream>
|
||||
|
||||
namespace Backends {
|
||||
|
||||
struct SHA1MappedFilesystem_InfoBuilderState {
|
||||
std::atomic_bool info_builder_dirty {false};
|
||||
std::mutex info_builder_queue_mutex;
|
||||
using InfoBuilderEntry = std::function<void(void)>;
|
||||
std::list<InfoBuilderEntry> info_builder_queue;
|
||||
};
|
||||
|
||||
SHA1MappedFilesystem::SHA1MappedFilesystem(
|
||||
ObjectStore2& os
|
||||
) : StorageBackendI::StorageBackendI(os), _ibs(std::make_unique<SHA1MappedFilesystem_InfoBuilderState>()) {
|
||||
}
|
||||
|
||||
SHA1MappedFilesystem::~SHA1MappedFilesystem(void) {
|
||||
}
|
||||
|
||||
void SHA1MappedFilesystem::tick(void) {
|
||||
if (_ibs->info_builder_dirty) {
|
||||
std::lock_guard l{_ibs->info_builder_queue_mutex};
|
||||
_ibs->info_builder_dirty = false; // set while holding lock
|
||||
|
||||
for (auto& it : _ibs->info_builder_queue) {
|
||||
it();
|
||||
}
|
||||
_ibs->info_builder_queue.clear();
|
||||
}
|
||||
}
|
||||
|
||||
ObjectHandle SHA1MappedFilesystem::newObject(ByteSpan id) {
|
||||
ObjectHandle o{_os.registry(), _os.registry().create()};
|
||||
|
||||
o.emplace<ObjComp::Ephemeral::Backend>(this);
|
||||
o.emplace<ObjComp::ID>(std::vector<uint8_t>{id});
|
||||
//o.emplace<ObjComp::Ephemeral::FilePath>(object_file_path.generic_u8string());
|
||||
|
||||
_os.throwEventConstruct(o);
|
||||
|
||||
return o;
|
||||
}
|
||||
|
||||
void SHA1MappedFilesystem::newFromFile(std::string_view file_name, std::string_view file_path, std::function<void(ObjectHandle o)>&& cb) {
|
||||
std::thread(std::move([
|
||||
this,
|
||||
ibs = _ibs.get(),
|
||||
cb = std::move(cb),
|
||||
file_name_ = std::string(file_name),
|
||||
file_path_ = std::string(file_path)
|
||||
]() mutable {
|
||||
// 0. open and fail
|
||||
std::unique_ptr<File2I> file_impl = construct_file2_rw_mapped(file_path_, -1);
|
||||
if (!file_impl->isGood()) {
|
||||
{
|
||||
std::lock_guard l{ibs->info_builder_queue_mutex};
|
||||
ibs->info_builder_queue.push_back([file_path_](){
|
||||
// back on iterate thread
|
||||
|
||||
std::cerr << "SHA1MF error: failed opening file '" << file_path_ << "'!\n";
|
||||
});
|
||||
ibs->info_builder_dirty = true; // still in scope, set before mutex unlock
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// 1. build info by hashing all chunks
|
||||
FT1InfoSHA1 sha1_info;
|
||||
// build info
|
||||
sha1_info.file_name = file_name_;
|
||||
sha1_info.file_size = file_impl->_file_size; // TODO: remove the reliance on implementation details
|
||||
|
||||
{ // build chunks
|
||||
// HACK: load file fully
|
||||
// ... its only a hack if its not memory mapped, but reading in chunk_sized chunks is probably a good idea anyway
|
||||
const auto file_data = file_impl->read(file_impl->_file_size, 0);
|
||||
size_t i = 0;
|
||||
for (; i + sha1_info.chunk_size < file_data.size; i += sha1_info.chunk_size) {
|
||||
sha1_info.chunks.push_back(hash_sha1(file_data.ptr+i, sha1_info.chunk_size));
|
||||
}
|
||||
|
||||
if (i < file_data.size) {
|
||||
sha1_info.chunks.push_back(hash_sha1(file_data.ptr+i, file_data.size-i));
|
||||
}
|
||||
}
|
||||
|
||||
file_impl.reset();
|
||||
|
||||
std::lock_guard l{ibs->info_builder_queue_mutex};
|
||||
ibs->info_builder_queue.push_back(std::move([
|
||||
this,
|
||||
file_name_,
|
||||
file_path_,
|
||||
sha1_info = std::move(sha1_info),
|
||||
cb = std::move(cb)
|
||||
]() mutable { //
|
||||
// executed on iterate thread
|
||||
|
||||
// reopen, cant move, since std::function needs to be copy consturctable (meh)
|
||||
std::unique_ptr<File2I> file_impl = construct_file2_rw_mapped(file_path_, sha1_info.file_size);
|
||||
if (!file_impl->isGood()) {
|
||||
std::cerr << "SHA1MF error: failed opening file '" << file_path_ << "'!\n";
|
||||
return;
|
||||
}
|
||||
|
||||
// 2. hash info
|
||||
std::vector<uint8_t> sha1_info_data;
|
||||
std::vector<uint8_t> sha1_info_hash;
|
||||
|
||||
std::cout << "SHA1MF info is: \n" << sha1_info;
|
||||
sha1_info_data = sha1_info.toBuffer();
|
||||
std::cout << "SHA1MF sha1_info size: " << sha1_info_data.size() << "\n";
|
||||
sha1_info_hash = hash_sha1(sha1_info_data.data(), sha1_info_data.size());
|
||||
std::cout << "SHA1MF sha1_info_hash: " << bin2hex(sha1_info_hash) << "\n";
|
||||
|
||||
ObjectHandle o;
|
||||
// check if content exists
|
||||
// TODO: store "info_to_content" in reg/backend, for better lookup speed
|
||||
// rn ok, bc this is rare
|
||||
for (const auto& [it_ov, it_ih] : _os.registry().view<Components::FT1InfoSHA1Hash>().each()) {
|
||||
if (it_ih.hash == sha1_info_hash) {
|
||||
o = {_os.registry(), it_ov};
|
||||
}
|
||||
}
|
||||
//if (self->info_to_content.count(sha1_info_hash)) {
|
||||
if (static_cast<bool>(o)) {
|
||||
//ce = self->info_to_content.at(sha1_info_hash);
|
||||
|
||||
// TODO: check if content is incomplete and use file instead
|
||||
if (!o.all_of<Components::FT1InfoSHA1>()) {
|
||||
o.emplace<Components::FT1InfoSHA1>(sha1_info);
|
||||
}
|
||||
if (!o.all_of<Components::FT1InfoSHA1Data>()) {
|
||||
o.emplace<Components::FT1InfoSHA1Data>(sha1_info_data);
|
||||
}
|
||||
|
||||
// hash has to be set already
|
||||
// Components::FT1InfoSHA1Hash
|
||||
|
||||
{ // lookup tables and have
|
||||
auto& cc = o.get_or_emplace<Components::FT1ChunkSHA1Cache>();
|
||||
cc.have_all = true;
|
||||
// skip have vec, since all
|
||||
//cc.have_chunk
|
||||
cc.have_count = sha1_info.chunks.size(); // need?
|
||||
|
||||
//self->_info_to_content[sha1_info_hash] = ce;
|
||||
cc.chunk_hash_to_index.clear(); // for cpy pst
|
||||
for (size_t i = 0; i < sha1_info.chunks.size(); i++) {
|
||||
//self->_chunks[sha1_info.chunks[i]] = ce;
|
||||
cc.chunk_hash_to_index[sha1_info.chunks[i]].push_back(i);
|
||||
}
|
||||
}
|
||||
|
||||
{ // file info
|
||||
// TODO: not overwrite fi? since same?
|
||||
auto& file_info = o.emplace_or_replace<Message::Components::Transfer::FileInfo>();
|
||||
file_info.file_list.emplace_back() = {std::string{file_name_}, file_impl->_file_size};
|
||||
file_info.total_size = file_impl->_file_size;
|
||||
|
||||
o.emplace_or_replace<Message::Components::Transfer::FileInfoLocal>(std::vector{std::string{file_path_}});
|
||||
}
|
||||
|
||||
// hmmm
|
||||
o.remove<Message::Components::Transfer::TagPaused>();
|
||||
|
||||
// we dont want the info anymore
|
||||
o.remove<Components::ReRequestInfoTimer>();
|
||||
} else {
|
||||
o = newObject(ByteSpan{sha1_info_hash});
|
||||
|
||||
o.emplace<Components::FT1InfoSHA1>(sha1_info);
|
||||
o.emplace<Components::FT1InfoSHA1Data>(sha1_info_data); // keep around? or file?
|
||||
o.emplace<Components::FT1InfoSHA1Hash>(sha1_info_hash);
|
||||
{ // lookup tables and have
|
||||
auto& cc = o.emplace<Components::FT1ChunkSHA1Cache>();
|
||||
cc.have_all = true;
|
||||
// skip have vec, since all
|
||||
cc.have_count = sha1_info.chunks.size(); // need?
|
||||
|
||||
cc.chunk_hash_to_index.clear(); // for cpy pst
|
||||
for (size_t i = 0; i < sha1_info.chunks.size(); i++) {
|
||||
cc.chunk_hash_to_index[sha1_info.chunks[i]].push_back(i);
|
||||
}
|
||||
}
|
||||
|
||||
{ // file info
|
||||
auto& file_info = o.emplace<Message::Components::Transfer::FileInfo>();
|
||||
file_info.file_list.emplace_back() = {std::string{file_name_}, file_impl->_file_size};
|
||||
file_info.total_size = file_impl->_file_size;
|
||||
|
||||
o.emplace<Message::Components::Transfer::FileInfoLocal>(std::vector{std::string{file_path_}});
|
||||
}
|
||||
}
|
||||
|
||||
o.emplace_or_replace<Message::Components::Transfer::File>(std::move(file_impl));
|
||||
|
||||
// TODO: replace with transfers stats
|
||||
if (!o.all_of<Message::Components::Transfer::BytesSent>()) {
|
||||
o.emplace<Message::Components::Transfer::BytesSent>(0u);
|
||||
}
|
||||
|
||||
cb(o);
|
||||
|
||||
// TODO: earlier?
|
||||
_os.throwEventUpdate(o);
|
||||
}));
|
||||
ibs->info_builder_dirty = true; // still in scope, set before mutex unlock
|
||||
})).detach();
|
||||
}
|
||||
|
||||
std::unique_ptr<File2I> SHA1MappedFilesystem::file2(Object ov, FILE2_FLAGS flags) {
|
||||
ObjectHandle o{_os.registry(), ov};
|
||||
|
||||
if (!static_cast<bool>(o)) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
if (!o.all_of<Message::Components::Transfer::FileInfoLocal>()) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
const auto& file_list = o.get<Message::Components::Transfer::FileInfoLocal>().file_list;
|
||||
if (file_list.empty()) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
auto res = construct_file2_rw_mapped(file_list.front(), -1);
|
||||
if (!res || !res->isGood()) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
} // Backends
|
||||
|
||||
@ -1,39 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include <solanaceae/object_store/object_store.hpp>
|
||||
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <memory>
|
||||
|
||||
namespace Backends {
|
||||
|
||||
// fwd to hide the threading headers
|
||||
struct SHA1MappedFilesystem_InfoBuilderState;
|
||||
|
||||
struct SHA1MappedFilesystem : public StorageBackendI {
|
||||
std::unique_ptr<SHA1MappedFilesystem_InfoBuilderState> _ibs;
|
||||
|
||||
SHA1MappedFilesystem(
|
||||
ObjectStore2& os
|
||||
);
|
||||
~SHA1MappedFilesystem(void);
|
||||
|
||||
// pull from info builder queue
|
||||
// call from main thread (os thread?) often
|
||||
void tick(void);
|
||||
|
||||
ObjectHandle newObject(ByteSpan id) override;
|
||||
|
||||
// performs async file hashing
|
||||
// create message in cb
|
||||
void newFromFile(std::string_view file_name, std::string_view file_path, std::function<void(ObjectHandle o)>&& cb/*, bool merge_preexisting = false*/);
|
||||
|
||||
// might return pre-existing?
|
||||
ObjectHandle newFromInfoHash(ByteSpan info_hash);
|
||||
|
||||
std::unique_ptr<File2I> file2(Object o, FILE2_FLAGS flags); // default does nothing
|
||||
};
|
||||
|
||||
} // Backends
|
||||
|
||||
@ -29,6 +29,7 @@
|
||||
|
||||
#include <iostream>
|
||||
#include <filesystem>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
|
||||
namespace Message::Components {
|
||||
@ -192,8 +193,7 @@ SHA1_NGCFT1::SHA1_NGCFT1(
|
||||
_nft(nft),
|
||||
_tcm(tcm),
|
||||
_tep(tep),
|
||||
_neep(neep),
|
||||
_mfb(os)
|
||||
_neep(neep)
|
||||
{
|
||||
// TODO: also create and destroy
|
||||
_rmm.subscribe(this, RegistryMessageModel_Event::message_updated);
|
||||
@ -221,7 +221,17 @@ SHA1_NGCFT1::SHA1_NGCFT1(
|
||||
|
||||
float SHA1_NGCFT1::iterate(float delta) {
|
||||
//std::cerr << "---------- new tick ----------\n";
|
||||
_mfb.tick(); // does not need to be called as often, once every sec would be enough, but the pointer deref + atomic bool should be very fast
|
||||
// info builder queue
|
||||
if (_info_builder_dirty) {
|
||||
std::lock_guard l{_info_builder_queue_mutex};
|
||||
_info_builder_dirty = false; // set while holding lock
|
||||
|
||||
for (auto& it : _info_builder_queue) {
|
||||
//it.fn();
|
||||
it();
|
||||
}
|
||||
_info_builder_queue.clear();
|
||||
}
|
||||
|
||||
entt::dense_map<Contact3, size_t> peer_open_requests;
|
||||
|
||||
@ -430,89 +440,6 @@ float SHA1_NGCFT1::iterate(float delta) {
|
||||
}
|
||||
}
|
||||
|
||||
// gets called back on main thread after a "new" file info got built on a different thread
|
||||
void SHA1_NGCFT1::onSendFileHashFinished(ObjectHandle o, Message3Registry* reg_ptr, Contact3 c, uint64_t ts) {
|
||||
// sanity
|
||||
if (!o.all_of<Components::FT1InfoSHA1, Components::FT1InfoSHA1Hash>()) {
|
||||
assert(false);
|
||||
return;
|
||||
}
|
||||
|
||||
// update content lookup
|
||||
const auto& info_hash = o.get<Components::FT1InfoSHA1Hash>().hash;
|
||||
_info_to_content[info_hash] = o;
|
||||
|
||||
// update chunk lookup
|
||||
const auto& cc = o.get<Components::FT1ChunkSHA1Cache>();
|
||||
const auto& info = o.get<Components::FT1InfoSHA1>();
|
||||
for (size_t i = 0; i < info.chunks.size(); i++) {
|
||||
_chunks[info.chunks[i]] = o;
|
||||
}
|
||||
|
||||
// remove from info request queue
|
||||
if (auto it = std::find(_queue_content_want_info.begin(), _queue_content_want_info.end(), o); it != _queue_content_want_info.end()) {
|
||||
_queue_content_want_info.erase(it);
|
||||
}
|
||||
|
||||
// TODO: we dont want chunks anymore
|
||||
// TODO: make sure to abort every receiving transfer (sending info and chunk should be fine, info uses copy and chunk handle)
|
||||
|
||||
// something happend, update all chunk pickers
|
||||
if (o.all_of<Components::SuspectedParticipants>()) {
|
||||
for (const auto& pcv : o.get<Components::SuspectedParticipants>().participants) {
|
||||
Contact3Handle pch{_cr, pcv};
|
||||
assert(static_cast<bool>(pch));
|
||||
pch.emplace_or_replace<ChunkPickerUpdateTag>();
|
||||
}
|
||||
}
|
||||
|
||||
// create message
|
||||
const auto c_self = _cr.get<Contact::Components::Self>(c).self;
|
||||
if (!_cr.valid(c_self)) {
|
||||
std::cerr << "SHA1_NGCFT1 error: failed to get self!\n";
|
||||
return;
|
||||
}
|
||||
|
||||
const auto msg_e = reg_ptr->create();
|
||||
reg_ptr->emplace<Message::Components::ContactTo>(msg_e, c);
|
||||
reg_ptr->emplace<Message::Components::ContactFrom>(msg_e, c_self);
|
||||
reg_ptr->emplace<Message::Components::Timestamp>(msg_e, ts); // reactive?
|
||||
reg_ptr->emplace<Message::Components::Read>(msg_e, ts);
|
||||
|
||||
reg_ptr->emplace<Message::Components::Transfer::TagHaveAll>(msg_e);
|
||||
reg_ptr->emplace<Message::Components::Transfer::TagSending>(msg_e);
|
||||
|
||||
o.get_or_emplace<Components::Messages>().messages.push_back({*reg_ptr, msg_e});
|
||||
|
||||
//reg_ptr->emplace<Message::Components::Transfer::FileKind>(e, file_kind);
|
||||
// file id would be sha1_info hash or something
|
||||
//reg_ptr->emplace<Message::Components::Transfer::FileID>(e, file_id);
|
||||
|
||||
if (_cr.any_of<Contact::Components::ToxGroupEphemeral>(c)) {
|
||||
const uint32_t group_number = _cr.get<Contact::Components::ToxGroupEphemeral>(c).group_number;
|
||||
uint32_t message_id = 0;
|
||||
|
||||
// TODO: check return
|
||||
_nft.NGC_FT1_send_message_public(group_number, message_id, static_cast<uint32_t>(NGCFT1_file_kind::HASH_SHA1_INFO), info_hash.data(), info_hash.size());
|
||||
reg_ptr->emplace<Message::Components::ToxGroupMessageID>(msg_e, message_id);
|
||||
} else if (
|
||||
// non online group
|
||||
_cr.any_of<Contact::Components::ToxGroupPersistent>(c)
|
||||
) {
|
||||
// create msg_id
|
||||
const uint32_t message_id = randombytes_random();
|
||||
reg_ptr->emplace<Message::Components::ToxGroupMessageID>(msg_e, message_id);
|
||||
} // TODO: else private message
|
||||
|
||||
reg_ptr->get_or_emplace<Message::Components::SyncedBy>(msg_e).ts.try_emplace(c_self, ts);
|
||||
reg_ptr->get_or_emplace<Message::Components::ReceivedBy>(msg_e).ts.try_emplace(c_self, ts);
|
||||
|
||||
_rmm.throwEventConstruct(*reg_ptr, msg_e);
|
||||
|
||||
// TODO: place in iterate?
|
||||
updateMessages(o);
|
||||
}
|
||||
|
||||
bool SHA1_NGCFT1::onEvent(const Message::Events::MessageUpdated& e) {
|
||||
// see tox_transfer_manager.cpp for reference
|
||||
if (!e.e.all_of<Message::Components::Transfer::ActionAccept, Message::Components::Content>()) {
|
||||
@ -1063,7 +990,6 @@ bool SHA1_NGCFT1::onEvent(const Events::NGCFT1_recv_done& e) {
|
||||
);
|
||||
}
|
||||
|
||||
#if 0
|
||||
if (!cc.have_all) { // debug print self have set
|
||||
std::cout << "DEBUG print have bitset: s:" << cc.have_chunk.size_bits();
|
||||
for (size_t i = 0; i < cc.have_chunk.size_bytes(); i++) {
|
||||
@ -1075,7 +1001,6 @@ bool SHA1_NGCFT1::onEvent(const Events::NGCFT1_recv_done& e) {
|
||||
}
|
||||
printf("\n");
|
||||
}
|
||||
#endif
|
||||
} else {
|
||||
std::cout << "SHA1_NGCFT1 warning: got chunk duplicate\n";
|
||||
}
|
||||
@ -1251,10 +1176,252 @@ bool SHA1_NGCFT1::sendFilePath(const Contact3 c, std::string_view file_name, std
|
||||
// get current time unix epoch utc
|
||||
uint64_t ts = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
|
||||
|
||||
_mfb.newFromFile(
|
||||
file_name, file_path,
|
||||
[this, reg_ptr, c, ts](ObjectHandle o) { onSendFileHashFinished(o, reg_ptr, c, ts); }
|
||||
);
|
||||
std::thread(std::move([
|
||||
// copy everything
|
||||
self = this,
|
||||
ts,
|
||||
c,
|
||||
reg_ptr,
|
||||
file_name_ = std::string(file_name),
|
||||
file_path_ = std::string(file_path)
|
||||
]() mutable {
|
||||
std::unique_ptr<File2I> file_impl = construct_file2_rw_mapped(file_path_, -1);
|
||||
if (!file_impl->isGood()) {
|
||||
{
|
||||
std::lock_guard l{self->_info_builder_queue_mutex};
|
||||
self->_info_builder_queue.push_back([file_path_](){
|
||||
// back on iterate thread
|
||||
|
||||
std::cerr << "SHA1_NGCFT1 error: failed opening file '" << file_path_ << "'!\n";
|
||||
});
|
||||
self->_info_builder_dirty = true; // still in scope, set before mutex unlock
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// 1. build info by hashing all chunks
|
||||
|
||||
FT1InfoSHA1 sha1_info;
|
||||
// build info
|
||||
sha1_info.file_name = file_name_;
|
||||
sha1_info.file_size = file_impl->_file_size; // TODO: remove the reliance on implementation details
|
||||
|
||||
{ // build chunks
|
||||
// HACK: load file fully
|
||||
// ... its only a hack if its not memory mapped, but reading in chunk_sized chunks is probably a good idea anyway
|
||||
const auto file_data = file_impl->read(file_impl->_file_size, 0);
|
||||
size_t i = 0;
|
||||
for (; i + sha1_info.chunk_size < file_data.size; i += sha1_info.chunk_size) {
|
||||
sha1_info.chunks.push_back(hash_sha1(file_data.ptr+i, sha1_info.chunk_size));
|
||||
}
|
||||
|
||||
if (i < file_data.size) {
|
||||
sha1_info.chunks.push_back(hash_sha1(file_data.ptr+i, file_data.size-i));
|
||||
}
|
||||
}
|
||||
|
||||
file_impl.reset();
|
||||
|
||||
{
|
||||
std::lock_guard l{self->_info_builder_queue_mutex};
|
||||
self->_info_builder_queue.push_back(std::move([
|
||||
self,
|
||||
ts,
|
||||
c,
|
||||
reg_ptr,
|
||||
file_name_,
|
||||
file_path_,
|
||||
sha1_info = std::move(sha1_info)
|
||||
]() mutable { //
|
||||
// back on iterate thread
|
||||
|
||||
std::unique_ptr<File2I> file_impl = construct_file2_rw_mapped(file_path_, sha1_info.file_size);
|
||||
if (!file_impl->isGood()) {
|
||||
std::cerr << "SHA1_NGCFT1 error: failed opening file '" << file_path_ << "'!\n";
|
||||
return;
|
||||
}
|
||||
|
||||
// 2. hash info
|
||||
std::vector<uint8_t> sha1_info_data;
|
||||
std::vector<uint8_t> sha1_info_hash;
|
||||
|
||||
std::cout << "SHA1_NGCFT1 info is: \n" << sha1_info;
|
||||
sha1_info_data = sha1_info.toBuffer();
|
||||
std::cout << "SHA1_NGCFT1 sha1_info size: " << sha1_info_data.size() << "\n";
|
||||
sha1_info_hash = hash_sha1(sha1_info_data.data(), sha1_info_data.size());
|
||||
std::cout << "SHA1_NGCFT1 sha1_info_hash: " << bin2hex(sha1_info_hash) << "\n";
|
||||
|
||||
// check if content exists
|
||||
ObjectHandle ce;
|
||||
if (self->_info_to_content.count(sha1_info_hash)) {
|
||||
ce = self->_info_to_content.at(sha1_info_hash);
|
||||
|
||||
// TODO: check if content is incomplete and use file instead
|
||||
if (!ce.all_of<Components::FT1InfoSHA1>()) {
|
||||
ce.emplace<Components::FT1InfoSHA1>(sha1_info);
|
||||
}
|
||||
if (!ce.all_of<Components::FT1InfoSHA1Data>()) {
|
||||
ce.emplace<Components::FT1InfoSHA1Data>(sha1_info_data);
|
||||
}
|
||||
|
||||
// hash has to be set already
|
||||
// Components::FT1InfoSHA1Hash
|
||||
|
||||
{ // lookup tables and have
|
||||
auto& cc = ce.get_or_emplace<Components::FT1ChunkSHA1Cache>();
|
||||
cc.have_all = true;
|
||||
// skip have vec, since all
|
||||
//cc.have_chunk
|
||||
cc.have_count = sha1_info.chunks.size(); // need?
|
||||
|
||||
self->_info_to_content[sha1_info_hash] = ce;
|
||||
cc.chunk_hash_to_index.clear(); // for cpy pst
|
||||
for (size_t i = 0; i < sha1_info.chunks.size(); i++) {
|
||||
self->_chunks[sha1_info.chunks[i]] = ce;
|
||||
cc.chunk_hash_to_index[sha1_info.chunks[i]].push_back(i);
|
||||
}
|
||||
}
|
||||
|
||||
{ // file info
|
||||
// TODO: not overwrite fi? since same?
|
||||
auto& file_info = ce.emplace_or_replace<Message::Components::Transfer::FileInfo>();
|
||||
file_info.file_list.emplace_back() = {std::string{file_name_}, file_impl->_file_size};
|
||||
file_info.total_size = file_impl->_file_size;
|
||||
|
||||
ce.emplace_or_replace<Message::Components::Transfer::FileInfoLocal>(std::vector{std::string{file_path_}});
|
||||
}
|
||||
|
||||
// cleanup file
|
||||
if (ce.all_of<Message::Components::Transfer::File>()) {
|
||||
// replace
|
||||
ce.remove<Message::Components::Transfer::File>();
|
||||
}
|
||||
ce.emplace<Message::Components::Transfer::File>(std::move(file_impl));
|
||||
|
||||
if (!ce.all_of<Message::Components::Transfer::BytesSent>()) {
|
||||
ce.emplace<Message::Components::Transfer::BytesSent>(0u);
|
||||
}
|
||||
|
||||
ce.remove<Message::Components::Transfer::TagPaused>();
|
||||
|
||||
// we dont want the info anymore
|
||||
ce.remove<Components::ReRequestInfoTimer>();
|
||||
if (auto it = std::find(self->_queue_content_want_info.begin(), self->_queue_content_want_info.end(), ce); it != self->_queue_content_want_info.end()) {
|
||||
self->_queue_content_want_info.erase(it);
|
||||
}
|
||||
|
||||
// TODO: we dont want chunks anymore
|
||||
// TODO: make sure to abort every receiving transfer (sending info and chunk should be fine, info uses copy and chunk handle)
|
||||
} else {
|
||||
// TODO: backend
|
||||
ce = {self->_os.registry(), self->_os.registry().create()};
|
||||
self->_info_to_content[sha1_info_hash] = ce;
|
||||
|
||||
ce.emplace<Components::FT1InfoSHA1>(sha1_info);
|
||||
ce.emplace<Components::FT1InfoSHA1Data>(sha1_info_data); // keep around? or file?
|
||||
ce.emplace<Components::FT1InfoSHA1Hash>(sha1_info_hash);
|
||||
{ // lookup tables and have
|
||||
auto& cc = ce.emplace<Components::FT1ChunkSHA1Cache>();
|
||||
cc.have_all = true;
|
||||
// skip have vec, since all
|
||||
//cc.have_chunk
|
||||
cc.have_count = sha1_info.chunks.size(); // need?
|
||||
|
||||
self->_info_to_content[sha1_info_hash] = ce;
|
||||
cc.chunk_hash_to_index.clear(); // for cpy pst
|
||||
for (size_t i = 0; i < sha1_info.chunks.size(); i++) {
|
||||
self->_chunks[sha1_info.chunks[i]] = ce;
|
||||
cc.chunk_hash_to_index[sha1_info.chunks[i]].push_back(i);
|
||||
}
|
||||
}
|
||||
|
||||
{ // file info
|
||||
auto& file_info = ce.emplace<Message::Components::Transfer::FileInfo>();
|
||||
//const auto& file = ce.get<Message::Components::Transfer::File>();
|
||||
file_info.file_list.emplace_back() = {std::string{file_name_}, file_impl->_file_size};
|
||||
file_info.total_size = file_impl->_file_size;
|
||||
|
||||
ce.emplace<Message::Components::Transfer::FileInfoLocal>(std::vector{std::string{file_path_}});
|
||||
}
|
||||
|
||||
ce.emplace<Message::Components::Transfer::File>(std::move(file_impl));
|
||||
|
||||
ce.emplace<Message::Components::Transfer::BytesSent>(0u);
|
||||
}
|
||||
|
||||
// something happend, update all chunk pickers
|
||||
if (ce.all_of<Components::SuspectedParticipants>()) {
|
||||
for (const auto& pcv : ce.get<Components::SuspectedParticipants>().participants) {
|
||||
Contact3Handle pch{self->_cr, pcv};
|
||||
assert(static_cast<bool>(pch));
|
||||
pch.emplace_or_replace<ChunkPickerUpdateTag>();
|
||||
}
|
||||
}
|
||||
|
||||
const auto c_self = self->_cr.get<Contact::Components::Self>(c).self;
|
||||
if (!self->_cr.valid(c_self)) {
|
||||
std::cerr << "SHA1_NGCFT1 error: failed to get self!\n";
|
||||
return;
|
||||
}
|
||||
|
||||
const auto msg_e = reg_ptr->create();
|
||||
reg_ptr->emplace<Message::Components::ContactTo>(msg_e, c);
|
||||
reg_ptr->emplace<Message::Components::ContactFrom>(msg_e, c_self);
|
||||
reg_ptr->emplace<Message::Components::Timestamp>(msg_e, ts); // reactive?
|
||||
reg_ptr->emplace<Message::Components::Read>(msg_e, ts);
|
||||
|
||||
reg_ptr->emplace<Message::Components::Transfer::TagHaveAll>(msg_e);
|
||||
reg_ptr->emplace<Message::Components::Transfer::TagSending>(msg_e);
|
||||
|
||||
ce.get_or_emplace<Components::Messages>().messages.push_back({*reg_ptr, msg_e});
|
||||
|
||||
|
||||
//reg_ptr->emplace<Message::Components::Transfer::FileKind>(e, file_kind);
|
||||
// file id would be sha1_info hash or something
|
||||
//reg_ptr->emplace<Message::Components::Transfer::FileID>(e, file_id);
|
||||
|
||||
// remove? done in updateMessages() anyway
|
||||
if (ce.all_of<Message::Components::Transfer::FileInfo>()) {
|
||||
reg_ptr->emplace<Message::Components::Transfer::FileInfo>(msg_e, ce.get<Message::Components::Transfer::FileInfo>());
|
||||
}
|
||||
if (ce.all_of<Message::Components::Transfer::FileInfoLocal>()) {
|
||||
reg_ptr->emplace<Message::Components::Transfer::FileInfoLocal>(msg_e, ce.get<Message::Components::Transfer::FileInfoLocal>());
|
||||
}
|
||||
if (ce.all_of<Message::Components::Transfer::BytesSent>()) {
|
||||
reg_ptr->emplace<Message::Components::Transfer::BytesSent>(msg_e, ce.get<Message::Components::Transfer::BytesSent>());
|
||||
}
|
||||
|
||||
// TODO: determine if this is true
|
||||
//reg_ptr->emplace<Message::Components::Transfer::TagPaused>(e);
|
||||
|
||||
if (self->_cr.any_of<Contact::Components::ToxGroupEphemeral>(c)) {
|
||||
const uint32_t group_number = self->_cr.get<Contact::Components::ToxGroupEphemeral>(c).group_number;
|
||||
uint32_t message_id = 0;
|
||||
|
||||
// TODO: check return
|
||||
self->_nft.NGC_FT1_send_message_public(group_number, message_id, static_cast<uint32_t>(NGCFT1_file_kind::HASH_SHA1_INFO), sha1_info_hash.data(), sha1_info_hash.size());
|
||||
reg_ptr->emplace<Message::Components::ToxGroupMessageID>(msg_e, message_id);
|
||||
} else if (
|
||||
// non online group
|
||||
self->_cr.any_of<Contact::Components::ToxGroupPersistent>(c)
|
||||
) {
|
||||
// create msg_id
|
||||
const uint32_t message_id = randombytes_random();
|
||||
reg_ptr->emplace<Message::Components::ToxGroupMessageID>(msg_e, message_id);
|
||||
}
|
||||
|
||||
reg_ptr->get_or_emplace<Message::Components::SyncedBy>(msg_e).ts.try_emplace(c_self, ts);
|
||||
reg_ptr->get_or_emplace<Message::Components::ReceivedBy>(msg_e).ts.try_emplace(c_self, ts);
|
||||
|
||||
self->_rmm.throwEventConstruct(*reg_ptr, msg_e);
|
||||
|
||||
// TODO: place in iterate?
|
||||
self->updateMessages(ce);
|
||||
|
||||
}));
|
||||
self->_info_builder_dirty = true; // still in scope, set before mutex unlock
|
||||
}
|
||||
})).detach();
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@ -13,12 +13,12 @@
|
||||
#include "./sending_transfers.hpp"
|
||||
#include "./receiving_transfers.hpp"
|
||||
|
||||
#include "./backends/sha1_mapped_filesystem.hpp"
|
||||
|
||||
#include <entt/container/dense_map.hpp>
|
||||
|
||||
#include <random>
|
||||
#include <chrono>
|
||||
#include <atomic>
|
||||
#include <mutex>
|
||||
#include <list>
|
||||
|
||||
class SHA1_NGCFT1 : public ToxEventI, public RegistryMessageModelEventI, public NGCFT1EventI, public NGCEXTEventI {
|
||||
ObjectStore2& _os;
|
||||
@ -30,8 +30,6 @@ class SHA1_NGCFT1 : public ToxEventI, public RegistryMessageModelEventI, public
|
||||
ToxEventProviderI& _tep;
|
||||
NGCEXTEventProvider& _neep;
|
||||
|
||||
Backends::SHA1MappedFilesystem _mfb;
|
||||
|
||||
std::minstd_rand _rng {1337*11};
|
||||
|
||||
using clock = std::chrono::steady_clock;
|
||||
@ -69,6 +67,11 @@ class SHA1_NGCFT1 : public ToxEventI, public RegistryMessageModelEventI, public
|
||||
// only used to remove participation on peer exit
|
||||
entt::dense_map<uint64_t, Contact3Handle> _tox_peer_to_contact;
|
||||
|
||||
std::atomic_bool _info_builder_dirty {false};
|
||||
std::mutex _info_builder_queue_mutex;
|
||||
using InfoBuilderEntry = std::function<void(void)>;
|
||||
std::list<InfoBuilderEntry> _info_builder_queue;
|
||||
|
||||
void updateMessages(ObjectHandle ce);
|
||||
|
||||
std::optional<std::pair<uint32_t, uint32_t>> selectPeerForRequest(ObjectHandle ce);
|
||||
@ -94,8 +97,6 @@ class SHA1_NGCFT1 : public ToxEventI, public RegistryMessageModelEventI, public
|
||||
|
||||
float iterate(float delta);
|
||||
|
||||
void onSendFileHashFinished(ObjectHandle o, Message3Registry* reg_ptr, Contact3 c, uint64_t ts);
|
||||
|
||||
protected: // rmm events (actions)
|
||||
bool onEvent(const Message::Events::MessageUpdated&) override;
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user