From 45a3915985e1b50a3629d0d0f2e5057f65817810 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Thu, 28 Dec 2023 00:51:34 +0100 Subject: [PATCH] low level sync stuff --- CMakeLists.txt | 3 - plugins/CMakeLists.txt | 12 + plugins/plugin_crdtnotes_toxsync.cpp | 97 +++++++ src/CMakeLists.txt | 29 +- src/solanaceae/crdtnotes/crdtnotes.hpp | 7 + .../crdtnotes_contact_sync_model.hpp | 50 ++++ .../crdtnotes_toxsync/crdtnotes_toxsync.cpp | 261 +++++++++++++++++- .../crdtnotes_toxsync/crdtnotes_toxsync.hpp | 52 +++- 8 files changed, 492 insertions(+), 19 deletions(-) create mode 100644 plugins/plugin_crdtnotes_toxsync.cpp create mode 100644 src/solanaceae/crdtnotes/crdtnotes_contact_sync_model.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 9087b8b..68a6054 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,11 +5,8 @@ project(solanaceae_crdtnotes) if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) set(SOLANACEAE_CRDTNOTES_STANDALONE ON) - # why the f do i need this >:( - set(NOT_SOLANACEAE_CRDTNOTES_STANDALONE OFF) else() set(SOLANACEAE_CRDTNOTES_STANDALONE OFF) - set(NOT_SOLANACEAE_CRDTNOTES_STANDALONE ON) endif() message("II SOLANACEAE_CRDTNOTES_STANDALONE " ${SOLANACEAE_CRDTNOTES_STANDALONE}) diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index d127b2c..6c8553d 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -22,3 +22,15 @@ target_link_libraries(plugin_crdtnotes_imgui PUBLIC ######################################## +if (TARGET solanaceae_crdtnotes_toxsync) + add_library(plugin_crdtnotes_toxsync SHARED + ./plugin_crdtnotes_toxsync.cpp + ) + target_compile_features(plugin_crdtnotes_toxsync PUBLIC cxx_std_17) + target_link_libraries(plugin_crdtnotes_toxsync PUBLIC + solanaceae_crdtnotes_toxsync + solanaceae_plugin + ) +endif() + +######################################## diff --git a/plugins/plugin_crdtnotes_toxsync.cpp b/plugins/plugin_crdtnotes_toxsync.cpp new file mode 100644 index 0000000..f4ce2be --- /dev/null +++ b/plugins/plugin_crdtnotes_toxsync.cpp @@ -0,0 +1,97 @@ +#include + +#include +#include +//#include +#include +#include +#include + +#include +#include + +#define RESOLVE_INSTANCE(x) static_cast(solana_api->resolveInstance(#x)) +#define PROVIDE_INSTANCE(x, p, v) solana_api->provideInstance(#x, p, static_cast(v)) + +static std::unique_ptr g_crdtn_ts = nullptr; + +extern "C" { + +SOLANA_PLUGIN_EXPORT const char* solana_plugin_get_name(void) { + return "CRDTNotesToxSync"; +} + +SOLANA_PLUGIN_EXPORT uint32_t solana_plugin_get_version(void) { + return SOLANA_PLUGIN_VERSION; +} + +SOLANA_PLUGIN_EXPORT uint32_t solana_plugin_start(struct SolanaAPI* solana_api) { + std::cout << "PLUGIN CRDTNTS START()\n"; + + if (solana_api == nullptr) { + return 1; + } + + //ConfigModelI* conf = nullptr; + CRDTNotes* notes = nullptr; + Contact3Registry* cr = nullptr; + ToxI* t = nullptr; + ToxEventProviderI* tep = nullptr; + + { // make sure required types are loaded + //conf = RESOLVE_INSTANCE(ConfigModelI); + notes = RESOLVE_INSTANCE(CRDTNotes); + cr = RESOLVE_INSTANCE(Contact3Registry); + t = RESOLVE_INSTANCE(ToxI); + tep = RESOLVE_INSTANCE(ToxEventProviderI); + + //if (conf == nullptr) { + //std::cerr << "PLUGIN CRDTN missing ConfigModelI\n"; + //return 2; + //} + + if (notes == nullptr) { + std::cerr << "PLUGIN CRDTNTS missing CRDTNotes\n"; + return 2; + } + + if (cr == nullptr) { + std::cerr << "PLUGIN CRDTNTS missing Contact3Registry\n"; + return 2; + } + + if (t == nullptr) { + std::cerr << "PLUGIN CRDTNTS missing ToxI\n"; + return 2; + } + + if (tep == nullptr) { + std::cerr << "PLUGIN CRDTNTS missing ToxEventProviderI\n"; + return 2; + } + } + + // static store, could be anywhere tho + // construct with fetched dependencies + g_crdtn_ts = std::make_unique(*notes, *cr, *t, *tep); + + // register types + PROVIDE_INSTANCE(CRDTNotesToxSync, "CRDTNotesToxSync", g_crdtn_ts.get()); + + return 0; +} + +SOLANA_PLUGIN_EXPORT void solana_plugin_stop(void) { + std::cout << "PLUGIN CRDTNTS STOP()\n"; + + g_crdtn_ts.reset(); +} + +SOLANA_PLUGIN_EXPORT void solana_plugin_tick(float delta) { + (void)delta; + //std::cout << "PLUGIN CRDTN TICK()\n"; + g_crdtn_ts->iterate(delta); +} + +} // extern C + diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 39feea7..4cc6a37 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -3,11 +3,13 @@ cmake_minimum_required(VERSION 3.24 FATAL_ERROR) add_library(solanaceae_crdtnotes ./solanaceae/crdtnotes/crdtnotes.hpp ./solanaceae/crdtnotes/crdtnotes.cpp + ./solanaceae/crdtnotes/crdtnotes_contact_sync_model.hpp ) target_include_directories(solanaceae_crdtnotes PUBLIC .) target_compile_features(solanaceae_crdtnotes PUBLIC cxx_std_17) target_link_libraries(solanaceae_crdtnotes PUBLIC crdt_version3 + solanaceae_contact #solanaceae_util ) @@ -29,18 +31,21 @@ target_link_libraries(solanaceae_crdtnotes_imgui PUBLIC ######################################## -add_library(solanaceae_crdtnotes_toxsync - ./solanaceae/crdtnotes_toxsync/crdtnotes_toxsync.hpp - ./solanaceae/crdtnotes_toxsync/crdtnotes_toxsync.cpp -) -target_include_directories(solanaceae_crdtnotes_toxsync PUBLIC .) -target_compile_features(solanaceae_crdtnotes_toxsync PUBLIC cxx_std_17) -target_link_libraries(solanaceae_crdtnotes_toxsync PUBLIC - solanaceae_crdtnotes - #solanaceae_util - solanaceae_contact +if (TARGET solanaceae_toxcore AND TARGET solanaceae_tox_contacts) + add_library(solanaceae_crdtnotes_toxsync + ./solanaceae/crdtnotes_toxsync/crdtnotes_toxsync.hpp + ./solanaceae/crdtnotes_toxsync/crdtnotes_toxsync.cpp + ) + target_include_directories(solanaceae_crdtnotes_toxsync PUBLIC .) + target_compile_features(solanaceae_crdtnotes_toxsync PUBLIC cxx_std_17) + target_link_libraries(solanaceae_crdtnotes_toxsync PUBLIC + solanaceae_crdtnotes + #solanaceae_util + solanaceae_contact - #tox -) + solanaceae_toxcore + solanaceae_tox_contacts + ) +endif() ######################################## diff --git a/src/solanaceae/crdtnotes/crdtnotes.hpp b/src/solanaceae/crdtnotes/crdtnotes.hpp index f838cb3..4e7ab1f 100644 --- a/src/solanaceae/crdtnotes/crdtnotes.hpp +++ b/src/solanaceae/crdtnotes/crdtnotes.hpp @@ -8,6 +8,9 @@ #include #include +// fwd +struct CRDTNotesContactSyncModelI; + using ID32 = std::array; template<> @@ -33,6 +36,10 @@ class CRDTNotes { using CRDTAgent = ID32; using DocID = ID32; using Doc = GreenCRDT::V3::TextDocument; + struct Frontier { // newest known seq for given agent + CRDTAgent agent; + uint64_t seq{0}; + }; private: // TODO: add metadata to docs diff --git a/src/solanaceae/crdtnotes/crdtnotes_contact_sync_model.hpp b/src/solanaceae/crdtnotes/crdtnotes_contact_sync_model.hpp new file mode 100644 index 0000000..d3e82dd --- /dev/null +++ b/src/solanaceae/crdtnotes/crdtnotes_contact_sync_model.hpp @@ -0,0 +1,50 @@ +#pragma once + +#include "./crdtnotes.hpp" + +#include + +// send api +struct CRDTNotesContactSyncModelI { + virtual ~CRDTNotesContactSyncModelI(void) {} + + // gossip + public: + // notify of doc existing + virtual void SendGossip( + Contact3Handle c, + const CRDTNotes::DocID& doc_id + ) = 0; + + virtual void SendGossip( + Contact3Handle c, + const CRDTNotes::DocID& doc_id, + const std::vector& selected_frontier + ) = 0; + + // fetch + public: + // causes the other peer to send gossip with all known frontiers (on cool down) + virtual void SendFetchCompleteFrontier( + Contact3Handle c, + const CRDTNotes::DocID& doc_id + ) = 0; + + // action range request + virtual void SendFetchOps( + Contact3Handle c, + const CRDTNotes::DocID& doc_id, + const CRDTNotes::CRDTAgent& agent, + const uint64_t seq_from, + const uint64_t seq_to + ) = 0; + + public: // ops response + virtual void SendOps( + Contact3Handle c, + const CRDTNotes::DocID& doc_id, + // TODO: optimize this + const std::vector& + ) = 0; +}; + diff --git a/src/solanaceae/crdtnotes_toxsync/crdtnotes_toxsync.cpp b/src/solanaceae/crdtnotes_toxsync/crdtnotes_toxsync.cpp index 9f3e9b7..ea811bd 100644 --- a/src/solanaceae/crdtnotes_toxsync/crdtnotes_toxsync.cpp +++ b/src/solanaceae/crdtnotes_toxsync/crdtnotes_toxsync.cpp @@ -1,6 +1,265 @@ #include "./crdtnotes_toxsync.hpp" +#include -CRDTNotesToxSync::CRDTNotesToxSync(CRDTNotes& notes, Contact3Registry& cr) : _notes(notes), _cr(cr) { +#include + +enum class NGCEXT_Event : uint8_t { + // - DocID + CRDTN_GOSSIP = 0x80 | 0x10, + + // - DocID + // - array [ + // - AgentID + // - seq (frontier) + // - ] + CRDTN_GOSSIP_FRONTIER, + + // - DocID + CRDTN_FETCH_COMPLETE_FRONTIER, + + // - DocID + // - AgentID + // - seq_from + // - seq_to + CRDTN_FETCH_OP_RANGE, + + // - DocID + // - array [ + // - seq + // - action date + // - ] + CRDTN_OPS, +}; + + +CRDTNotesToxSync::CRDTNotesToxSync( + CRDTNotes& notes, + Contact3Registry& cr, + ToxI& t, + ToxEventProviderI& tep +) : _notes(notes), _cr(cr), _t(t), _tep(tep) { +} + +CRDTNotesToxSync::~CRDTNotesToxSync(void) { +} + +float CRDTNotesToxSync::iterate(float time_delta) { + return 1.f; // TODO: 1sec for now, needs better logic +} + +void CRDTNotesToxSync::SendGossip( + Contact3Handle c, + const CRDTNotes::DocID& doc_id +) { + if (!c.all_of()) { + return; + } + + std::vector pkg; + + pkg.push_back(static_cast(NGCEXT_Event::CRDTN_GOSSIP)); + + for (const uint8_t v : doc_id) { + pkg.push_back(v); + } + + // send + const auto& gp = c.get(); + _t.toxGroupSendCustomPrivatePacket( + gp.group_number, gp.peer_number, + true, + pkg + ); +} + +void CRDTNotesToxSync::SendGossip( + Contact3Handle c, + const CRDTNotes::DocID& doc_id, + const std::vector& selected_frontier +) { + if (!c.all_of()) { + return; + } + + std::vector pkg; + + pkg.push_back(static_cast(NGCEXT_Event::CRDTN_GOSSIP_FRONTIER)); + // 1 + + for (const uint8_t v : doc_id) { + pkg.push_back(v); + } + // +32 + + for (const auto& [f_id, f_seq] : selected_frontier) { + for (const uint8_t v : f_id) { + pkg.push_back(v); + } + // +32 + + for (size_t i = 0; i < sizeof(f_seq); i++) { + pkg.push_back((f_seq >> i*8) & 0xff); + } + // +8 + } + // +40 + + // send + const auto& gp = c.get(); + _t.toxGroupSendCustomPrivatePacket( + gp.group_number, gp.peer_number, + true, + pkg + ); +} + +void CRDTNotesToxSync::SendFetchCompleteFrontier( + Contact3Handle c, + const CRDTNotes::DocID& doc_id +) { + if (!c.all_of()) { + return; + } + + std::vector pkg; + + pkg.push_back(static_cast(NGCEXT_Event::CRDTN_FETCH_COMPLETE_FRONTIER)); + + for (const uint8_t v : doc_id) { + pkg.push_back(v); + } + + // send + const auto& gp = c.get(); + _t.toxGroupSendCustomPrivatePacket( + gp.group_number, gp.peer_number, + true, + pkg + ); +} + +void CRDTNotesToxSync::SendFetchOps( + Contact3Handle c, + const CRDTNotes::DocID& doc_id, + const CRDTNotes::CRDTAgent& agent, + const uint64_t seq_from, + const uint64_t seq_to +) { + if (!c.all_of()) { + return; + } + + std::vector pkg; + + pkg.push_back(static_cast(NGCEXT_Event::CRDTN_FETCH_OP_RANGE)); + + for (const uint8_t v : doc_id) { + pkg.push_back(v); + } + + for (const uint8_t v : agent) { + pkg.push_back(v); + } + + for (size_t i = 0; i < sizeof(seq_from); i++) { + pkg.push_back((seq_from >> i*8) & 0xff); + } + // +8 + + for (size_t i = 0; i < sizeof(seq_to); i++) { + pkg.push_back((seq_to >> i*8) & 0xff); + } + // +8 + + + // send + const auto& gp = c.get(); + _t.toxGroupSendCustomPrivatePacket( + gp.group_number, gp.peer_number, + true, + pkg + ); +} + +void CRDTNotesToxSync::SendOps( + Contact3Handle c, + const CRDTNotes::DocID& doc_id, + const std::vector& ops +) { + // ideally this is a file transfer/stream + + if (!c.all_of()) { + return; + } + + std::vector pkg; + + pkg.push_back(static_cast(NGCEXT_Event::CRDTN_OPS)); + + for (const uint8_t v : doc_id) { + pkg.push_back(v); + } + + // this is very inefficent + // a full add op is 124bytes like this + for (const auto& op : ops) { + if(std::holds_alternative(op)) { + const auto& add_op = std::get(op); + pkg.push_back(0x00); // wasteful 1 byte for 1 bit + + for (const uint8_t v : add_op.id.id) { + pkg.push_back(v); + } + + for (size_t i = 0; i < sizeof(add_op.id.seq); i++) { + pkg.push_back((add_op.id.seq >> i*8) & 0xff); + } + pkg.push_back(add_op.value); // what we actually care for + + if (add_op.parent_left.has_value()) { + // exists + pkg.push_back(0x01); // wasteful 1 byte for 1 bit + for (const uint8_t v : add_op.parent_left.value().id) { + pkg.push_back(v); + } + for (size_t i = 0; i < sizeof(add_op.parent_left.value().seq); i++) { + pkg.push_back((add_op.parent_left.value().seq >> i*8) & 0xff); + } + } else { + pkg.push_back(0x00); // wasteful 1 byte for 1 bit + } + + if (add_op.parent_right.has_value()) { + // exists + pkg.push_back(0x01); // wasteful 1 byte for 1 bit + for (const uint8_t v : add_op.parent_right.value().id) { + pkg.push_back(v); + } + for (size_t i = 0; i < sizeof(add_op.parent_right.value().seq); i++) { + pkg.push_back((add_op.parent_right.value().seq >> i*8) & 0xff); + } + } else { + pkg.push_back(0x00); // wasteful 1 byte for 1 bit + } + } else if (std::holds_alternative(op)) { + const auto& del_op = std::get(op); + pkg.push_back(0x01); // wasteful 1 byte for 1 bit + for (const uint8_t v : del_op.id.id) { + pkg.push_back(v); + } + for (size_t i = 0; i < sizeof(del_op.id.seq); i++) { + pkg.push_back((del_op.id.seq >> i*8) & 0xff); + } + } + } + + // send + const auto& gp = c.get(); + _t.toxGroupSendCustomPrivatePacket( + gp.group_number, gp.peer_number, + true, + pkg + ); } diff --git a/src/solanaceae/crdtnotes_toxsync/crdtnotes_toxsync.hpp b/src/solanaceae/crdtnotes_toxsync/crdtnotes_toxsync.hpp index 11b8695..b23b0e3 100644 --- a/src/solanaceae/crdtnotes_toxsync/crdtnotes_toxsync.hpp +++ b/src/solanaceae/crdtnotes_toxsync/crdtnotes_toxsync.hpp @@ -1,15 +1,61 @@ #pragma once #include +#include #include +#include -class CRDTNotesToxSync { +// fwd +struct ToxI; +struct ToxEventProviderI; + +// implements CRDTNotesContactSyncModelI and attaches itself to tox contacts +class CRDTNotesToxSync : public CRDTNotesContactSyncModelI, public ToxEventI { CRDTNotes& _notes; Contact3Registry& _cr; + ToxI& _t; + ToxEventProviderI& _tep; public: - CRDTNotesToxSync(CRDTNotes& notes, Contact3Registry& cr); + CRDTNotesToxSync( + CRDTNotes& notes, + Contact3Registry& cr, + ToxI& t, + ToxEventProviderI& tep + ); + ~CRDTNotesToxSync(void); - float iterate(void); + float iterate(float time_delta); + + public: // sync api + void SendGossip( + Contact3Handle c, + const CRDTNotes::DocID& doc_id + ) override; + + void SendGossip( + Contact3Handle c, + const CRDTNotes::DocID& doc_id, + const std::vector& selected_frontier + ) override; + + void SendFetchCompleteFrontier( + Contact3Handle c, + const CRDTNotes::DocID& doc_id + ) override; + + void SendFetchOps( + Contact3Handle c, + const CRDTNotes::DocID& doc_id, + const CRDTNotes::CRDTAgent& agent, + const uint64_t seq_from, + const uint64_t seq_to + ) override; + + void SendOps( + Contact3Handle c, + const CRDTNotes::DocID& doc_id, + const std::vector& + ) override; };