handle more packets, fix major flaw in diffing algo (again) and add extra tests

This commit is contained in:
Green Sky 2022-12-20 18:46:44 +01:00
parent cb0c2642f8
commit ed72b27808
No known key found for this signature in database
3 changed files with 184 additions and 23 deletions

View File

@ -214,7 +214,6 @@ struct TextDocument {
if (!differ && list_start == state.list.size() && text_start == text.size()) { if (!differ && list_start == state.list.size() && text_start == text.size()) {
return {}; return {};
} }
//std::cout << "list.size: " << state.list.size() << "(" << getText().size() << ")" << " text.size: " << text.size() << "\n"; //std::cout << "list.size: " << state.list.size() << "(" << getText().size() << ")" << " text.size: " << text.size() << "\n";
//std::cout << "list_start: " << list_start << " text_start: " << text_start << "\n"; //std::cout << "list_start: " << list_start << " text_start: " << text_start << "\n";
@ -224,7 +223,9 @@ struct TextDocument {
//for (; list_end > 0 && text_end > 0 && list_end >= list_start && text_end >= text_start;) { //for (; list_end > 0 && text_end > 0 && list_end >= list_start && text_end >= text_start;) {
//while (list_end >= list_start && text_end >= text_start) { //while (list_end >= list_start && text_end >= text_start) {
size_t list_end_counted = 0; size_t list_end_counted = 0;
while (list_start_counted - list_end_counted > state.doc_size && text_end >= text_start) { differ = false; // var reuse
//while (list_start_counted - list_end_counted > state.doc_size && text_end >= text_start) {
while (state.doc_size - list_start_counted > list_end_counted && text_end >= text_start) {
// jump over tombstones // jump over tombstones
if (!state.list[list_end-1].value.has_value()) { if (!state.list[list_end-1].value.has_value()) {
list_end--; list_end--;
@ -232,6 +233,7 @@ struct TextDocument {
} }
if (state.list[list_end-1].value.value() != text[text_end-1]) { if (state.list[list_end-1].value.value() != text[text_end-1]) {
differ = true;
break; break;
} }
@ -240,20 +242,29 @@ struct TextDocument {
list_end_counted++; list_end_counted++;
} }
if (!differ && text_start == text_end+1) {
// we ran into eachother without seeing the different char
// TODO: do we need to increment list_end? text_end?
list_end++;
}
//std::cout << "list_end: " << list_end << " text_end: " << text_end << "\n"; //std::cout << "list_end: " << list_end << " text_end: " << text_end << "\n";
//std::cout << "substring before: " << text.substr(text_start, text.size() - state.doc_size) << "\n";
std::vector<Op> ops; std::vector<Op> ops;
// 1. clear range (del all list_start - list_end) // 1. clear range (del all list_start - list_end)
if (list_start <= list_end && list_start < state.list.size()) { if (list_start <= list_end && list_start < state.list.size()) {
//list_end += list_start == list_end;
ops = delRange( ops = delRange(
state.list[list_start].id, state.list[list_start].id,
(list_start == list_end ? list_end+1 : list_end) < state.list.size() ? std::make_optional(state.list[list_end].id) : std::nullopt list_end < state.list.size() ? std::make_optional(state.list[list_end].id) : std::nullopt
); );
//std::cout << "deleted: " << ops.size() << "\n"; //std::cout << "deleted: " << ops.size() << "\n";
} }
//std::cout << "text between: " << getText() << "\n"; //std::cout << "text between: " << getText() << "\n";
//std::cout << "substring between: " << text.substr(text_start, text.size() - state.doc_size) << "\n";
// 2. add range (add all text_start - text_end) // 2. add range (add all text_start - text_end)
if (state.doc_size < text.size()) { if (state.doc_size < text.size()) {
@ -266,7 +277,6 @@ struct TextDocument {
ops.insert(ops.end(), tmp_add_ops.begin(), tmp_add_ops.end()); ops.insert(ops.end(), tmp_add_ops.begin(), tmp_add_ops.end());
} }
//assert(false && "implement me");
return ops; return ops;
} }
}; };

View File

@ -418,8 +418,9 @@ void testBugDoubleDel(void) {
{ {
std::string_view new_text{"a"}; std::string_view new_text{"a"};
doc.merge(new_text); const auto ops = doc.merge(new_text);
assert(doc.getText() == new_text); assert(doc.getText() == new_text);
assert(ops.size() == 1);
} }
{ {
@ -445,20 +446,23 @@ void testBugSameDel(void) {
{ {
std::string_view new_text{"a"}; std::string_view new_text{"a"};
doc.merge(new_text); const auto ops = doc.merge(new_text);
assert(doc.getText() == new_text); assert(doc.getText() == new_text);
assert(ops.size() == 1);
} }
{ {
std::string_view new_text{"aa"}; std::string_view new_text{"aa"};
const auto ops = doc.merge(new_text); const auto ops = doc.merge(new_text);
assert(doc.getText() == new_text); assert(doc.getText() == new_text);
assert(ops.size() == 1);
} }
{ {
std::string_view new_text{"a"}; std::string_view new_text{"a"};
const auto ops = doc.merge(new_text); const auto ops = doc.merge(new_text);
assert(doc.getText() == new_text); assert(doc.getText() == new_text);
assert(ops.size() == 1);
} }
} }
@ -468,32 +472,122 @@ void testBugSameDel2(void) {
{ {
std::string_view new_text{"a"}; std::string_view new_text{"a"};
doc.merge(new_text); const auto ops = doc.merge(new_text);
assert(doc.getText() == new_text); assert(doc.getText() == new_text);
assert(ops.size() == 1);
} }
{ {
std::string_view new_text{"aa"}; std::string_view new_text{"aa"};
const auto ops = doc.merge(new_text); const auto ops = doc.merge(new_text);
assert(doc.getText() == new_text); assert(doc.getText() == new_text);
assert(ops.size() == 1);
} }
{ {
std::string_view new_text{"aaa"}; std::string_view new_text{"aaa"};
const auto ops = doc.merge(new_text); const auto ops = doc.merge(new_text);
assert(doc.getText() == new_text); assert(doc.getText() == new_text);
assert(ops.size() == 1);
} }
{ {
std::string_view new_text{"aa"}; std::string_view new_text{"aa"};
const auto ops = doc.merge(new_text); const auto ops = doc.merge(new_text);
assert(doc.getText() == new_text); assert(doc.getText() == new_text);
assert(ops.size() == 1);
} }
{ {
std::string_view new_text{"a"}; std::string_view new_text{"a"};
const auto ops = doc.merge(new_text); const auto ops = doc.merge(new_text);
assert(doc.getText() == new_text); assert(doc.getText() == new_text);
assert(ops.size() == 1);
}
}
void testMulti1(void) {
Doc docA;
docA.local_agent = 'A';
Doc docB;
docB.local_agent = 'B';
// state A
{
std::string_view new_text{"iiiiiii"};
const auto ops = docA.merge(new_text);
assert(docA.getText() == new_text);
assert(docB.apply(ops));
assert(docB.getText() == new_text);
assert(docB.state.doc_size == docA.state.doc_size);
assert(docB.state.list.size() == docA.state.list.size());
}
// now B inserts b
{
std::string_view new_text{"iiibiiii"};
const auto ops = docB.merge(new_text);
assert(docB.getText() == new_text);
assert(ops.size() == 1); // 1 new inserted char, nothing to delete
assert(docA.apply(ops));
assert(docA.getText() == new_text);
}
}
void testPaste1(void) {
Doc docA;
docA.local_agent = 'A';
{
std::string_view new_text{"iiiiiii"};
const auto ops = docA.merge(new_text);
assert(ops.size() == 7);
assert(docA.getText() == new_text);
}
{
std::string_view new_text{"iiiiiii\n"};
const auto ops = docA.merge(new_text);
assert(ops.size() == 1);
assert(docA.getText() == new_text);
}
{
std::string_view new_text{"iiiiiii\niiiiiii"};
const auto ops = docA.merge(new_text);
assert(ops.size() == 7);
assert(docA.getText() == new_text);
}
}
void testPaste2(void) {
Doc docA;
docA.local_agent = 'A';
{
std::string_view new_text{"aiiiiib"};
const auto ops = docA.merge(new_text);
assert(ops.size() == 7);
assert(docA.getText() == new_text);
}
{
std::string_view new_text{"aiiiiib\n"};
const auto ops = docA.merge(new_text);
assert(ops.size() == 1);
assert(docA.getText() == new_text);
}
{
std::string_view new_text{"aiiiiib\naiiiiib"};
const auto ops = docA.merge(new_text);
assert(ops.size() == 7);
assert(docA.getText() == new_text);
} }
} }
@ -580,6 +674,27 @@ int main(void) {
testBugSameDel2(); testBugSameDel2();
} }
std::cout << std::string(40, '=') << "\n";
{
std::cout << "testMulti1:\n";
testMulti1();
}
std::cout << std::string(40, '=') << "\n";
{
std::cout << "testPaste1:\n";
testPaste1();
}
std::cout << std::string(40, '=') << "\n";
{
std::cout << "testPaste2:\n";
testPaste2();
}
return 0; return 0;
} }

View File

@ -1,4 +1,3 @@
#include "toxcore/tox.h"
#include <crdt/text_document.hpp> #include <crdt/text_document.hpp>
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
@ -286,6 +285,26 @@ echo 'setup done'
} // namespace vim } // namespace vim
// visibility hack
struct RequestCommands {
Agent agent;
uint64_t after_seq{0};
uint64_t until_seq{0};
};
NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(RequestCommands,
agent,
after_seq,
until_seq
)
// hash for unordered_set
template<>
struct std::hash<std::pair<uint32_t, Agent>> {
std::size_t operator()(std::pair<uint32_t, Agent> const& s) const noexcept {
return std::hash<uint32_t>{}(s.first) << 3 ^ std::hash<Agent>{}(s.second);
}
};
struct SharedContext { struct SharedContext {
std::atomic_bool should_quit {false}; std::atomic_bool should_quit {false};
@ -310,6 +329,12 @@ struct SharedContext {
std::unordered_set<ToxPubKey> should_gossip_remote; // list of ids we have new seq for (only modified by tox thread) std::unordered_set<ToxPubKey> should_gossip_remote; // list of ids we have new seq for (only modified by tox thread)
std::unordered_map<ToxPubKey, uint64_t> heard_gossip; // seq frontiers we have heard about std::unordered_map<ToxPubKey, uint64_t> heard_gossip; // seq frontiers we have heard about
// peer ids that requested the last known seq for agent
std::unordered_set<std::pair<uint32_t, Agent>> requested_frontier;
// peer ids that requested a command (range)
std::vector<std::pair<uint32_t, RequestCommands>> requested_commands;
Tox* tox {nullptr}; Tox* tox {nullptr};
bool tox_dht_online {false}; bool tox_dht_online {false};
bool tox_group_online {false}; bool tox_group_online {false};
@ -350,16 +375,19 @@ namespace pkg {
using Command = ::Command; using Command = ::Command;
// request every command for agent after seq (inclusive) // request every command for agent after_seq - until_seq (inclusive)
struct RequestCommands { //struct RequestCommands {
Agent agent; //Agent agent;
uint64_t seq{0}; //uint64_t after_seq{0};
}; //uint64_t until_seq{0};
//};
using RequestCommands = ::RequestCommands;
NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(RequestCommands, //NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(RequestCommands,
agent, //agent,
seq //after_seq,
) //until_seq
//)
} // namespace pkg } // namespace pkg
@ -560,8 +588,10 @@ void toxThread(SharedContext* ctx) {
// prepend pkgid // prepend pkgid
data.emplace(data.begin(), static_cast<uint8_t>(pkg::PKGID::FRONTIER)); data.emplace(data.begin(), static_cast<uint8_t>(pkg::PKGID::FRONTIER));
if (!tox_group_send_custom_packet(ctx->tox, 0, true, data.data(), data.size(), nullptr)) { Tox_Err_Group_Send_Custom_Packet send_err{TOX_ERR_GROUP_SEND_CUSTOM_PACKET_OK};
std::cerr << "failed to send gossip packet of local agent\n"; if (!tox_group_send_custom_packet(ctx->tox, 0, true, data.data(), data.size(), &send_err)) {
std::cerr << "failed to send gossip packet of local agent" << send_err << "\n";
assert(send_err != TOX_ERR_GROUP_SEND_CUSTOM_PACKET_TOO_LONG);
// TODO: set should_gossip_local back to true? // TODO: set should_gossip_local back to true?
} else { } else {
std::cout << "sent gossip of local agent\n"; std::cout << "sent gossip of local agent\n";
@ -572,8 +602,10 @@ void toxThread(SharedContext* ctx) {
// prepend pkgid // prepend pkgid
data.emplace(data.begin(), static_cast<uint8_t>(pkg::PKGID::COMMAND)); data.emplace(data.begin(), static_cast<uint8_t>(pkg::PKGID::COMMAND));
if (!tox_group_send_custom_packet(ctx->tox, 0, true, data.data(), data.size(), nullptr)) { Tox_Err_Group_Send_Custom_Packet send_err{TOX_ERR_GROUP_SEND_CUSTOM_PACKET_OK};
if (!tox_group_send_custom_packet(ctx->tox, 0, true, data.data(), data.size(), &send_err)) {
std::cerr << "failed to send command packet of local agent\n"; std::cerr << "failed to send command packet of local agent\n";
assert(send_err != TOX_ERR_GROUP_SEND_CUSTOM_PACKET_TOO_LONG);
} else { } else {
std::cout << "sent command of local agent\n"; std::cout << "sent command of local agent\n";
} }
@ -963,7 +995,7 @@ static void self_connection_status_cb(Tox*, TOX_CONNECTION connection_status, vo
std::cout << "self_connection_status_cb " << connection_status << "\n"; std::cout << "self_connection_status_cb " << connection_status << "\n";
} }
static void handle_pkg(SharedContext& ctx, const uint8_t* data, size_t length) { static void handle_pkg(SharedContext& ctx, const uint8_t* data, size_t length, uint32_t peer_id) {
if (length < 2) { if (length < 2) {
std::cerr << "got too short pkg " << length << "\n"; std::cerr << "got too short pkg " << length << "\n";
return; return;
@ -990,6 +1022,7 @@ static void handle_pkg(SharedContext& ctx, const uint8_t* data, size_t length) {
} }
case pkg::PKGID::REQUEST_FRONTIER: { case pkg::PKGID::REQUEST_FRONTIER: {
pkg::RequestFrontier pkg = p_j; pkg::RequestFrontier pkg = p_j;
ctx.requested_frontier.emplace(peer_id, pkg.agent);
break; break;
} }
case pkg::PKGID::COMMAND: { case pkg::PKGID::COMMAND: {
@ -1005,6 +1038,9 @@ static void handle_pkg(SharedContext& ctx, const uint8_t* data, size_t length) {
} }
case pkg::PKGID::REQUEST_COMMANDS: { case pkg::PKGID::REQUEST_COMMANDS: {
pkg::RequestCommands pkg = p_j; pkg::RequestCommands pkg = p_j;
// TODO: this can lead to double requests
// TODO: maybe settle for single seq requests for now?, since they are indivitual packets anyway
ctx.requested_commands.push_back(std::make_pair(peer_id, pkg));
break; break;
} }
default: default:
@ -1016,13 +1052,13 @@ static void handle_pkg(SharedContext& ctx, const uint8_t* data, size_t length) {
static void group_custom_packet_cb(Tox*, uint32_t group_number, uint32_t peer_id, const uint8_t* data, size_t length, void* user_data) { static void group_custom_packet_cb(Tox*, uint32_t group_number, uint32_t peer_id, const uint8_t* data, size_t length, void* user_data) {
std::cout << "group_custom_packet_cb\n"; std::cout << "group_custom_packet_cb\n";
SharedContext& ctx = *static_cast<SharedContext*>(user_data); SharedContext& ctx = *static_cast<SharedContext*>(user_data);
handle_pkg(ctx, data, length); handle_pkg(ctx, data, length, peer_id);
} }
static void group_custom_private_packet_cb(Tox*, uint32_t group_number, uint32_t peer_id, const uint8_t* data, size_t length, void* user_data) { static void group_custom_private_packet_cb(Tox*, uint32_t group_number, uint32_t peer_id, const uint8_t* data, size_t length, void* user_data) {
std::cout << "group_custom_private_packet_cb\n"; std::cout << "group_custom_private_packet_cb\n";
SharedContext& ctx = *static_cast<SharedContext*>(user_data); SharedContext& ctx = *static_cast<SharedContext*>(user_data);
handle_pkg(ctx, data, length); handle_pkg(ctx, data, length, peer_id);
} }
} }