Tha_14/tomato-testing
1
0
Fork 0
forked from Green-Sky/tomato
Code Pull Requests Actions Packages Releases Activity

Merge pull request #8 from Green-Sky/fragment_store

Browse Source 
ObjectStore backed MessageFragments
This commit is contained in:
Erik Scholz 2024-04-14 14:38:49 +02:00 committed by GitHub
commit 8c24234126
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
17 changed files with 179 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitmodules vendored
View File

@ -23,3 +23,6 @@
[submodule "external/solanaceae_object_store"]
path = external/solanaceae_object_store
url = https://github.com/Green-Sky/solanaceae_object_store.git
[submodule "external/solanaceae_message_serializer"]
path = external/solanaceae_message_serializer
url = https://github.com/Green-Sky/solanaceae_message_serializer.git

4
CMakeLists.txt
View File

@ -23,8 +23,8 @@ option(TOMATO_ASAN "Build tomato with asan (gcc/clang/msvc)" OFF)
if (TOMATO_ASAN)
if (${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU" OR ${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang")
if (NOT WIN32) # exclude mingw
link_libraries(-fsanitize=address)
#link_libraries(-fsanitize=address,undefined)
#link_libraries(-fsanitize=address)
link_libraries(-fsanitize=address,undefined)
#link_libraries(-fsanitize=undefined)
message("II enabled ASAN")
else()

3
external/CMakeLists.txt vendored
View File

@ -1,10 +1,11 @@
cmake_minimum_required(VERSION 3.9 FATAL_ERROR)
cmake_minimum_required(VERSION 3.14...3.24 FATAL_ERROR)
add_subdirectory(./entt)
add_subdirectory(./solanaceae_util)
add_subdirectory(./solanaceae_contact)
add_subdirectory(./solanaceae_message3)
add_subdirectory(./solanaceae_message_serializer)
add_subdirectory(./solanaceae_plugin)

2
external/solanaceae_message3 vendored

@ -1 +1 @@
Subproject commit 7c28b232a46ebede9d6f09bc6eafb49bacfa99ea
Subproject commit f9f70a05b1d248e84198c83abeda3579107d09bb

1
external/solanaceae_message_serializer vendored Submodule

@ -0,0 +1 @@
Subproject commit 1409485ef1ee4a2bcf38d7f4631f33e8646d8718

2
external/solanaceae_object_store vendored

@ -1 +1 @@
Subproject commit 46955795b034ed4b058958bba620bd8965ce91f7
Subproject commit e26959c380bc76e8c01a517019c4c0a569156b1d

2
external/solanaceae_tox vendored

@ -1 +1 @@
Subproject commit ce81ef7cf7cea2fe2091912c9eafe787cbba6100
Subproject commit d5c1bf07db96143939d47e3c49cbc4fef308b3d3

20
flake.lock generated
View File

@ -34,10 +34,28 @@
"type": "github"
}
},
"nlohmann-json": {
"flake": false,
"locked": {
"lastModified": 1701207391,
"narHash": "sha256-7F0Jon+1oWL7uqet5i1IgHX0fUw/+z0QwEcA3zs5xHg=",
"owner": "nlohmann",
"repo": "json",
"rev": "9cca280a4d0ccf0c08f47a99aa71d1b0e52f8d03",
"type": "github"
},
"original": {
"owner": "nlohmann",
"ref": "v3.11.3",
"repo": "json",
"type": "github"
}
},
"root": {
"inputs": {
"flake-utils": "flake-utils",
"nixpkgs": "nixpkgs"
"nixpkgs": "nixpkgs",
"nlohmann-json": "nlohmann-json"
}
},
"systems": {

23
flake.nix
View File

@ -6,19 +6,25 @@
inputs = {
nixpkgs.url = "github:NixOS/nixpkgs/release-23.11";
flake-utils.url = "github:numtide/flake-utils";
nlohmann-json = {
url = "github:nlohmann/json/v3.11.3"; # TODO: read version from file
flake = false;
};
};
outputs = { self, nixpkgs, flake-utils }:
outputs = { self, nixpkgs, flake-utils, nlohmann-json }:
flake-utils.lib.eachDefaultSystem (system:
let
pkgs = import nixpkgs { inherit system; };
stdenv = (pkgs.stdenvAdapters.keepDebugInfo pkgs.stdenv);
in {
packages.default = pkgs.stdenv.mkDerivation {
#packages.default = pkgs.stdenv.mkDerivation {
packages.default = stdenv.mkDerivation {
pname = "tomato";
version = "0.0.0";
src = ./.;
submodules = 1;
submodules = 1; # does nothing
nativeBuildInputs = with pkgs; [
cmake
@ -58,9 +64,10 @@
cmakeFlags = [
"-DTOMATO_ASAN=OFF"
"-DCMAKE_BUILD_TYPE=RelWithDebInfo"
"-DFETCHCONTENT_SOURCE_DIR_JSON=${pkgs.nlohmann_json.src}" # we care less about version here
# do we really care less about the version? do we need a stable abi?
"-DFETCHCONTENT_SOURCE_DIR_ZSTD=${pkgs.zstd.src}" # TODO: use package instead
"-DFETCHCONTENT_SOURCE_DIR_JSON=${nlohmann-json}" # we care about the version
# TODO: use package instead
"-DFETCHCONTENT_SOURCE_DIR_ZSTD=${pkgs.zstd.src}" # we dont care about the version (we use 1.4.x features)
];
# TODO: replace with install command
@ -69,7 +76,7 @@
mv bin/tomato $out/bin
'';
dontStrip = true;
dontStrip = true; # does nothing
# copied from nixpkgs's SDL2 default.nix
# SDL is weird in that instead of just dynamically linking with
@ -96,6 +103,8 @@
'';
};
#packages.debug = pkgs.enableDebugging self.packages.${system}.default;
devShells.${system}.default = pkgs.mkShell {
#inputsFrom = with pkgs; [ SDL2 ];
buildInputs = [ self.packages.${system}.default ]; # this makes a prebuild tomato available in the shell, do we want this?

3
src/CMakeLists.txt
View File

@ -1,5 +1,7 @@
cmake_minimum_required(VERSION 3.9 FATAL_ERROR)
########################################
add_executable(tomato
./main.cpp
./icon.rc
@ -75,6 +77,7 @@ target_link_libraries(tomato PUBLIC
solanaceae_util
solanaceae_contact
solanaceae_message3
solanaceae_message_serializer
solanaceae_plugin

144
src/chat_gui4.cpp
View File

@ -47,6 +47,18 @@ namespace Components {
} // Components
namespace Context {
// TODO: move back to chat log window and keep per window instead of per contact
struct CGView {
// set to the ts of the newest rendered msg
Message3Handle begin{};
// set to the ts of the oldest rendered msg
Message3Handle end{};
};
} // Context
static constexpr float lerp(float a, float b, float t) {
return a + t * (b - a);
}
@ -269,28 +281,6 @@ float ChatGui4::render(float time_delta) {
auto* msg_reg_ptr = _rmm.get(*_selected_contact);
if (msg_reg_ptr != nullptr) {
const auto& mm = *msg_reg_ptr;
//const auto& unread_storage = mm.storage<Message::Components::TagUnread>();
if (const auto* unread_storage = mm.storage<Message::Components::TagUnread>(); unread_storage != nullptr && !unread_storage->empty()) {
//assert(unread_storage->size() == 0);
//assert(unread_storage.cbegin() == unread_storage.cend());
#if 0
std::cout << "UNREAD ";
Message3 prev_ent = entt::null;
for (const Message3 e : mm.view<Message::Components::TagUnread>()) {
std::cout << entt::to_integral(e) << " ";
if (prev_ent == e) {
assert(false && "dup");
}
prev_ent = e;
}
std::cout << "\n";
#endif
}
}
constexpr ImGuiTableFlags table_flags =
ImGuiTableFlags_BordersInnerV |
ImGuiTableFlags_RowBg |
@ -303,6 +293,9 @@ float ChatGui4::render(float time_delta) {
ImGui::TableSetupColumn("timestamp");
ImGui::TableSetupColumn("extra_info", _show_chat_extra_info ? ImGuiTableColumnFlags_None : ImGuiTableColumnFlags_Disabled);
Message3Handle message_view_oldest; // oldest visible message
Message3Handle message_view_newest; // last visible message
// very hacky, and we have variable hight entries
//ImGuiListClipper clipper;
@ -389,12 +382,26 @@ float ChatGui4::render(float time_delta) {
}
// use username as visibility test
if (ImGui::IsItemVisible() && msg_reg.all_of<Message::Components::TagUnread>(e)) {
// get time now
const uint64_t ts_now = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
msg_reg.emplace_or_replace<Message::Components::Read>(e, ts_now);
msg_reg.remove<Message::Components::TagUnread>(e);
msg_reg.emplace_or_replace<Components::UnreadFade>(e, 1.f);
if (ImGui::IsItemVisible()) {
if (msg_reg.all_of<Message::Components::TagUnread>(e)) {
// get time now
const uint64_t ts_now = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
msg_reg.emplace_or_replace<Message::Components::Read>(e, ts_now);
msg_reg.remove<Message::Components::TagUnread>(e);
msg_reg.emplace_or_replace<Components::UnreadFade>(e, 1.f);
// we remove the unread tag here
_rmm.throwEventUpdate(msg_reg, e);
}
// track view
if (!static_cast<bool>(message_view_oldest)) {
message_view_oldest = {msg_reg, e};
message_view_newest = {msg_reg, e};
} else if (static_cast<bool>(message_view_newest)) {
// update to latest
message_view_newest = {msg_reg, e};
}
}
// highlight self
@ -559,9 +566,90 @@ float ChatGui4::render(float time_delta) {
//ImGui::TableNextRow(0, TEXT_BASE_HEIGHT);
//ImGui::TableNextRow(0, TEXT_BASE_HEIGHT);
{ // update view cursers
if (!msg_reg.ctx().contains<Context::CGView>()) {
msg_reg.ctx().emplace<Context::CGView>();
}
auto& cg_view = msg_reg.ctx().get<Context::CGView>();
// any message in view
if (!static_cast<bool>(message_view_oldest)) {
// no message in view, we setup a view at current time, so the next frags are loaded
if (!static_cast<bool>(cg_view.begin) || !static_cast<bool>(cg_view.end)) {
// fix invalid state
if (static_cast<bool>(cg_view.begin)) {
cg_view.begin.destroy();
_rmm.throwEventDestroy(cg_view.begin);
}
if (static_cast<bool>(cg_view.end)) {
cg_view.end.destroy();
_rmm.throwEventDestroy(cg_view.end);
}
// create new
cg_view.begin = {msg_reg, msg_reg.create()};
cg_view.end = {msg_reg, msg_reg.create()};
cg_view.begin.emplace_or_replace<Message::Components::ViewCurserBegin>(cg_view.end);
cg_view.end.emplace_or_replace<Message::Components::ViewCurserEnd>(cg_view.begin);
cg_view.begin.get_or_emplace<Message::Components::Timestamp>().ts = Message::getTimeMS();
cg_view.end.get_or_emplace<Message::Components::Timestamp>().ts = Message::getTimeMS();
std::cout << "CG: created view FRONT begin ts\n";
_rmm.throwEventConstruct(cg_view.begin);
std::cout << "CG: created view FRONT end ts\n";
_rmm.throwEventConstruct(cg_view.end);
} // else? we do nothing?
} else {
bool begin_created {false};
if (!static_cast<bool>(cg_view.begin)) {
cg_view.begin = {msg_reg, msg_reg.create()};
begin_created = true;
}
bool end_created {false};
if (!static_cast<bool>(cg_view.end)) {
cg_view.end = {msg_reg, msg_reg.create()};
end_created = true;
}
cg_view.begin.emplace_or_replace<Message::Components::ViewCurserBegin>(cg_view.end);
cg_view.end.emplace_or_replace<Message::Components::ViewCurserEnd>(cg_view.begin);
{
auto& old_begin_ts = cg_view.begin.get_or_emplace<Message::Components::Timestamp>().ts;
if (old_begin_ts != message_view_newest.get<Message::Components::Timestamp>().ts) {
old_begin_ts = message_view_newest.get<Message::Components::Timestamp>().ts;
if (begin_created) {
std::cout << "CG: created view begin ts with " << old_begin_ts << "\n";
_rmm.throwEventConstruct(cg_view.begin);
} else {
//std::cout << "CG: updated view begin ts to " << old_begin_ts << "\n";
_rmm.throwEventUpdate(cg_view.begin);
}
}
}
{
auto& old_end_ts = cg_view.end.get_or_emplace<Message::Components::Timestamp>().ts;
if (old_end_ts != message_view_oldest.get<Message::Components::Timestamp>().ts) {
old_end_ts = message_view_oldest.get<Message::Components::Timestamp>().ts;
if (end_created) {
std::cout << "CG: created view end ts with " << old_end_ts << "\n";
_rmm.throwEventConstruct(cg_view.end);
} else {
//std::cout << "CG: updated view end ts to " << old_end_ts << "\n";
_rmm.throwEventUpdate(cg_view.end);
}
}
}
}
}
ImGui::EndTable();
}
if (ImGui::GetScrollY() >= ImGui::GetScrollMaxY()) {
ImGui::SetScrollHereY(1.f);
}

1
src/chat_gui4.hpp
View File

@ -32,6 +32,7 @@ class ChatGui4 {
FileSelector _fss;
SendImagePopup _sip;
// TODO: refactor this to allow multiple open contacts
std::optional<Contact3> _selected_contact;
// TODO: per contact

72
src/fragment_store/README.md
View File

@ -1,72 +0,0 @@
# Fragment Store
Fragments are are pieces of information split into Metadata and Data.
They can be stored seperated or together.
They can be used as a Transport protocol/logic too.
# Store types
### Object Store
Fragment files are stored with the first 2 hex chars as sub folders:
eg:
`objects/` (object store root)
- `5f/` (first 2hex subfolder)
- `4fffffff` (the fragment file without the first 2 hexchars)
### Split Object Store
Same as Object Store, but medadata and data stored in seperate files.
Metadata files have the `.meta` suffix. They also have a filetype specific suffix, like `.json`, `.msgpack` etc.
### Memory Store
Just keeps the Fragments in memory.
# File formats
Files can be compressed and encrypted. Since compression needs the data structure to funcion, it is applied before it is encrypted.
### Text Json
Text json only makes sense for metadata if it's neither compressed nor encrypted. (otherwise its binary on disk anyway, so why waste bytes).
Since the content of data is not looked at, nothing stops you from using text json and ecrypt it, but atleast basic compression is advised.
A Metadata json object has the following keys:
- `enc` (uint) Encryption type of the data, if any
- `comp` (uint) Compression type of the data, if any
- `metadata` (obj) the
## Binary file headers
### Split Metadata
file magic bytes `SOLMET` (6 bytes)
1 byte encryption type (`0x00` is none)
1 byte compression type (`0x00` is none)
...metadata here...
note that the encryption and compression are for the metadata only.
The metadata itself contains encryption and compression info about the data.
### Split Data
(none) all the data is in the metadata file.
This is mostly to allow direct storage for files in the Fragment store without excessive duplication.
Keep in mind to not use the actual file name as the data/meta file name.
### Single fragment
file magic bytes `SOLFIL` (6 bytes)
1 byte encryption type (`0x00` is none)
1 byte compression type (`0x00` is none)
...metadata here...
...data here...

10
src/main_screen.cpp
View File

@ -1,5 +1,8 @@
#include "./main_screen.hpp"
#include <solanaceae/message3/nj/message_components_serializer.hpp>
#include <solanaceae/tox_messages/nj/tox_message_components_serializer.hpp>
#include <solanaceae/contact/components.hpp>
#include <imgui/imgui.h>
@ -12,6 +15,7 @@
MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::string save_password, std::vector<std::string> plugins) :
renderer(renderer_),
rmm(cr),
msnj{cr, {}, {}},
mts(rmm),
tc(save_path, save_password),
tpi(tc.getTox()),
@ -34,6 +38,9 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri
{
tel.subscribeAll(tc);
registerMessageComponents(msnj);
registerToxMessageComponents(msnj);
conf.set("tox", "save_file_path", save_path);
{ // name stuff
@ -54,6 +61,7 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri
g_provideInstance<ConfigModelI>("ConfigModelI", "host", &conf);
g_provideInstance<Contact3Registry>("Contact3Registry", "1", "host", &cr);
g_provideInstance<RegistryMessageModel>("RegistryMessageModel", "host", &rmm);
g_provideInstance<MessageSerializerNJ>("MessageSerializerNJ", "host", &msnj);
g_provideInstance<ToxI>("ToxI", "host", &tc);
g_provideInstance<ToxPrivateI>("ToxPrivateI", "host", &tpi);
@ -417,7 +425,7 @@ Screen* MainScreen::tick(float time_delta, bool& quit) {
tdch.tick(time_delta); // compute
mts.iterate(); // compute
mts.iterate(); // compute (after mfs)
_min_tick_interval = std::min<float>(
// HACK: pow by 1.6 to increase 50 -> ~500 (~522)

2
src/main_screen.hpp
View File

@ -7,6 +7,7 @@
#include <solanaceae/contact/contact_model3.hpp>
#include <solanaceae/message3/registry_message_model.hpp>
#include <solanaceae/message3/message_time_sort.hpp>
#include <solanaceae/message3/message_serializer.hpp>
#include <solanaceae/plugin/plugin_manager.hpp>
#include <solanaceae/toxcore/tox_event_logger.hpp>
#include "./tox_private_impl.hpp"
@ -49,6 +50,7 @@ struct MainScreen final : public Screen {
SimpleConfigModel conf;
Contact3Registry cr;
RegistryMessageModel rmm;
MessageSerializerNJ msnj;
MessageTimeSort mts;
ToxEventLogger tel{std::cout};

1
src/tox_client.hpp
View File

@ -48,4 +48,3 @@ class ToxClient : public ToxDefaultImpl, public ToxEventProviderBase {
void saveToxProfile(void);
};

3
src/tox_friend_faux_offline_messaging.cpp
View File

@ -120,7 +120,8 @@ ToxFriendFauxOfflineMessaging::dfmc_Ret ToxFriendFauxOfflineMessaging::doFriendM
// require
if (!mr->all_of<
Message::Components::MessageText, // text only for now
Message::Components::ContactTo
Message::Components::ContactTo,
Message::Components::ToxFriendMessageID // yes, needs fake ids
>(msg)
) {
continue; // skip