Merge pull request #8 from Green-Sky/fragment_store
ObjectStore backed MessageFragments
This commit is contained in:
commit
8c24234126
3
.gitmodules
vendored
3
.gitmodules
vendored
@ -23,3 +23,6 @@
|
||||
[submodule "external/solanaceae_object_store"]
|
||||
path = external/solanaceae_object_store
|
||||
url = https://github.com/Green-Sky/solanaceae_object_store.git
|
||||
[submodule "external/solanaceae_message_serializer"]
|
||||
path = external/solanaceae_message_serializer
|
||||
url = https://github.com/Green-Sky/solanaceae_message_serializer.git
|
||||
|
@ -23,8 +23,8 @@ option(TOMATO_ASAN "Build tomato with asan (gcc/clang/msvc)" OFF)
|
||||
if (TOMATO_ASAN)
|
||||
if (${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU" OR ${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang")
|
||||
if (NOT WIN32) # exclude mingw
|
||||
link_libraries(-fsanitize=address)
|
||||
#link_libraries(-fsanitize=address,undefined)
|
||||
#link_libraries(-fsanitize=address)
|
||||
link_libraries(-fsanitize=address,undefined)
|
||||
#link_libraries(-fsanitize=undefined)
|
||||
message("II enabled ASAN")
|
||||
else()
|
||||
|
3
external/CMakeLists.txt
vendored
3
external/CMakeLists.txt
vendored
@ -1,10 +1,11 @@
|
||||
cmake_minimum_required(VERSION 3.9 FATAL_ERROR)
|
||||
cmake_minimum_required(VERSION 3.14...3.24 FATAL_ERROR)
|
||||
|
||||
add_subdirectory(./entt)
|
||||
|
||||
add_subdirectory(./solanaceae_util)
|
||||
add_subdirectory(./solanaceae_contact)
|
||||
add_subdirectory(./solanaceae_message3)
|
||||
add_subdirectory(./solanaceae_message_serializer)
|
||||
|
||||
add_subdirectory(./solanaceae_plugin)
|
||||
|
||||
|
2
external/solanaceae_message3
vendored
2
external/solanaceae_message3
vendored
@ -1 +1 @@
|
||||
Subproject commit 7c28b232a46ebede9d6f09bc6eafb49bacfa99ea
|
||||
Subproject commit f9f70a05b1d248e84198c83abeda3579107d09bb
|
1
external/solanaceae_message_serializer
vendored
Submodule
1
external/solanaceae_message_serializer
vendored
Submodule
@ -0,0 +1 @@
|
||||
Subproject commit 1409485ef1ee4a2bcf38d7f4631f33e8646d8718
|
2
external/solanaceae_object_store
vendored
2
external/solanaceae_object_store
vendored
@ -1 +1 @@
|
||||
Subproject commit 46955795b034ed4b058958bba620bd8965ce91f7
|
||||
Subproject commit e26959c380bc76e8c01a517019c4c0a569156b1d
|
2
external/solanaceae_tox
vendored
2
external/solanaceae_tox
vendored
@ -1 +1 @@
|
||||
Subproject commit ce81ef7cf7cea2fe2091912c9eafe787cbba6100
|
||||
Subproject commit d5c1bf07db96143939d47e3c49cbc4fef308b3d3
|
20
flake.lock
generated
20
flake.lock
generated
@ -34,10 +34,28 @@
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nlohmann-json": {
|
||||
"flake": false,
|
||||
"locked": {
|
||||
"lastModified": 1701207391,
|
||||
"narHash": "sha256-7F0Jon+1oWL7uqet5i1IgHX0fUw/+z0QwEcA3zs5xHg=",
|
||||
"owner": "nlohmann",
|
||||
"repo": "json",
|
||||
"rev": "9cca280a4d0ccf0c08f47a99aa71d1b0e52f8d03",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nlohmann",
|
||||
"ref": "v3.11.3",
|
||||
"repo": "json",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"root": {
|
||||
"inputs": {
|
||||
"flake-utils": "flake-utils",
|
||||
"nixpkgs": "nixpkgs"
|
||||
"nixpkgs": "nixpkgs",
|
||||
"nlohmann-json": "nlohmann-json"
|
||||
}
|
||||
},
|
||||
"systems": {
|
||||
|
23
flake.nix
23
flake.nix
@ -6,19 +6,25 @@
|
||||
inputs = {
|
||||
nixpkgs.url = "github:NixOS/nixpkgs/release-23.11";
|
||||
flake-utils.url = "github:numtide/flake-utils";
|
||||
nlohmann-json = {
|
||||
url = "github:nlohmann/json/v3.11.3"; # TODO: read version from file
|
||||
flake = false;
|
||||
};
|
||||
};
|
||||
|
||||
outputs = { self, nixpkgs, flake-utils }:
|
||||
outputs = { self, nixpkgs, flake-utils, nlohmann-json }:
|
||||
flake-utils.lib.eachDefaultSystem (system:
|
||||
let
|
||||
pkgs = import nixpkgs { inherit system; };
|
||||
stdenv = (pkgs.stdenvAdapters.keepDebugInfo pkgs.stdenv);
|
||||
in {
|
||||
packages.default = pkgs.stdenv.mkDerivation {
|
||||
#packages.default = pkgs.stdenv.mkDerivation {
|
||||
packages.default = stdenv.mkDerivation {
|
||||
pname = "tomato";
|
||||
version = "0.0.0";
|
||||
|
||||
src = ./.;
|
||||
submodules = 1;
|
||||
submodules = 1; # does nothing
|
||||
|
||||
nativeBuildInputs = with pkgs; [
|
||||
cmake
|
||||
@ -58,9 +64,10 @@
|
||||
cmakeFlags = [
|
||||
"-DTOMATO_ASAN=OFF"
|
||||
"-DCMAKE_BUILD_TYPE=RelWithDebInfo"
|
||||
"-DFETCHCONTENT_SOURCE_DIR_JSON=${pkgs.nlohmann_json.src}" # we care less about version here
|
||||
# do we really care less about the version? do we need a stable abi?
|
||||
"-DFETCHCONTENT_SOURCE_DIR_ZSTD=${pkgs.zstd.src}" # TODO: use package instead
|
||||
|
||||
"-DFETCHCONTENT_SOURCE_DIR_JSON=${nlohmann-json}" # we care about the version
|
||||
# TODO: use package instead
|
||||
"-DFETCHCONTENT_SOURCE_DIR_ZSTD=${pkgs.zstd.src}" # we dont care about the version (we use 1.4.x features)
|
||||
];
|
||||
|
||||
# TODO: replace with install command
|
||||
@ -69,7 +76,7 @@
|
||||
mv bin/tomato $out/bin
|
||||
'';
|
||||
|
||||
dontStrip = true;
|
||||
dontStrip = true; # does nothing
|
||||
|
||||
# copied from nixpkgs's SDL2 default.nix
|
||||
# SDL is weird in that instead of just dynamically linking with
|
||||
@ -96,6 +103,8 @@
|
||||
'';
|
||||
};
|
||||
|
||||
#packages.debug = pkgs.enableDebugging self.packages.${system}.default;
|
||||
|
||||
devShells.${system}.default = pkgs.mkShell {
|
||||
#inputsFrom = with pkgs; [ SDL2 ];
|
||||
buildInputs = [ self.packages.${system}.default ]; # this makes a prebuild tomato available in the shell, do we want this?
|
||||
|
@ -1,5 +1,7 @@
|
||||
cmake_minimum_required(VERSION 3.9 FATAL_ERROR)
|
||||
|
||||
########################################
|
||||
|
||||
add_executable(tomato
|
||||
./main.cpp
|
||||
./icon.rc
|
||||
@ -75,6 +77,7 @@ target_link_libraries(tomato PUBLIC
|
||||
solanaceae_util
|
||||
solanaceae_contact
|
||||
solanaceae_message3
|
||||
solanaceae_message_serializer
|
||||
|
||||
solanaceae_plugin
|
||||
|
||||
|
@ -47,6 +47,18 @@ namespace Components {
|
||||
|
||||
} // Components
|
||||
|
||||
namespace Context {
|
||||
|
||||
// TODO: move back to chat log window and keep per window instead of per contact
|
||||
struct CGView {
|
||||
// set to the ts of the newest rendered msg
|
||||
Message3Handle begin{};
|
||||
// set to the ts of the oldest rendered msg
|
||||
Message3Handle end{};
|
||||
};
|
||||
|
||||
} // Context
|
||||
|
||||
static constexpr float lerp(float a, float b, float t) {
|
||||
return a + t * (b - a);
|
||||
}
|
||||
@ -269,28 +281,6 @@ float ChatGui4::render(float time_delta) {
|
||||
|
||||
auto* msg_reg_ptr = _rmm.get(*_selected_contact);
|
||||
|
||||
if (msg_reg_ptr != nullptr) {
|
||||
const auto& mm = *msg_reg_ptr;
|
||||
//const auto& unread_storage = mm.storage<Message::Components::TagUnread>();
|
||||
if (const auto* unread_storage = mm.storage<Message::Components::TagUnread>(); unread_storage != nullptr && !unread_storage->empty()) {
|
||||
//assert(unread_storage->size() == 0);
|
||||
//assert(unread_storage.cbegin() == unread_storage.cend());
|
||||
|
||||
#if 0
|
||||
std::cout << "UNREAD ";
|
||||
Message3 prev_ent = entt::null;
|
||||
for (const Message3 e : mm.view<Message::Components::TagUnread>()) {
|
||||
std::cout << entt::to_integral(e) << " ";
|
||||
if (prev_ent == e) {
|
||||
assert(false && "dup");
|
||||
}
|
||||
prev_ent = e;
|
||||
}
|
||||
std::cout << "\n";
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
constexpr ImGuiTableFlags table_flags =
|
||||
ImGuiTableFlags_BordersInnerV |
|
||||
ImGuiTableFlags_RowBg |
|
||||
@ -303,6 +293,9 @@ float ChatGui4::render(float time_delta) {
|
||||
ImGui::TableSetupColumn("timestamp");
|
||||
ImGui::TableSetupColumn("extra_info", _show_chat_extra_info ? ImGuiTableColumnFlags_None : ImGuiTableColumnFlags_Disabled);
|
||||
|
||||
Message3Handle message_view_oldest; // oldest visible message
|
||||
Message3Handle message_view_newest; // last visible message
|
||||
|
||||
// very hacky, and we have variable hight entries
|
||||
//ImGuiListClipper clipper;
|
||||
|
||||
@ -389,12 +382,26 @@ float ChatGui4::render(float time_delta) {
|
||||
}
|
||||
|
||||
// use username as visibility test
|
||||
if (ImGui::IsItemVisible() && msg_reg.all_of<Message::Components::TagUnread>(e)) {
|
||||
// get time now
|
||||
const uint64_t ts_now = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
|
||||
msg_reg.emplace_or_replace<Message::Components::Read>(e, ts_now);
|
||||
msg_reg.remove<Message::Components::TagUnread>(e);
|
||||
msg_reg.emplace_or_replace<Components::UnreadFade>(e, 1.f);
|
||||
if (ImGui::IsItemVisible()) {
|
||||
if (msg_reg.all_of<Message::Components::TagUnread>(e)) {
|
||||
// get time now
|
||||
const uint64_t ts_now = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
|
||||
msg_reg.emplace_or_replace<Message::Components::Read>(e, ts_now);
|
||||
msg_reg.remove<Message::Components::TagUnread>(e);
|
||||
msg_reg.emplace_or_replace<Components::UnreadFade>(e, 1.f);
|
||||
|
||||
// we remove the unread tag here
|
||||
_rmm.throwEventUpdate(msg_reg, e);
|
||||
}
|
||||
|
||||
// track view
|
||||
if (!static_cast<bool>(message_view_oldest)) {
|
||||
message_view_oldest = {msg_reg, e};
|
||||
message_view_newest = {msg_reg, e};
|
||||
} else if (static_cast<bool>(message_view_newest)) {
|
||||
// update to latest
|
||||
message_view_newest = {msg_reg, e};
|
||||
}
|
||||
}
|
||||
|
||||
// highlight self
|
||||
@ -559,9 +566,90 @@ float ChatGui4::render(float time_delta) {
|
||||
//ImGui::TableNextRow(0, TEXT_BASE_HEIGHT);
|
||||
//ImGui::TableNextRow(0, TEXT_BASE_HEIGHT);
|
||||
|
||||
{ // update view cursers
|
||||
if (!msg_reg.ctx().contains<Context::CGView>()) {
|
||||
msg_reg.ctx().emplace<Context::CGView>();
|
||||
}
|
||||
|
||||
auto& cg_view = msg_reg.ctx().get<Context::CGView>();
|
||||
|
||||
// any message in view
|
||||
if (!static_cast<bool>(message_view_oldest)) {
|
||||
// no message in view, we setup a view at current time, so the next frags are loaded
|
||||
if (!static_cast<bool>(cg_view.begin) || !static_cast<bool>(cg_view.end)) {
|
||||
// fix invalid state
|
||||
if (static_cast<bool>(cg_view.begin)) {
|
||||
cg_view.begin.destroy();
|
||||
_rmm.throwEventDestroy(cg_view.begin);
|
||||
}
|
||||
if (static_cast<bool>(cg_view.end)) {
|
||||
cg_view.end.destroy();
|
||||
_rmm.throwEventDestroy(cg_view.end);
|
||||
}
|
||||
|
||||
// create new
|
||||
cg_view.begin = {msg_reg, msg_reg.create()};
|
||||
cg_view.end = {msg_reg, msg_reg.create()};
|
||||
|
||||
cg_view.begin.emplace_or_replace<Message::Components::ViewCurserBegin>(cg_view.end);
|
||||
cg_view.end.emplace_or_replace<Message::Components::ViewCurserEnd>(cg_view.begin);
|
||||
|
||||
cg_view.begin.get_or_emplace<Message::Components::Timestamp>().ts = Message::getTimeMS();
|
||||
cg_view.end.get_or_emplace<Message::Components::Timestamp>().ts = Message::getTimeMS();
|
||||
|
||||
std::cout << "CG: created view FRONT begin ts\n";
|
||||
_rmm.throwEventConstruct(cg_view.begin);
|
||||
std::cout << "CG: created view FRONT end ts\n";
|
||||
_rmm.throwEventConstruct(cg_view.end);
|
||||
} // else? we do nothing?
|
||||
} else {
|
||||
bool begin_created {false};
|
||||
if (!static_cast<bool>(cg_view.begin)) {
|
||||
cg_view.begin = {msg_reg, msg_reg.create()};
|
||||
begin_created = true;
|
||||
}
|
||||
bool end_created {false};
|
||||
if (!static_cast<bool>(cg_view.end)) {
|
||||
cg_view.end = {msg_reg, msg_reg.create()};
|
||||
end_created = true;
|
||||
}
|
||||
cg_view.begin.emplace_or_replace<Message::Components::ViewCurserBegin>(cg_view.end);
|
||||
cg_view.end.emplace_or_replace<Message::Components::ViewCurserEnd>(cg_view.begin);
|
||||
|
||||
{
|
||||
auto& old_begin_ts = cg_view.begin.get_or_emplace<Message::Components::Timestamp>().ts;
|
||||
if (old_begin_ts != message_view_newest.get<Message::Components::Timestamp>().ts) {
|
||||
old_begin_ts = message_view_newest.get<Message::Components::Timestamp>().ts;
|
||||
if (begin_created) {
|
||||
std::cout << "CG: created view begin ts with " << old_begin_ts << "\n";
|
||||
_rmm.throwEventConstruct(cg_view.begin);
|
||||
} else {
|
||||
//std::cout << "CG: updated view begin ts to " << old_begin_ts << "\n";
|
||||
_rmm.throwEventUpdate(cg_view.begin);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
auto& old_end_ts = cg_view.end.get_or_emplace<Message::Components::Timestamp>().ts;
|
||||
if (old_end_ts != message_view_oldest.get<Message::Components::Timestamp>().ts) {
|
||||
old_end_ts = message_view_oldest.get<Message::Components::Timestamp>().ts;
|
||||
if (end_created) {
|
||||
std::cout << "CG: created view end ts with " << old_end_ts << "\n";
|
||||
_rmm.throwEventConstruct(cg_view.end);
|
||||
} else {
|
||||
//std::cout << "CG: updated view end ts to " << old_end_ts << "\n";
|
||||
_rmm.throwEventUpdate(cg_view.end);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ImGui::EndTable();
|
||||
}
|
||||
|
||||
|
||||
if (ImGui::GetScrollY() >= ImGui::GetScrollMaxY()) {
|
||||
ImGui::SetScrollHereY(1.f);
|
||||
}
|
||||
|
@ -32,6 +32,7 @@ class ChatGui4 {
|
||||
FileSelector _fss;
|
||||
SendImagePopup _sip;
|
||||
|
||||
// TODO: refactor this to allow multiple open contacts
|
||||
std::optional<Contact3> _selected_contact;
|
||||
|
||||
// TODO: per contact
|
||||
|
@ -1,72 +0,0 @@
|
||||
# Fragment Store
|
||||
|
||||
Fragments are are pieces of information split into Metadata and Data.
|
||||
They can be stored seperated or together.
|
||||
They can be used as a Transport protocol/logic too.
|
||||
|
||||
# Store types
|
||||
|
||||
### Object Store
|
||||
|
||||
Fragment files are stored with the first 2 hex chars as sub folders:
|
||||
eg:
|
||||
`objects/` (object store root)
|
||||
- `5f/` (first 2hex subfolder)
|
||||
- `4fffffff` (the fragment file without the first 2 hexchars)
|
||||
|
||||
### Split Object Store
|
||||
|
||||
Same as Object Store, but medadata and data stored in seperate files.
|
||||
Metadata files have the `.meta` suffix. They also have a filetype specific suffix, like `.json`, `.msgpack` etc.
|
||||
|
||||
### Memory Store
|
||||
|
||||
Just keeps the Fragments in memory.
|
||||
|
||||
# File formats
|
||||
|
||||
Files can be compressed and encrypted. Since compression needs the data structure to funcion, it is applied before it is encrypted.
|
||||
|
||||
### Text Json
|
||||
|
||||
Text json only makes sense for metadata if it's neither compressed nor encrypted. (otherwise its binary on disk anyway, so why waste bytes).
|
||||
Since the content of data is not looked at, nothing stops you from using text json and ecrypt it, but atleast basic compression is advised.
|
||||
|
||||
A Metadata json object has the following keys:
|
||||
- `enc` (uint) Encryption type of the data, if any
|
||||
- `comp` (uint) Compression type of the data, if any
|
||||
- `metadata` (obj) the
|
||||
|
||||
## Binary file headers
|
||||
|
||||
### Split Metadata
|
||||
|
||||
file magic bytes `SOLMET` (6 bytes)
|
||||
|
||||
1 byte encryption type (`0x00` is none)
|
||||
|
||||
1 byte compression type (`0x00` is none)
|
||||
|
||||
...metadata here...
|
||||
|
||||
note that the encryption and compression are for the metadata only.
|
||||
The metadata itself contains encryption and compression info about the data.
|
||||
|
||||
### Split Data
|
||||
|
||||
(none) all the data is in the metadata file.
|
||||
This is mostly to allow direct storage for files in the Fragment store without excessive duplication.
|
||||
Keep in mind to not use the actual file name as the data/meta file name.
|
||||
|
||||
### Single fragment
|
||||
|
||||
file magic bytes `SOLFIL` (6 bytes)
|
||||
|
||||
1 byte encryption type (`0x00` is none)
|
||||
|
||||
1 byte compression type (`0x00` is none)
|
||||
|
||||
...metadata here...
|
||||
|
||||
...data here...
|
||||
|
@ -1,5 +1,8 @@
|
||||
#include "./main_screen.hpp"
|
||||
|
||||
#include <solanaceae/message3/nj/message_components_serializer.hpp>
|
||||
#include <solanaceae/tox_messages/nj/tox_message_components_serializer.hpp>
|
||||
|
||||
#include <solanaceae/contact/components.hpp>
|
||||
|
||||
#include <imgui/imgui.h>
|
||||
@ -12,6 +15,7 @@
|
||||
MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::string save_password, std::vector<std::string> plugins) :
|
||||
renderer(renderer_),
|
||||
rmm(cr),
|
||||
msnj{cr, {}, {}},
|
||||
mts(rmm),
|
||||
tc(save_path, save_password),
|
||||
tpi(tc.getTox()),
|
||||
@ -34,6 +38,9 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri
|
||||
{
|
||||
tel.subscribeAll(tc);
|
||||
|
||||
registerMessageComponents(msnj);
|
||||
registerToxMessageComponents(msnj);
|
||||
|
||||
conf.set("tox", "save_file_path", save_path);
|
||||
|
||||
{ // name stuff
|
||||
@ -54,6 +61,7 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri
|
||||
g_provideInstance<ConfigModelI>("ConfigModelI", "host", &conf);
|
||||
g_provideInstance<Contact3Registry>("Contact3Registry", "1", "host", &cr);
|
||||
g_provideInstance<RegistryMessageModel>("RegistryMessageModel", "host", &rmm);
|
||||
g_provideInstance<MessageSerializerNJ>("MessageSerializerNJ", "host", &msnj);
|
||||
|
||||
g_provideInstance<ToxI>("ToxI", "host", &tc);
|
||||
g_provideInstance<ToxPrivateI>("ToxPrivateI", "host", &tpi);
|
||||
@ -417,7 +425,7 @@ Screen* MainScreen::tick(float time_delta, bool& quit) {
|
||||
|
||||
tdch.tick(time_delta); // compute
|
||||
|
||||
mts.iterate(); // compute
|
||||
mts.iterate(); // compute (after mfs)
|
||||
|
||||
_min_tick_interval = std::min<float>(
|
||||
// HACK: pow by 1.6 to increase 50 -> ~500 (~522)
|
||||
|
@ -7,6 +7,7 @@
|
||||
#include <solanaceae/contact/contact_model3.hpp>
|
||||
#include <solanaceae/message3/registry_message_model.hpp>
|
||||
#include <solanaceae/message3/message_time_sort.hpp>
|
||||
#include <solanaceae/message3/message_serializer.hpp>
|
||||
#include <solanaceae/plugin/plugin_manager.hpp>
|
||||
#include <solanaceae/toxcore/tox_event_logger.hpp>
|
||||
#include "./tox_private_impl.hpp"
|
||||
@ -49,6 +50,7 @@ struct MainScreen final : public Screen {
|
||||
SimpleConfigModel conf;
|
||||
Contact3Registry cr;
|
||||
RegistryMessageModel rmm;
|
||||
MessageSerializerNJ msnj;
|
||||
MessageTimeSort mts;
|
||||
|
||||
ToxEventLogger tel{std::cout};
|
||||
|
@ -48,4 +48,3 @@ class ToxClient : public ToxDefaultImpl, public ToxEventProviderBase {
|
||||
void saveToxProfile(void);
|
||||
};
|
||||
|
||||
|
||||
|
@ -120,7 +120,8 @@ ToxFriendFauxOfflineMessaging::dfmc_Ret ToxFriendFauxOfflineMessaging::doFriendM
|
||||
// require
|
||||
if (!mr->all_of<
|
||||
Message::Components::MessageText, // text only for now
|
||||
Message::Components::ContactTo
|
||||
Message::Components::ContactTo,
|
||||
Message::Components::ToxFriendMessageID // yes, needs fake ids
|
||||
>(msg)
|
||||
) {
|
||||
continue; // skip
|
||||
|
Loading…
Reference in New Issue
Block a user