Green-Sky/tomato
Green-Sky/tomato
1
0
Fork 1
Code Issues Pull Requests 1 Actions 2 Packages Projects Releases 1 Wiki Activity

refactoring, add to mainscreen

Browse Source
This commit is contained in:
Green Sky 2024-02-13 12:30:29 +01:00
parent 98ab974515
commit e67d7d37b5
No known key found for this signature in database
11 changed files with 214 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/CMakeLists.txt
View File

@ -2,6 +2,11 @@ cmake_minimum_required(VERSION 3.9 FATAL_ERROR)
add_library(fragment_store
./fragment_store/fragment_store_i.hpp
./fragment_store/types.hpp
./fragment_store/meta_components.hpp
./fragment_store/meta_components_id.inl
./fragment_store/serializer.hpp
./fragment_store/serializer.cpp
./fragment_store/fragment_store.hpp
./fragment_store/fragment_store.cpp
)
@ -106,6 +111,8 @@ target_link_libraries(tomato PUBLIC
solanaceae_tox_contacts
solanaceae_tox_messages
fragment_store
SDL3::SDL3
imgui

59
src/fragment_store/fragment_store.cpp
View File

@ -51,7 +51,7 @@ FragmentStore::FragmentStore(
registerSerializers();
}
entt::basic_handle<entt::basic_registry<FragmentID>> FragmentStore::fragmentHandle(FragmentID fid) {
FragmentStore::FragmentHandle FragmentStore::fragmentHandle(FragmentID fid) {
return {_reg, fid};
}
@ -107,7 +107,7 @@ FragmentID FragmentStore::newFragmentMemoryOwned(
const auto new_frag = _reg.create();
_reg.emplace<Components::ID>(new_frag, id);
_reg.emplace<FragComp::ID>(new_frag, id);
// TODO: memory comp
_reg.emplace<std::unique_ptr<std::vector<uint8_t>>>(new_frag) = std::move(new_data);
@ -149,12 +149,12 @@ FragmentID FragmentStore::newFragmentFile(
const auto new_frag = _reg.create();
_reg.emplace<Components::ID>(new_frag, id);
_reg.emplace<FragComp::ID>(new_frag, id);
// file (info) comp
_reg.emplace<Components::Ephemeral::FilePath>(new_frag, fragment_file_path.generic_u8string());
_reg.emplace<FragComp::Ephemeral::FilePath>(new_frag, fragment_file_path.generic_u8string());
_reg.emplace<Components::Ephemeral::MetaFileType>(new_frag, mft);
_reg.emplace<FragComp::Ephemeral::MetaFileType>(new_frag, mft);
// meta needs to be synced to file
std::function<write_to_storage_fetch_data_cb> empty_data_cb = [](const uint8_t*, uint64_t) -> uint64_t { return 0; };
@ -177,7 +177,7 @@ FragmentID FragmentStore::getFragmentByID(
) {
// TODO: accelerate
// maybe keep it sorted and binary search? hash table lookup?
for (const auto& [frag, id_comp] : _reg.view<Components::ID>().each()) {
for (const auto& [frag, id_comp] : _reg.view<FragComp::ID>().each()) {
if (id == id_comp.v) {
return frag;
}
@ -206,7 +206,7 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function<write_to_storage
return false;
}
if (!_reg.all_of<Components::Ephemeral::FilePath>(fid)) {
if (!_reg.all_of<FragComp::Ephemeral::FilePath>(fid)) {
// not a file fragment?
return false;
}
@ -214,20 +214,20 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function<write_to_storage
// split object storage
MetaFileType meta_type = MetaFileType::TEXT_JSON; // TODO: better defaults
if (_reg.all_of<Components::Ephemeral::MetaFileType>(fid)) {
meta_type = _reg.get<Components::Ephemeral::MetaFileType>(fid).type;
if (_reg.all_of<FragComp::Ephemeral::MetaFileType>(fid)) {
meta_type = _reg.get<FragComp::Ephemeral::MetaFileType>(fid).type;
}
Encryption meta_enc = Encryption::NONE; // TODO: better defaults
Compression meta_comp = Compression::NONE; // TODO: better defaults
if (meta_type != MetaFileType::TEXT_JSON) {
if (_reg.all_of<Components::Ephemeral::MetaEncryptionType>(fid)) {
meta_enc = _reg.get<Components::Ephemeral::MetaEncryptionType>(fid).enc;
if (_reg.all_of<FragComp::Ephemeral::MetaEncryptionType>(fid)) {
meta_enc = _reg.get<FragComp::Ephemeral::MetaEncryptionType>(fid).enc;
}
if (_reg.all_of<Components::Ephemeral::MetaCompressionType>(fid)) {
meta_comp = _reg.get<Components::Ephemeral::MetaCompressionType>(fid).comp;
if (_reg.all_of<FragComp::Ephemeral::MetaCompressionType>(fid)) {
meta_comp = _reg.get<FragComp::Ephemeral::MetaCompressionType>(fid).comp;
}
} else {
// we cant have encryption or compression
@ -236,15 +236,15 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function<write_to_storage
// TODO: forcing for testing
//if (_reg.all_of<Components::Ephemeral::MetaEncryptionType>(fid)) {
_reg.emplace_or_replace<Components::Ephemeral::MetaEncryptionType>(fid, Encryption::NONE);
_reg.emplace_or_replace<FragComp::Ephemeral::MetaEncryptionType>(fid, Encryption::NONE);
//}
//if (_reg.all_of<Components::Ephemeral::MetaCompressionType>(fid)) {
_reg.emplace_or_replace<Components::Ephemeral::MetaCompressionType>(fid, Compression::NONE);
_reg.emplace_or_replace<FragComp::Ephemeral::MetaCompressionType>(fid, Compression::NONE);
//}
}
std::ofstream meta_file{
_reg.get<Components::Ephemeral::FilePath>(fid).path + ".meta" + metaFileTypeSuffix(meta_type),
_reg.get<FragComp::Ephemeral::FilePath>(fid).path + ".meta" + metaFileTypeSuffix(meta_type),
std::ios::out | std::ios::trunc | std::ios::binary // always binary, also for text
};
@ -253,7 +253,7 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function<write_to_storage
}
std::ofstream data_file{
_reg.get<Components::Ephemeral::FilePath>(fid).path,
_reg.get<FragComp::Ephemeral::FilePath>(fid).path,
std::ios::out | std::ios::trunc | std::ios::binary // always binary, also for text
};
@ -320,19 +320,32 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function<write_to_storage
// TODO: use temp files and move to old location
if (_reg.all_of<Components::Ephemeral::DirtyTag>(fid)) {
_reg.remove<Components::Ephemeral::DirtyTag>(fid);
if (_reg.all_of<FragComp::Ephemeral::DirtyTag>(fid)) {
_reg.remove<FragComp::Ephemeral::DirtyTag>(fid);
}
return true;
}
bool FragmentStore::syncToStorage(FragmentID fid, const uint8_t* data, const uint64_t data_size) {
std::function<FragmentStore::write_to_storage_fetch_data_cb> fn_cb = [read = 0ull, data, data_size](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t {
uint64_t i = 0;
for (; i+read < data_size && i < buffer_size; i++) {
request_buffer[i] = data[i+read];
}
read += i;
return i;
};
return syncToStorage(fid, fn_cb);
}
static bool serl_json_data_enc_type(void* comp, nlohmann::json& out) {
if (comp == nullptr) {
return false;
}
auto& r_comp = *reinterpret_cast<Components::DataEncryptionType*>(comp);
auto& r_comp = *reinterpret_cast<FragComp::DataEncryptionType*>(comp);
out = static_cast<std::underlying_type_t<Encryption>>(r_comp.enc);
@ -344,7 +357,7 @@ static bool serl_json_data_comp_type(void* comp, nlohmann::json& out) {
return false;
}
auto& r_comp = *reinterpret_cast<Components::DataCompressionType*>(comp);
auto& r_comp = *reinterpret_cast<FragComp::DataCompressionType*>(comp);
out = static_cast<std::underlying_type_t<Compression>>(r_comp.comp);
@ -352,8 +365,8 @@ static bool serl_json_data_comp_type(void* comp, nlohmann::json& out) {
}
void FragmentStore::registerSerializers(void) {
_sc.registerSerializerJson<Components::DataEncryptionType>(serl_json_data_enc_type);
_sc.registerSerializerJson<Components::DataCompressionType>(serl_json_data_comp_type);
_sc.registerSerializerJson<FragComp::DataEncryptionType>(serl_json_data_enc_type);
_sc.registerSerializerJson<FragComp::DataCompressionType>(serl_json_data_comp_type);
std::cout << "registered serl text json cbs:\n";
for (const auto& [type_id, _] : _sc._serl_json) {

103
src/fragment_store/fragment_store.hpp
View File

@ -2,101 +2,24 @@
#include "./fragment_store_i.hpp"
#include <entt/core/fwd.hpp>
#include "./types.hpp"
#include "./meta_components.hpp"
#include "./serializer.hpp"
#include <entt/core/type_info.hpp>
#include <entt/entity/registry.hpp>
#include <entt/container/dense_map.hpp>
#include <nlohmann/json_fwd.hpp>
#include <utility>
#include <vector>
#include <array>
#include <cstdint>
#include <random>
enum class Encryption : uint8_t {
NONE = 0x00,
};
enum class Compression : uint8_t {
NONE = 0x00,
};
enum class MetaFileType : uint8_t {
TEXT_JSON,
//BINARY_ARB,
BINARY_MSGPACK,
};
namespace Components {
// TODO: is this special and should this be saved to meta or not (its already in the file name on disk)
struct ID {
std::vector<uint8_t> v;
};
struct DataEncryptionType {
Encryption enc {Encryption::NONE};
};
struct DataCompressionType {
Compression comp {Compression::NONE};
};
// meta that is not written to (meta-)file
namespace Ephemeral {
// excluded from file meta
struct FilePath {
// contains store path, if any
std::string path;
};
// TODO: seperate into remote and local?
// (remote meaning eg. the file on disk was changed by another program)
struct DirtyTag {};
// type as comp
struct MetaFileType {
::MetaFileType type {::MetaFileType::TEXT_JSON};
};
struct MetaEncryptionType {
Encryption enc {Encryption::NONE};
};
struct MetaCompressionType {
Compression comp {Compression::NONE};
};
} // Ephemeral
} // Components
struct SerializerCallbacks {
// nlohmann
// json/msgpack
using serialize_json_fn = bool(*)(void* comp, nlohmann::json& out);
entt::dense_map<entt::id_type, serialize_json_fn> _serl_json;
using deserialize_json_fn = bool(*)(void* comp, const nlohmann::json& in);
entt::dense_map<entt::id_type, deserialize_json_fn> _deserl_json;
void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info) {
_serl_json[type_info.hash()] = fn;
}
template<typename CompType>
void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info = entt::type_id<CompType>()) { registerSerializerJson(fn, type_info); }
void registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info) {
_deserl_json[type_info.hash()] = fn;
}
template<typename CompType>
void registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info = entt::type_id<CompType>()) { registerDeSerializerJson(fn, type_info); }
};
struct FragmentStore : public FragmentStoreI {
using FragmentHandle = entt::basic_handle<entt::basic_registry<FragmentID>>;
entt::basic_registry<FragmentID> _reg;
std::minstd_rand _rng{std::random_device{}()};
@ -113,10 +36,11 @@ struct FragmentStore : public FragmentStoreI {
FragmentStore(std::array<uint8_t, 8> session_uuid_namespace);
// HACK: get access to the reg
entt::basic_handle<entt::basic_registry<FragmentID>> fragmentHandle(FragmentID fid);
FragmentHandle fragmentHandle(FragmentID fid);
// TODO: make the frags ref counted
// TODO: check for exising
std::vector<uint8_t> generateNewUID(std::array<uint8_t, 8>& uuid_namespace);
std::vector<uint8_t> generateNewUID(void);
@ -154,21 +78,20 @@ struct FragmentStore : public FragmentStoreI {
);
// remove fragment?
// unload?
// syncs fragment to file
// ========== sync fragment to storage ==========
using write_to_storage_fetch_data_cb = uint64_t(uint8_t* request_buffer, uint64_t buffer_size);
// calls data_cb with a buffer to be filled in, cb returns actual count of data. if returned < max, its the last buffer.
bool syncToStorage(FragmentID fid, std::function<write_to_storage_fetch_data_cb>& data_cb);
// unload frags?
// if frags are file backed, we can close the file if not needed
bool syncToStorage(FragmentID fid, const uint8_t* data, const uint64_t data_size);
// fragment discovery?
private:
void registerSerializers(void); // internal comps
// internal actual backends
// TODO: seperate out
bool syncToMemory(FragmentID fid, std::function<write_to_storage_fetch_data_cb>& data_cb);
bool syncToFile(FragmentID fid, std::function<write_to_storage_fetch_data_cb>& data_cb);
};

60
src/fragment_store/meta_components.hpp Normal file
View File

@ -0,0 +1,60 @@
#pragma once
#include "./types.hpp"
#include <vector>
#include <string>
#include <cstdint>
namespace Fragment::Components {
// TODO: is this special and should this be saved to meta or not (its already in the file name on disk)
struct ID {
std::vector<uint8_t> v;
};
struct DataEncryptionType {
Encryption enc {Encryption::NONE};
};
struct DataCompressionType {
Compression comp {Compression::NONE};
};
// meta that is not written to (meta-)file
namespace Ephemeral {
// excluded from file meta
struct FilePath {
// contains store path, if any
std::string path;
};
// TODO: seperate into remote and local?
// (remote meaning eg. the file on disk was changed by another program)
struct DirtyTag {};
// type as comp
struct MetaFileType {
::MetaFileType type {::MetaFileType::TEXT_JSON};
};
struct MetaEncryptionType {
Encryption enc {Encryption::NONE};
};
struct MetaCompressionType {
Compression comp {Compression::NONE};
};
} // Ephemeral
} // Components
// shortened to save bytes (until I find a way to save by ID in msgpack)
namespace FragComp = Fragment::Components;
#include "./meta_components_id.inl"

26
src/fragment_store/meta_components_id.inl Normal file
View File

@ -0,0 +1,26 @@
#pragma once
#include "./meta_components.hpp"
#include <entt/core/type_info.hpp>
// TODO: move more central
#define DEFINE_COMP_ID(x) \
template<> \
constexpr entt::id_type entt::type_hash<x>::value() noexcept { \
using namespace entt::literals; \
return #x##_hs; \
} \
template<> \
constexpr std::string_view entt::type_name<x>::value() noexcept { \
return #x; \
}
// cross compiler stable ids
DEFINE_COMP_ID(FragComp::DataEncryptionType)
DEFINE_COMP_ID(FragComp::DataCompressionType)
#undef DEFINE_COMP_ID

10
src/fragment_store/serializer.cpp Normal file
View File

@ -0,0 +1,10 @@
#include "./serializer.hpp"
void SerializerCallbacks::registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info) {
_serl_json[type_info.hash()] = fn;
}
void SerializerCallbacks::registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info) {
_deserl_json[type_info.hash()] = fn;
}

25
src/fragment_store/serializer.hpp Normal file
View File

@ -0,0 +1,25 @@
#pragma once
#include <entt/core/type_info.hpp>
#include <entt/container/dense_map.hpp>
#include <nlohmann/json_fwd.hpp>
struct SerializerCallbacks {
// nlohmann
// json/msgpack
using serialize_json_fn = bool(*)(void* comp, nlohmann::json& out);
entt::dense_map<entt::id_type, serialize_json_fn> _serl_json;
using deserialize_json_fn = bool(*)(void* comp, const nlohmann::json& in);
entt::dense_map<entt::id_type, deserialize_json_fn> _deserl_json;
void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info);
template<typename CompType>
void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info = entt::type_id<CompType>()) { registerSerializerJson(fn, type_info); }
void registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info);
template<typename CompType>
void registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info = entt::type_id<CompType>()) { registerDeSerializerJson(fn, type_info); }
};

17
src/fragment_store/test_fragstore.cpp
View File

@ -43,8 +43,8 @@ int main(void) {
{
auto frag0h = fs.fragmentHandle(frag0);
frag0h.emplace_or_replace<Components::DataCompressionType>();
frag0h.emplace_or_replace<Components::DataEncryptionType>();
frag0h.emplace_or_replace<FragComp::DataCompressionType>();
frag0h.emplace_or_replace<FragComp::DataEncryptionType>();
frag0h.emplace_or_replace<Components::MessagesTimestampRange>();
std::function<FragmentStore::write_to_storage_fetch_data_cb> fn_cb = [read = 0ul](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t {
@ -62,8 +62,8 @@ int main(void) {
{
auto frag1h = fs.fragmentHandle(frag1);
frag1h.emplace_or_replace<Components::DataCompressionType>();
frag1h.emplace_or_replace<Components::DataEncryptionType>();
frag1h.emplace_or_replace<FragComp::DataCompressionType>();
frag1h.emplace_or_replace<FragComp::DataEncryptionType>();
std::function<FragmentStore::write_to_storage_fetch_data_cb> fn_cb = [read = 0ul](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t {
static constexpr std::string_view text = "This is some random data";
@ -78,6 +78,15 @@ int main(void) {
fs.syncToStorage(frag1, fn_cb);
}
{
auto frag2h = fs.fragmentHandle(frag2);
frag2h.emplace_or_replace<FragComp::DataCompressionType>();
frag2h.emplace_or_replace<FragComp::DataEncryptionType>();
static constexpr std::string_view text = "This is more random data";
fs.syncToStorage(frag2, reinterpret_cast<const uint8_t*>(text.data()), text.size());
}
return 0;
}

16
src/fragment_store/types.hpp Normal file
View File

@ -0,0 +1,16 @@
#pragma once
#include <cstdint>
enum class Encryption : uint8_t {
NONE = 0x00,
};
enum class Compression : uint8_t {
NONE = 0x00,
};
enum class MetaFileType : uint8_t {
TEXT_JSON,
//BINARY_ARB,
BINARY_MSGPACK,
};

5
src/main_screen.cpp
View File

@ -1,4 +1,5 @@
#include "./main_screen.hpp"
#include "fragment_store/fragment_store.hpp"
#include <solanaceae/contact/components.hpp>
@ -49,6 +50,10 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri
std::cout << "own address: " << tc.toxSelfGetAddressStr() << "\n";
{ // setup plugin instances
// TODO: make interface useful
g_provideInstance<FragmentStoreI>("FragmentStoreI", "host", &fs);
g_provideInstance<FragmentStore>("FragmentStore", "host", &fs);
g_provideInstance<ConfigModelI>("ConfigModelI", "host", &conf);
g_provideInstance<Contact3Registry>("Contact3Registry", "1", "host", &cr);
g_provideInstance<RegistryMessageModel>("RegistryMessageModel", "host", &rmm);

3
src/main_screen.hpp
View File

@ -2,6 +2,7 @@
#include "./screen.hpp"
#include "./fragment_store/fragment_store.hpp"
#include <solanaceae/util/simple_config_model.hpp>
#include <solanaceae/contact/contact_model3.hpp>
#include <solanaceae/message3/registry_message_model.hpp>
@ -43,6 +44,8 @@ extern "C" {
struct MainScreen final : public Screen {
SDL_Renderer* renderer;
FragmentStore fs;
SimpleConfigModel conf;
Contact3Registry cr;
RegistryMessageModel rmm;