Tha_14/tomato-testing
1
0
Fork 0
forked from Green-Sky/tomato
Code Pull Requests Actions Packages Releases Activity

move os and backend to sub

Browse Source
This commit is contained in:
Green Sky 2024-04-12 13:34:20 +02:00
parent 2597edd579
commit dfbb1dea68
No known key found for this signature in database
22 changed files with 28 additions and 1943 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitmodules vendored
View File

@ -20,3 +20,6 @@
[submodule "external/solanaceae_plugin"] [submodule "external/solanaceae_plugin"]
path = external/solanaceae_plugin path = external/solanaceae_plugin
url = https://github.com/Green-Sky/solanaceae_plugin.git url = https://github.com/Green-Sky/solanaceae_plugin.git
[submodule "external/solanaceae_object_store"]
path = external/solanaceae_object_store
url = https://github.com/Green-Sky/solanaceae_object_store.git

24
external/CMakeLists.txt vendored
View File

@ -12,6 +12,8 @@ add_subdirectory(./toxcore)
add_subdirectory(./solanaceae_toxcore) add_subdirectory(./solanaceae_toxcore)
add_subdirectory(./solanaceae_tox) add_subdirectory(./solanaceae_tox)
add_subdirectory(./solanaceae_object_store)
add_subdirectory(./sdl) add_subdirectory(./sdl)
add_subdirectory(./imgui) add_subdirectory(./imgui)
@ -28,25 +30,3 @@ if (NOT TARGET nlohmann_json::nlohmann_json)
FetchContent_MakeAvailable(json) FetchContent_MakeAvailable(json)
endif() endif()
if (NOT TARGET zstd::zstd)
# TODO: try find_package() first
# TODO: try pkg-config next (will work on most distros)
set(ZSTD_BUILD_STATIC ON)
set(ZSTD_BUILD_SHARED OFF)
set(ZSTD_BUILD_PROGRAMS OFF)
set(ZSTD_BUILD_CONTRIB OFF)
set(ZSTD_BUILD_TESTS OFF)
FetchContent_Declare(zstd
URL "https://github.com/facebook/zstd/releases/download/v1.5.5/zstd-1.5.5.tar.gz"
DOWNLOAD_EXTRACT_TIMESTAMP TRUE
SOURCE_SUBDIR build/cmake
EXCLUDE_FROM_ALL
)
FetchContent_MakeAvailable(zstd)
add_library(zstd INTERFACE) # somehow zstd fkd this up
target_include_directories(zstd INTERFACE ${zstd_SOURCE_DIR}/lib/)
target_link_libraries(zstd INTERFACE libzstd_static)
add_library(zstd::zstd ALIAS zstd)
endif()

1
external/solanaceae_object_store vendored Submodule

@ -0,0 +1 @@
Subproject commit 4d3ffb8192623740f6e170855ee1cffd428b78da

58
src/CMakeLists.txt
View File

@ -1,51 +1,14 @@
cmake_minimum_required(VERSION 3.9 FATAL_ERROR) cmake_minimum_required(VERSION 3.9 FATAL_ERROR)
add_library(fragment_store
./fragment_store/file2_zstd.hpp
./fragment_store/file2_zstd.cpp
./fragment_store/uuid_generator.hpp
./fragment_store/uuid_generator.cpp
./fragment_store/types.hpp
./fragment_store/meta_components.hpp
./fragment_store/meta_components_id.inl
./fragment_store/file2_stack.hpp
./fragment_store/file2_stack.cpp
./fragment_store/serializer_json.hpp
./fragment_store/object_store.hpp
./fragment_store/object_store.cpp
./fragment_store/backends/filesystem_storage.hpp
./fragment_store/backends/filesystem_storage.cpp
./json/message_components.hpp # TODO: move
./json/tox_message_components.hpp # TODO: move
)
target_link_libraries(fragment_store PUBLIC
nlohmann_json::nlohmann_json
EnTT::EnTT
solanaceae_util
solanaceae_file2
zstd::zstd
solanaceae_tox_messages # TODO: move
)
########################################
add_executable(test_file_zstd
fragment_store/test_file_zstd.cpp
)
target_link_libraries(test_file_zstd PUBLIC
fragment_store
)
######################################## ########################################
add_library(message_fragment_store add_library(message_fragment_store
./fragment_store/uuid_generator.hpp
./fragment_store/uuid_generator.cpp
./json/message_components.hpp # TODO: move
./json/tox_message_components.hpp # TODO: move
./fragment_store/message_serializer.hpp ./fragment_store/message_serializer.hpp
./fragment_store/message_serializer.cpp ./fragment_store/message_serializer.cpp
./fragment_store/messages_meta_components.hpp ./fragment_store/messages_meta_components.hpp
@ -60,8 +23,9 @@ add_library(message_fragment_store
) )
target_compile_features(message_fragment_store PRIVATE cxx_std_20) target_compile_features(message_fragment_store PRIVATE cxx_std_20)
target_link_libraries(message_fragment_store PUBLIC target_link_libraries(message_fragment_store PUBLIC
fragment_store solanaceae_object_store
solanaceae_message3 solanaceae_message3
solanaceae_tox_messages # TODO: move
) )
######################################## ########################################
@ -71,7 +35,8 @@ add_executable(convert_message_object_store
) )
target_link_libraries(convert_message_object_store PUBLIC target_link_libraries(convert_message_object_store PUBLIC
fragment_store solanaceae_object_store
solanaceae_object_store_backend_filesystem
message_fragment_store message_fragment_store
) )
@ -158,7 +123,8 @@ target_link_libraries(tomato PUBLIC
solanaceae_tox_contacts solanaceae_tox_contacts
solanaceae_tox_messages solanaceae_tox_messages
fragment_store solanaceae_object_store
solanaceae_object_store_backend_filesystem
message_fragment_store message_fragment_store
SDL3::SDL3 SDL3::SDL3

627
src/fragment_store/backends/filesystem_storage.cpp
View File

@ -1,627 +0,0 @@
#include "./filesystem_storage.hpp"
#include "../meta_components.hpp"
#include "../serializer_json.hpp"
#include <solanaceae/util/utils.hpp>
#include <entt/container/dense_set.hpp>
#include <nlohmann/json.hpp>
#include <solanaceae/file/file2_std.hpp>
#include <solanaceae/file/file2_mem.hpp>
#include "../file2_stack.hpp"
#include <filesystem>
#include <iostream>
static const char* metaFileTypeSuffix(MetaFileType mft) {
switch (mft) {
case MetaFileType::TEXT_JSON: return ".json";
//case MetaFileType::BINARY_ARB: return ".bin";
case MetaFileType::BINARY_MSGPACK: return ".msgpack";
}
return ""; // .unk?
}
// TODO: move to ... somewhere. (span? file2i?)
static ByteSpan spanFromRead(const std::variant<ByteSpan, std::vector<uint8_t>>& data_var) {
if (std::holds_alternative<std::vector<uint8_t>>(data_var)) {
auto& vec = std::get<std::vector<uint8_t>>(data_var);
return {vec.data(), vec.size()};
} else if (std::holds_alternative<ByteSpan>(data_var)) {
return std::get<ByteSpan>(data_var);
} else {
assert(false);
return {};
}
}
namespace backend {
FilesystemStorage::FilesystemStorage(
ObjectStore2& os,
std::string_view storage_path,
MetaFileType mft_new
) : StorageBackendI::StorageBackendI(os), _storage_path(storage_path), _mft_new(mft_new) {
}
FilesystemStorage::~FilesystemStorage(void) {
}
ObjectHandle FilesystemStorage::newObject(ByteSpan id) {
{ // first check if id is already used (TODO: solve the multi obj/backend problem)
auto exising_oh = _os.getOneObjectByID(id);
if (static_cast<bool>(exising_oh)) {
return {};
}
}
std::filesystem::create_directories(_storage_path);
if (!std::filesystem::is_directory(_storage_path)) {
std::cerr << "FS error: failed to create storage path dir '" << _storage_path << "'\n";
return {};
}
const auto id_hex = bin2hex(id);
std::filesystem::path object_file_path;
// TODO: refactor this magic somehow
if (id_hex.size() < 6) {
object_file_path = std::filesystem::path{_storage_path}/id_hex;
} else {
// use the first 2hex (1byte) as a subfolder
std::filesystem::create_directories(std::string{_storage_path} + id_hex.substr(0, 2));
object_file_path = std::filesystem::path{std::string{_storage_path} + id_hex.substr(0, 2)} / id_hex.substr(2);
}
if (std::filesystem::exists(object_file_path)) {
std::cerr << "FS error: object already exists in path '" << id_hex << "'\n";
return {};
}
ObjectHandle oh{_os.registry(), _os.registry().create()};
oh.emplace<ObjComp::Ephemeral::Backend>(this);
oh.emplace<ObjComp::ID>(std::vector<uint8_t>{id});
oh.emplace<ObjComp::Ephemeral::FilePath>(object_file_path.generic_u8string());
oh.emplace<ObjComp::Ephemeral::MetaFileType>(_mft_new);
// meta needs to be synced to file
std::function<write_to_storage_fetch_data_cb> empty_data_cb = [](auto*, auto) -> uint64_t { return 0; };
if (!write(oh, empty_data_cb)) {
std::cerr << "FS error: write failed while creating new object file\n";
oh.destroy();
return {};
}
// while new metadata might be created here, making sure the file could be created is more important
_os.throwEventConstruct(oh);
return oh;
}
bool FilesystemStorage::write(Object o, std::function<write_to_storage_fetch_data_cb>& data_cb) {
auto& reg = _os.registry();
if (!reg.valid(o)) {
return false;
}
ObjectHandle oh {reg, o};
if (!oh.all_of<ObjComp::Ephemeral::FilePath>()) {
// not a file fragment?
return false;
}
// split object storage (meta and data are 2 files)
MetaFileType meta_type = MetaFileType::TEXT_JSON; // TODO: better defaults?
if (reg.all_of<ObjComp::Ephemeral::MetaFileType>(o)) {
meta_type = oh.get<ObjComp::Ephemeral::MetaFileType>().type;
}
Encryption meta_enc = Encryption::NONE; // TODO: better defaults?
Compression meta_comp = Compression::NONE; // TODO: better defaults?
if (meta_type != MetaFileType::TEXT_JSON) {
if (oh.all_of<ObjComp::Ephemeral::MetaEncryptionType>()) {
meta_enc = oh.get<ObjComp::Ephemeral::MetaEncryptionType>().enc;
}
if (oh.all_of<ObjComp::Ephemeral::MetaCompressionType>()) {
meta_comp = oh.get<ObjComp::Ephemeral::MetaCompressionType>().comp;
}
} else {
// we cant have encryption or compression
// so we force NONE for TEXT JSON
oh.emplace_or_replace<ObjComp::Ephemeral::MetaEncryptionType>(Encryption::NONE);
oh.emplace_or_replace<ObjComp::Ephemeral::MetaCompressionType>(Compression::NONE);
}
std::filesystem::path meta_tmp_path = oh.get<ObjComp::Ephemeral::FilePath>().path + ".meta" + metaFileTypeSuffix(meta_type) + ".tmp";
meta_tmp_path.replace_filename("." + meta_tmp_path.filename().generic_u8string());
// TODO: make meta comp work with mem compressor
//auto meta_file_stack = buildFileStackWrite(std::string_view{meta_tmp_path.generic_u8string()}, meta_enc, meta_comp);
std::stack<std::unique_ptr<File2I>> meta_file_stack;
meta_file_stack.push(std::make_unique<File2WFile>(std::string_view{meta_tmp_path.generic_u8string()}));
if (meta_file_stack.empty()) {
std::cerr << "FS error: failed to create temporary meta file stack\n";
std::filesystem::remove(meta_tmp_path); // might have created an empty file
return false;
}
Encryption data_enc = Encryption::NONE; // TODO: better defaults
Compression data_comp = Compression::NONE; // TODO: better defaults
if (oh.all_of<ObjComp::DataEncryptionType>()) {
data_enc = oh.get<ObjComp::DataEncryptionType>().enc;
}
if (oh.all_of<ObjComp::DataCompressionType>()) {
data_comp = oh.get<ObjComp::DataCompressionType>().comp;
}
std::filesystem::path data_tmp_path = oh.get<ObjComp::Ephemeral::FilePath>().path + ".tmp";
data_tmp_path.replace_filename("." + data_tmp_path.filename().generic_u8string());
auto data_file_stack = buildFileStackWrite(std::string_view{data_tmp_path.generic_u8string()}, data_enc, data_comp);
if (data_file_stack.empty()) {
while (!meta_file_stack.empty()) { meta_file_stack.pop(); }
std::filesystem::remove(meta_tmp_path);
std::cerr << "FS error: failed to create temporary data file stack\n";
return false;
}
try { // TODO: properly sanitize strings, so this cant throw
// sharing code between binary msgpack and text json for now
nlohmann::json meta_data_j = nlohmann::json::object(); // metadata needs to be an object, null not allowed
// metadata file
auto& sjc = _os.registry().ctx().get<SerializerJsonCallbacks<Object>>();
// TODO: refactor extract to OS
for (const auto& [type_id, storage] : reg.storage()) {
if (!storage.contains(o)) {
continue;
}
//std::cout << "storage type: type_id:" << type_id << " name:" << storage.type().name() << "\n";
// use type_id to find serializer
auto s_cb_it = sjc._serl.find(type_id);
if (s_cb_it == sjc._serl.end()) {
// could not find serializer, not saving
continue;
}
// noooo, why cant numbers be keys
//if (meta_type == MetaFileType::BINARY_MSGPACK) { // msgpack uses the hash id instead
//s_cb_it->second(storage.value(fid), meta_data[storage.type().hash()]);
//} else if (meta_type == MetaFileType::TEXT_JSON) {
s_cb_it->second(oh, meta_data_j[storage.type().name()]);
//}
}
if (meta_type == MetaFileType::BINARY_MSGPACK) { // binary metadata file
std::vector<uint8_t> binary_meta_data;
{
std::stack<std::unique_ptr<File2I>> binary_writer_stack;
binary_writer_stack.push(std::make_unique<File2MemW>(binary_meta_data));
if (!buildStackWrite(binary_writer_stack, meta_enc, meta_comp)) {
while (!meta_file_stack.empty()) { meta_file_stack.pop(); }
std::filesystem::remove(meta_tmp_path);
while (!data_file_stack.empty()) { data_file_stack.pop(); }
std::filesystem::remove(data_tmp_path);
std::cerr << "FS error: binary writer creation failed '" << oh.get<ObjComp::Ephemeral::FilePath>().path << "'\n";
return false;
}
{
const std::vector<uint8_t> meta_data = nlohmann::json::to_msgpack(meta_data_j);
if (!binary_writer_stack.top()->write(ByteSpan{meta_data})) {
// i feel like exceptions or refactoring would be nice here
while (!meta_file_stack.empty()) { meta_file_stack.pop(); }
std::filesystem::remove(meta_tmp_path);
while (!data_file_stack.empty()) { data_file_stack.pop(); }
std::filesystem::remove(data_tmp_path);
std::cerr << "FS error: binary writer failed '" << oh.get<ObjComp::Ephemeral::FilePath>().path << "'\n";
return false;
}
}
}
//// the meta file is itself msgpack data
nlohmann::json meta_header_j = nlohmann::json::array();
meta_header_j.emplace_back() = "SOLMET";
meta_header_j.push_back(meta_enc);
meta_header_j.push_back(meta_comp);
// with a custom msgpack impl like cmp, we can be smarter here and dont need an extra buffer
meta_header_j.push_back(nlohmann::json::binary(binary_meta_data));
const auto meta_header_data = nlohmann::json::to_msgpack(meta_header_j);
meta_file_stack.top()->write({meta_header_data.data(), meta_header_data.size()});
} else if (meta_type == MetaFileType::TEXT_JSON) {
// cant be compressed or encrypted
const auto meta_file_json_str = meta_data_j.dump(2, ' ', true);
meta_file_stack.top()->write({reinterpret_cast<const uint8_t*>(meta_file_json_str.data()), meta_file_json_str.size()});
}
} catch (...) {
while (!meta_file_stack.empty()) { meta_file_stack.pop(); } // destroy stack // TODO: maybe work with scope?
std::filesystem::remove(meta_tmp_path);
while (!data_file_stack.empty()) { data_file_stack.pop(); } // destroy stack // TODO: maybe work with scope?
std::filesystem::remove(data_tmp_path);
std::cerr << "FS error: failed to serialize json data\n";
return false;
}
// now data
// for zstd compression, chunk size is frame size. (no cross frame referencing)
// TODO: add buffering steam layer
static constexpr int64_t chunk_size{1024*1024}; // 1MiB should be enough
std::vector<uint8_t> buffer(chunk_size);
uint64_t buffer_actual_size {0};
do {
buffer_actual_size = data_cb(buffer.data(), buffer.size());
if (buffer_actual_size == 0) {
break;
}
if (buffer_actual_size > buffer.size()) {
// wtf
break;
}
data_file_stack.top()->write({buffer.data(), buffer_actual_size});
} while (buffer_actual_size == buffer.size());
// flush // TODO: use scope
while (!meta_file_stack.empty()) { meta_file_stack.pop(); } // destroy stack // TODO: maybe work with scope?
while (!data_file_stack.empty()) { data_file_stack.pop(); } // destroy stack // TODO: maybe work with scope?
std::filesystem::rename(
meta_tmp_path,
oh.get<ObjComp::Ephemeral::FilePath>().path + ".meta" + metaFileTypeSuffix(meta_type)
);
std::filesystem::rename(
data_tmp_path,
oh.get<ObjComp::Ephemeral::FilePath>().path
);
// TODO: check return value of renames
if (oh.all_of<ObjComp::Ephemeral::DirtyTag>()) {
oh.remove<ObjComp::Ephemeral::DirtyTag>();
}
return true;
}
bool FilesystemStorage::read(Object o, std::function<read_from_storage_put_data_cb>& data_cb) {
auto& reg = _os.registry();
if (!reg.valid(o)) {
return false;
}
ObjectHandle oh {reg, o};
if (!oh.all_of<ObjComp::Ephemeral::FilePath>()) {
// not a file
return false;
}
const auto& obj_path = oh.get<ObjComp::Ephemeral::FilePath>().path;
// TODO: check if metadata dirty?
// TODO: what if file changed on disk?
std::cout << "FS: loading fragment '" << obj_path << "'\n";
Compression data_comp = Compression::NONE;
if (oh.all_of<ObjComp::DataCompressionType>()) {
data_comp = oh.get<ObjComp::DataCompressionType>().comp;
}
auto data_file_stack = buildFileStackRead(std::string_view{obj_path}, Encryption::NONE, data_comp);
if (data_file_stack.empty()) {
return false;
}
// TODO: make it read in a single chunk instead?
static constexpr int64_t chunk_size {1024 * 1024}; // 1MiB should be good for read
do {
auto data_var = data_file_stack.top()->read(chunk_size);
ByteSpan data = spanFromRead(data_var);
if (data.empty()) {
// error or probably eof
break;
}
data_cb(data);
if (data.size < chunk_size) {
// eof
break;
}
} while (data_file_stack.top()->isGood());
return true;
}
void FilesystemStorage::scanAsync(void) {
scanPathAsync(_storage_path);
}
size_t FilesystemStorage::scanPath(std::string_view path) {
// TODO: extract so async can work (or/and make iteratable generator)
if (path.empty() || !std::filesystem::is_directory(path)) {
std::cerr << "FS error: scan path not a directory '" << path << "'\n";
return 0;
}
// step 1: make snapshot of files, validate metafiles and save id/path+meta.ext
// can be extra thread (if non vfs)
struct ObjFileEntry {
std::string id_str;
std::filesystem::path obj_path;
std::string meta_ext;
bool operator==(const ObjFileEntry& other) const {
// only compare by id
return id_str == other.id_str;
}
};
struct ObjFileEntryHash {
size_t operator()(const ObjFileEntry& it) const {
return entt::hashed_string(it.id_str.data(), it.id_str.size());
}
};
entt::dense_set<ObjFileEntry, ObjFileEntryHash> file_obj_list;
std::filesystem::path storage_path{path};
auto handle_file = [&](const std::filesystem::path& file_path) {
if (!std::filesystem::is_regular_file(file_path)) {
return;
}
// handle file
if (file_path.has_extension()) {
// skip over metadata, assuming only metafiles have extentions (might be wrong?)
// also skips temps
return;
}
auto relative_path = std::filesystem::proximate(file_path, storage_path);
std::string id_str = relative_path.generic_u8string();
// delete all '/'
id_str.erase(std::remove(id_str.begin(), id_str.end(), '/'), id_str.end());
if (id_str.size() % 2 != 0) {
std::cerr << "FS error: non hex object uid detected: '" << id_str << "'\n";
}
if (file_obj_list.contains(ObjFileEntry{id_str, {}, ""})) {
std::cerr << "FS error: object duplicate detected: '" << id_str << "'\n";
return; // skip
}
const char* meta_ext = ".meta.msgpack";
{ // find meta
// TODO: this as to know all possible extentions
bool has_meta_msgpack = std::filesystem::is_regular_file(file_path.generic_u8string() + ".meta.msgpack");
bool has_meta_json = std::filesystem::is_regular_file(file_path.generic_u8string() + ".meta.json");
const size_t meta_sum =
(has_meta_msgpack?1:0) +
(has_meta_json?1:0)
;
if (meta_sum > 1) { // has multiple
std::cerr << "FS error: object with multiple meta files detected: " << id_str << "\n";
return; // skip
}
if (meta_sum == 0) {
std::cerr << "FS error: object missing meta file detected: " << id_str << "\n";
return; // skip
}
if (has_meta_json) {
meta_ext = ".meta.json";
}
}
file_obj_list.emplace(ObjFileEntry{
std::move(id_str),
file_path,
meta_ext
});
};
for (const auto& outer_path : std::filesystem::directory_iterator(storage_path)) {
if (std::filesystem::is_regular_file(outer_path)) {
handle_file(outer_path);
} else if (std::filesystem::is_directory(outer_path)) {
// subdir, part of id
for (const auto& inner_path : std::filesystem::directory_iterator(outer_path)) {
//if (std::filesystem::is_regular_file(inner_path)) {
//// handle file
//} // TODO: support deeper recursion?
handle_file(inner_path);
}
}
}
std::cout << "FS: scan found:\n";
for (const auto& it : file_obj_list) {
std::cout << " " << it.id_str << "\n";
}
// step 2: check if files preexist in reg
// main thread
// (merge into step 3 and do more error checking?)
// TODO: properly handle duplicates, dups form different backends might be legit
// important
for (auto it = file_obj_list.begin(); it != file_obj_list.end();) {
auto id = hex2bin(it->id_str);
auto oh = _os.getOneObjectByID(ByteSpan{id});
if (static_cast<bool>(oh)) {
// pre exising (handle differently??)
// check if store differs?
it = file_obj_list.erase(it);
} else {
it++;
}
}
auto& sjc = _os.registry().ctx().get<SerializerJsonCallbacks<Object>>();
std::vector<Object> scanned_objs;
// step 3: parse meta and insert into reg of non preexising
// main thread
// TODO: check timestamps of preexisting and reload? mark external/remote dirty?
for (const auto& it : file_obj_list) {
MetaFileType mft {MetaFileType::TEXT_JSON};
Encryption meta_enc {Encryption::NONE};
Compression meta_comp {Compression::NONE};
nlohmann::json j;
if (it.meta_ext == ".meta.msgpack") {
std::ifstream file(it.obj_path.generic_u8string() + it.meta_ext, std::ios::in | std::ios::binary);
if (!file.is_open()) {
std::cout << "FS error: failed opening meta " << it.obj_path << "\n";
continue;
}
mft = MetaFileType::BINARY_MSGPACK;
// file is a msgpack within a msgpack
std::vector<uint8_t> full_meta_data;
{ // read meta file
// figure out size
file.seekg(0, file.end);
uint64_t file_size = file.tellg();
file.seekg(0, file.beg);
full_meta_data.resize(file_size);
file.read(reinterpret_cast<char*>(full_meta_data.data()), full_meta_data.size());
}
const auto meta_header_j = nlohmann::json::from_msgpack(full_meta_data, true, false);
if (!meta_header_j.is_array() || meta_header_j.size() < 4) {
std::cerr << "FS error: broken binary meta " << it.obj_path << "\n";
continue;
}
if (meta_header_j.at(0) != "SOLMET") {
std::cerr << "FS error: wrong magic '" << meta_header_j.at(0) << "' in meta " << it.obj_path << "\n";
continue;
}
meta_enc = meta_header_j.at(1);
if (meta_enc != Encryption::NONE) {
std::cerr << "FS error: unknown encryption " << it.obj_path << "\n";
continue;
}
meta_comp = meta_header_j.at(2);
if (meta_comp != Compression::NONE && meta_comp != Compression::ZSTD) {
std::cerr << "FS error: unknown compression " << it.obj_path << "\n";
continue;
}
//const auto& meta_data_ref = meta_header_j.at(3).is_binary()?meta_header_j.at(3):meta_header_j.at(3).at("data");
if (!meta_header_j.at(3).is_binary()) {
std::cerr << "FS error: meta data not binary " << it.obj_path << "\n";
continue;
}
const nlohmann::json::binary_t& meta_data_ref = meta_header_j.at(3);
std::stack<std::unique_ptr<File2I>> binary_reader_stack;
binary_reader_stack.push(std::make_unique<File2MemR>(ByteSpan{meta_data_ref.data(), meta_data_ref.size()}));
if (!buildStackRead(binary_reader_stack, meta_enc, meta_comp)) {
std::cerr << "FS error: binary reader creation failed " << it.obj_path << "\n";
continue;
}
// HACK: read fixed amout of data, but this way if we have neither enc nor comp we pass the span through
auto binary_read_value = binary_reader_stack.top()->read(10*1024*1024); // is 10MiB large enough for meta?
const auto binary_read_span = spanFromRead(binary_read_value);
assert(binary_read_span.size < 10*1024*1024);
j = nlohmann::json::from_msgpack(binary_read_span, true, false);
} else if (it.meta_ext == ".meta.json") {
std::ifstream file(it.obj_path.generic_u8string() + it.meta_ext, std::ios::in | std::ios::binary);
if (!file.is_open()) {
std::cerr << "FS error: failed opening meta " << it.obj_path << "\n";
continue;
}
mft = MetaFileType::TEXT_JSON;
file >> j;
} else {
assert(false);
}
if (!j.is_object()) {
std::cerr << "FS error: json in meta is broken " << it.id_str << "\n";
continue;
}
// TODO: existing fragment file
//newFragmentFile();
ObjectHandle oh{_os.registry(), _os.registry().create()};
oh.emplace<ObjComp::Ephemeral::Backend>(this);
oh.emplace<ObjComp::ID>(hex2bin(it.id_str));
oh.emplace<ObjComp::Ephemeral::MetaFileType>(mft);
oh.emplace<ObjComp::Ephemeral::MetaEncryptionType>(meta_enc);
oh.emplace<ObjComp::Ephemeral::MetaCompressionType>(meta_comp);
oh.emplace<ObjComp::Ephemeral::FilePath>(it.obj_path.generic_u8string());
for (const auto& [k, v] : j.items()) {
// type id from string hash
const auto type_id = entt::hashed_string(k.data(), k.size());
const auto deserl_fn_it = sjc._deserl.find(type_id);
if (deserl_fn_it != sjc._deserl.cend()) {
// TODO: check return value
deserl_fn_it->second(oh, v);
} else {
std::cerr << "FS warning: missing deserializer for meta key '" << k << "'\n";
}
}
scanned_objs.push_back(oh);
}
// TODO: mutex and move code to async and return this list ?
// throw new frag event here, after loading them all
for (const Object o : scanned_objs) {
_os.throwEventConstruct(o);
}
return scanned_objs.size();
}
void FilesystemStorage::scanPathAsync(std::string path) {
// add path to queue
// HACK: if path is known/any fragment is in the path, this operation blocks (non async)
scanPath(path); // TODO: make async and post result
}
} // backend

40
src/fragment_store/backends/filesystem_storage.hpp
View File

@ -1,40 +0,0 @@
#pragma once
#include "../types.hpp"
#include "../object_store.hpp"
#include <string>
namespace backend {
struct FilesystemStorage : public StorageBackendI {
FilesystemStorage(
ObjectStore2& os,
std::string_view storage_path = "test_obj_store",
MetaFileType mft_new = MetaFileType::BINARY_MSGPACK
);
~FilesystemStorage(void);
// TODO: fix the path for this specific fs?
// for now we assume a single storage path per backend (there can be multiple per type)
std::string _storage_path;
// meta file type for new objects
MetaFileType _mft_new {MetaFileType::BINARY_MSGPACK};
ObjectHandle newObject(ByteSpan id) override;
bool write(Object o, std::function<write_to_storage_fetch_data_cb>& data_cb) override;
bool read(Object o, std::function<read_from_storage_put_data_cb>& data_cb) override;
//// convenience function
//nlohmann::json loadFromStorageNJ(FragmentID fid);
void scanAsync(void);
private:
size_t scanPath(std::string_view path);
void scanPathAsync(std::string path);
};
} // backend

8
src/fragment_store/convert_frag_to_obj.cpp
View File

@ -1,7 +1,7 @@
#include "./object_store.hpp" #include <solanaceae/object_store/object_store.hpp>
#include "./backends/filesystem_storage.hpp" #include <solanaceae/object_store/backends/filesystem_storage.hpp>
#include "./meta_components.hpp" #include <solanaceae/object_store/meta_components.hpp>
#include "./serializer_json.hpp" #include <solanaceae/object_store/serializer_json.hpp>
#include "./message_fragment_store.hpp" #include "./message_fragment_store.hpp"
#include <solanaceae/util/utils.hpp> #include <solanaceae/util/utils.hpp>

93
src/fragment_store/file2_stack.cpp
View File

@ -1,93 +0,0 @@
#include "./file2_stack.hpp"
#include <solanaceae/file/file2_std.hpp>
#include "./file2_zstd.hpp"
#include <cassert>
#include <iostream>
// add enc and comp file layers
// assumes a file is already in the stack
bool buildStackRead(std::stack<std::unique_ptr<File2I>>& file_stack, Encryption encryption, Compression compression) {
assert(!file_stack.empty());
// TODO: decrypt here
assert(encryption == Encryption::NONE);
// add layer based on enum
if (compression == Compression::ZSTD) {
file_stack.push(std::make_unique<File2ZSTDR>(*file_stack.top().get()));
} else {
// TODO: make error instead
assert(compression == Compression::NONE);
}
if (!file_stack.top()->isGood()) {
std::cerr << "FS error: file failed to add " << (int)compression << " decompression layer\n";
return false;
}
return true;
}
// do i need this?
std::stack<std::unique_ptr<File2I>> buildFileStackRead(std::string_view file_path, Encryption encryption, Compression compression) {
std::stack<std::unique_ptr<File2I>> file_stack;
file_stack.push(std::make_unique<File2RFile>(file_path));
if (!file_stack.top()->isGood()) {
std::cerr << "FS error: opening file for reading '" << file_path << "'\n";
return {};
}
if (!buildStackRead(file_stack, encryption, compression)) {
std::cerr << "FS error: file failed to add layers for '" << file_path << "'\n";
return {};
}
return file_stack;
}
// add enc and comp file layers
// assumes a file is already in the stack
bool buildStackWrite(std::stack<std::unique_ptr<File2I>>& file_stack, Encryption encryption, Compression compression) {
assert(!file_stack.empty());
// TODO: encrypt here
assert(encryption == Encryption::NONE);
// add layer based on enum
if (compression == Compression::ZSTD) {
file_stack.push(std::make_unique<File2ZSTDW>(*file_stack.top().get()));
} else {
// TODO: make error instead
assert(compression == Compression::NONE);
}
if (!file_stack.top()->isGood()) {
std::cerr << "FS error: file failed to add " << (int)compression << " compression layer\n";
return false;
}
return true;
}
// do i need this?
std::stack<std::unique_ptr<File2I>> buildFileStackWrite(std::string_view file_path, Encryption encryption, Compression compression) {
std::stack<std::unique_ptr<File2I>> file_stack;
file_stack.push(std::make_unique<File2WFile>(file_path));
if (!file_stack.top()->isGood()) {
std::cerr << "FS error: opening file for writing '" << file_path << "'\n";
return {};
}
if (!buildStackWrite(file_stack, encryption, compression)) {
std::cerr << "FS error: file failed to add layers for '" << file_path << "'\n";
return {};
}
return file_stack;
}

23
src/fragment_store/file2_stack.hpp
View File

@ -1,23 +0,0 @@
#pragma once
#include "./types.hpp"
#include <solanaceae/file/file2.hpp>
#include <stack>
#include <memory>
#include <string_view>
// add enc and comp file layers
// assumes a file is already in the stack
[[nodiscard]] bool buildStackRead(std::stack<std::unique_ptr<File2I>>& file_stack, Encryption encryption, Compression compression);
// do i need this?
[[nodiscard]] std::stack<std::unique_ptr<File2I>> buildFileStackRead(std::string_view file_path, Encryption encryption, Compression compression);
// add enc and comp file layers
// assumes a file is already in the stack
[[nodiscard]] bool buildStackWrite(std::stack<std::unique_ptr<File2I>>& file_stack, Encryption encryption, Compression compression);
// do i need this?
[[nodiscard]] std::stack<std::unique_ptr<File2I>> buildFileStackWrite(std::string_view file_path, Encryption encryption, Compression compression);

209
src/fragment_store/file2_zstd.cpp
View File

@ -1,209 +0,0 @@
#include "./file2_zstd.hpp"
#include <cstdint>
#include <iostream>
#include <variant>
#include <vector>
#include <cassert>
File2ZSTDW::File2ZSTDW(File2I& real) :
File2I(true, false),
_real_file(real)
{
ZSTD_CCtx_setParameter(_cctx.get(), ZSTD_c_compressionLevel, 0); // default (3)
ZSTD_CCtx_setParameter(_cctx.get(), ZSTD_c_checksumFlag, 1); // add extra checksums (to frames?)
}
File2ZSTDW::~File2ZSTDW(void) {
// flush remaining data (and maybe header)
// actually nvm, write will always flush all data, so only on empty files this would be an issue
}
bool File2ZSTDW::isGood(void) {
return _real_file.isGood();
}
bool File2ZSTDW::write(const ByteSpan data, int64_t pos) {
if (pos != -1) {
return false;
}
if (data.empty()) {
return false; // return true?
}
if (data.size < 16) {
std::cout << "F2ZSTD warning: each write is a zstd frame and compression suffers significantly for small frames.\n";
}
std::vector<uint8_t> compressed_buffer(ZSTD_CStreamOutSize());
ZSTD_inBuffer input = { data.ptr, data.size, 0 };
size_t remaining_ret {0};
do {
// remaining data in input < compressed_buffer size (heuristic)
bool const lastChunk = (input.size - input.pos) <= compressed_buffer.size();
ZSTD_EndDirective const mode = lastChunk ? ZSTD_e_end : ZSTD_e_continue;
ZSTD_outBuffer output = { compressed_buffer.data(), compressed_buffer.size(), 0 };
remaining_ret = ZSTD_compressStream2(_cctx.get(), &output , &input, mode);
if (ZSTD_isError(remaining_ret)) {
std::cerr << "F2WRZSTD error: compressing data failed\n";
break;
}
_real_file.write(ByteSpan{compressed_buffer.data(), output.pos});
} while ((input.pos < input.size || remaining_ret != 0) && _real_file.isGood());
return _real_file.isGood();
}
std::variant<ByteSpan, std::vector<uint8_t>> File2ZSTDW::read(uint64_t, int64_t) {
return {};
}
// ######################################### decompression
File2ZSTDR::File2ZSTDR(File2I& real) :
File2I(false, true),
_real_file(real),
// 64kib
_in_buffer(ZSTD_DStreamInSize()),
_out_buffer(ZSTD_DStreamOutSize())
{
}
bool File2ZSTDR::isGood(void) {
return _real_file.isGood();
}
bool File2ZSTDR::write(const ByteSpan, int64_t) {
return false;
}
std::variant<ByteSpan, std::vector<uint8_t>> File2ZSTDR::read(uint64_t size, int64_t pos) {
if (pos != -1) {
// error, only support streaming (for now)
return {};
}
std::vector<uint8_t> ret_data;
// actually first we check previous data
if (!_decompressed_buffer.empty()) {
uint64_t required_size = std::min<uint64_t>(size, _decompressed_buffer.size());
ret_data.insert(ret_data.end(), _decompressed_buffer.cbegin(), _decompressed_buffer.cbegin() + required_size);
_decompressed_buffer.erase(_decompressed_buffer.cbegin(), _decompressed_buffer.cbegin() + required_size);
}
bool eof {false};
// outerloop here
while (ret_data.size() < size && !eof) {
// first make sure we have data in input
if (_z_input.src == nullptr || _z_input.pos == _z_input.size) {
const auto request_size = _in_buffer.size();
if (!feedInput(_real_file.read(request_size, -1))) {
return ret_data;
}
// if _z_input.size < _in_buffer.size() -> assume eof?
if (_z_input.size < request_size) {
eof = true;
//std::cout << "---- eof\n";
}
}
do {
ZSTD_outBuffer output = { _out_buffer.data(), _out_buffer.size(), 0 };
size_t const ret = ZSTD_decompressStream(_dctx.get(), &output , &_z_input);
if (ZSTD_isError(ret)) {
// error <.<
std::cerr << "---- error: decompression error\n";
return ret_data;
}
// no new decomp data?
if (output.pos == 0) {
if (ret != 0) {
// if not error and not 0, indicates that
// there is additional flushing needed
continue;
}
assert(eof || ret == 0);
break;
}
int64_t returning_size = std::min<int64_t>(int64_t(size) - int64_t(ret_data.size()), output.pos);
assert(returning_size >= 0);
if (returning_size > 0) {
ret_data.insert(
ret_data.end(),
reinterpret_cast<const uint8_t*>(output.dst),
reinterpret_cast<const uint8_t*>(output.dst) + returning_size
);
}
// make sure we keep excess decompressed data
if (returning_size < int64_t(output.pos)) {
//const auto remaining_size = output.pos - returning_size;
_decompressed_buffer.insert(
_decompressed_buffer.cend(),
reinterpret_cast<const uint8_t*>(output.dst) + returning_size,
reinterpret_cast<const uint8_t*>(output.dst) + output.pos
);
}
} while (_z_input.pos < _z_input.size);
}
return ret_data;
}
bool File2ZSTDR::feedInput(std::variant<ByteSpan, std::vector<uint8_t>>&& read_buff) {
// TODO: optimize, we copy the buffer, but we might not need to
if (std::holds_alternative<ByteSpan>(read_buff)) {
const auto& span = std::get<ByteSpan>(read_buff);
if (span.size > _in_buffer.size()) {
// error, how did we read more than we asked for??
return {};
}
if (span.empty()) {
_z_input = { _in_buffer.data(), 0, 0 };
} else {
// cpy
_in_buffer = static_cast<std::vector<uint8_t>>(span);
_z_input = {
_in_buffer.data(),
span.size,
0
};
}
} else if (std::holds_alternative<std::vector<uint8_t>>(read_buff)) {
auto& vec = std::get<std::vector<uint8_t>>(read_buff);
if (vec.size() > _in_buffer.size()) {
// error, how did we read more than we asked for??
return {};
}
// cpy
_in_buffer = vec;
_z_input = {
_in_buffer.data(),
_in_buffer.size(),
0
};
} else {
// error, unsupported return value of read??
return false;
}
return true;
}

51
src/fragment_store/file2_zstd.hpp
View File

@ -1,51 +0,0 @@
#pragma once
#include <solanaceae/file/file2.hpp>
#include <memory>
#include <zstd.h>
// zstd compression wrapper over another file
// WARNING: only supports sequential writes
struct File2ZSTDW : public File2I {
File2I& _real_file;
// TODO: hide this detail?
std::unique_ptr<ZSTD_CCtx, decltype(&ZSTD_freeCCtx)> _cctx{ZSTD_createCCtx(), &ZSTD_freeCCtx};
File2ZSTDW(File2I& real);
virtual ~File2ZSTDW(void);
bool isGood(void) override;
// for simplicity and potential future seekability each write is its own frame
bool write(const ByteSpan data, int64_t pos = -1) override;
std::variant<ByteSpan, std::vector<uint8_t>> read(uint64_t size, int64_t pos = -1) override;
};
// zstd decompression wrapper over another file
// WARNING: only supports sequential reads
// TODO: add seeking support (use frames)
struct File2ZSTDR : public File2I {
File2I& _real_file;
// TODO: hide this detail?
std::unique_ptr<ZSTD_DCtx, decltype(&ZSTD_freeDCtx)> _dctx{ZSTD_createDCtx(), &ZSTD_freeDCtx};
std::vector<uint8_t> _in_buffer;
std::vector<uint8_t> _out_buffer;
std::vector<uint8_t> _decompressed_buffer; // retains decompressed unread data between read() calls
ZSTD_inBuffer _z_input{nullptr, 0, 0};
File2ZSTDR(File2I& real);
virtual ~File2ZSTDR(void) {}
bool isGood(void) override;
bool write(const ByteSpan data, int64_t pos = -1) override;
std::variant<ByteSpan, std::vector<uint8_t>> read(uint64_t size, int64_t pos = -1) override;
private:
bool feedInput(std::variant<ByteSpan, std::vector<uint8_t>>&& read_buff);
};

2
src/fragment_store/message_fragment_store.cpp
View File

@ -1,6 +1,6 @@
#include "./message_fragment_store.hpp" #include "./message_fragment_store.hpp"
#include "./serializer_json.hpp" #include <solanaceae/object_store/serializer_json.hpp>
#include "../json/message_components.hpp" #include "../json/message_components.hpp"

4
src/fragment_store/message_fragment_store.hpp
View File

@ -1,7 +1,7 @@
#pragma once #pragma once
#include "./meta_components.hpp" #include <solanaceae/object_store/object_store.hpp>
#include "./object_store.hpp" #include <solanaceae/object_store/meta_components.hpp>
#include "./uuid_generator.hpp" #include "./uuid_generator.hpp"

2
src/fragment_store/messages_meta_components.hpp
View File

@ -1,6 +1,6 @@
#pragma once #pragma once
#include "./meta_components.hpp" #include <solanaceae/object_store/meta_components.hpp>
namespace ObjectStore::Components { namespace ObjectStore::Components {
struct MessagesVersion { struct MessagesVersion {

77
src/fragment_store/meta_components.hpp
View File

@ -1,77 +0,0 @@
#pragma once
#include "./types.hpp"
#include "./object_store.hpp"
#include <vector>
#include <string>
#include <cstdint>
namespace ObjectStore::Components {
// TODO: is this special and should this be saved to meta or not (its already in the file name on disk)
struct ID {
std::vector<uint8_t> v;
};
struct DataEncryptionType {
Encryption enc {Encryption::NONE};
};
struct DataCompressionType {
Compression comp {Compression::NONE};
};
// meta that is not written to (meta-)file
namespace Ephemeral {
// TODO: move, backend specific
struct MetaFileType {
::MetaFileType type {::MetaFileType::TEXT_JSON};
};
struct MetaEncryptionType {
Encryption enc {Encryption::NONE};
};
struct MetaCompressionType {
Compression comp {Compression::NONE};
};
struct Backend {
// TODO: shared_ptr instead??
StorageBackendI* ptr;
};
// excluded from file meta
// TODO: move to backend specific
struct FilePath {
// contains store path, if any
std::string path;
};
// TODO: seperate into remote and local?
// (remote meaning eg. the file on disk was changed by another program)
struct DirtyTag {};
} // Ephemeral
} // Components
// shortened to save bytes (until I find a way to save by ID in msgpack)
namespace ObjComp = ObjectStore::Components;
// old names from frag era
namespace Fragment::Components {
//struct ID {};
//struct DataEncryptionType {};
//struct DataCompressionType {};
struct ID : public ObjComp::ID {};
struct DataEncryptionType : public ObjComp::DataEncryptionType {};
struct DataCompressionType : public ObjComp::DataCompressionType {};
}
namespace FragComp = Fragment::Components;
#include "./meta_components_id.inl"

30
src/fragment_store/meta_components_id.inl
View File

@ -1,30 +0,0 @@
#pragma once
#include "./meta_components.hpp"
#include <entt/core/type_info.hpp>
// TODO: move more central
#define DEFINE_COMP_ID(x) \
template<> \
constexpr entt::id_type entt::type_hash<x>::value() noexcept { \
using namespace entt::literals; \
return #x##_hs; \
} \
template<> \
constexpr std::string_view entt::type_name<x>::value() noexcept { \
return #x; \
}
// cross compiler stable ids
DEFINE_COMP_ID(ObjComp::DataEncryptionType)
DEFINE_COMP_ID(ObjComp::DataCompressionType)
// old stuff
DEFINE_COMP_ID(FragComp::DataEncryptionType)
DEFINE_COMP_ID(FragComp::DataCompressionType)
#undef DEFINE_COMP_ID

140
src/fragment_store/object_store.cpp
View File

@ -1,140 +0,0 @@
#include "./object_store.hpp"
#include "./meta_components.hpp"
#include "./serializer_json.hpp"
#include <nlohmann/json.hpp> // this sucks
#include <iostream>
// TODO: move somewhere else
static bool serl_json_data_enc_type(const ObjectHandle oh, nlohmann::json& out) {
if (!oh.all_of<ObjComp::DataEncryptionType>()) {
return false;
}
out = static_cast<std::underlying_type_t<Encryption>>(
oh.get<ObjComp::DataEncryptionType>().enc
);
return true;
}
static bool deserl_json_data_enc_type(ObjectHandle oh, const nlohmann::json& in) {
oh.emplace_or_replace<ObjComp::DataEncryptionType>(
static_cast<Encryption>(
static_cast<std::underlying_type_t<Encryption>>(in)
)
);
return true;
}
static bool serl_json_data_comp_type(const ObjectHandle oh, nlohmann::json& out) {
if (!oh.all_of<ObjComp::DataCompressionType>()) {
return false;
}
out = static_cast<std::underlying_type_t<Compression>>(
oh.get<ObjComp::DataCompressionType>().comp
);
return true;
}
static bool deserl_json_data_comp_type(ObjectHandle oh, const nlohmann::json& in) {
oh.emplace_or_replace<ObjComp::DataCompressionType>(
static_cast<Compression>(
static_cast<std::underlying_type_t<Compression>>(in)
)
);
return true;
}
StorageBackendI::StorageBackendI(ObjectStore2& os) : _os(os) {
}
ObjectHandle StorageBackendI::newObject(ByteSpan) {
//return {_os.registry(), entt::null};
return {};
}
bool StorageBackendI::write(Object o, const ByteSpan data) {
std::function<write_to_storage_fetch_data_cb> fn_cb = [read = 0ull, data](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t {
uint64_t i = 0;
for (; i+read < data.size && i < buffer_size; i++) {
request_buffer[i] = data[i+read];
}
read += i;
return i;
};
return write(o, fn_cb);
}
ObjectStore2::ObjectStore2(void) {
// HACK: set them up independently
auto& sjc = _reg.ctx().emplace<SerializerJsonCallbacks<Object>>();
sjc.registerSerializer<ObjComp::DataEncryptionType>(serl_json_data_enc_type);
sjc.registerDeSerializer<ObjComp::DataEncryptionType>(deserl_json_data_enc_type);
sjc.registerSerializer<ObjComp::DataCompressionType>(serl_json_data_comp_type);
sjc.registerDeSerializer<ObjComp::DataCompressionType>(deserl_json_data_comp_type);
// old stuff
sjc.registerSerializer<FragComp::DataEncryptionType>(serl_json_data_enc_type);
sjc.registerDeSerializer<FragComp::DataEncryptionType>(deserl_json_data_enc_type);
sjc.registerSerializer<FragComp::DataCompressionType>(serl_json_data_comp_type);
sjc.registerDeSerializer<FragComp::DataCompressionType>(deserl_json_data_comp_type);
}
ObjectStore2::~ObjectStore2(void) {
}
ObjectRegistry& ObjectStore2::registry(void) {
return _reg;
}
ObjectHandle ObjectStore2::objectHandle(const Object o) {
return {_reg, o};
}
ObjectHandle ObjectStore2::getOneObjectByID(const ByteSpan id) {
// TODO: accelerate
// maybe keep it sorted and binary search? hash table lookup?
for (const auto& [obj, id_comp] : _reg.view<ObjComp::ID>().each()) {
if (id == ByteSpan{id_comp.v}) {
return {_reg, obj};
}
}
return {_reg, entt::null};
}
void ObjectStore2::throwEventConstruct(const Object o) {
std::cout << "OS debug: event construct " << entt::to_integral(o) << "\n";
dispatch(
ObjectStore_Event::object_construct,
ObjectStore::Events::ObjectConstruct{
ObjectHandle{_reg, o}
}
);
}
void ObjectStore2::throwEventUpdate(const Object o) {
std::cout << "OS debug: event update " << entt::to_integral(o) << "\n";
dispatch(
ObjectStore_Event::object_update,
ObjectStore::Events::ObjectUpdate{
ObjectHandle{_reg, o}
}
);
}
void ObjectStore2::throwEventDestroy(const Object o) {
std::cout << "OS debug: event destroy " << entt::to_integral(o) << "\n";
dispatch(
ObjectStore_Event::object_destroy,
ObjectStore::Events::ObjectUpdate{
ObjectHandle{_reg, o}
}
);
}

95
src/fragment_store/object_store.hpp
View File

@ -1,95 +0,0 @@
#pragma once
#include <solanaceae/util/event_provider.hpp>
#include <solanaceae/util/span.hpp>
#include <entt/entity/registry.hpp>
#include <entt/entity/handle.hpp>
#include <cstdint>
// internal id
enum class Object : uint32_t {};
using ObjectRegistry = entt::basic_registry<Object>;
using ObjectHandle = entt::basic_handle<ObjectRegistry>;
// fwd
struct ObjectStore2;
struct StorageBackendI {
// OR or OS ?
ObjectStore2& _os;
StorageBackendI(ObjectStore2& os);
// default impl fails, acting like a read only store
virtual ObjectHandle newObject(ByteSpan id);
// ========== write object to storage ==========
using write_to_storage_fetch_data_cb = uint64_t(uint8_t* request_buffer, uint64_t buffer_size);
// calls data_cb with a buffer to be filled in, cb returns actual count of data. if returned < max, its the last buffer.
virtual bool write(Object o, std::function<write_to_storage_fetch_data_cb>& data_cb) = 0;
bool write(Object o, const ByteSpan data);
// ========== read object from storage ==========
using read_from_storage_put_data_cb = void(const ByteSpan buffer);
virtual bool read(Object o, std::function<read_from_storage_put_data_cb>& data_cb) = 0;
};
namespace ObjectStore::Events {
struct ObjectConstruct {
const ObjectHandle e;
};
struct ObjectUpdate {
const ObjectHandle e;
};
struct ObjectDestory {
const ObjectHandle e;
};
} // ObjectStore::Events
enum class ObjectStore_Event : uint16_t {
object_construct,
object_update,
object_destroy,
MAX
};
struct ObjectStoreEventI {
using enumType = ObjectStore_Event;
virtual ~ObjectStoreEventI(void) {}
virtual bool onEvent(const ObjectStore::Events::ObjectConstruct&) { return false; }
virtual bool onEvent(const ObjectStore::Events::ObjectUpdate&) { return false; }
virtual bool onEvent(const ObjectStore::Events::ObjectDestory&) { return false; }
};
using ObjectStoreEventProviderI = EventProviderI<ObjectStoreEventI>;
struct ObjectStore2 : public ObjectStoreEventProviderI {
static constexpr const char* version {"2"};
ObjectRegistry _reg;
// TODO: default backend?
ObjectStore2(void);
virtual ~ObjectStore2(void);
ObjectRegistry& registry(void);
ObjectHandle objectHandle(const Object o);
// TODO: properly think about multiple objects witht he same id / different backends
ObjectHandle getOneObjectByID(const ByteSpan id);
// sync?
void throwEventConstruct(const Object o);
void throwEventUpdate(const Object o);
void throwEventDestroy(const Object o);
};

67
src/fragment_store/serializer_json.hpp
View File

@ -1,67 +0,0 @@
#pragma once
#include <entt/core/type_info.hpp>
#include <entt/container/dense_map.hpp>
#include <entt/entity/handle.hpp>
#include <nlohmann/json_fwd.hpp>
// nlohmann
template<typename EntityType = entt::entity>
struct SerializerJsonCallbacks {
using Registry = entt::basic_registry<EntityType>;
using Handle = entt::basic_handle<Registry>;
using serialize_fn = bool(*)(const Handle h, nlohmann::json& out);
entt::dense_map<entt::id_type, serialize_fn> _serl;
using deserialize_fn = bool(*)(Handle h, const nlohmann::json& in);
entt::dense_map<entt::id_type, deserialize_fn> _deserl;
template<typename T>
static bool component_get_json(const Handle h, nlohmann::json& j) {
if (h.template all_of<T>()) {
if constexpr (!std::is_empty_v<T>) {
j = h.template get<T>();
}
return true;
}
return false;
}
template<typename T>
static bool component_emplace_or_replace_json(Handle h, const nlohmann::json& j) {
if constexpr (std::is_empty_v<T>) {
h.template emplace_or_replace<T>(); // assert empty json?
} else {
h.template emplace_or_replace<T>(static_cast<T>(j));
}
return true;
}
void registerSerializer(serialize_fn fn, const entt::type_info& type_info) {
_serl[type_info.hash()] = fn;
}
template<typename CompType>
void registerSerializer(
serialize_fn fn = component_get_json<CompType>,
const entt::type_info& type_info = entt::type_id<CompType>()
) {
registerSerializer(fn, type_info);
}
void registerDeSerializer(deserialize_fn fn, const entt::type_info& type_info) {
_deserl[type_info.hash()] = fn;
}
template<typename CompType>
void registerDeSerializer(
deserialize_fn fn = component_emplace_or_replace_json<CompType>,
const entt::type_info& type_info = entt::type_id<CompType>()
) {
registerDeSerializer(fn, type_info);
}
};

394
src/fragment_store/test_file_zstd.cpp
View File

@ -1,394 +0,0 @@
#include "./file2_zstd.hpp"
#include <solanaceae/util/span.hpp>
#include <solanaceae/file/file2_mem.hpp>
#include <solanaceae/file/file2_std.hpp>
#include <filesystem>
#include <iostream>
#include <variant>
#include <algorithm>
#include <vector>
#include <random>
#include <cassert>
const static std::string_view test_text1{"test1 1234 1234 :) 1234 5678 88888888\n"};
const static ByteSpan data_test_text1{
reinterpret_cast<const uint8_t*>(test_text1.data()),
test_text1.size()
};
const static std::string_view test_text2{"test2 1234 1234 :) 1234 5678 88888888\n"};
const static ByteSpan data_test_text2{
reinterpret_cast<const uint8_t*>(test_text2.data()),
test_text2.size()
};
const static std::string_view test_text3{
"00000000000000000000000000000000000000000000000000000 test 00000000000000000000000000000000000000\n"
"00000000000000000000000000000000000000000000000000000 test 00000000000000000000000000000000000000\n"
"00000000000000000000000000000000000000000000000000000 test 00000000000000000000000000000000000000\n"
"00000000000000000000000000000000000000000000000000000 test 00000000000000000000000000000000000000\n"
"00000000000000000000000000000000000000000000000000000 test 00000000000000000000000000000000000000\n"
"00000000000000000000000000000000000000000000000000000 test 00000000000000000000000000000000000000\n"
};
const static ByteSpan data_test_text3{
reinterpret_cast<const uint8_t*>(test_text3.data()),
test_text3.size()
};
int main(void) {
{ // first do a simple mem backed test
std::vector<uint8_t> buffer;
{ // write
File2MemW f_w_mem{buffer};
assert(f_w_mem.isGood());
File2ZSTDW f_w_zstd{f_w_mem};
assert(f_w_zstd.isGood());
bool res = f_w_zstd.write(data_test_text1);
assert(res);
assert(f_w_zstd.isGood());
// write another frame of the same data
res = f_w_zstd.write(data_test_text2);
assert(res);
assert(f_w_zstd.isGood());
// write larger frame
res = f_w_zstd.write(data_test_text3);
assert(res);
assert(f_w_zstd.isGood());
}
std::cout << "in mem size: " << buffer.size() << "\n";
{ // read
File2MemR f_r_mem{ByteSpan{buffer}};
assert(f_r_mem.isGood());
File2ZSTDR f_r_zstd{f_r_mem};
assert(f_r_zstd.isGood());
// reads return owning buffers
{ // readback data_test_text1
auto r_res_var = f_r_zstd.read(data_test_text1.size);
//assert(f_r_zstd.isGood());
//assert(f_r_file.isGood());
assert(std::holds_alternative<std::vector<uint8_t>>(r_res_var));
const auto& r_res_vec = std::get<std::vector<uint8_t>>(r_res_var);
//std::cout << "decomp: " << std::string_view{reinterpret_cast<const char*>(r_res_vec.data()), r_res_vec.size()};
assert(r_res_vec.size() == data_test_text1.size);
assert(std::equal(data_test_text1.cbegin(), data_test_text1.cend(), r_res_vec.cbegin()));
}
{ // readback data_test_text2
auto r_res_var = f_r_zstd.read(data_test_text2.size);
//assert(f_r_zstd.isGood());
//assert(f_r_file.isGood());
assert(std::holds_alternative<std::vector<uint8_t>>(r_res_var));
const auto& r_res_vec = std::get<std::vector<uint8_t>>(r_res_var);
//std::cout << "decomp: " << std::string_view{reinterpret_cast<const char*>(r_res_vec.data()), r_res_vec.size()};
assert(r_res_vec.size() == data_test_text2.size);
assert(std::equal(
data_test_text2.cbegin(),
data_test_text2.cend(),
r_res_vec.cbegin()
));
}
{ // readback data_test_text3
auto r_res_var = f_r_zstd.read(data_test_text3.size);
//assert(f_r_zstd.isGood());
//assert(f_r_file.isGood());
assert(std::holds_alternative<std::vector<uint8_t>>(r_res_var));
const auto& r_res_vec = std::get<std::vector<uint8_t>>(r_res_var);
//std::cout << "decomp: " << std::string_view{reinterpret_cast<const char*>(r_res_vec.data()), r_res_vec.size()};
assert(r_res_vec.size() == data_test_text3.size);
assert(std::equal(
data_test_text3.cbegin(),
data_test_text3.cend(),
r_res_vec.cbegin()
));
}
{ // assert eof somehow
// since its eof, reading a single byte should return a zero sized buffer
auto r_res_var = f_r_zstd.read(1);
if (std::holds_alternative<std::vector<uint8_t>>(r_res_var)) {
assert(std::get<std::vector<uint8_t>>(r_res_var).empty());
} else if (std::holds_alternative<ByteSpan>(r_res_var)) {
assert(std::get<ByteSpan>(r_res_var).empty());
} else {
assert(false);
}
}
}
}
const auto temp_dir = std::filesystem::temp_directory_path() / "file2_zstd_tests";
std::filesystem::create_directories(temp_dir); // making sure
assert(std::filesystem::exists(temp_dir));
std::cout << "test temp dir: " << temp_dir << "\n";
const auto test1_file_path = temp_dir / "testfile1.zstd";
{ // simple write test
File2WFile f_w_file{std::string_view{test1_file_path.u8string()}, true};
assert(f_w_file.isGood());
File2ZSTDW f_w_zstd{f_w_file};
assert(f_w_zstd.isGood());
assert(f_w_file.isGood());
//bool res = f_w_file.write(data_test_text1);
bool res = f_w_zstd.write(data_test_text1);
assert(res);
assert(f_w_zstd.isGood());
assert(f_w_file.isGood());
// write another frame of the same data
res = f_w_zstd.write(data_test_text2);
assert(res);
assert(f_w_zstd.isGood());
assert(f_w_file.isGood());
// write larger frame
res = f_w_zstd.write(data_test_text3);
assert(res);
assert(f_w_zstd.isGood());
assert(f_w_file.isGood());
}
// after flush
assert(std::filesystem::file_size(test1_file_path) != 0);
{ // simple read test (using write test created file)
File2RFile f_r_file{std::string_view{test1_file_path.u8string()}};
assert(f_r_file.isGood());
File2ZSTDR f_r_zstd{f_r_file};
assert(f_r_zstd.isGood());
assert(f_r_file.isGood());
// reads return owning buffers
{ // readback data_test_text1
auto r_res_var = f_r_zstd.read(data_test_text1.size);
//assert(f_r_zstd.isGood());
//assert(f_r_file.isGood());
assert(std::holds_alternative<std::vector<uint8_t>>(r_res_var));
const auto& r_res_vec = std::get<std::vector<uint8_t>>(r_res_var);
//std::cout << "decomp: " << std::string_view{reinterpret_cast<const char*>(r_res_vec.data()), r_res_vec.size()};
assert(r_res_vec.size() == data_test_text1.size);
assert(std::equal(data_test_text1.cbegin(), data_test_text1.cend(), r_res_vec.cbegin()));
}
{ // readback data_test_text2
auto r_res_var = f_r_zstd.read(data_test_text2.size);
//assert(f_r_zstd.isGood());
//assert(f_r_file.isGood());
assert(std::holds_alternative<std::vector<uint8_t>>(r_res_var));
const auto& r_res_vec = std::get<std::vector<uint8_t>>(r_res_var);
//std::cout << "decomp: " << std::string_view{reinterpret_cast<const char*>(r_res_vec.data()), r_res_vec.size()};
assert(r_res_vec.size() == data_test_text2.size);
assert(std::equal(
data_test_text2.cbegin(),
data_test_text2.cend(),
r_res_vec.cbegin()
));
}
{ // readback data_test_text3
auto r_res_var = f_r_zstd.read(data_test_text3.size);
//assert(f_r_zstd.isGood());
//assert(f_r_file.isGood());
assert(std::holds_alternative<std::vector<uint8_t>>(r_res_var));
const auto& r_res_vec = std::get<std::vector<uint8_t>>(r_res_var);
//std::cout << "decomp: " << std::string_view{reinterpret_cast<const char*>(r_res_vec.data()), r_res_vec.size()};
assert(r_res_vec.size() == data_test_text3.size);
assert(std::equal(
data_test_text3.cbegin(),
data_test_text3.cend(),
r_res_vec.cbegin()
));
}
{ // assert eof somehow
// since its eof, reading a single byte should return a zero sized buffer
auto r_res_var = f_r_zstd.read(1);
if (std::holds_alternative<std::vector<uint8_t>>(r_res_var)) {
assert(std::get<std::vector<uint8_t>>(r_res_var).empty());
} else if (std::holds_alternative<ByteSpan>(r_res_var)) {
assert(std::get<ByteSpan>(r_res_var).empty());
} else {
assert(false);
}
}
}
const auto test2_file_path = temp_dir / "testfile2.zstd";
{ // write and read a single frame with increasing size
for (size_t fslog = 1; fslog <= 25; fslog++) {
const size_t frame_size = 1<<fslog;
//std::cerr << "fs: " << frame_size << "\n";
{ // write
std::minstd_rand rng_data{11*1337};
File2WFile f_w_file{std::string_view{test2_file_path.u8string()}, true};
assert(f_w_file.isGood());
File2ZSTDW f_w_zstd{f_w_file};
assert(f_w_zstd.isGood());
assert(f_w_file.isGood());
std::vector<uint8_t> tmp_data(frame_size);
for (auto& e : tmp_data) {
e = uint8_t(rng_data() & 0xff); // cutoff bad but good enough
}
assert(tmp_data.size() == frame_size);
bool res = f_w_zstd.write(ByteSpan{tmp_data});
assert(res);
assert(f_w_zstd.isGood());
assert(f_w_file.isGood());
}
{ // read
std::minstd_rand rng_data{11*1337};
File2RFile f_r_file{std::string_view{test2_file_path.u8string()}};
assert(f_r_file.isGood());
File2ZSTDR f_r_zstd{f_r_file};
assert(f_r_zstd.isGood());
assert(f_r_file.isGood());
{ // read frame
auto r_res_var = f_r_zstd.read(frame_size);
assert(std::holds_alternative<std::vector<uint8_t>>(r_res_var));
const auto& r_res_vec = std::get<std::vector<uint8_t>>(r_res_var);
assert(r_res_vec.size() == frame_size);
// assert equal
for (auto& e : r_res_vec) {
assert(e == uint8_t(rng_data() & 0xff));
}
}
{ // eof test
auto r_res_var = f_r_zstd.read(1);
if (std::holds_alternative<std::vector<uint8_t>>(r_res_var)) {
assert(std::get<std::vector<uint8_t>>(r_res_var).empty());
} else if (std::holds_alternative<ByteSpan>(r_res_var)) {
assert(std::get<ByteSpan>(r_res_var).empty());
} else {
assert(false);
}
}
}
// since we spam file, we immediatly remove them
std::filesystem::remove(test2_file_path);
}
}
const auto test3_file_path = temp_dir / "testfile3.zstd";
{ // large file test write
File2WFile f_w_file{std::string_view{test3_file_path.u8string()}, true};
assert(f_w_file.isGood());
File2ZSTDW f_w_zstd{f_w_file};
assert(f_w_zstd.isGood());
assert(f_w_file.isGood());
std::minstd_rand rng{11*1337};
std::minstd_rand rng_data{11*1337}; // make investigating easier
size_t total_raw_size {0};
for (size_t i = 0; i < 2000; i++) {
const size_t frame_size = (rng() % ((2<<19) - 1)) + 1;
std::vector<uint8_t> tmp_data(frame_size);
for (auto& e : tmp_data) {
e = uint8_t(rng_data() & 0xff); // cutoff bad but good enough
}
bool res = f_w_zstd.write(ByteSpan{tmp_data});
assert(res);
assert(f_w_zstd.isGood());
assert(f_w_file.isGood());
total_raw_size += frame_size;
}
std::cout << "t3 total raw size: " << total_raw_size << "\n";
}
// after flush
std::cout << "t3 size on disk: " << std::filesystem::file_size(test3_file_path) << "\n";
{ // large file test read
File2RFile f_r_file{std::string_view{test3_file_path.u8string()}};
assert(f_r_file.isGood());
File2ZSTDR f_r_zstd{f_r_file};
assert(f_r_zstd.isGood());
assert(f_r_file.isGood());
// using same rng state as write to compare
std::minstd_rand rng{11*1337};
std::minstd_rand rng_data{11*1337};
for (size_t i = 0; i < 2000; i++) {
const size_t frame_size = (rng() % ((2<<19) - 1)) + 1;
//std::cerr << "f: " << i << " fs: " << frame_size << "\n";
auto r_res_var = f_r_zstd.read(frame_size);
assert(std::holds_alternative<std::vector<uint8_t>>(r_res_var));
const auto& r_res_vec = std::get<std::vector<uint8_t>>(r_res_var);
assert(r_res_vec.size() == frame_size);
// assert equal
for (auto& e : r_res_vec) {
assert(e == uint8_t(rng_data() & 0xff));
}
}
{ // eof test
auto r_res_var = f_r_zstd.read(1);
if (std::holds_alternative<std::vector<uint8_t>>(r_res_var)) {
assert(std::get<std::vector<uint8_t>>(r_res_var).empty());
} else if (std::holds_alternative<ByteSpan>(r_res_var)) {
assert(std::get<ByteSpan>(r_res_var).empty());
} else {
assert(false);
}
}
}
// cleanup
std::filesystem::remove_all(temp_dir);
}

19
src/fragment_store/types.hpp
View File

@ -1,19 +0,0 @@
#pragma once
#include <cstdint>
enum class Encryption : uint8_t {
NONE = 0x00,
};
enum class Compression : uint8_t {
NONE = 0x00,
ZSTD = 0x01,
// TODO: zstd without magic
// TODO: zstd meta dict
// TODO: zstd data(message) dict
};
enum class MetaFileType : uint8_t {
TEXT_JSON,
BINARY_MSGPACK, // msgpacked msgpack
};

4
src/main_screen.hpp
View File

@ -2,8 +2,8 @@
#include "./screen.hpp" #include "./screen.hpp"
#include "./fragment_store/object_store.hpp" #include <solanaceae/object_store/object_store.hpp>
#include "./fragment_store/backends/filesystem_storage.hpp" #include <solanaceae/object_store/backends/filesystem_storage.hpp>
#include <solanaceae/util/simple_config_model.hpp> #include <solanaceae/util/simple_config_model.hpp>
#include <solanaceae/contact/contact_model3.hpp> #include <solanaceae/contact/contact_model3.hpp>
#include <solanaceae/message3/registry_message_model.hpp> #include <solanaceae/message3/registry_message_model.hpp>