make writing safe (by using a tmp file and moving to actual location)
This commit is contained in:
parent
67c6f9adb0
commit
0896038dd6
@ -186,15 +186,17 @@ FragmentID FragmentStore::newFragmentFile(
|
|||||||
|
|
||||||
_reg.emplace<FragComp::Ephemeral::MetaFileType>(new_frag, mft);
|
_reg.emplace<FragComp::Ephemeral::MetaFileType>(new_frag, mft);
|
||||||
|
|
||||||
throwEventConstruct(new_frag);
|
|
||||||
|
|
||||||
// meta needs to be synced to file
|
// meta needs to be synced to file
|
||||||
std::function<write_to_storage_fetch_data_cb> empty_data_cb = [](const uint8_t*, uint64_t) -> uint64_t { return 0; };
|
std::function<write_to_storage_fetch_data_cb> empty_data_cb = [](const uint8_t*, uint64_t) -> uint64_t { return 0; };
|
||||||
if (!syncToStorage(new_frag, empty_data_cb)) {
|
if (!syncToStorage(new_frag, empty_data_cb)) {
|
||||||
|
std::cerr << "FS error: syncToStorage failed while creating new fragment file\n";
|
||||||
_reg.destroy(new_frag);
|
_reg.destroy(new_frag);
|
||||||
return entt::null;
|
return entt::null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// while new metadata might be created here, making sure the file could be created is more important
|
||||||
|
throwEventConstruct(new_frag);
|
||||||
|
|
||||||
return new_frag;
|
return new_frag;
|
||||||
}
|
}
|
||||||
FragmentID FragmentStore::newFragmentFile(
|
FragmentID FragmentStore::newFragmentFile(
|
||||||
@ -266,12 +268,15 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function<write_to_storage
|
|||||||
//}
|
//}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
std::filesystem::path meta_tmp_path = _reg.get<FragComp::Ephemeral::FilePath>(fid).path + ".meta" + metaFileTypeSuffix(meta_type) + ".tmp";
|
||||||
|
meta_tmp_path.replace_filename("." + meta_tmp_path.filename().generic_u8string());
|
||||||
std::ofstream meta_file{
|
std::ofstream meta_file{
|
||||||
_reg.get<FragComp::Ephemeral::FilePath>(fid).path + ".meta" + metaFileTypeSuffix(meta_type),
|
meta_tmp_path,
|
||||||
std::ios::out | std::ios::trunc | std::ios::binary // always binary, also for text
|
std::ios::out | std::ios::trunc | std::ios::binary // always binary, also for text
|
||||||
};
|
};
|
||||||
|
|
||||||
if (!meta_file.is_open()) {
|
if (!meta_file.is_open()) {
|
||||||
|
std::cerr << "FS error: failed to create temporary meta file\n";
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -280,12 +285,17 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function<write_to_storage
|
|||||||
data_comp = _reg.get<FragComp::DataCompressionType>(fid).comp;
|
data_comp = _reg.get<FragComp::DataCompressionType>(fid).comp;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
std::filesystem::path data_tmp_path = _reg.get<FragComp::Ephemeral::FilePath>(fid).path + ".tmp";
|
||||||
|
data_tmp_path.replace_filename("." + data_tmp_path.filename().generic_u8string());
|
||||||
std::ofstream data_file{
|
std::ofstream data_file{
|
||||||
_reg.get<FragComp::Ephemeral::FilePath>(fid).path,
|
data_tmp_path,
|
||||||
std::ios::out | std::ios::trunc | std::ios::binary // always binary, also for text
|
std::ios::out | std::ios::trunc | std::ios::binary // always binary, also for text
|
||||||
};
|
};
|
||||||
|
|
||||||
if (!data_file.is_open()) {
|
if (!data_file.is_open()) {
|
||||||
|
meta_file.close();
|
||||||
|
std::filesystem::remove(meta_tmp_path);
|
||||||
|
std::cerr << "FS error: failed to create temporary data file\n";
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -419,9 +429,21 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function<write_to_storage
|
|||||||
}
|
}
|
||||||
|
|
||||||
meta_file.flush();
|
meta_file.flush();
|
||||||
|
meta_file.close();
|
||||||
data_file.flush();
|
data_file.flush();
|
||||||
|
data_file.close();
|
||||||
|
|
||||||
// TODO: use temp files and move to old location
|
std::filesystem::rename(
|
||||||
|
meta_tmp_path,
|
||||||
|
_reg.get<FragComp::Ephemeral::FilePath>(fid).path + ".meta" + metaFileTypeSuffix(meta_type)
|
||||||
|
);
|
||||||
|
|
||||||
|
std::filesystem::rename(
|
||||||
|
data_tmp_path,
|
||||||
|
_reg.get<FragComp::Ephemeral::FilePath>(fid).path
|
||||||
|
);
|
||||||
|
|
||||||
|
// TODO: check return value of renames
|
||||||
|
|
||||||
if (_reg.all_of<FragComp::Ephemeral::DirtyTag>(fid)) {
|
if (_reg.all_of<FragComp::Ephemeral::DirtyTag>(fid)) {
|
||||||
_reg.remove<FragComp::Ephemeral::DirtyTag>(fid);
|
_reg.remove<FragComp::Ephemeral::DirtyTag>(fid);
|
||||||
|
@ -105,8 +105,8 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) {
|
|||||||
if (fragment_uid.empty()) {
|
if (fragment_uid.empty()) {
|
||||||
for (auto& [ts_begin, ts_end, fid] : fuid_open) {
|
for (auto& [ts_begin, ts_end, fid] : fuid_open) {
|
||||||
const int64_t frag_range = int64_t(ts_end) - int64_t(ts_begin);
|
const int64_t frag_range = int64_t(ts_end) - int64_t(ts_begin);
|
||||||
//constexpr static int64_t max_frag_ts_extent {1000*60*60};
|
constexpr static int64_t max_frag_ts_extent {1000*60*60};
|
||||||
constexpr static int64_t max_frag_ts_extent {1000*60*3}; // 3min for testing
|
//constexpr static int64_t max_frag_ts_extent {1000*60*3}; // 3min for testing
|
||||||
const int64_t possible_extention = max_frag_ts_extent - frag_range;
|
const int64_t possible_extention = max_frag_ts_extent - frag_range;
|
||||||
|
|
||||||
// which direction
|
// which direction
|
||||||
|
@ -36,6 +36,9 @@ namespace Fragment::Components {
|
|||||||
struct MessagesContact {
|
struct MessagesContact {
|
||||||
std::vector<uint8_t> id;
|
std::vector<uint8_t> id;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// TODO: add src contact (self id)
|
||||||
|
|
||||||
} // Fragment::Components
|
} // Fragment::Components
|
||||||
|
|
||||||
// handles fragments for messages
|
// handles fragments for messages
|
||||||
|
Loading…
Reference in New Issue
Block a user