loading logic implemented but broken (very funky and sometimes even out of contact)

flake.nix

@@ -12,13 +12,15 @@
     flake-utils.lib.eachDefaultSystem (system:
     let
       pkgs = import nixpkgs { inherit system; };
+      stdenv = (pkgs.stdenvAdapters.keepDebugInfo pkgs.stdenv);
     in {
-      packages.default = pkgs.stdenv.mkDerivation {
+      #packages.default = pkgs.stdenv.mkDerivation {
+      packages.default = stdenv.mkDerivation {
         pname = "tomato";
         version = "0.0.0";
 
         src = ./.;
-        submodules = 1;
+        submodules = 1; # does nothing
 
         nativeBuildInputs = with pkgs; [
           cmake
@@ -70,7 +72,7 @@
           mv bin/tomato $out/bin
         '';
 
-        dontStrip = true;
+        dontStrip = true; # does nothing
 
         # copied from nixpkgs's SDL2 default.nix
         # SDL is weird in that instead of just dynamically linking with
@@ -97,6 +99,8 @@
           '';
       };
 
+      #packages.debug = pkgs.enableDebugging self.packages.${system}.default;
+
       devShells.${system}.default = pkgs.mkShell {
         #inputsFrom = with pkgs; [ SDL2 ];
         buildInputs = [ self.packages.${system}.default ]; # this makes a prebuild tomato available in the shell, do we want this?

src/chat_gui4.cpp

@@ -269,28 +269,6 @@ float ChatGui4::render(float time_delta) {
 
 					auto* msg_reg_ptr = _rmm.get(*_selected_contact);
 
-					if (msg_reg_ptr != nullptr) {
-						const auto& mm = *msg_reg_ptr;
-						//const auto& unread_storage = mm.storage<Message::Components::TagUnread>();
-						if (const auto* unread_storage = mm.storage<Message::Components::TagUnread>(); unread_storage != nullptr && !unread_storage->empty()) {
-							//assert(unread_storage->size() == 0);
-							//assert(unread_storage.cbegin() == unread_storage.cend());
-
-#if 0
-							std::cout << "UNREAD ";
-							Message3 prev_ent = entt::null;
-							for (const Message3 e : mm.view<Message::Components::TagUnread>()) {
-								std::cout << entt::to_integral(e) << " ";
-								if (prev_ent == e) {
-									assert(false && "dup");
-								}
-								prev_ent = e;
-							}
-							std::cout << "\n";
-#endif
-						}
-					}
-
 					constexpr ImGuiTableFlags table_flags =
 						ImGuiTableFlags_BordersInnerV |
 						ImGuiTableFlags_RowBg |
@@ -303,6 +281,9 @@ float ChatGui4::render(float time_delta) {
 						ImGui::TableSetupColumn("timestamp");
 						ImGui::TableSetupColumn("extra_info", _show_chat_extra_info ? ImGuiTableColumnFlags_None : ImGuiTableColumnFlags_Disabled);
 
+						Message3Handle message_view_oldest; // oldest visible message
+						Message3Handle message_view_newest; // last visible message
+
 						// very hacky, and we have variable hight entries
 						//ImGuiListClipper clipper;
 
@@ -389,7 +370,8 @@ float ChatGui4::render(float time_delta) {
 								}
 
 								// use username as visibility test
-								if (ImGui::IsItemVisible() && msg_reg.all_of<Message::Components::TagUnread>(e)) {
+								if (ImGui::IsItemVisible()) {
+									if (msg_reg.all_of<Message::Components::TagUnread>(e)) {
 										// get time now
 										const uint64_t ts_now = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
 										msg_reg.emplace_or_replace<Message::Components::Read>(e, ts_now);
@@ -397,6 +379,16 @@ float ChatGui4::render(float time_delta) {
 										msg_reg.emplace_or_replace<Components::UnreadFade>(e, 1.f);
 									}
 
+									// track view
+									if (!static_cast<bool>(message_view_oldest)) {
+										message_view_oldest = {msg_reg, e};
+										message_view_newest = {msg_reg, e};
+									} else if (static_cast<bool>(message_view_newest)) {
+										// update to latest
+										message_view_newest = {msg_reg, e};
+									}
+								}
+
 								// highlight self
 								if (_cr.any_of<Contact::Components::TagSelfWeak, Contact::Components::TagSelfStrong>(c_from.c)) {
 									ImU32 cell_bg_color = ImGui::GetColorU32(ImVec4(0.3f, 0.7f, 0.3f, 0.20f));
@@ -559,9 +551,88 @@ float ChatGui4::render(float time_delta) {
 						//ImGui::TableNextRow(0, TEXT_BASE_HEIGHT);
 						//ImGui::TableNextRow(0, TEXT_BASE_HEIGHT);
 
+						{ // update view cursers
+							// any message in view
+							if (!static_cast<bool>(message_view_oldest)) {
+								// no message in view? should we setup a view at current time?
+
+								if (static_cast<bool>(_view_end)) {
+									// TODO: throwEventDestroy
+									_view_end.destroy();
+								}
+								//if (static_cast<bool>(_view_begin)) {
+									//// TODO: throwEventDestroy
+									//_view_begin.destroy();
+								//}
+
+								// HACK: create begin curser with current time until someone else manages that
+								if (!static_cast<bool>(_view_begin) || _view_begin.registry() != msg_reg_ptr) {
+									_view_begin = {msg_reg, msg_reg.create()};
+
+									_view_begin.emplace_or_replace<Message::Components::ViewCurserBegin>(entt::null);
+									// TODO: this needs to be saved somewhere?
+									_view_begin.get_or_emplace<Message::Components::Timestamp>().ts = Message::getTimeMS();
+
+									std::cout << "CG: created view FRONT begin ts\n";
+									_rmm.throwEventConstruct(_view_begin);
+								}
+
+							} else {
+								// clean up old view
+								// TODO: properly handle this on contact transition (will be removed once multi chat refactor lands)
+								if (static_cast<bool>(_view_end) && _view_end.registry() != msg_reg_ptr) {
+									// TODO: throwEventDestroy
+									_view_end.destroy();
+								}
+								if (static_cast<bool>(_view_begin) && _view_begin.registry() != msg_reg_ptr) {
+									// TODO: throwEventDestroy
+									_view_begin.destroy();
+								}
+
+								bool end_created {false};
+								if (!static_cast<bool>(_view_end)) {
+									_view_end = {msg_reg, msg_reg.create()};
+									end_created = true;
+								}
+								bool begin_created {false};
+								if (!static_cast<bool>(_view_begin)) {
+									_view_begin = {msg_reg, msg_reg.create()};
+									begin_created = true;
+								}
+								_view_end.emplace_or_replace<Message::Components::ViewCurserEnd>(_view_begin);
+								_view_begin.emplace_or_replace<Message::Components::ViewCurserBegin>(_view_end);
+
+								auto& old_end_ts = _view_end.get_or_emplace<Message::Components::Timestamp>().ts;
+								auto& old_begin_ts = _view_begin.get_or_emplace<Message::Components::Timestamp>().ts;
+
+								if (old_end_ts != message_view_oldest.get<Message::Components::Timestamp>().ts) {
+									old_end_ts = message_view_oldest.get<Message::Components::Timestamp>().ts;
+									if (end_created) {
+										std::cout << "CG: created view end ts with " << old_end_ts << "\n";
+										_rmm.throwEventConstruct(_view_end);
+									} else {
+										std::cout << "CG: updated view end ts to " << old_end_ts << "\n";
+										_rmm.throwEventUpdate(_view_end);
+									}
+								}
+
+								if (old_begin_ts != message_view_newest.get<Message::Components::Timestamp>().ts) {
+									old_begin_ts = message_view_newest.get<Message::Components::Timestamp>().ts;
+									if (begin_created) {
+										std::cout << "CG: created view begin ts with " << old_begin_ts << "\n";
+										_rmm.throwEventConstruct(_view_begin);
+									} else {
+										std::cout << "CG: updated view begin ts to " << old_begin_ts << "\n";
+										_rmm.throwEventUpdate(_view_begin);
+									}
+								}
+							}
+						}
+
 						ImGui::EndTable();
 					}
 
+
 					if (ImGui::GetScrollY() >= ImGui::GetScrollMaxY()) {
 						ImGui::SetScrollHereY(1.f);
 					}

src/chat_gui4.hpp

@@ -10,6 +10,9 @@
 #include "./file_selector.hpp"
 #include "./send_image_popup.hpp"
 
+// HACK: move to public msg api?
+#include "./fragment_store/message_fragment_store.hpp"
+
 #include <entt/container/dense_map.hpp>
 
 #include <cstdint>
@@ -32,7 +35,12 @@ class ChatGui4 {
 	FileSelector _fss;
 	SendImagePopup _sip;
 
+	// TODO: refactor this to allow multiple open contacts
 	std::optional<Contact3> _selected_contact;
+	// set to the ts of the newest rendered msg
+	Message3Handle _view_begin{};
+	// set to the ts of the oldest rendered msg
+	Message3Handle _view_end{};
 
 	// TODO: per contact
 	std::string _text_input_buffer;

src/fragment_store/message_fragment_store.cpp

@@ -63,41 +63,51 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) {
 		return; // we only handle msg with ts
 	}
 
+	_potentially_dirty_contacts.emplace(m.registry()->ctx().get<Contact3>()); // always mark dirty here
+	if (m.any_of<Message::Components::ViewCurserBegin, Message::Components::ViewCurserEnd>()) {
+		// not an actual message, but we probalby need to check and see if we need to load fragments
+		std::cout << "MFS: new or updated curser\n";
+		return;
+	}
+
+	if (!m.all_of<Message::Components::FUID>()) {
+		std::cout << "MFS: new msg missing FUID\n";
 		if (!m.registry()->ctx().contains<Message::Components::OpenFragments>()) {
-		// first message in this reg
 			m.registry()->ctx().emplace<Message::Components::OpenFragments>();
 
 			// TODO: move this to async
-		// new reg -> load all fragments for this contact (for now, ranges later)
+			// TODO: move this to tick and just respect the dirty
-		for (const auto& [fid, tsrange, fmc] : _fs._reg.view<FragComp::MessagesTSRange, FragComp::MessagesContact>().each()) {
+			FragmentHandle most_recent_fag;
-			Contact3 frag_contact = entt::null;
+			uint64_t most_recent_ts{0};
-			// TODO: id lookup table, this is very inefficent
+			if (m.registry()->ctx().contains<Message::Components::ContactFragments>()) {
-			for (const auto& [c_it, id_it] : _cr.view<Contact::Components::ID>().each()) {
+				for (const auto fid : m.registry()->ctx().get<Message::Components::ContactFragments>().frags) {
-				if (fmc.id == id_it.data) {
+					auto fh = _fs.fragmentHandle(fid);
-					//h.emplace_or_replace<Message::Components::ContactTo>(c_it);
+					if (!static_cast<bool>(fh) || !fh.all_of<FragComp::MessagesTSRange, FragComp::MessagesContact>()) {
-					//return true;
+						// TODO: remove at this point?
-					frag_contact = c_it;
-					break;
-				}
-			}
-			if (!_cr.valid(frag_contact)) {
-				// unkown contact
 						continue;
 					}
 
-			// registry is the same as the one the message event is for
+					const uint64_t f_ts = fh.get<FragComp::MessagesTSRange>().begin;
-			if (static_cast<const RegistryMessageModel&>(_rmm).get(frag_contact) == m.registry()) {
+					if (f_ts > most_recent_ts) {
-				// TODO: make dirty instead (they already are)
+						if (m.registry()->ctx().contains<Message::Components::LoadedContactFragments>()) {
-				loadFragment(*m.registry(), FragmentHandle{_fs._reg, fid});
+							if (m.registry()->ctx().get<Message::Components::LoadedContactFragments>().frags.contains(fh)) {
+								continue; // already loaded
 							}
 						}
+						most_recent_ts = f_ts;
+						most_recent_fag = {_fs._reg, fid};
+					}
+				}
+			}
+
+			if (static_cast<bool>(most_recent_fag)) {
+				loadFragment(*m.registry(), most_recent_fag);
+			}
 		}
 
 		auto& fuid_open = m.registry()->ctx().get<Message::Components::OpenFragments>().fuid_open;
 
 		const auto msg_ts = m.get<Message::Components::Timestamp>().ts;
-
-	if (!m.all_of<Message::Components::FUID>()) {
 		// missing fuid
 		// find closesed non-sealed off fragment
 
@@ -380,6 +390,23 @@ static bool rangeVisible(uint64_t range_begin, uint64_t range_end, const Message
 	return false;
 }
 
+static bool isLess(const std::vector<uint8_t>& lhs, const std::vector<uint8_t>& rhs) {
+	size_t i = 0;
+	for (; i < lhs.size() && i < rhs.size(); i++) {
+		if (lhs[i] < rhs[i]) {
+			return true;
+		} else if (lhs[i] > rhs[i]) {
+			return false;
+		}
+		// else continue
+	}
+
+	// here we have equality of common lenths
+
+	// we define smaller arrays to be less
+	return lhs.size() < rhs.size();
+}
+
 float MessageFragmentStore::tick(float time_delta) {
 	// sync dirty fragments here
 	if (!_fuid_save_queue.empty()) {
@@ -497,21 +524,27 @@ float MessageFragmentStore::tick(float time_delta) {
 		// first do collision check agains every contact associated fragment
 		// that is not already loaded !!
 		if (msg_reg->ctx().contains<Message::Components::ContactFragments>()) {
-			for (const FragmentID fid : msg_reg->ctx().get<Message::Components::ContactFragments>().frags) {
+			if (!msg_reg->ctx().contains<Message::Components::LoadedContactFragments>()) {
-				// TODO: better ctx caching code?
+				msg_reg->ctx().emplace<Message::Components::LoadedContactFragments>();
-				if (msg_reg->ctx().contains<Message::Components::LoadedContactFragments>()) {
-					if (msg_reg->ctx().get<Message::Components::LoadedContactFragments>().frags.contains(fid)) {
-						continue;
 			}
+			const auto& loaded_frags = msg_reg->ctx().get<Message::Components::LoadedContactFragments>().frags;
+
+			for (const FragmentID fid : msg_reg->ctx().get<Message::Components::ContactFragments>().frags) {
+				if (loaded_frags.contains(fid)) {
+					continue;
 				}
 
 				auto fh = _fs.fragmentHandle(fid);
 
 				if (!static_cast<bool>(fh)) {
+					// WHAT
+					msg_reg->ctx().get<Message::Components::ContactFragments>().frags.erase(fid);
 					return 0.05f;
 				}
 
 				if (!fh.all_of<FragComp::MessagesTSRange>()) {
+					// ????
+					msg_reg->ctx().get<Message::Components::ContactFragments>().frags.erase(fid);
 					return 0.05f;
 				}
 
@@ -526,7 +559,113 @@ float MessageFragmentStore::tick(float time_delta) {
 			// no new visible fragment
 
 			// now, finally, check for adjecent fragments that need to be loaded
-			// we do this by finding a fragment in a rage
+			// we do this by finding the outermost fragment in a rage, and extend it by one
+
+			// TODO: this is all extreamly unperformant code !!!!!!
+			// rewrite using some bounding range tree to perform collision checks !!!
+
+			// for each view
+			auto c_b_view = msg_reg->view<Message::Components::Timestamp, Message::Components::ViewCurserBegin>();
+			c_b_view.use<Message::Components::ViewCurserBegin>();
+			for (const auto& [m, ts_begin_comp, vcb] : c_b_view.each()) {
+				// track down both in the same run
+				FragmentID frag_newest {entt::null};
+				uint64_t frag_newest_ts {};
+				FragmentID frag_oldest {entt::null};
+				uint64_t frag_oldest_ts {};
+
+				// find newest frag in range
+				for (const FragmentID fid : msg_reg->ctx().get<Message::Components::ContactFragments>().frags) {
+					// we want to find the last and first fragment of the range (if not all hits are loaded, we did something wrong)
+					if (!loaded_frags.contains(fid)) {
+						continue;
+					}
+
+					// not checking frags for validity here, we checked above
+					auto fh = _fs.fragmentHandle(fid);
+
+					const auto [range_begin, range_end] = fh.get<FragComp::MessagesTSRange>();
+
+					// perf, only check begin curser fist
+					if (ts_begin_comp.ts < range_end) {
+					//if (ts_begin_comp.ts < range_end || ts_end > range_begin) {
+						continue;
+					}
+
+					if (ts_begin_comp.ts < range_begin) {
+						// begin curser does not hit the frag, but end might still hit/contain it
+						// if has curser end, check that
+						if (!msg_reg->valid(vcb.curser_end) || !msg_reg->all_of<Message::Components::ViewCurserEnd, Message::Components::Timestamp>(vcb.curser_end)) {
+							// no end, save no hit
+							continue;
+						}
+						const auto ts_end = msg_reg->get<Message::Components::Timestamp>(vcb.curser_end).ts;
+
+						if (ts_end > range_begin) {
+							continue;
+						}
+					}
+
+					// save hit
+					if (!_fs._reg.valid(frag_newest)) {
+						frag_newest = fid;
+						frag_newest_ts = range_begin; // new only compare against begin
+					} else {
+						// now we check if >= prev
+						if (range_begin < frag_newest_ts) {
+							continue;
+						}
+
+						if (range_begin == frag_newest_ts) {
+							// equal ts -> fallback to id
+							if (isLess(fh.get<FragComp::ID>().v, _fs._reg.get<FragComp::ID>(frag_newest).v)) {
+								// we dont "care" about equality here, since that *should* never happen
+								continue;
+							}
+						}
+
+						frag_newest = fid;
+						frag_newest_ts = range_begin; // new only compare against begin
+					}
+
+					if (!_fs._reg.valid(frag_oldest)) {
+						frag_oldest = fid;
+						frag_oldest_ts = range_end; // old only compare against end
+					}
+
+					if (fid != frag_oldest && fid != frag_newest) {
+						// now check against old
+						if (range_end > frag_oldest_ts) {
+							continue;
+						}
+
+						if (range_end == frag_oldest_ts) {
+							// equal ts -> fallback to id
+							if (!isLess(fh.get<FragComp::ID>().v, _fs._reg.get<FragComp::ID>(frag_oldest).v)) {
+								// we dont "care" about equality here, since that *should* never happen
+								continue;
+							}
+						}
+
+						frag_oldest = fid;
+						frag_oldest_ts = range_end; // old only compare against end
+					}
+				}
+
+				auto frag_after = _fs.fragmentHandle(fragmentAfter(frag_newest));
+				if (static_cast<bool>(frag_after) && !loaded_frags.contains(frag_after)) {
+					std::cout << "MFS: loading frag after newest\n";
+					loadFragment(*msg_reg, frag_after);
+					return 0.05f;
+				}
+
+				auto frag_before = _fs.fragmentHandle(fragmentBefore(frag_oldest));
+				if (static_cast<bool>(frag_before) && !loaded_frags.contains(frag_before)) {
+					std::cout << "MFS: loading frag before oldest\n";
+					loadFragment(*msg_reg, frag_before);
+					return 0.05f;
+				}
+			}
 		} else {
 			// contact has no fragments, skip
 		}
@@ -544,23 +683,6 @@ void MessageFragmentStore::triggerScan(void) {
 	_fs.scanStoragePath("test_message_store/");
 }
 
-static bool isLess(const std::vector<uint8_t>& lhs, const std::vector<uint8_t>& rhs) {
-	size_t i = 0;
-	for (; i < lhs.size() && i < rhs.size(); i++) {
-		if (lhs[i] < rhs[i]) {
-			return true;
-		} else if (lhs[i] > rhs[i]) {
-			return false;
-		}
-		// else continue
-	}
-
-	// here we have equality of common lenths
-
-	// we define smaller arrays to be less
-	return lhs.size() < rhs.size();
-}
-
 FragmentID MessageFragmentStore::fragmentBefore(FragmentID fid) {
 	auto fh = _fs.fragmentHandle(fid);
 	if (!fh.all_of<FragComp::MessagesTSRange>()) {