tomato/toxcore/sort.c

183 lines
5.9 KiB
C
Raw Normal View History

Squashed 'external/toxcore/c-toxcore/' changes from 55752a2e2ef..11ab1d2a723 11ab1d2a723 fix: reduce memory usage in group chats by 75% Significantly reduced the memory usage of groups since all message slots are preallocated for every peer for send and receive buffers of buffer size (hundreds of MiB peak when save contained alot of peers to try to connect to) 4f09f4e147c chore: Fix tsan build by moving it to GitHub CI. 6460c25c9e0 refactor: Use `merge_sort` instead of `qsort` for sorting. c660bbe8c95 test: Fix crypto_test to initialise its plain text buffer. 0204db6184b cleanup: Fix layering check warnings. df2211e1548 refactor: Use tox memory allocator for temporary buffers in crypto. ac812871a2e feat: implement the last 2 missing network struct functions and make use of them 29d1043be0b test: friend request test now tests min/max message sizes 93aafd78c1f fix: friend requests with very long messages are no longer dropped 819aa2b2618 feat: Add option to disable DNS lookups in toxcore. 0ac23cee035 fix: windows use of REUSEADDR 7d2811d302d chore(ci): make bazel server shutdown faster 1dc399ba20d chore: Use vcpkg instead of conan in the MSVC build. 14d823165d9 chore: Migrate to conan 2. bdd17c16787 cleanup: Allocate logger using tox memory allocator. b396c061515 chore(deps): bump third_party/cmp from `2ac6bca` to `52bfcfa` 2e94da60d09 feat(net): add missing connect to network struct 41fb1839c7b chore: Add check to ensure version numbers agree. 934a8301113 chore: Release 0.2.20 3acef4bf044 fix: Add missing free in dht_get_nodes_response event. git-subtree-dir: external/toxcore/c-toxcore git-subtree-split: 11ab1d2a7232eee19b51ce126ccce267d6578903
2024-12-19 16:27:40 +01:00
/* SPDX-License-Identifier: GPL-3.0-or-later
Squashed 'external/toxcore/c-toxcore/' changes from 11ab1d2a723..d9b8fa6098d d9b8fa6098d fix: Fake broadcast address for 127.x.x.x aa649165a57 chore: Add code for future netprof TCP testing 9e5693de5ac chore: add to_string functions for netprof enums 52d915e6a90 cleanup: Heap allocate network profile objects 80fabd4a729 feat: Implement Tox network profiler 05abe083cb6 cleanup: Some random cleanups, mostly related to mem. 5cca24513b8 cleanup: Check that onion IP/Port packing worked. e092ecd1244 cleanup: Use tox memory allocator in some more places. 3cfe41c7587 fix: Avoid `memcpy`-ing structs into onion ping id data. e32ac001938 fix: Add more information on why the frame was not sent. ab887003687 fix: Allow TCP connections to fail `connect` calls. 7603170e663 refactor: Use tox memory in group connection allocations. 5bd8a85eb89 cleanup: Align internal logger with external on type of source line. e9bf524d9e1 cleanup: Add missing `#include` to sort_test.cc. d10c966b998 feat: Add `to_string` functions for toxencryptsave errors. 7bfd0dc8003 docs: Update the docs for group join functions 380dde9f2ae test: Add more logging to TCP connection constructor. 0f12f384c8c cleanup: Reduce stack frame sizes to below 4096 bytes. bc43cec0626 chore: Happy new year! fbe78f1702e cleanup: Add a `TOX_HIDE_DEPRECATED` check to hide deprecated symbols. 44d9da07e77 refactor: Use tox memory for group moderation/pack allocations. 7f26d520168 refactor: Use tox memory in group chats allocations. 2f62f3d0e77 refactor: Use tox Memory for group allocations. 8a968162041 chore: Add dispatch/events headers to bazel export. 2bbfb35abf6 docs: Output the error code string instead of int. in toxav logging d55d0e4eaef cleanup: Remove redundant code for checking if group exists 2a6dc643338 chore: Upgrade dependencies for websockify. fc0650601c1 fix: Allow peers to reconnect to group chats using a password git-subtree-dir: external/toxcore/c-toxcore git-subtree-split: d9b8fa6098de6c074038b6664d2572627540b148
2025-01-18 15:53:06 +01:00
* Copyright © 2023-2025 The TokTok team.
Squashed 'external/toxcore/c-toxcore/' changes from 55752a2e2ef..11ab1d2a723 11ab1d2a723 fix: reduce memory usage in group chats by 75% Significantly reduced the memory usage of groups since all message slots are preallocated for every peer for send and receive buffers of buffer size (hundreds of MiB peak when save contained alot of peers to try to connect to) 4f09f4e147c chore: Fix tsan build by moving it to GitHub CI. 6460c25c9e0 refactor: Use `merge_sort` instead of `qsort` for sorting. c660bbe8c95 test: Fix crypto_test to initialise its plain text buffer. 0204db6184b cleanup: Fix layering check warnings. df2211e1548 refactor: Use tox memory allocator for temporary buffers in crypto. ac812871a2e feat: implement the last 2 missing network struct functions and make use of them 29d1043be0b test: friend request test now tests min/max message sizes 93aafd78c1f fix: friend requests with very long messages are no longer dropped 819aa2b2618 feat: Add option to disable DNS lookups in toxcore. 0ac23cee035 fix: windows use of REUSEADDR 7d2811d302d chore(ci): make bazel server shutdown faster 1dc399ba20d chore: Use vcpkg instead of conan in the MSVC build. 14d823165d9 chore: Migrate to conan 2. bdd17c16787 cleanup: Allocate logger using tox memory allocator. b396c061515 chore(deps): bump third_party/cmp from `2ac6bca` to `52bfcfa` 2e94da60d09 feat(net): add missing connect to network struct 41fb1839c7b chore: Add check to ensure version numbers agree. 934a8301113 chore: Release 0.2.20 3acef4bf044 fix: Add missing free in dht_get_nodes_response event. git-subtree-dir: external/toxcore/c-toxcore git-subtree-split: 11ab1d2a7232eee19b51ce126ccce267d6578903
2024-12-19 16:27:40 +01:00
*/
#include "sort.h"
#include <assert.h>
#include "attributes.h"
#include "ccompat.h"
#include "util.h"
/**
* @brief Threshold for when to switch to insertion sort.
*
* This is a trade-off between the complexity of insertion sort and the
* overhead of merge sort. The threshold is chosen to be the smallest value
* that gives a measurable speedup for insertion sort over merge sort. This is
* based on measurements done in sort_bench.cc. Starting from 32 elements,
* merge sort is faster than insertion sort in all our tests (both unsorted
* and mostly-sorted).
*
* Toxcore has a lot of small arrays it wants to sort, so this optimisation
* makes sense.
*/
#define SMALL_ARRAY_THRESHOLD 16
non_null()
static void merge_sort_merge_back(
void *arr,
const void *l_arr, uint32_t l_arr_size,
const void *r_arr, uint32_t r_arr_size,
uint32_t left_start,
const void *object, const Sort_Funcs *funcs)
{
uint32_t li = 0;
uint32_t ri = 0;
uint32_t k = left_start;
while (li < l_arr_size && ri < r_arr_size) {
const void *l = funcs->get_callback(l_arr, li);
const void *r = funcs->get_callback(r_arr, ri);
// !(r < l) <=> (r >= l) <=> (l <= r)
if (!funcs->less_callback(object, r, l)) {
funcs->set_callback(arr, k, l);
++li;
} else {
funcs->set_callback(arr, k, r);
++ri;
}
++k;
}
/* Copy the remaining elements of `l_arr[]`, if there are any. */
while (li < l_arr_size) {
funcs->set_callback(arr, k, funcs->get_callback(l_arr, li));
++li;
++k;
}
/* Copy the remaining elements of `r_arr[]`, if there are any. */
while (ri < r_arr_size) {
funcs->set_callback(arr, k, funcs->get_callback(r_arr, ri));
++ri;
++k;
}
}
/** Function to merge the two haves `arr[left_start..mid]` and `arr[mid+1..right_end]` of array `arr[]`. */
non_null()
static void merge_sort_merge(
void *arr, uint32_t left_start, uint32_t mid, uint32_t right_end, void *tmp,
const void *object, const Sort_Funcs *funcs)
{
const uint32_t l_arr_size = mid - left_start + 1;
const uint32_t r_arr_size = right_end - mid;
/* Temporary arrays, using the tmp buffer created in `merge_sort` below. */
void *l_arr = funcs->subarr_callback(tmp, 0, l_arr_size);
void *r_arr = funcs->subarr_callback(tmp, l_arr_size, r_arr_size);
/* Copy data to temp arrays `l_arr[]` and `r_arr[]`.
*
* This is iterating and repeatedly calling `get` and `set`, which sounds
* slow, but is only marginally slower than having a `copy` callback. With
* a `copy` callback, we'd save 3-4% in time.
*/
for (uint32_t i = 0; i < l_arr_size; ++i) {
funcs->set_callback(l_arr, i, funcs->get_callback(arr, left_start + i));
}
for (uint32_t i = 0; i < r_arr_size; ++i) {
funcs->set_callback(r_arr, i, funcs->get_callback(arr, mid + 1 + i));
}
/* Merge the temp arrays back into `arr[left_start..right_end]`. */
merge_sort_merge_back(arr, l_arr, l_arr_size, r_arr, r_arr_size, left_start, object, funcs);
}
non_null()
static void insertion_sort_step(void *arr, void *tmp, uint32_t i, const void *object, const Sort_Funcs *funcs)
{
funcs->set_callback(tmp, 0, funcs->get_callback(arr, i));
uint32_t j = i;
while (j > 0) {
if (!funcs->less_callback(object, tmp, funcs->get_callback(arr, j - 1))) {
break;
}
funcs->set_callback(arr, j, funcs->get_callback(arr, j - 1));
--j;
}
funcs->set_callback(arr, j, tmp);
}
non_null()
static void insertion_sort_with_buf(void *arr, uint32_t arr_size, void *tmp, uint32_t tmp_size, const void *object, const Sort_Funcs *funcs)
{
for (uint32_t i = 1; i < arr_size; ++i) {
insertion_sort_step(arr, tmp, i, object, funcs);
}
}
non_null()
static bool insertion_sort(void *arr, uint32_t arr_size, const void *object, const Sort_Funcs *funcs)
{
void *tmp = funcs->alloc_callback(object, 1);
if (tmp == nullptr) {
return false;
}
insertion_sort_with_buf(arr, arr_size, tmp, 1, object, funcs);
funcs->delete_callback(object, tmp, 1);
return true;
}
void merge_sort_with_buf(void *arr, uint32_t arr_size, void *tmp, uint32_t tmp_size, const void *object, const Sort_Funcs *funcs)
{
assert(tmp_size >= arr_size);
if (arr_size <= SMALL_ARRAY_THRESHOLD) {
assert(tmp_size >= 1);
insertion_sort_with_buf(arr, arr_size, tmp, tmp_size, object, funcs);
return;
}
// Merge subarrays in bottom up manner. First merge subarrays of
// size 1 to create sorted subarrays of size 2, then merge subarrays
// of size 2 to create sorted subarrays of size 4, and so on.
for (uint32_t curr_size = 1; curr_size <= arr_size - 1; curr_size = 2 * curr_size) {
// Pick starting point of different subarrays of current size
for (uint32_t left_start = 0; left_start < arr_size - 1; left_start += 2 * curr_size) {
// Find ending point of left subarray. mid+1 is starting
// point of right
const uint32_t mid = min_u32(left_start + curr_size - 1, arr_size - 1);
const uint32_t right_end = min_u32(left_start + 2 * curr_size - 1, arr_size - 1);
// Merge Subarrays arr[left_start...mid] & arr[mid+1...right_end]
merge_sort_merge(arr, left_start, mid, right_end, tmp, object, funcs);
}
}
}
bool merge_sort(void *arr, uint32_t arr_size, const void *object, const Sort_Funcs *funcs)
{
if (arr_size <= SMALL_ARRAY_THRESHOLD) {
return insertion_sort(arr, arr_size, object, funcs);
}
void *tmp = funcs->alloc_callback(object, arr_size);
if (tmp == nullptr) {
return false;
}
merge_sort_with_buf(arr, arr_size, tmp, arr_size, object, funcs);
funcs->delete_callback(object, tmp, arr_size);
return true;
}