diff --git a/.github/workflows/build-macos.yaml b/.github/workflows/build-macos.yaml index d53872d79..d0beb544b 100644 --- a/.github/workflows/build-macos.yaml +++ b/.github/workflows/build-macos.yaml @@ -36,10 +36,10 @@ jobs: strategy: matrix: build_type: [RelWithDebugInfo] - cxx: [clang++-15] + cxx: [clang++-19] include: - - cxx: clang++-15 - package: llvm@15 + - cxx: clang++-19 + package: llvm@19 cc_name: clang cxx_name: clang++ needs_prefix: true diff --git a/bindings/python/include/svs/python/manager.h b/bindings/python/include/svs/python/manager.h index 3583c6836..68188c243 100644 --- a/bindings/python/include/svs/python/manager.h +++ b/bindings/python/include/svs/python/manager.h @@ -44,7 +44,10 @@ pybind11::tuple py_search( matrix_view(result_idx), matrix_view(result_dists) ); - svs::index::search_batch_into(self, q_result, query_data.cview()); + { + pybind11::gil_scoped_release release; + svs::index::search_batch_into(self, q_result, query_data.cview()); + } return pybind11::make_tuple(result_idx, result_dists); } @@ -86,7 +89,7 @@ void add_threading_interface(pybind11::class_& manager) { "num_threads", &Manager::get_num_threads, [](Manager& self, int num_threads) { - self.set_threadpool(svs::threads::DefaultThreadPool(num_threads)); + self.set_threadpool(svs::threads::SwitchNativeThreadPool(num_threads)); }, "Read/Write (int): Get and set the number of threads used to process queries." ); diff --git a/bindings/python/src/dynamic_vamana.cpp b/bindings/python/src/dynamic_vamana.cpp index 1b24eca1b..9361ea611 100644 --- a/bindings/python/src/dynamic_vamana.cpp +++ b/bindings/python/src/dynamic_vamana.cpp @@ -174,7 +174,12 @@ void add_points( "Expected IDs to be the same length as the number of rows in points!" ); } - index.add_points(data_view(py_data), std::span(ids.data(), ids.size()), reuse_empty); + auto data = data_view(py_data); + auto id_span = std::span(ids.data(), ids.size()); + { + py::gil_scoped_release release; + index.add_points(data, id_span, reuse_empty); + } } const char* ADD_POINTS_DOCSTRING = R"( @@ -381,8 +386,18 @@ void wrap(py::module& m) { add_dynamic_vamana_properties(vamana); - vamana.def("consolidate", &svs::DynamicVamana::consolidate, CONSOLIDATE_DOCSTRING); - vamana.def("compact", &svs::DynamicVamana::compact, COMPACT_DOCSTRING); + vamana.def( + "consolidate", + &svs::DynamicVamana::consolidate, + py::call_guard(), + CONSOLIDATE_DOCSTRING + ); + vamana.def( + "compact", + &svs::DynamicVamana::compact, + py::call_guard(), + COMPACT_DOCSTRING + ); // Reloading vamana.def( @@ -435,7 +450,11 @@ void wrap(py::module& m) { vamana.def( "delete", [](svs::DynamicVamana& index, const py_contiguous_array_t& ids) { - index.delete_points(as_span(ids)); + auto id_span = as_span(ids); + { + py::gil_scoped_release release; + index.delete_points(id_span); + } }, py::arg("ids"), DELETE_DOCSTRING diff --git a/bindings/python/src/python_bindings.cpp b/bindings/python/src/python_bindings.cpp index e7c14bf6f..f83057fc1 100644 --- a/bindings/python/src/python_bindings.cpp +++ b/bindings/python/src/python_bindings.cpp @@ -156,7 +156,7 @@ class ScopedModuleNameOverride { } // namespace -PYBIND11_MODULE(_svs, m) { +PYBIND11_MODULE(_svs, m, py::mod_gil_not_used()) { // Internally, the top level `__init__.py` imports everything from the C++ module named // `_svs`. // diff --git a/include/svs/concepts/graph.h b/include/svs/concepts/graph.h index d3e139eca..07fbc6f86 100644 --- a/include/svs/concepts/graph.h +++ b/include/svs/concepts/graph.h @@ -50,6 +50,15 @@ namespace svs::graphs { +/// Outcome of `MemoryGraph::add_edge(src, dst)`. Distinguishes three cases so callers +/// can route dropped edges (e.g. to a backedge buffer) without a TOCTOU race between a +/// pre-check and the insert. +enum class AddEdgeResult : uint8_t { + Added, // Edge was inserted. + AlreadyExists, // Edge was already present (or self-loop). Not inserted. + Full, // Node's adjacency list is at max_degree. Edge NOT inserted. +}; + // clang-format off /// @@ -135,12 +144,13 @@ template using index_type_t = typename G::index_type; /// @code{.cpp} /// template /// concept MemoryGraph = requires(T& g, const T& const_g) { -/// // Add an edge to the graph. -/// // Must return the out degree of `src` after adding the edge `src -> dst`. -/// // If adding the edge would result in the graph exceeding its maximum degree, -/// // implementations are free to not add this edge. +/// // Add an edge to the graph atomically. Returns an AddEdgeResult indicating: +/// // Added - edge was inserted +/// // AlreadyExists - edge was already present (or self-loop); no insert +/// // Full - node is at max_degree; edge NOT inserted (caller should +/// // route to an overflow buffer if needed) /// requires requires(index_type_t src, index_type_t dst) { -/// { g.add_edge(src, dst) } -> std::convertible_to; +/// { g.add_edge(src, dst) } -> std::convertible_to; /// }; /// /// // Completely clear the adjacency list for vertex ``i``. @@ -164,7 +174,7 @@ template concept MemoryGraph = requires(T& g, const T& const_g) { // Adding an edge. requires requires(index_type_t src, index_type_t dst) { - { g.add_edge(src, dst) } -> std::convertible_to; + { g.add_edge(src, dst) } -> std::convertible_to; }; // Clear adjacency list. diff --git a/include/svs/core/graph/graph.h b/include/svs/core/graph/graph.h index 89e456e07..9518c825f 100644 --- a/include/svs/core/graph/graph.h +++ b/include/svs/core/graph/graph.h @@ -20,8 +20,12 @@ #include "svs/core/data/simple.h" #include "svs/lib/algorithms.h" #include "svs/lib/boundscheck.h" +#include "svs/lib/concurrency/atomic_span.h" +#include "svs/lib/concurrency/seqlock.h" #include "svs/lib/saveload.h" +#include "svs/lib/spinlock.h" +#include #include #include #include @@ -57,12 +61,12 @@ template class SimpleGrap /// The integer representation used to represent vertices in this graph. using index_type = Idx; using value_type = std::span; - using const_value_type = std::span; + using const_value_type = AtomicSpan; /// Type used to represent mutable adjacency lists externally. using reference = std::span; /// Type used to represent constant adjacency lists externally. - using const_reference = std::span; + using const_reference = AtomicSpan; /// /// @brief Construct an empty graph of the desired size. @@ -75,7 +79,9 @@ template class SimpleGrap /// explicit SimpleGraphBase(size_t num_nodes, size_t max_degree) : data_{num_nodes, max_degree + 1} - , max_degree_{lib::narrow(max_degree)} { + , max_degree_{lib::narrow(max_degree)} + , seq_counters_(num_nodes) + , node_locks_(num_nodes) { reset(); } @@ -85,15 +91,19 @@ template class SimpleGrap size_t num_nodes, size_t max_degree, const Allocator& allocator ) : data_{num_nodes, max_degree + 1, allocator} - , max_degree_{lib::narrow(max_degree)} { + , max_degree_{lib::narrow(max_degree)} + , seq_counters_(num_nodes) + , node_locks_(num_nodes) { reset(); } explicit SimpleGraphBase(data_type data) : data_{std::move(data)} - , max_degree_{lib::narrow(data_.dimensions() - 1)} {} + , max_degree_{lib::narrow(data_.dimensions() - 1)} + , seq_counters_(data_.size()) + , node_locks_(data_.size()) {} - const_reference raw_row(Idx i) const { return data_.get_datum(i); } + std::span raw_row(Idx i) const { return data_.get_datum(i); } /// /// @brief Return the outward adjacency list for vertex ``i``. @@ -103,14 +113,17 @@ template class SimpleGrap const_reference get_node(Idx i) const { // Get the raw data. std::span raw_data = data_.get_datum(i); - auto num_neighbors = raw_data.front(); + Idx num_neighbors = std::atomic_ref(const_cast(raw_data.front())) + .load(std::memory_order_relaxed); + // Clamp to max_degree to safely handle torn reads of the length field. + num_neighbors = std::min(num_neighbors, max_degree_); - // Maybe prefetch the rest of the adjacncy list. + // Maybe prefetch the rest of the adjacency list. size_t bytes = (1 + num_neighbors) * sizeof(Idx); if (bytes > lib::CACHELINE_BYTES) { lib::prefetch(std::as_bytes(raw_data).subspan(lib::CACHELINE_BYTES)); } - return raw_data.subspan(1, num_neighbors); + return AtomicSpan(raw_data.data() + 1, num_neighbors); } /// @@ -119,16 +132,28 @@ template class SimpleGrap /// Complexity: Linear in the maximum degree. /// bool has_edge(Idx src, Idx dst) const { - const auto& list = get_node(src); - auto begin = list.begin(); - auto end = list.end(); - return (std::find(begin, end, dst) != end); + for (;;) { + auto maybe_seq = seq_counters_[src].read_begin(); + if (!maybe_seq) { + detail::pause(); + continue; + } + const auto& list = get_node(src); + bool found = (std::find(list.begin(), list.end(), dst) != list.end()); + if (seq_counters_[src].read_validate(*maybe_seq)) { + return found; + } + detail::pause(); + } } /// /// @brief Return the current out degree of vertex ``i``. /// - size_t get_node_degree(Idx i) const { return data_.get_datum(i).front(); } + size_t get_node_degree(Idx i) const { + return std::atomic_ref(const_cast(data_.get_datum(i).front())) + .load(std::memory_order_relaxed); + } /// /// @brief Prefetch the adjacency list for node ``i`` into the L1 cache. @@ -144,8 +169,11 @@ template class SimpleGrap /// The complexity of this operation is `O(1)`. /// void clear_node(Idx i) { - Idx& num_neighbors = data_.get_datum(i).front(); - num_neighbors = 0; + std::lock_guard lock{node_locks_[i]}; + auto seq = seq_counters_[i].begin_write(); + std::atomic_ref(data_.get_datum(i).front()) + .store(0, std::memory_order_relaxed); + seq_counters_[i].end_write(seq); } /// @@ -177,6 +205,7 @@ template class SimpleGrap /// @copydoc replace_node(Idx,const std::vector&) void replace_node(Idx i, std::span new_neighbors) { + std::lock_guard lock{node_locks_[i]}; std::span raw_data = data_.get_datum(i); // Clamp the number of elements to copy to the maximum out degree to correctly @@ -186,13 +215,31 @@ template class SimpleGrap Idx elements_to_copy = std::min(max_degree_, lib::narrow_cast(new_neighbors.size())); - std::span adjusted_neighbors = new_neighbors.first(elements_to_copy); - value_type adjacency_list = raw_data.subspan(1, elements_to_copy); + auto seq = seq_counters_[i].begin_write(); + for (Idx j = 0; j < elements_to_copy; ++j) { + std::atomic_ref(raw_data[1 + j]) + .store(new_neighbors[j], std::memory_order_relaxed); + } + std::atomic_ref(raw_data[0]) + .store(elements_to_copy, std::memory_order_relaxed); + seq_counters_[i].end_write(seq); + } - std::copy( - adjusted_neighbors.begin(), adjusted_neighbors.end(), adjacency_list.begin() - ); - raw_data.front() = elements_to_copy; + /// @copydoc replace_node(Idx,const std::vector&) + void replace_node(Idx i, AtomicSpan new_neighbors) { + std::lock_guard lock{node_locks_[i]}; + std::span raw_data = data_.get_datum(i); + Idx elements_to_copy = + std::min(max_degree_, lib::narrow_cast(new_neighbors.size())); + + auto seq = seq_counters_[i].begin_write(); + for (Idx j = 0; j < elements_to_copy; ++j) { + std::atomic_ref(raw_data[1 + j]) + .store(new_neighbors[j], std::memory_order_relaxed); + } + std::atomic_ref(raw_data[0]) + .store(elements_to_copy, std::memory_order_relaxed); + seq_counters_[i].end_write(seq); } /// @@ -208,10 +255,10 @@ template class SimpleGrap /// * ``get_node_degree(src) == max_degree()`` (adjacency list is already full) /// * ``dst`` is already an out-neighbor of ``src``. /// - size_t add_edge(Idx src, Idx dst) { + AddEdgeResult add_edge(Idx src, Idx dst) { // Don't assign a node as its own neighbor. if (src == dst) { - return get_node_degree(src); + return AddEdgeResult::AlreadyExists; } if constexpr (checkbounds_v) { @@ -225,11 +272,15 @@ template class SimpleGrap } } + // Acquire lock — all reads and writes under the lock to prevent + // concurrent writers from seeing stale state. + std::lock_guard lock{node_locks_[src]}; + // Check if there's room for the new node. std::span raw_data = data_.get_datum(src); Idx current_size = raw_data.front(); if (current_size == max_degree_) { - return current_size; + return AddEdgeResult::Full; } // At this point, we know there is room. @@ -248,17 +299,22 @@ template class SimpleGrap auto it = std::find(begin, end - 1, dst); // auto it = std::lower_bound(begin, end - 1, dst); if (it != end - 1 && (*it == dst)) { - return current_size; + return AddEdgeResult::AlreadyExists; } - // Insert at the new location. - std::copy_backward(it, end - 1, end); - (*it) = dst; + auto seq = seq_counters_[src].begin_write(); - // // Assign the new edge and update the number of neighbors. - // adjacency_list.back() = dst; - raw_data.front() = new_size; - return new_size; + // Insert at the new location using atomic stores. + for (auto dst_it = end - 1, src_it = end - 2; dst_it != it; --dst_it, --src_it) { + std::atomic_ref(*dst_it).store(*src_it, std::memory_order_relaxed); + } + std::atomic_ref(*it).store(dst, std::memory_order_relaxed); + + // Update the number of neighbors. + std::atomic_ref(raw_data.front()).store(new_size, std::memory_order_relaxed); + + seq_counters_[src].end_write(seq); + return AddEdgeResult::Added; } /// Return the maximum out-degree this graph is capable of containing. @@ -270,9 +326,16 @@ template class SimpleGrap data_type& get_data() { return data_; } // Resizeable API - void unsafe_resize(size_t new_size) { data_.resize(new_size); } + void unsafe_resize(size_t new_size) { + data_.resize(new_size); + seq_counters_.resize(new_size); + node_locks_.resize(new_size); + } void add_node() { unsafe_resize(n_nodes() + 1); } + /// @brief Access the per-node sequence lock counters for concurrent read validation. + const SeqLockArray& seq_counters() const { return seq_counters_; } + ///// Saving static constexpr lib::Version save_version = lib::Version(0, 0, 0); static constexpr std::string_view serialization_schema = "default_graph"; @@ -370,6 +433,8 @@ template class SimpleGrap protected: data_type data_; Idx max_degree_; + SeqLockArray seq_counters_; + std::vector node_locks_; }; ///// diff --git a/include/svs/core/translation.h b/include/svs/core/translation.h index 4b91a031f..0904d3e6b 100644 --- a/include/svs/core/translation.h +++ b/include/svs/core/translation.h @@ -144,6 +144,59 @@ class IDTranslator { internal_to_external_[internal_id] = lib::narrow(external_id); } + /// + /// @brief Insert mappings, replacing any existing mapping whose current internal + /// ID is considered stale by the caller (e.g. the associated slot has been + /// marked for deletion but the translator entry was not yet cleaned up). + /// + /// Throws if an external ID already maps to a non-stale internal ID (i.e. the + /// user is trying to re-insert a live mapping). In this case, the translator is + /// left partially modified — prior successful replacements are not rolled back. + /// + /// @param external Container of external IDs to insert. + /// @param internal Container of internal IDs to insert (same length as external). + /// @param is_stale Callable ``bool(internal_id_type)``. Returns true if the given + /// internal ID is stale and its existing mapping may be overwritten. + /// + template + void replace_stale_and_insert( + const External& external, const Internal& internal, IsStale&& is_stale + ) { + auto ext_begin = external.begin(); + auto ext_end = external.end(); + auto int_begin = internal.begin(); + auto int_end = internal.end(); + + if (std::distance(ext_begin, ext_end) != std::distance(int_begin, int_end)) { + throw ANNEXCEPTION( + "Length of external IDs is {} while the length of internal IDs is {}!", + std::distance(ext_begin, ext_end), + std::distance(int_begin, int_end) + ); + } + if (!lib::all_unique(ext_begin, ext_end)) { + throw ANNEXCEPTION("External IDs contain repeat elements!"); + } + if (!lib::all_unique(int_begin, int_end)) { + throw ANNEXCEPTION("Internal IDs contain repeat elements!"); + } + + auto e = ext_begin; + auto i = int_begin; + for (; e != ext_end; ++e, ++i) { + auto found = external_to_internal_.find(*e); + if (found != external_to_internal_.end()) { + auto old_internal = found->second; + if (!is_stale(old_internal)) { + throw ANNEXCEPTION("Index already contains external ID {}!", *e); + } + // Stale — erase reverse mapping; forward will be overwritten below. + internal_to_external_.erase(old_internal); + } + insert_translation(*e, *i); + } + } + /// /// @brief Return whether the external ID exists. /// @@ -180,6 +233,16 @@ class IDTranslator { return internal_to_external_.at(i); } + /// @brief Return the external ID, or a default if not found. + external_id_type + get_external_or(internal_id_type i, external_id_type default_val) const { + auto it = internal_to_external_.find(i); + if (it == internal_to_external_.end()) { + return default_val; + } + return it->second; + } + /// /// @brief Return a start forward iterator over the external->internal IDs. /// diff --git a/include/svs/index/vamana/consolidate.h b/include/svs/index/vamana/consolidate.h index 3a16410f9..e507d9197 100644 --- a/include/svs/index/vamana/consolidate.h +++ b/include/svs/index/vamana/consolidate.h @@ -196,8 +196,21 @@ class GraphConsolidator { all_candidates.clear(); for (auto dst : neighbors) { if (is_deleted(dst)) { - const auto& others = graph_.get_node(dst); - all_candidates.insert(others.begin(), others.end()); + // SeqLock retry: a concurrent consolidate may be writing dst's + // neighbors if dst is not deleted in the other consolidate's view. + for (;;) { + auto maybe_seq = graph_.seq_counters()[dst].read_begin(); + if (!maybe_seq) { + svs::detail::pause(); + continue; + } + const auto& others = graph_.get_node(dst); + all_candidates.insert(others.begin(), others.end()); + if (graph_.seq_counters()[dst].read_validate(*maybe_seq)) { + break; + } + svs::detail::pause(); + } } else { all_candidates.insert(dst); } @@ -250,40 +263,62 @@ class GraphConsolidator { continue; } - // Determine if any of the neighbors of this node are deleted. - const auto& neighbors = graph_.get_node(src); - if (std::none_of(neighbors.begin(), neighbors.end(), is_deleted)) { - continue; - } + // SeqLock retry: a concurrent consolidate's apply_updates may be + // writing src's neighbors while we read them. + for (;;) { + auto maybe_seq = graph_.seq_counters()[src].read_begin(); + if (!maybe_seq) { + svs::detail::pause(); + continue; + } - // Add all neighbors and neighbors-of-deleted-neighbors. - populate_candidates(all_candidates, neighbors, is_deleted); - - // Insert non-deleted candidates into the vector to prepare for pruning. - filter_candidates( - valid_candidates, - all_candidates, - accessor(data_, src), - accessor, - general_distance, - is_deleted - ); + // Determine if any of the neighbors of this node are deleted. + const auto& neighbors = graph_.get_node(src); + if (std::none_of(neighbors.begin(), neighbors.end(), is_deleted)) { + if (graph_.seq_counters()[src].read_validate(*maybe_seq)) { + break; + } + svs::detail::pause(); + continue; + } - size_t new_candidate_size = - std::min(valid_candidates.size(), params_.max_candidate_pool_size); - valid_candidates.resize(new_candidate_size); - heuristic_prune_neighbors( - prune_strategy(distance_), - params_.prune_to, - params_.alpha, - data_, - accessor, - general_distance, - src, - lib::as_const_span(valid_candidates), - final_candidates - ); - update_buffer.insert(i, final_candidates); + // Add all neighbors and neighbors-of-deleted-neighbors. + populate_candidates(all_candidates, neighbors, is_deleted); + + // Insert non-deleted candidates into the vector to prepare for + // pruning. + filter_candidates( + valid_candidates, + all_candidates, + accessor(data_, src), + accessor, + general_distance, + is_deleted + ); + + size_t new_candidate_size = + std::min(valid_candidates.size(), params_.max_candidate_pool_size); + valid_candidates.resize(new_candidate_size); + heuristic_prune_neighbors( + prune_strategy(distance_), + params_.prune_to, + params_.alpha, + data_, + accessor, + general_distance, + src, + lib::as_const_span(valid_candidates), + final_candidates + ); + + if (graph_.seq_counters()[src].read_validate(*maybe_seq)) { + // Consistent read — commit the results. + update_buffer.insert(i, final_candidates); + break; + } + svs::detail::pause(); + // Retry: discard stale candidates, recompute on next iteration. + } } } diff --git a/include/svs/index/vamana/dynamic_index.h b/include/svs/index/vamana/dynamic_index.h index 5f0ce7c16..d9d3e6ba0 100644 --- a/include/svs/index/vamana/dynamic_index.h +++ b/include/svs/index/vamana/dynamic_index.h @@ -17,7 +17,10 @@ #pragma once // stdlib +#include #include +#include +#include // Include the flat index to spin-up exhaustive searches on demand. #include "svs/index/flat/flat.h" @@ -64,7 +67,15 @@ class MultiMutableVamanaIndex; /// /// Only used for `MutableVamanaIndex`. /// -enum class SlotMetadata : uint8_t { Empty = 0x00, Valid = 0x01, Deleted = 0x02 }; +enum class SlotMetadata : uint8_t { + Empty = 0x00, + Valid = 0x01, + Deleted = 0x02, + // Reserved by an in-flight add_points: slot owned by the adder, vector + // copied, adjacency list being built. Invisible to search, consolidate, + // and subsequent add_points until promoted to Valid. + Pending = 0x04, +}; template inline constexpr std::string_view name(); template <> inline constexpr std::string_view name() { @@ -76,6 +87,9 @@ template <> inline constexpr std::string_view name() { template <> inline constexpr std::string_view name() { return "Deleted"; } +template <> inline constexpr std::string_view name() { + return "Pending"; +} // clang-format off inline constexpr std::string_view name(SlotMetadata metadata) { @@ -84,6 +98,7 @@ inline constexpr std::string_view name(SlotMetadata metadata) { SVS_SWITCH_RETURN(SlotMetadata::Empty) SVS_SWITCH_RETURN(SlotMetadata::Valid) SVS_SWITCH_RETURN(SlotMetadata::Deleted) + SVS_SWITCH_RETURN(SlotMetadata::Pending) } #undef SVS_SWITCH_RETURN throw ANNEXCEPTION("Unreachable!"); @@ -97,7 +112,13 @@ class ValidBuilder { template constexpr PredicatedSearchNeighbor operator()(I i, float distance) const { - bool invalid = getindex(status_, i) == SlotMetadata::Deleted; + // A neighbor is returnable only if its slot is Valid. Deleted slots + // must be skipped; Pending slots are reserved by an in-flight add and + // their vectors/edges are not yet fully published. Empty slots should + // never be reached via a valid edge, but we defend anyway. + bool invalid = + std::atomic_ref(const_cast(getindex(status_, i))) + .load(std::memory_order_acquire) != SlotMetadata::Valid; // This neighbor should be skipped if the metadata corresponding to the given index // marks this slot as deleted. return PredicatedSearchNeighbor(i, distance, !invalid); @@ -153,6 +174,14 @@ class MutableVamanaIndex { std::vector status_; size_t first_empty_ = 0; IDTranslator translator_; + // Count of Valid slots. Maintained atomically in add_points/delete_entry. + // Wrapped in unique_ptr because std::atomic is not movable. + std::unique_ptr> num_valid_{ + std::make_unique>(0)}; + // Protects translator access: exclusive for writes (add/consolidate/compact), + // shared for reads (delete/search). Wrapped in unique_ptr for movability. + std::unique_ptr translator_mutex_{ + std::make_unique()}; // Thread local data structures. distance_type distance_; @@ -192,6 +221,7 @@ class MutableVamanaIndex { , status_(data_.size(), SlotMetadata::Valid) , first_empty_{data_.size()} , translator_() + , num_valid_{std::make_unique>(data_.size())} , distance_{std::move(distance_function)} , threadpool_{threads::as_threadpool(std::move(threadpool_proto))} , search_parameters_{vamana::construct_default_search_parameters(data_)} @@ -219,6 +249,7 @@ class MutableVamanaIndex { , status_(data_.size(), SlotMetadata::Valid) , first_empty_{data_.size()} , translator_() + , num_valid_{std::make_unique>(data_.size())} , distance_(std::move(distance_function)) , threadpool_(threads::as_threadpool(std::move(threadpool_proto))) , search_parameters_(vamana::construct_default_search_parameters(data_)) @@ -285,6 +316,7 @@ class MutableVamanaIndex { , status_{data_.size(), SlotMetadata::Valid} , first_empty_{data_.size()} , translator_{std::move(translator)} + , num_valid_{std::make_unique>(data_.size())} , distance_{distance_function} , threadpool_{std::move(threadpool)} , search_parameters_{config.search_parameters} @@ -360,7 +392,15 @@ class MutableVamanaIndex { /// /// @brief Check whether the external ID `e` exists in the index. /// - bool has_id(size_t e) const { return translator_.has_external(e); } + bool has_id(size_t e) const { + if (!translator_.has_external(e)) { + return false; + } + // Check slot is not Deleted (deferred translator cleanup). + auto internal = translator_.get_internal(e); + return std::atomic_ref(const_cast(status_[internal])) + .load(std::memory_order_acquire) == SlotMetadata::Valid; + } /// /// @brief Get the external ID mapped to be `i`. @@ -369,7 +409,11 @@ class MutableVamanaIndex { /// /// Requires that mapping for `i` exists. Otherwise, all bets are off. /// - size_t translate_internal_id(Idx i) const { return translator_.get_external(i); } + size_t translate_internal_id(Idx i) const { + // Use get_external_or to handle concurrent consolidate erasing entries. + // If the entry was erased, return the internal ID as-is (stale result). + return translator_.get_external_or(i, static_cast(i)); + } /// /// @brief Call the functor with all external IDs in the index. @@ -378,8 +422,13 @@ class MutableVamanaIndex { /// each external ID in the index. /// template void on_ids(F&& f) const { + // Skip entries whose slot is Deleted (deferred translator cleanup). for (auto pair : translator_) { - f(pair.first); + auto internal = pair.second; + if (std::atomic_ref(const_cast(status_[internal])) + .load(std::memory_order_acquire) == SlotMetadata::Valid) { + f(pair.first); + } } } @@ -393,11 +442,7 @@ class MutableVamanaIndex { } /// @brief Return the number of **valid** (non-deleted) entries in the index. - size_t size() const { - // NB: Index translation should always be kept in-sync with the number of valid - // elements. - return translator_.size(); - } + size_t size() const { return num_valid_->load(std::memory_order_acquire); } /// /// @brief Translate in-place a collection of internal IDs to external IDs. @@ -421,7 +466,7 @@ class MutableVamanaIndex { template requires(std::tuple_size_v == 2) void translate_to_external(DenseArray& ids) { - // N.B.: lib::narrow_cast should be valid because the origin of the IDs is internal. + std::shared_lock lock{*translator_mutex_}; threads::parallel_for( threadpool_, threads::StaticPartition{getsize<0>(ids)}, @@ -641,55 +686,79 @@ class MutableVamanaIndex { ); } - // Gather all empty slots. + // Phase 1: Bookkeeping under lock — slot allocation, resize, translator, + // data copy. This serializes concurrent add_points() calls for the brief + // bookkeeping phase (~5-50μs). std::vector slots{}; - slots.reserve(num_points); - bool have_room = false; - - size_t s = reuse_empty ? 0 : first_empty_; - size_t smax = status_.size(); - for (; s < smax; ++s) { - if (status_[s] == SlotMetadata::Empty) { - slots.push_back(s); + { + std::lock_guard lock{*translator_mutex_}; + + // Gather all empty slots. + slots.reserve(num_points); + bool have_room = false; + + size_t s = reuse_empty ? 0 : first_empty_; + size_t smax = status_.size(); + for (; s < smax; ++s) { + if (status_[s] == SlotMetadata::Empty) { + slots.push_back(s); + } + if (slots.size() == num_points) { + have_room = true; + break; + } } - if (slots.size() == num_points) { - have_room = true; - break; + + // Check if we have enough indices. If we don't, we need to resize. + if (!have_room) { + size_t needed = num_points - slots.size(); + size_t current_size = data_.size(); + size_t new_size = current_size + needed; + data_.resize(new_size); + graph_.unsafe_resize(new_size); + status_.resize(new_size, SlotMetadata::Empty); + + threads::UnitRange extra_points{ + current_size, current_size + needed}; + slots.insert(slots.end(), extra_points.begin(), extra_points.end()); + } + assert(slots.size() == num_points); + + // Update the id translation. An existing mapping is stale only if + // the old slot is Deleted (delete_entries defers translator + // cleanup; consolidate normally drains those entries). A Pending + // slot belongs to an in-flight adder and MUST NOT be treated as + // stale — that would clobber the other adder's mapping. + translator_ + .replace_stale_and_insert(external_ids, slots, [this](auto internal) { + return std::atomic_ref( + const_cast(status_[internal]) + ) + .load(std::memory_order_acquire) == SlotMetadata::Deleted; + }); + + // Copy data and clear adjacency lists. + copy_points(points, slots); + clear_lists(slots); + + // Stamp the reserved slots as Pending before releasing the lock. + // Pending signals "reserved by an in-flight add; do not touch" + // to concurrent consolidate, delete, and add. Promoted to Valid + // after VamanaBuilder::construct finishes below. + for (auto s : slots) { + std::atomic_ref(status_[s]) + .store(SlotMetadata::Pending, std::memory_order_release); } - } - // Check if we have enough indices. If we don't, we need to resize the data and - // the graph. - if (!have_room) { - size_t needed = num_points - slots.size(); - size_t current_size = data_.size(); - size_t new_size = current_size + needed; - data_.resize(new_size); - - // Graph resizing marked as un-safe because graph contain internal references - // and thus it's not a good idea to go around shrinking the graph without care. - // - // However, we are only growing here, so resizing will not change any - // invariants. - graph_.unsafe_resize(new_size); - status_.resize(new_size, SlotMetadata::Empty); - - // Append the correct number of extra slots. - threads::UnitRange extra_points{current_size, current_size + needed}; - slots.insert(slots.end(), extra_points.begin(), extra_points.end()); + if (!slots.empty()) { + first_empty_ = std::max(first_empty_, slots.back() + 1); + } } - assert(slots.size() == num_points); - // Try to update the id translation now that we have internal ids. - // If this fails, we still haven't mutated the index data structure so we're safe - // to throw an exception. - translator_.insert(external_ids, slots); - - // Copy the given points into the data and clear the adjacency lists for the graph. - copy_points(points, slots); - clear_lists(slots); - - // Patch in the new neighbors. + // Phase 2: Graph construction — runs WITHOUT lock. + // VamanaBuilder::construct() is thread-safe via per-node spinlock+seqlock. + // NOTE: VamanaBuilder constructor asserts graph_.n_nodes() == data_.size(). + // Both are grown together under the lock above, so this is always consistent. auto parameters = VamanaBuildParameters{ alpha_, graph_.max_degree(), @@ -711,14 +780,14 @@ class MutableVamanaIndex { logger_, logging::Level::Trace}; builder.construct(alpha_, entry_point(), slots, logging::Level::Trace, logger_); - // Mark all added entries as valid. + + // Mark added entries as valid (unique slots per thread, no lock needed). for (const auto& i : slots) { - status_[i] = SlotMetadata::Valid; + std::atomic_ref(status_[i]) + .store(SlotMetadata::Valid, std::memory_order_release); } + num_valid_->fetch_add(slots.size(), std::memory_order_acq_rel); - if (!slots.empty()) { - first_empty_ = std::max(first_empty_, slots.back() + 1); - } return slots; } @@ -745,21 +814,57 @@ class MutableVamanaIndex { /// graph. /// template size_t delete_entries(const T& ids) { - translator_.check_external_exist(ids.begin(), ids.end()); + std::shared_lock lock{*translator_mutex_}; + size_t deleted = 0; for (auto i : ids) { + if (!translator_.has_external(i)) { + continue; // Already deleted + consolidated, or never existed. + } delete_entry(translator_.get_internal(i)); + ++deleted; } - translator_.delete_external(ids); - return ids.size(); + // Don't erase translator entries here — concurrent search may still + // need them for translate_to_external(). Cleanup happens in + // consolidate()/compact() when deleted slots become empty. + return deleted; } void delete_entry(size_t i) { - SlotMetadata& meta = getindex(status_, i); - assert(meta == SlotMetadata::Valid); - meta = SlotMetadata::Deleted; + auto& meta = getindex(status_, i); + auto ref = std::atomic_ref(meta); + // CAS Valid → Deleted. If the slot is Pending (concurrent adder still + // in phase 2), wait for the adder to promote it to Valid before we + // can soft-delete; otherwise the delete would be silently lost. Only + // the thread that successfully transitions decrements num_valid_; + // double-deletes silently no-op. + for (;;) { + SlotMetadata expected = SlotMetadata::Valid; + if (ref.compare_exchange_strong( + expected, + SlotMetadata::Deleted, + std::memory_order_acq_rel, + std::memory_order_relaxed + )) { + num_valid_->fetch_sub(1, std::memory_order_acq_rel); + return; + } + if (expected != SlotMetadata::Pending) { + // Already Deleted or Empty — no-op. + return; + } + // Pending: adder's Pending → Valid store is imminent; spin. + svs::detail::pause(); + } } - bool is_deleted(size_t i) const { return status_[i] != SlotMetadata::Valid; } + bool is_deleted(size_t i) const { + // True only for slots that have been soft-deleted. Pending (in-flight + // add) and Empty are NOT deleted: consolidate must not prune them out + // of other nodes' adjacency lists, and search already filters + // non-Valid slots via ValidBuilder. + return std::atomic_ref(const_cast(status_[i])) + .load(std::memory_order_acquire) == SlotMetadata::Deleted; + } Idx entry_point() const { assert(entry_point_.size() == 1); @@ -767,15 +872,21 @@ class MutableVamanaIndex { } /// - /// @brief Return all the non-missing internal IDs. - /// - /// This includes both valid and soft-deleted entries. + /// @brief Return all internal IDs whose slot is Valid (live). /// + /// Used by compact() to pick the surviving set. Pending slots (in-flight + /// adds) are excluded — compact is only safe to run when the caller has + /// ensured no Pending slots exist (compact holds translator_mutex_ + /// exclusive, which prevents a new add from entering phase 1, but an add + /// that reached phase 2 before compact grabbed the lock may still be + /// publishing status Pending → Valid; the compact caller must quiesce + /// these adds first). std::vector nonmissing_indices() const { auto indices = std::vector(); indices.reserve(size()); for (size_t i = 0, imax = status_.size(); i < imax; ++i) { - if (!is_deleted(i)) { + if (std::atomic_ref(const_cast(status_[i])) + .load(std::memory_order_acquire) == SlotMetadata::Valid) { indices.push_back(i); } } @@ -862,30 +973,32 @@ class MutableVamanaIndex { } ///// Finishing steps. - // Resize the graph and data. - graph_.unsafe_resize(max_index); - data_.resize(max_index); - first_empty_ = max_index; - - // Compact metadata and ID remapping. - for (size_t new_id = 0; new_id < max_index; ++new_id) { - auto old_id = getindex(new_to_old_id_map, new_id); - // No work to be done if there was no remapping. - if (new_id == old_id) { - continue; - } + { + std::lock_guard lock{*translator_mutex_}; + // Resize the graph and data. + graph_.unsafe_resize(max_index); + data_.resize(max_index); + first_empty_ = max_index; + + // Compact metadata and ID remapping. + for (size_t new_id = 0; new_id < max_index; ++new_id) { + auto old_id = getindex(new_to_old_id_map, new_id); + if (new_id == old_id) { + continue; + } - auto status = getindex(status_, old_id); - status_[new_id] = status; - if (status == SlotMetadata::Valid) { - translator_.remap_internal_id(old_id, new_id); + auto status = getindex(status_, old_id); + status_[new_id] = status; + if (status == SlotMetadata::Valid) { + translator_.remap_internal_id(old_id, new_id); + } } - } - status_.resize(max_index); + status_.resize(max_index); - // Update entry points. - for (auto& ep : entry_point_) { - ep = old_to_new_id_map.at(ep); + // Update entry points. + for (auto& ep : entry_point_) { + ep = old_to_new_id_map.at(ep); + } } } @@ -978,10 +1091,26 @@ class MutableVamanaIndex { check_is_deleted ); - // After consolidation - set all `Deleted` slots to `Empty`. - for (auto& status : status_) { - if (status == SlotMetadata::Deleted) { - status = SlotMetadata::Empty; + // After consolidation - clean up deleted slots under lock. + { + std::lock_guard lock{*translator_mutex_}; + // Erase translator entries for deleted slots (deferred from delete_entries). + // Skip entries already absent — add_points with replace_stale_and_insert + // may have reassigned the external ID and erased the stale reverse entry. + std::vector deleted_internal_ids; + for (size_t i = 0, imax = status_.size(); i < imax; ++i) { + if (status_[i] == SlotMetadata::Deleted && translator_.has_internal(i)) { + deleted_internal_ids.push_back(i); + } + } + if (!deleted_internal_ids.empty()) { + translator_.delete_internal(deleted_internal_ids, false); + } + // Set all `Deleted` slots to `Empty`. + for (auto& status : status_) { + if (status == SlotMetadata::Deleted) { + status = SlotMetadata::Empty; + } } } } @@ -1239,6 +1368,11 @@ class MutableVamanaIndex { case SlotMetadata::Empty: { return false; } + case SlotMetadata::Pending: { + // In-flight add: edges may be only partially published. + // Treat as not-yet-live for consistency checking. + return false; + } } // Make GCC happy. return false; diff --git a/include/svs/index/vamana/greedy_search.h b/include/svs/index/vamana/greedy_search.h index f12c0129f..ac91331fe 100644 --- a/include/svs/index/vamana/greedy_search.h +++ b/include/svs/index/vamana/greedy_search.h @@ -20,6 +20,8 @@ #include "svs/concepts/distance.h" #include "svs/concepts/graph.h" #include "svs/index/vamana/search_buffer.h" +#include "svs/lib/concurrency/seqlock.h" +#include "svs/lib/spinlock.h" #include #include @@ -159,45 +161,62 @@ void greedy_search( const auto& node = search_buffer.next(); auto node_id = node.id(); - // Get the adjacency list for this vertex and prepare prefetching logic. - auto neighbors = graph.get_node(node_id); - const size_t num_neighbors = neighbors.size(); - search_tracker.visited(Neighbor{node}, neighbors.size()); - - auto prefetcher = lib::make_prefetcher( - lib::PrefetchParameters{ - prefetch_parameters.lookahead, prefetch_parameters.step}, - num_neighbors, - [&](size_t i) { accessor.prefetch(dataset, neighbors[i]); }, - [&](size_t i) { - // Perform the visited set enabled check just once. - if (search_buffer.visited_set_enabled()) { - // Prefetch next bucket so it's (hopefully) in the cache when we next - // consult the visited filter. - if (i + 1 < num_neighbors) { - search_buffer.unsafe_prefetch_visited(neighbors[i + 1]); + for (;;) { // SeqLock retry loop + auto maybe_seq = graph.seq_counters()[node_id].read_begin(); + if (!maybe_seq) { + detail::pause(); + continue; + } + + // Get the adjacency list for this vertex and prepare prefetching logic. + auto neighbors = graph.get_node(node_id); + const size_t num_neighbors = neighbors.size(); + search_tracker.visited(Neighbor{node}, num_neighbors); + + auto prefetcher = lib::make_prefetcher( + lib::PrefetchParameters{ + prefetch_parameters.lookahead, prefetch_parameters.step}, + num_neighbors, + [&](size_t i) { accessor.prefetch(dataset, neighbors[i]); }, + [&](size_t i) { + // Perform the visited set enabled check just once. + if (search_buffer.visited_set_enabled()) { + // Prefetch next bucket so it's (hopefully) in the cache when + // we next consult the visited filter. + if (i + 1 < num_neighbors) { + search_buffer.unsafe_prefetch_visited(neighbors[i + 1]); + } + return !search_buffer.unsafe_is_visited(neighbors[i]); } - return !search_buffer.unsafe_is_visited(neighbors[i]); + + // Otherwise, always prefetch the next data item. + return true; } + ); - // Otherwise, always prefetch the next data item. - return true; - } - ); + ///// Neighbor expansion. + prefetcher(); + for (auto id : neighbors) { + if (search_buffer.emplace_visited(id)) { + continue; + } - ///// Neighbor expansion. - prefetcher(); - for (auto id : neighbors) { - if (search_buffer.emplace_visited(id)) { - continue; - } + // Run the prefetcher. + prefetcher(); - // Run the prefetcher. - prefetcher(); + // Compute distance and update search buffer. + auto dist = + distance::compute(distance_function, query, accessor(dataset, id)); + search_buffer.insert(builder(id, dist)); + } - // Compute distance and update search buffer. - auto dist = distance::compute(distance_function, query, accessor(dataset, id)); - search_buffer.insert(builder(id, dist)); + // Validate that no concurrent write occurred during the read. + if (graph.seq_counters()[node_id].read_validate(*maybe_seq)) { + break; // Consistent read — proceed to the next node. + } + detail::pause(); + // Retry: stale entries from the invalid read remain in the search buffer. + // They have valid IDs and distances, and insert() deduplicates by ID. } } } diff --git a/include/svs/index/vamana/vamana_build.h b/include/svs/index/vamana/vamana_build.h index 77d5ceadb..a7b9b7946 100644 --- a/include/svs/index/vamana/vamana_build.h +++ b/include/svs/index/vamana/vamana_build.h @@ -194,7 +194,6 @@ class VamanaBuilder { , params_{params} , prefetch_hint_{prefetch_hint} , threadpool_{threadpool} - , vertex_locks_(data.size()) , backedge_buffer_{data.size(), 1000} { // Print all parameters svs::logging::log( @@ -210,12 +209,10 @@ class VamanaBuilder { params.window_size, params.use_full_search_history ); - // Check class invariants. - if (graph_.n_nodes() != data_.size()) { - throw ANNEXCEPTION( - "Expected graph to be pre-allocated with {} vertices!", data_.size() - ); - } + // Note: graph/data size invariant (graph_.n_nodes() == data_.size()) is + // maintained under mutation_mutex_ in add_points(). During concurrent + // add_points() calls, sizes may temporarily differ between lock release + // and this point, but both are always >= the slots we will operate on. } void construct( @@ -494,10 +491,11 @@ class VamanaBuilder { [&](const auto& is, uint64_t SVS_UNUSED(tid)) { for (auto node_id : is) { for (auto other_id : graph_.get_node(node_id)) { - std::lock_guard lock{vertex_locks_[other_id]}; - if (graph_.get_node_degree(other_id) < params_.graph_max_degree) { - graph_.add_edge(other_id, node_id); - } else { + // graph_.add_edge is atomic under node_locks_[other_id]. + // If it reports Full, route to the overflow buffer — + // no TOCTOU race between a pre-check and the insert. + if (graph_.add_edge(other_id, node_id) == + graphs::AddEdgeResult::Full) { backedge_buffer_.add_edge(other_id, node_id); } } @@ -591,8 +589,6 @@ class VamanaBuilder { GreedySearchPrefetchParameters prefetch_hint_; /// Worker threadpool. Pool& threadpool_; - /// Per-vertex locks. - std::vector vertex_locks_; /// Overflow backedge buffer. BackedgeBuffer backedge_buffer_; }; diff --git a/include/svs/lib/concurrency/atomic_span.h b/include/svs/lib/concurrency/atomic_span.h new file mode 100644 index 000000000..4d7d4ae50 --- /dev/null +++ b/include/svs/lib/concurrency/atomic_span.h @@ -0,0 +1,93 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +namespace svs { + +/// +/// @brief A non-owning, zero-copy view over a contiguous range of ``T`` that performs +/// atomic loads on every element access. +/// +/// Each dereference uses ``std::atomic_ref::load(std::memory_order_relaxed)``. +/// On x86, this compiles to a plain MOV instruction — identical to non-atomic access. +/// +/// This type is designed to be used as a drop-in replacement for ``std::span`` +/// when concurrent reads and writes are possible, ensuring no undefined behavior +/// while maintaining zero-copy semantics. +/// +template class AtomicSpan { + public: + using value_type = std::remove_const_t; + + class iterator { + public: + using value_type = AtomicSpan::value_type; + using difference_type = std::ptrdiff_t; + using iterator_category = std::input_iterator_tag; + + explicit iterator(const T* p) + : ptr_(p) {} + + value_type operator*() const { + return std::atomic_ref(const_cast(*ptr_)) + .load(std::memory_order_relaxed); + } + + iterator& operator++() { + ++ptr_; + return *this; + } + + iterator operator++(int) { + auto tmp = *this; + ++ptr_; + return tmp; + } + + bool operator==(const iterator& other) const { return ptr_ == other.ptr_; } + bool operator!=(const iterator& other) const { return ptr_ != other.ptr_; } + + private: + const T* ptr_; + }; + + AtomicSpan(const T* data, size_t size) + : data_(data) + , size_(size) {} + + size_t size() const { return size_; } + bool empty() const { return size_ == 0; } + const T* data() const { return data_; } + + value_type operator[](size_t i) const { + return std::atomic_ref(const_cast(data_[i])) + .load(std::memory_order_relaxed); + } + + iterator begin() const { return iterator{data_}; } + iterator end() const { return iterator{data_ + size_}; } + + private: + const T* data_; + size_t size_; +}; + +} // namespace svs diff --git a/include/svs/lib/concurrency/seqlock.h b/include/svs/lib/concurrency/seqlock.h new file mode 100644 index 000000000..9dd21131f --- /dev/null +++ b/include/svs/lib/concurrency/seqlock.h @@ -0,0 +1,135 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +namespace svs { + +/// +/// @brief Per-element sequence lock counter for reader-writer synchronization. +/// +/// Uses a uint8_t counter: odd values indicate a write in progress, even values indicate +/// a stable state. +/// +/// **Writer-writer serialization is the caller's responsibility.** Only one writer +/// at a time may call ``begin_write``/``end_write`` on a given counter. Use an external +/// lock (e.g., per-node ``SpinLock``) to serialize concurrent writers to the same element. +/// +class SeqLockCounter { + using counter_type = uint8_t; + + public: + SeqLockCounter() = default; + + SeqLockCounter(const SeqLockCounter& other) + : seq_(other.seq_.load(std::memory_order_relaxed)) {} + + SeqLockCounter& operator=(const SeqLockCounter& other) { + seq_.store(other.seq_.load(std::memory_order_relaxed), std::memory_order_relaxed); + return *this; + } + + SeqLockCounter(SeqLockCounter&& other) noexcept + : seq_(other.seq_.load(std::memory_order_relaxed)) {} + + SeqLockCounter& operator=(SeqLockCounter&& other) noexcept { + seq_.store(other.seq_.load(std::memory_order_relaxed), std::memory_order_relaxed); + return *this; + } + + /// + /// @brief Begin a write operation. Returns the pre-write sequence value. + /// + /// Increments the counter to an odd value, signaling to readers that a write is in + /// progress. The returned value must be passed to ``end_write``. + /// + counter_type begin_write() { + auto seq = seq_.load(std::memory_order_relaxed); + seq_.store(seq + 1, std::memory_order_relaxed); + std::atomic_thread_fence(std::memory_order_release); + return seq; + } + + /// + /// @brief End a write operation. + /// + /// @param seq The value returned by the corresponding ``begin_write`` call. + /// + /// Increments the counter to an even value, signaling that the write is complete + /// and data is consistent. + /// + void end_write(counter_type seq) { seq_.store(seq + 2, std::memory_order_release); } + + /// + /// @brief Begin a read operation. + /// + /// @returns The current sequence value if it is even (no write in progress), + /// or ``std::nullopt`` if a write is in progress. + /// + /// The returned value (if present) must be passed to ``read_validate`` after the + /// read is complete. + /// + std::optional read_begin() const { + auto seq = seq_.load(std::memory_order_acquire); + if (seq % 2 > 0) { + return std::nullopt; + } + return seq; + } + + /// + /// @brief Validate that no write occurred during the read. + /// + /// @param seq The value returned by ``read_begin``. + /// + /// @returns ``true`` if the data read between ``read_begin`` and ``read_validate`` + /// is consistent (no concurrent write occurred). + /// + bool read_validate(counter_type seq) const { + std::atomic_thread_fence(std::memory_order_acquire); + return seq_.load(std::memory_order_relaxed) == seq; + } + + private: + std::atomic seq_{0}; +}; + +/// +/// @brief Array of SeqLock counters, one per element (e.g., one per graph node). +/// +class SeqLockArray { + public: + SeqLockArray() = default; + explicit SeqLockArray(size_t n) + : counters_(n) {} + + SeqLockCounter& operator[](size_t i) { return counters_[i]; } + const SeqLockCounter& operator[](size_t i) const { return counters_[i]; } + + void resize(size_t n) { counters_.resize(n); } + size_t size() const { return counters_.size(); } + + private: + std::vector counters_; +}; + +} // namespace svs diff --git a/include/svs/lib/spinlock.h b/include/svs/lib/spinlock.h index 3f3b85486..b4b644dbe 100644 --- a/include/svs/lib/spinlock.h +++ b/include/svs/lib/spinlock.h @@ -37,6 +37,13 @@ class SpinLock { public: SpinLock() = default; + SpinLock(const SpinLock& /*unused*/) + : value_{false} {} + SpinLock& operator=(const SpinLock& /*unused*/) { return *this; } + SpinLock(SpinLock&& /*unused*/) noexcept + : value_{false} {} + SpinLock& operator=(SpinLock&& /*unused*/) noexcept { return *this; } + /// /// Implement C++ named requirements "Lockable" ///