fix(hgraph): backport duplicate handling to v0.18

docs/hgraph.md

@@ -225,10 +225,21 @@ means that the index uses PQ quantization with 64 subspaces, enables reordering
 - **Optional Values**: 1 to INT_MAX
 - **Default Value**: Must be provided (no default value)
 
+### max_duplicates_per_group
+- **Parameter Type**: int
+- **Parameter Description**: Maximum number of additional duplicate IDs returned for each
+  duplicate group during KNN graph search
+- **Optional Values**: -1 (unlimited), 0 (do not expand duplicate IDs), or any positive integer
+- **Default Value**: -1
+- **Notes**: Only applies when `support_duplicate` is enabled; range search keeps its existing
+  expansion behavior
+
 ## Examples for Search Parameter String
 ```json
 "hgraph": {
-    "ef_search": 200
+    "ef_search": 200,
+    "max_duplicates_per_group": 2
 }
 ```
-means that the search will use an ef_search value of 200 to control the search quality and performance trade-off.
+means that the search uses an ef_search value of 200 and returns at most two additional IDs from
+each duplicate group.

include/vsag/constants.h

@@ -184,6 +184,7 @@ extern const char* const HGRAPH_PRECISE_IO_TYPE;
 extern const char* const HGRAPH_PRECISE_FILE_PATH;
 extern const char* const HGRAPH_PARAMETER_EF_RUNTIME;
 extern const char* const HGRAPH_PARAMETER_HOPS_LIMIT;
+extern const char* const HGRAPH_PARAMETER_MAX_DUPLICATES_PER_GROUP;
 extern const char* const HGRAPH_EXTRA_INFO_SIZE;
 extern const char* const HGRAPH_SUPPORT_DUPLICATE;
 extern const char* const HGRAPH_DUPLICATE_DISTANCE_THRESHOLD;

src/algorithm/hgraph.cpp

@@ -44,6 +44,18 @@
 
 namespace vsag {
 
+namespace {
+
+// Layout: [magic][version][duplicate records][end magic]. It is appended after the complete v0.14
+// payload so legacy readers can ignore it without shifting existing fields. The end magic differs
+// from the native-format footer magic so Footer::Parse keeps using the legacy branch.
+constexpr uint64_t V0_14_DUPLICATE_EXTENSION_MAGIC = 0x3150554447415356ULL;
+constexpr uint64_t V0_14_DUPLICATE_EXTENSION_END_MAGIC = 0x5653414744555031ULL;
+constexpr uint64_t V0_14_DUPLICATE_EXTENSION_VERSION = 1;
+constexpr uint64_t V0_14_DUPLICATE_EXTENSION_FIXED_SIZE = sizeof(uint64_t) * 3;
+
+}  // namespace
+
 static DatasetPtr
 make_empty_dataset_with_stats() {
     SearchStatistics stats;
@@ -933,10 +945,20 @@ HGraph::KnnSearch(const DatasetPtr& query,
     auto search_result = DistanceHeap::MakeInstanceBySize<true, false>(ctx.alloc, k);
     const auto* query_data = get_data(query);
     if (is_last_filter) {
+        if (const auto* pending_duplicates = iter_filter_ctx->GetPendingDuplicates();
+            pending_duplicates != nullptr) {
+            for (const auto& [pending_id, pending_dist] : *pending_duplicates) {
+                if (iter_filter_ctx->CheckPoint(pending_id)) {
+                    search_result->Push(pending_dist, pending_id);
+                }
+            }
+        }
         while (!iter_filter_ctx->Empty()) {
             uint32_t cur_inner_id = iter_filter_ctx->GetTopID();
             float cur_dist = iter_filter_ctx->GetTopDist();
-            search_result->Push(cur_dist, cur_inner_id);
+            if (not iter_filter_ctx->IsPendingDuplicate(cur_inner_id)) {
+                search_result->Push(cur_dist, cur_inner_id);
+            }
             iter_filter_ctx->PopDiscard();
         }
     } else {
@@ -964,6 +986,7 @@ HGraph::KnnSearch(const DatasetPtr& query,
         search_param.is_inner_id_allowed = ft;
         search_param.topk = static_cast<int64_t>(search_param.ef);
         search_param.consider_duplicate = this->label_table_->CompressDuplicateData();
+        search_param.max_duplicates_per_group = params.max_duplicates_per_group;
         search_param.parallel_search_thread_count = params.parallel_search_thread_count;
         search_param.min_distance = params.min_distance;
 
@@ -1283,6 +1306,70 @@ HGraph::deserialize_basic_info_v0_14(StreamReader& reader) {
     this->label_table_->total_count_.store(static_cast<int64_t>(size));
 }
 
+void
+HGraph::serialize_duplicate_info_v0_14(StreamWriter& writer) const {
+    if (not this->label_table_->CompressDuplicateData() or
+        this->label_table_->duplicate_count_ == 0) {
+        return;
+    }
+
+    StreamWriter::WriteObj(writer, V0_14_DUPLICATE_EXTENSION_MAGIC);
+    StreamWriter::WriteObj(writer, V0_14_DUPLICATE_EXTENSION_VERSION);
+    this->label_table_->SerializeDuplicateRecords(writer);
+    StreamWriter::WriteObj(writer, V0_14_DUPLICATE_EXTENSION_END_MAGIC);
+}
+
+void
+HGraph::deserialize_duplicate_info_v0_14(StreamReader& reader) const {
+    const uint64_t extension_start = reader.GetCursor();
+    const uint64_t stream_size = reader.Length();
+    if (extension_start >= stream_size) {
+        return;
+    }
+
+    const uint64_t remaining_size = stream_size - extension_start;
+    if (remaining_size < sizeof(uint64_t)) {
+        return;
+    }
+
+    uint64_t begin_magic = 0;
+    StreamReader::ReadObj(reader, begin_magic);
+    uint64_t end_magic = 0;
+    reader.PushSeek(stream_size - sizeof(uint64_t));
+    StreamReader::ReadObj(reader, end_magic);
+    reader.PopSeek();
+
+    if (begin_magic != V0_14_DUPLICATE_EXTENSION_MAGIC) {
+        reader.Seek(extension_start);
+        if (end_magic == V0_14_DUPLICATE_EXTENSION_END_MAGIC) {
+            throw VsagException(ErrorType::INVALID_BINARY,
+                                "invalid v0.14 duplicate extension header");
+        }
+        return;
+    }
+    if (end_magic != V0_14_DUPLICATE_EXTENSION_END_MAGIC or
+        remaining_size < V0_14_DUPLICATE_EXTENSION_FIXED_SIZE) {
+        throw VsagException(ErrorType::INVALID_BINARY, "invalid v0.14 duplicate extension framing");
+    }
+
+    uint64_t version = 0;
+    StreamReader::ReadObj(reader, version);
+    if (version != V0_14_DUPLICATE_EXTENSION_VERSION) {
+        throw VsagException(
+            ErrorType::INVALID_BINARY,
+            fmt::format("unsupported v0.14 duplicate extension version {}", version));
+    }
+
+    const uint64_t payload_size = remaining_size - V0_14_DUPLICATE_EXTENSION_FIXED_SIZE;
+    auto payload_reader = reader.Slice(payload_size);
+    this->label_table_->DeserializeDuplicateRecords(payload_reader);
+    if (payload_reader.GetCursor() != payload_reader.Length()) {
+        throw VsagException(ErrorType::INVALID_BINARY,
+                            "v0.14 duplicate extension payload size mismatch");
+    }
+    StreamReader::ReadObj(reader, end_magic);
+}
+
 #define TO_JSON_BASE64(json_obj, var) json_obj[#var].SetString(base64_encode_obj(this->var##_));
 
 JsonType
@@ -1410,6 +1497,7 @@ HGraph::Serialize(StreamWriter& writer) const {
         if (this->use_attribute_filter_ and this->attr_filter_index_ != nullptr) {
             this->attr_filter_index_->Serialize(writer);
         }
+        this->serialize_duplicate_info_v0_14(writer);
         return;
     }
 
@@ -1474,6 +1562,7 @@ HGraph::Deserialize(StreamReader& reader) {
         if (this->use_attribute_filter_ and this->attr_filter_index_ != nullptr) {
             this->attr_filter_index_->Deserialize(reader);
         }
+        this->deserialize_duplicate_info_v0_14(reader);
     } else {  // create like `else if ( ver in [v0.15, v0.17] )` here if need in the future
         logger::debug("parse with new version format");
 
@@ -2222,6 +2311,7 @@ HGraph::SearchWithRequest(const SearchRequest& request) const {
             search_param.topk, static_cast<int64_t>(static_cast<float>(k) * params.topk_factor));
     }
     search_param.consider_duplicate = true;
+    search_param.max_duplicates_per_group = params.max_duplicates_per_group;
     if (params.enable_time_record) {
         search_param.time_cost = std::make_shared<Timer>();
         search_param.time_cost->SetThreshold(params.timeout_ms);

src/algorithm/hgraph.h

@@ -292,6 +292,12 @@ class HGraph : public InnerIndexInterface {
     void
     deserialize_basic_info_v0_14(StreamReader& reader);
 
+    void
+    serialize_duplicate_info_v0_14(StreamWriter& writer) const;
+
+    void
+    deserialize_duplicate_info_v0_14(StreamReader& reader) const;
+
 private:
     void
     reorder(const void* query,

src/algorithm/hgraph_parameter.cpp

@@ -15,6 +15,8 @@
 
 #include "hgraph_parameter.h"
 
+#include <nlohmann/json.hpp>
+
 #include "datacell/extra_info_datacell_parameter.h"
 #include "datacell/flatten_datacell_parameter.h"
 #include "datacell/graph_datacell_parameter.h"
@@ -223,6 +225,22 @@ HGraphSearchParameters::FromJson(const std::string& json_string) {
     if (params[INDEX_TYPE_HGRAPH].Contains("min_distance")) {
         obj.min_distance = params[INDEX_TYPE_HGRAPH]["min_distance"].GetFloat();
     }
+    if (params[INDEX_TYPE_HGRAPH].Contains(HGRAPH_PARAMETER_MAX_DUPLICATES_PER_GROUP)) {
+        const auto& max_duplicates =
+            params[INDEX_TYPE_HGRAPH][HGRAPH_PARAMETER_MAX_DUPLICATES_PER_GROUP];
+        CHECK_ARGUMENT(max_duplicates.IsNumberInteger(),
+                       "max_duplicates_per_group must be an integer");
+        const auto* max_duplicates_json = max_duplicates.GetInnerJson();
+        if (max_duplicates_json->is_number_unsigned()) {
+            CHECK_ARGUMENT(max_duplicates_json->get<uint64_t>() <=
+                               static_cast<uint64_t>(std::numeric_limits<int64_t>::max()),
+                           "max_duplicates_per_group exceeds int64_t range");
+        }
+        obj.max_duplicates_per_group = max_duplicates.GetInt();
+        CHECK_ARGUMENT(obj.max_duplicates_per_group >= -1,
+                       fmt::format("max_duplicates_per_group({}) must be >= -1",
+                                   obj.max_duplicates_per_group));
+    }
 
     return obj;
 }

src/algorithm/hgraph_parameter.h

@@ -82,6 +82,9 @@ class HGraphSearchParameters : public IndexSearchParameter {
     bool use_reorder{false};
     bool use_extra_info_filter{false};
     float min_distance{std::numeric_limits<float>::lowest()};
+    // Maximum additional duplicate IDs returned per group during KNN graph search.
+    // -1 means unlimited; 0 disables duplicate expansion.
+    int64_t max_duplicates_per_group{-1};
 
 private:
     HGraphSearchParameters() = default;

src/algorithm/hgraph_parameter_test.cpp

@@ -204,6 +204,24 @@ TEST_CASE("HGraph maps support_duplicate to graph parameter", "[ut][HGraphParame
     REQUIRE(typed_param->duplicate_distance_threshold == 0.25F);
 }
 
+TEST_CASE("HGraph Search Parameters validate max_duplicates_per_group",
+          "[ut][HGraphSearchParameters][duplicate]") {
+    REQUIRE(vsag::HGraphSearchParameters::FromJson(R"({"hgraph": {"ef_search": 32}})")
+                .max_duplicates_per_group == -1);
+    REQUIRE(vsag::HGraphSearchParameters::FromJson(
+                R"({"hgraph": {"ef_search": 32, "max_duplicates_per_group": 0}})")
+                .max_duplicates_per_group == 0);
+    REQUIRE(vsag::HGraphSearchParameters::FromJson(
+                R"({"hgraph": {"ef_search": 32, "max_duplicates_per_group": 2}})")
+                .max_duplicates_per_group == 2);
+    REQUIRE_THROWS(vsag::HGraphSearchParameters::FromJson(
+        R"({"hgraph": {"ef_search": 32, "max_duplicates_per_group": -2}})"));
+    REQUIRE_THROWS(vsag::HGraphSearchParameters::FromJson(
+        R"({"hgraph": {"ef_search": 32, "max_duplicates_per_group": 1.5}})"));
+    REQUIRE_THROWS(vsag::HGraphSearchParameters::FromJson(
+        R"({"hgraph": {"ef_search": 32, "max_duplicates_per_group": 9223372036854775808}})"));
+}
+
 TEST_CASE("HGraph maps label_remap_type to inner index parameter", "[ut][HGraphParameter]") {
     auto param = vsag::JsonType::Parse(R"({
         "base_quantization_type": "fp32",

src/constants.cpp

@@ -164,6 +164,7 @@ const char* const HGRAPH_PRECISE_IO_TYPE = "precise_io_type";
 const char* const HGRAPH_PRECISE_FILE_PATH = "precise_file_path";
 const char* const HGRAPH_PARAMETER_EF_RUNTIME = "ef_search";
 const char* const HGRAPH_PARAMETER_HOPS_LIMIT = "hops_limit";
+const char* const HGRAPH_PARAMETER_MAX_DUPLICATES_PER_GROUP = "max_duplicates_per_group";
 const char* const HGRAPH_EXTRA_INFO_SIZE = "extra_info_size";
 const char* const HGRAPH_SUPPORT_DUPLICATE = "support_duplicate";
 const char* const HGRAPH_DUPLICATE_DISTANCE_THRESHOLD = "duplicate_distance_threshold";

src/impl/inner_search_param.h

@@ -64,6 +64,7 @@ class InnerSearchParam {
 
     // use in search process with duplicate ids
     bool consider_duplicate{false};
+    int64_t max_duplicates_per_group{-1};
 
     // skip results with dist <= min_distance (for search iterator)
     float min_distance{std::numeric_limits<float>::lowest()};
@@ -87,6 +88,8 @@ class InnerSearchParam {
             factor = other.factor;
             first_order_scan_ratio = other.first_order_scan_ratio;
             parallel_search_thread_count = other.parallel_search_thread_count;
+            consider_duplicate = other.consider_duplicate;
+            max_duplicates_per_group = other.max_duplicates_per_group;
         }
         return *this;
     }