From 9dcb64e9a98631493878a364f696d3c5d7548c4d Mon Sep 17 00:00:00 2001 From: Sanhaoji2 Date: Tue, 13 Jan 2026 17:04:13 +0800 Subject: [PATCH 1/2] separate memory data file path --- include/abstract_index.h | 2 ++ include/index.h | 3 +++ include/parameters.h | 38 +++++++++++++++++++++++++++++++ src/index.cpp | 49 +++++++++++++++++++++++++++++----------- 4 files changed, 79 insertions(+), 13 deletions(-) diff --git a/include/abstract_index.h b/include/abstract_index.h index 525759e34..3a4a81737 100644 --- a/include/abstract_index.h +++ b/include/abstract_index.h @@ -53,6 +53,8 @@ class AbstractIndex #ifdef EXEC_ENV_OLS virtual void load(AlignedFileReader &reader, uint32_t num_threads, uint32_t search_l) = 0; #else + virtual void load(const IndexLoadParams& index_load_params) = 0; + virtual void load(const char *index_file, uint32_t num_threads, uint32_t search_l, LabelFormatType label_format_type = LabelFormatType::String) = 0; #endif diff --git a/include/index.h b/include/index.h index 59577e53c..25255db1e 100644 --- a/include/index.h +++ b/include/index.h @@ -81,7 +81,10 @@ template clas #ifdef EXEC_ENV_OLS DISKANN_DLLEXPORT void load(AlignedFileReader &reader, uint32_t num_threads, uint32_t search_l); #else + DISKANN_DLLEXPORT void load(const IndexLoadParams& load_params); + DISKANN_DLLEXPORT void load(const char *index_file, uint32_t num_threads, uint32_t search_l, LabelFormatType label_format_type = LabelFormatType::String); + #endif // get some private variables diff --git a/include/parameters.h b/include/parameters.h index 2b08ade86..951976ba6 100644 --- a/include/parameters.h +++ b/include/parameters.h @@ -146,4 +146,42 @@ class IndexWriteParametersBuilder uint32_t _num_diverse_build{ defaults::NUM_DIVERSE_BUILD }; }; +struct IndexLoadParams +{ + std::string index_file_path; + uint32_t num_threads{defaults::NUM_THREADS}; + uint32_t search_list_size{defaults::SEARCH_LIST_SIZE}; + LabelFormatType label_format_type{LabelFormatType::String}; + + // Optional file paths - if empty, will be derived from index_file_path + std::string data_file_path; + std::string tags_file_path; + std::string delete_set_file_path; + std::string graph_file_path; + std::string labels_file_path; + std::string labels_to_medoids_file_path; + std::string labels_map_file_path; + std::string seller_file_path; + std::string bitmask_label_file_path; + std::string integer_label_file_path; + + IndexLoadParams() = default; + + IndexLoadParams(const std::string &index_file_path, uint32_t num_threads = defaults::NUM_THREADS, + uint32_t search_list_size = defaults::SEARCH_LIST_SIZE, + LabelFormatType label_format_type = LabelFormatType::String) + : index_file_path(index_file_path), num_threads(num_threads), search_list_size(search_list_size), + label_format_type(label_format_type) + { + } + + IndexLoadParams(const char *index_file_path, uint32_t num_threads = defaults::NUM_THREADS, + uint32_t search_list_size = defaults::SEARCH_LIST_SIZE, + LabelFormatType label_format_type = LabelFormatType::String) + : index_file_path(index_file_path), num_threads(num_threads), search_list_size(search_list_size), + label_format_type(label_format_type) + { + } +}; + } // namespace diskann diff --git a/src/index.cpp b/src/index.cpp index 8c4df137c..7974b3b2a 100644 --- a/src/index.cpp +++ b/src/index.cpp @@ -581,13 +581,26 @@ size_t Index::load_delete_set(const std::string &filename) // load the index from file and update the max_degree, cur (navigating // node loc), and _final_graph (adjacency list) -template + #ifdef EXEC_ENV_OLS +template void Index::load(AlignedFileReader &reader, uint32_t num_threads, uint32_t search_l) { #else +template void Index::load(const char *filename, uint32_t num_threads, uint32_t search_l, LabelFormatType label_format_type) { + IndexLoadParams load_params(filename, num_threads, search_l, label_format_type); + load(load_params); +} + +template +void Index::load(const IndexLoadParams & load_params) +{ + const std::string &filename = load_params.index_file_path; + const uint32_t num_threads = load_params.num_threads; + const uint32_t search_l = load_params.search_list_size; + const LabelFormatType label_format_type = load_params.label_format_type; #endif std::unique_lock ul(_update_lock); std::unique_lock cl(_consolidate_lock); @@ -598,20 +611,27 @@ void Index::load(const char *filename, uint32_t num_threads, ui size_t tags_file_num_pts = 0, graph_num_pts = 0, data_file_num_pts = 0, label_num_pts = 0; - std::string mem_index_file(filename); - std::string labels_file = mem_index_file + "_labels.txt"; - std::string labels_to_medoids = mem_index_file + "_labels_to_medoids.txt"; - std::string labels_map_file = mem_index_file + "_labels_map.txt"; + // Use file paths from load_params if provided, otherwise derive from index_file_path + std::string labels_file = load_params.labels_file_path.empty() + ? filename + "_labels.txt" : load_params.labels_file_path; + std::string labels_to_medoids = load_params.labels_to_medoids_file_path.empty() + ? filename + "_labels_to_medoids.txt" : load_params.labels_to_medoids_file_path; + std::string labels_map_file = load_params.labels_map_file_path.empty() + ? filename + "_labels_map.txt" : load_params.labels_map_file_path; if (!_save_as_one_file) { // For DLVS Store, we will not support saving the index in multiple // files. #ifndef EXEC_ENV_OLS - std::string data_file = std::string(filename) + ".data"; - std::string tags_file = std::string(filename) + ".tags"; - std::string delete_set_file = std::string(filename) + ".del"; - std::string graph_file = std::string(filename); + std::string data_file = load_params.data_file_path.empty() + ? filename + ".data" : load_params.data_file_path; + std::string tags_file = load_params.tags_file_path.empty() + ? filename + ".tags" : load_params.tags_file_path; + std::string delete_set_file = load_params.delete_set_file_path.empty() + ? filename + ".del" : load_params.delete_set_file_path; + std::string graph_file = load_params.graph_file_path.empty() + ? filename : load_params.graph_file_path; data_file_num_pts = load_data(data_file); this->_table_stats.node_count = data_file_num_pts; this->_table_stats.node_mem_usage = this->_data_store->get_data_size(); @@ -646,8 +666,9 @@ void Index::load(const char *filename, uint32_t num_threads, ui throw diskann::ANNException(stream.str(), -1, __FUNCSIG__, __FILE__, __LINE__); } - std::string index_seller_file = std::string(filename) + "_sellers.bin"; - std::string old_index_seller_file = std::string(filename) + "_sellers.txt"; + std::string index_seller_file = load_params.seller_file_path.empty() + ? filename + "_sellers.bin" : load_params.seller_file_path; + std::string old_index_seller_file = filename + "_sellers.txt"; if (file_exists(index_seller_file)) { //uint64_t nrows_seller_file; @@ -678,8 +699,10 @@ void Index::load(const char *filename, uint32_t num_threads, ui _diverse_index = true; } - std::string bitmask_label_file = std::string(filename) + "_bitmask_labels.bin"; - std::string integer_label_file = std::string(filename) + "_integer_labels.bin"; + std::string bitmask_label_file = load_params.bitmask_label_file_path.empty() + ? filename + "_bitmask_labels.bin" : load_params.bitmask_label_file_path; + std::string integer_label_file = load_params.integer_label_file_path.empty() + ? filename + "_integer_labels.bin" : load_params.integer_label_file_path; if (file_exists(labels_file) || file_exists(bitmask_label_file) || file_exists(integer_label_file)) From bf987ee0a1690a9acf11316b0823fe246367f34e Mon Sep 17 00:00:00 2001 From: Sanhaoji2 Date: Tue, 13 Jan 2026 18:17:07 +0800 Subject: [PATCH 2/2] fix unv file path --- include/parameters.h | 1 + src/index.cpp | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/include/parameters.h b/include/parameters.h index 951976ba6..ae13621d8 100644 --- a/include/parameters.h +++ b/include/parameters.h @@ -164,6 +164,7 @@ struct IndexLoadParams std::string seller_file_path; std::string bitmask_label_file_path; std::string integer_label_file_path; + std::string universal_label_file_path; IndexLoadParams() = default; diff --git a/src/index.cpp b/src/index.cpp index 7974b3b2a..5f1db52b9 100644 --- a/src/index.cpp +++ b/src/index.cpp @@ -764,8 +764,8 @@ void Index::load(const IndexLoadParams & load_params) _label_to_start_id.clear(); label_helper().load_label_medoids(labels_to_medoids, _label_to_start_id); - std::string universal_label_file(filename); - universal_label_file += "_universal_label.txt"; + std::string universal_label_file = load_params.universal_label_file_path.empty() + ? filename + "_universal_label.txt" : load_params.universal_label_file_path; if (file_exists(universal_label_file)) { std::ifstream universal_label_reader(universal_label_file);