From 351f03c9aafdd7159e9fc374920787dba72c3a2f Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Wed, 4 Jun 2025 13:47:49 -0700 Subject: [PATCH 01/38] update to vc143 --- AnnService/Aggregator.vcxproj | 8 ++++---- AnnService/BalancedDataPartition.vcxproj | 8 ++++---- AnnService/Client.vcxproj | 8 ++++---- AnnService/CoreLibrary.vcxproj | 8 ++++---- AnnService/IndexBuilder.vcxproj | 8 ++++---- AnnService/IndexSearcher.vcxproj | 8 ++++---- AnnService/Quantizer.vcxproj | 8 ++++---- AnnService/SSDServing.vcxproj | 8 ++++---- AnnService/Server.vcxproj | 8 ++++---- AnnService/SocketLib.vcxproj | 8 ++++---- Test/Test.vcxproj | 8 ++++---- Test/WinRTTest/WinRTTest.vcxproj | 8 ++++---- Wrappers/CLRCore.vcxproj | 8 ++++---- Wrappers/CsharpClient.vcxproj | 8 ++++---- Wrappers/CsharpCore.vcxproj | 8 ++++---- Wrappers/JavaClient.vcxproj | 8 ++++---- Wrappers/JavaCore.vcxproj | 8 ++++---- Wrappers/PythonClient.vcxproj | 8 ++++---- Wrappers/PythonCore.vcxproj | 10 +++++----- 19 files changed, 77 insertions(+), 77 deletions(-) diff --git a/AnnService/Aggregator.vcxproj b/AnnService/Aggregator.vcxproj index 4946c13f9..aa38cabe7 100644 --- a/AnnService/Aggregator.vcxproj +++ b/AnnService/Aggregator.vcxproj @@ -29,26 +29,26 @@ Application true - v142 + v143 MultiByte Application false - v142 + v143 true MultiByte Application true - v142 + v143 MultiByte Application false - v142 + v143 true MultiByte diff --git a/AnnService/BalancedDataPartition.vcxproj b/AnnService/BalancedDataPartition.vcxproj index 7751af309..085fa3b65 100644 --- a/AnnService/BalancedDataPartition.vcxproj +++ b/AnnService/BalancedDataPartition.vcxproj @@ -29,26 +29,26 @@ Application true - v142 + v143 MultiByte Application false - v142 + v143 true MultiByte Application true - v142 + v143 MultiByte Application false - v142 + v143 true MultiByte diff --git a/AnnService/Client.vcxproj b/AnnService/Client.vcxproj index 9381af598..90d8c9125 100644 --- a/AnnService/Client.vcxproj +++ b/AnnService/Client.vcxproj @@ -29,26 +29,26 @@ Application true - v142 + v143 MultiByte Application false - v142 + v143 true MultiByte Application true - v142 + v143 MultiByte Application false - v142 + v143 true MultiByte diff --git a/AnnService/CoreLibrary.vcxproj b/AnnService/CoreLibrary.vcxproj index 615d7cca3..d86127e1d 100644 --- a/AnnService/CoreLibrary.vcxproj +++ b/AnnService/CoreLibrary.vcxproj @@ -30,26 +30,26 @@ Application true - v142 + v143 MultiByte Application false - v142 + v143 true MultiByte StaticLibrary true - v142 + v143 MultiByte StaticLibrary false - v142 + v143 true MultiByte diff --git a/AnnService/IndexBuilder.vcxproj b/AnnService/IndexBuilder.vcxproj index 0900590cb..891e8d37b 100644 --- a/AnnService/IndexBuilder.vcxproj +++ b/AnnService/IndexBuilder.vcxproj @@ -29,26 +29,26 @@ Application true - v142 + v143 MultiByte Application false - v142 + v143 true MultiByte Application true - v142 + v143 MultiByte Application false - v142 + v143 true MultiByte diff --git a/AnnService/IndexSearcher.vcxproj b/AnnService/IndexSearcher.vcxproj index 6d1378379..867d6bb2d 100644 --- a/AnnService/IndexSearcher.vcxproj +++ b/AnnService/IndexSearcher.vcxproj @@ -30,26 +30,26 @@ Application true - v142 + v143 MultiByte Application false - v142 + v143 true MultiByte Application true - v142 + v143 MultiByte Application false - v142 + v143 true MultiByte diff --git a/AnnService/Quantizer.vcxproj b/AnnService/Quantizer.vcxproj index 942e55e1d..a09b38b8a 100644 --- a/AnnService/Quantizer.vcxproj +++ b/AnnService/Quantizer.vcxproj @@ -39,26 +39,26 @@ Application true - v142 + v143 MultiByte Application false - v142 + v143 true MultiByte Application true - v142 + v143 MultiByte Application false - v142 + v143 true MultiByte diff --git a/AnnService/SSDServing.vcxproj b/AnnService/SSDServing.vcxproj index 5ff13c19a..40b1d0765 100644 --- a/AnnService/SSDServing.vcxproj +++ b/AnnService/SSDServing.vcxproj @@ -40,26 +40,26 @@ Application true - v142 + v143 MultiByte Application false - v142 + v143 true MultiByte StaticLibrary true - v142 + v143 MultiByte StaticLibrary false - v142 + v143 true MultiByte diff --git a/AnnService/Server.vcxproj b/AnnService/Server.vcxproj index 3b38afe40..b98a6751c 100644 --- a/AnnService/Server.vcxproj +++ b/AnnService/Server.vcxproj @@ -29,26 +29,26 @@ Application true - v142 + v143 MultiByte Application false - v142 + v143 true MultiByte Application true - v142 + v143 MultiByte Application false - v142 + v143 true MultiByte diff --git a/AnnService/SocketLib.vcxproj b/AnnService/SocketLib.vcxproj index d29ac8ede..a4f10f845 100644 --- a/AnnService/SocketLib.vcxproj +++ b/AnnService/SocketLib.vcxproj @@ -28,26 +28,26 @@ StaticLibrary true - v142 + v143 MultiByte StaticLibrary false - v142 + v143 true MultiByte StaticLibrary true - v142 + v143 MultiByte StaticLibrary false - v142 + v143 true MultiByte diff --git a/Test/Test.vcxproj b/Test/Test.vcxproj index edf5d4541..9e6ca7dde 100644 --- a/Test/Test.vcxproj +++ b/Test/Test.vcxproj @@ -29,26 +29,26 @@ Application true - v142 + v143 MultiByte Application false - v142 + v143 true MultiByte Application true - v142 + v143 MultiByte Application false - v142 + v143 true MultiByte diff --git a/Test/WinRTTest/WinRTTest.vcxproj b/Test/WinRTTest/WinRTTest.vcxproj index 47b719755..72457cda7 100644 --- a/Test/WinRTTest/WinRTTest.vcxproj +++ b/Test/WinRTTest/WinRTTest.vcxproj @@ -30,26 +30,26 @@ Application true - v142 + v143 Unicode Application false - v142 + v143 true Unicode Application true - v142 + v143 Unicode Application false - v142 + v143 true Unicode diff --git a/Wrappers/CLRCore.vcxproj b/Wrappers/CLRCore.vcxproj index 4b215a512..d055eba4e 100644 --- a/Wrappers/CLRCore.vcxproj +++ b/Wrappers/CLRCore.vcxproj @@ -33,28 +33,28 @@ DynamicLibrary true - v142 + v143 true Unicode DynamicLibrary false - v142 + v143 true Unicode DynamicLibrary true - v142 + v143 true MultiByte DynamicLibrary false - v142 + v143 true MultiByte diff --git a/Wrappers/CsharpClient.vcxproj b/Wrappers/CsharpClient.vcxproj index 63ca24e29..7781ee5cd 100644 --- a/Wrappers/CsharpClient.vcxproj +++ b/Wrappers/CsharpClient.vcxproj @@ -29,26 +29,26 @@ DynamicLibrary true - v142 + v143 MultiByte DynamicLibrary false - v142 + v143 true MultiByte DynamicLibrary true - v142 + v143 MultiByte DynamicLibrary false - v142 + v143 true MultiByte diff --git a/Wrappers/CsharpCore.vcxproj b/Wrappers/CsharpCore.vcxproj index 13cd32bec..84de82ebf 100644 --- a/Wrappers/CsharpCore.vcxproj +++ b/Wrappers/CsharpCore.vcxproj @@ -29,26 +29,26 @@ Application true - v142 + v143 MultiByte Application false - v142 + v143 true MultiByte DynamicLibrary true - v142 + v143 MultiByte DynamicLibrary false - v142 + v143 true MultiByte diff --git a/Wrappers/JavaClient.vcxproj b/Wrappers/JavaClient.vcxproj index d977ac424..a551d6344 100644 --- a/Wrappers/JavaClient.vcxproj +++ b/Wrappers/JavaClient.vcxproj @@ -29,26 +29,26 @@ DynamicLibrary true - v142 + v143 MultiByte DynamicLibrary false - v142 + v143 true MultiByte DynamicLibrary true - v142 + v143 MultiByte DynamicLibrary false - v142 + v143 true MultiByte diff --git a/Wrappers/JavaCore.vcxproj b/Wrappers/JavaCore.vcxproj index a3efa65ad..16d00ff83 100644 --- a/Wrappers/JavaCore.vcxproj +++ b/Wrappers/JavaCore.vcxproj @@ -29,26 +29,26 @@ Application true - v142 + v143 MultiByte Application false - v142 + v143 true MultiByte DynamicLibrary true - v142 + v143 MultiByte DynamicLibrary false - v142 + v143 true MultiByte diff --git a/Wrappers/PythonClient.vcxproj b/Wrappers/PythonClient.vcxproj index 89a6ac506..5250ff70c 100644 --- a/Wrappers/PythonClient.vcxproj +++ b/Wrappers/PythonClient.vcxproj @@ -30,26 +30,26 @@ DynamicLibrary true - v142 + v143 MultiByte DynamicLibrary false - v142 + v143 true MultiByte DynamicLibrary true - v142 + v143 MultiByte DynamicLibrary false - v142 + v143 true MultiByte diff --git a/Wrappers/PythonCore.vcxproj b/Wrappers/PythonCore.vcxproj index 59d3d2a85..1f4217485 100644 --- a/Wrappers/PythonCore.vcxproj +++ b/Wrappers/PythonCore.vcxproj @@ -30,26 +30,26 @@ Application true - v142 + v143 MultiByte Application false - v142 + v143 true MultiByte DynamicLibrary true - v142 + v143 MultiByte DynamicLibrary false - v142 + v143 true MultiByte @@ -117,7 +117,7 @@ - + From e762fd50000ebd7db0ec5e15eab075ab3208df05 Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Fri, 21 Nov 2025 21:30:21 -0800 Subject: [PATCH 02/38] move mergepostings to refineindex --- AnnService/inc/Core/Common/Dataset.h | 7 +- .../inc/Core/SPANN/ExtraDynamicSearcher.h | 73 ++++++++++++++----- AnnService/inc/Core/SPANN/IExtraSearcher.h | 2 +- AnnService/src/Core/SPANN/SPANNIndex.cpp | 14 ++-- AnnService/src/Core/VectorIndex.cpp | 58 +++++++-------- Test/src/SPFreshTest.cpp | 68 +---------------- 6 files changed, 95 insertions(+), 127 deletions(-) diff --git a/AnnService/inc/Core/Common/Dataset.h b/AnnService/inc/Core/Common/Dataset.h index a5abaf743..86dc8bcd7 100644 --- a/AnnService/inc/Core/Common/Dataset.h +++ b/AnnService/inc/Core/Common/Dataset.h @@ -265,6 +265,7 @@ namespace SPTAG } rows = rows_; + incRows = 0; if (rowEnd_ >= colStart_) cols = rowEnd_; else cols = cols_ * sizeof(T); data = (char*)data_; @@ -390,11 +391,7 @@ namespace SPTAG IOBINARY(pInput, ReadBinary, sizeof(SizeType), (char*)&(r)); IOBINARY(pInput, ReadBinary, sizeof(DimensionType), (char*)(&c)); - if (data == nullptr) Initialize(r, c, blockSize, capacity); - else if (r > rows + incRows) { - SPTAGLIB_LOG(Helper::LogLevel::LL_Warning, "Read more data (%d, %d) than before (%d, %d)!\n", r, c, rows + incRows, mycols); - if(AddBatch(r - rows - incRows) != ErrorCode::Success) return ErrorCode::MemoryOverFlow; - } + if (data == nullptr || r != rows + incRows) Initialize(r, c, blockSize, capacity); for (SizeType i = 0; i < r; i++) { IOBINARY(pInput, ReadBinary, sizeof(T) * mycols, (char*)At(i)); diff --git a/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h b/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h index d5b88a8ae..57b101557 100644 --- a/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h +++ b/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h @@ -298,7 +298,7 @@ namespace SPTAG::SPANN { { int assumptionBrokenNum = 0; int postVectorNum = postingList.size() / m_vectorInfoSize; - uint8_t* postingP = reinterpret_cast(&postingList.front()); + uint8_t* postingP = reinterpret_cast(postingList.data()); float minDist; float maxDist; float avgDist = 0; @@ -486,7 +486,7 @@ namespace SPTAG::SPANN { } // TODO - ErrorCode RefineIndex(std::shared_ptr p_index, + ErrorCode RefineIndex(std::shared_ptr& p_index, bool p_prereassign, std::vector *p_headmapping, std::vector *p_mapping) override { SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "Begin RefineIndex\n"); @@ -501,6 +501,7 @@ namespace SPTAG::SPANN { p_index->m_iDataBlockSize, p_index->m_iDataCapacity); } std::atomic_bool doneReassign = false; + Helper::Concurrent::ConcurrentSet mergelist; while (!doneReassign) { auto preReassignTimeBegin = std::chrono::high_resolution_clock::now(); std::atomic finalcode = ErrorCode::Success; @@ -552,10 +553,10 @@ namespace SPTAG::SPANN { (int)(postingList.size()), (int)(ret == ErrorCode::Success)); PrintErrorInPosting(postingList, index); finalcode = ret; - return; + //return; } SizeType postVectorNum = (SizeType)(postingList.size() / m_vectorInfoSize); - auto *postingP = reinterpret_cast(&postingList.front()); + auto *postingP = reinterpret_cast(postingList.data()); uint8_t *vectorId = postingP; int vectorCount = 0; for (int j = 0; j < postVectorNum; @@ -575,6 +576,8 @@ namespace SPTAG::SPANN { vectorCount++; } + if (vectorCount <= m_mergeThreshold) mergelist.insert(p_headmapping->at(index)); + postingList.resize(vectorCount * m_vectorInfoSize); new_postingSizes.UpdateSize(p_headmapping->at(index), vectorCount); *new_checkSums[p_headmapping->at(index)] = @@ -615,8 +618,8 @@ namespace SPTAG::SPANN { if (p_prereassign) { - p_index->SaveIndex(m_opt->m_indexDirectory + FolderSep + m_opt->m_headIndexFolder); Checkpoint(m_opt->m_indexDirectory); + p_index->SaveIndex(m_opt->m_indexDirectory + FolderSep + m_opt->m_headIndexFolder); CalculatePostingDistribution(p_index.get()); } else @@ -627,6 +630,37 @@ namespace SPTAG::SPANN { std::string p_checksumPath = m_opt->m_indexDirectory + FolderSep + m_opt->m_checksumFile; new_checkSums.Save(p_checksumPath); db->Checkpoint(m_opt->m_indexDirectory); + + if ((finalcode = m_postingSizes.Load(p_persistenRecord, p_index->m_iDataBlockSize, + p_index->m_iDataCapacity)) != ErrorCode::Success) + return finalcode; + if ((finalcode = m_checkSums.Load(p_checksumPath, p_index->m_iDataBlockSize, + p_index->m_iDataCapacity)) != ErrorCode::Success) + return finalcode; + + if ((finalcode = m_versionMap->Load(m_opt->m_indexDirectory + FolderSep + m_opt->m_deleteIDFile, + p_index->m_iDataBlockSize, p_index->m_iDataCapacity)) != + ErrorCode::Success) + return finalcode; + if ((finalcode = m_vectorTranslateMap->Load( + m_opt->m_indexDirectory + FolderSep + m_opt->m_headIDFile, p_index->m_iDataBlockSize, + p_index->m_iDataCapacity)) != ErrorCode::Success) + return finalcode; + if ((finalcode = + VectorIndex::LoadIndex(m_opt->m_indexDirectory + FolderSep + m_opt->m_headIndexFolder, + p_index)) != ErrorCode::Success) + return finalcode; + + if (mergelist.size() > 0) + { + for (SizeType pid : mergelist) + { + MergeAsync(p_index.get(), pid); + } + Checkpoint(m_opt->m_indexDirectory); + p_index->SaveIndex(m_opt->m_indexDirectory + FolderSep + m_opt->m_headIndexFolder); + m_vectorTranslateMap->Save(m_opt->m_indexDirectory + FolderSep + m_opt->m_headIDFile); + } } SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "SPFresh: ReWriting SSD Info\n"); @@ -651,7 +685,7 @@ namespace SPTAG::SPANN { std::string postingList; auto splitGetBegin = std::chrono::high_resolution_clock::now(); if ((ret=db->Get(headID, &postingList, MaxTimeout, &(p_exWorkSpace->m_diskRequests))) != - ErrorCode::Success)// || !m_checkSum.ValidateChecksum(postingList.c_str(), (int)(postingList.size()), *m_checkSums[headID])) + ErrorCode::Success || !m_checkSum.ValidateChecksum(postingList.c_str(), (int)(postingList.size()), *m_checkSums[headID])) { SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Split fail to get oversized postings: key=%d size=%d\n", headID, m_postingSizes.GetSize(headID)); return ret; @@ -660,7 +694,7 @@ namespace SPTAG::SPANN { elapsedMSeconds = std::chrono::duration_cast(splitGetEnd - splitGetBegin).count(); m_stat.m_getCost += elapsedMSeconds; // reinterpret postingList to vectors and IDs - auto* postingP = reinterpret_cast(&postingList.front()); + auto* postingP = reinterpret_cast(postingList.data()); SizeType postVectorNum = (SizeType)(postingList.size() / m_vectorInfoSize); //SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "DEBUG: db get Posting %d successfully with length %d real length:%d vectorNum:%d\n", headID, (int)(postingList.size()), m_postingSizes.GetSize(headID), postVectorNum); @@ -926,7 +960,7 @@ namespace SPTAG::SPANN { return ret; } - auto* postingP = reinterpret_cast(¤tPostingList.front()); + auto* postingP = reinterpret_cast(currentPostingList.data()); size_t postVectorNum = currentPostingList.size() / m_vectorInfoSize; int currentLength = 0; uint8_t* vectorId = postingP; @@ -1076,7 +1110,7 @@ namespace SPTAG::SPANN { if (currentLength > nextLength) { /* ReAssign queryResult->VID*/ - postingP = reinterpret_cast(&nextPostingList.front()); + postingP = reinterpret_cast(nextPostingList.data()); for (int j = 0; j < nextLength; j++) { uint8_t* vectorId = postingP + j * m_vectorInfoSize; // SizeType vid = *(reinterpret_cast(vectorId)); @@ -1090,7 +1124,7 @@ namespace SPTAG::SPANN { } else { /* ReAssign headID*/ - postingP = reinterpret_cast(¤tPostingList.front()); + postingP = reinterpret_cast(currentPostingList.data()); for (int j = 0; j < currentLength; j++) { uint8_t* vectorId = postingP + j * m_vectorInfoSize; // SizeType vid = *(reinterpret_cast(vectorId)); @@ -1110,7 +1144,7 @@ namespace SPTAG::SPANN { { std::shared_ptr vectorinfo = std::make_shared(m_vectorInfoSize, ' '); - Serialize(&(vectorinfo->front()), vid, m_versionMap->GetVersion(vid), + Serialize(vectorinfo->data(), vid, m_versionMap->GetVersion(vid), p_index->GetSample(deletedHead)); ReassignAsync(p_index, vectorinfo, -1); } @@ -1197,7 +1231,7 @@ namespace SPTAG::SPANN { if (!m_versionMap->Deleted(vid)) { std::shared_ptr vectorinfo = std::make_shared(m_vectorInfoSize, ' '); - Serialize(&(vectorinfo->front()), vid, m_versionMap->GetVersion(vid), headVector); + Serialize(vectorinfo->data(), vid, m_versionMap->GetVersion(vid), headVector); ReassignAsync(p_index, vectorinfo, -1); } } @@ -1208,7 +1242,7 @@ namespace SPTAG::SPANN { for (int i = 0; i < postingLists.size(); i++) { auto& postingList = postingLists[i]; size_t postVectorNum = postingList.size() / m_vectorInfoSize; - auto* postingP = reinterpret_cast(&postingList.front()); + auto* postingP = reinterpret_cast(postingList.data()); for (int j = 0; j < postVectorNum; j++) { uint8_t* vectorId = postingP + j * m_vectorInfoSize; SizeType vid = *(reinterpret_cast(vectorId)); @@ -1566,7 +1600,7 @@ namespace SPTAG::SPANN { if (vectorNum > m_postingSizeLimit) vectorNum = m_postingSizeLimit; - auto *postingP = reinterpret_cast(&tempPosting.front()); + auto *postingP = reinterpret_cast(tempPosting.data()); std::string newPosting(m_vectorInfoSize * vectorNum, '\0'); char *ptr = (char *)(newPosting.c_str()); for (int j = 0; j < vectorNum; ++j, ptr += m_vectorInfoSize) @@ -1680,7 +1714,7 @@ namespace SPTAG::SPANN { Helper::LogLevel::LL_Error, "ValidatePostings fail: posting id:%d, required size:%d, buffer size:%d, checksum:%d\n", pids[i], (int)(m_postingSizes.GetSize(pids[i]) * m_vectorInfoSize), (int)(postings[i].GetAvailableSize()), (int)(*m_checkSums[pids[i]])); - //return false; + return false; } } return true; @@ -1688,8 +1722,7 @@ namespace SPTAG::SPANN { bool ValidatePostings(std::vector &pids, std::vector &postings) { - if (!m_opt->m_checksumInRead) - return true; + if (!m_opt->m_checksumInRead) return true; ErrorCode ret; for (int i = 0; i < pids.size(); i++) @@ -1702,7 +1735,8 @@ namespace SPTAG::SPANN { "ValidatePostings fail: posting id:%d, required size:%d, buffer size:%d, checksum:%d\n", pids[i], (int)(m_postingSizes.GetSize(pids[i]) * m_vectorInfoSize), (int)(postings[i].size()), (int)(*m_checkSums[pids[i]])); - // return false; + PrintErrorInPosting(postings[i], pids[i]); + return false; } } return true; @@ -1794,7 +1828,7 @@ namespace SPTAG::SPANN { queryResults.AddPoint(vectorID, distance2leaf); } auto compEnd = std::chrono::high_resolution_clock::now(); - if (realNum <= m_mergeThreshold) MergeAsync(p_index.get(), curPostingID); // TODO: Control merge + //if (realNum <= m_mergeThreshold) MergeAsync(p_index.get(), curPostingID); // TODO: Control merge compLatency += ((double)std::chrono::duration_cast(compEnd - compStart).count()); @@ -2485,6 +2519,7 @@ namespace SPTAG::SPANN { !m_checkSum.ValidateChecksum(posting.c_str(), (int)(posting.size()), *m_checkSums[postingID])) { SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "[CheckPosting] Get checksum fail %d!\n", postingID); + PrintErrorInPosting(posting, postingID); return ret; } } diff --git a/AnnService/inc/Core/SPANN/IExtraSearcher.h b/AnnService/inc/Core/SPANN/IExtraSearcher.h index 95d171854..6e8aad0fa 100644 --- a/AnnService/inc/Core/SPANN/IExtraSearcher.h +++ b/AnnService/inc/Core/SPANN/IExtraSearcher.h @@ -313,7 +313,7 @@ namespace SPTAG { virtual ErrorCode GetPostingDebug(ExtraWorkSpace* p_exWorkSpace, std::shared_ptr p_index, SizeType vid, std::vector& VIDs, std::shared_ptr& vecs) = 0; - virtual ErrorCode RefineIndex(std::shared_ptr p_index, bool p_prereassign = true, + virtual ErrorCode RefineIndex(std::shared_ptr& p_index, bool p_prereassign = true, std::vector *p_headmapping = nullptr, std::vector *p_mapping = nullptr) { diff --git a/AnnService/src/Core/SPANN/SPANNIndex.cpp b/AnnService/src/Core/SPANN/SPANNIndex.cpp index 3c52c08f2..90340b8d4 100644 --- a/AnnService/src/Core/SPANN/SPANNIndex.cpp +++ b/AnnService/src/Core/SPANN/SPANNIndex.cpp @@ -1387,18 +1387,22 @@ ErrorCode Index::RefineIndex(const std::vectorRefineIndex(m_index, false, &headOldtoNew, p_mapping)) != ErrorCode::Success) - return ret; - if (nullptr != m_pMetadata) { if (p_indexStreams.size() < GetIndexFiles()->size() + 2) return ErrorCode::LackOfInputs; if ((ret = m_pMetadata->RefineMetadata(NewtoOld, p_indexStreams[GetIndexFiles()->size()], - p_indexStreams[GetIndexFiles()->size()+1])) != - ErrorCode::Success) + p_indexStreams[GetIndexFiles()->size() + 1])) != ErrorCode::Success) return ret; } + for (int i = 0; i < p_indexStreams.size(); i++) + { + p_indexStreams[i]->ShutDown(); + } + + if ((ret = m_extraSearcher->RefineIndex(m_index, false, &headOldtoNew, p_mapping)) != ErrorCode::Success) + return ret; + return ret; } diff --git a/AnnService/src/Core/VectorIndex.cpp b/AnnService/src/Core/VectorIndex.cpp index e329b4051..2f8ebfd13 100644 --- a/AnnService/src/Core/VectorIndex.cpp +++ b/AnnService/src/Core/VectorIndex.cpp @@ -392,6 +392,27 @@ ErrorCode VectorIndex::SaveIndex(const std::string &p_folderPath) if (!m_bReady || GetNumSamples() - GetNumDeleted() == 0) return ErrorCode::EmptyIndex; + if (GetIndexAlgoType() == IndexAlgoType::SPANN && GetParameter("IndexDirectory", "Base") != p_folderPath) + { + std::string oldFolder = GetParameter("IndexDirectory", "Base"); + ErrorCode ret = SaveIndex(oldFolder); + if (ret != ErrorCode::Success || !copydirectory(oldFolder, p_folderPath)) + { + SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Failed to copy index directory contents to %s!\n", + p_folderPath.c_str()); + return ErrorCode::Fail; + } + + SetParameter("IndexDirectory", p_folderPath, "Base"); + auto configFile = SPTAG::f_createIO(); + if (configFile == nullptr || !configFile->Initialize((p_folderPath + FolderSep + "indexloader.ini").c_str(), std::ios::out)) + return ErrorCode::FailedCreateFile; + if ((ret = SaveIndexConfig(configFile)) != ErrorCode::Success) + return ret; + + return ErrorCode::Success; + } + std::string folderPath(p_folderPath); if (!folderPath.empty() && *(folderPath.rbegin()) != FolderSep) { @@ -402,29 +423,6 @@ ErrorCode VectorIndex::SaveIndex(const std::string &p_folderPath) mkdir(folderPath.c_str()); } - if (GetIndexAlgoType() == IndexAlgoType::SPANN && GetParameter("IndexDirectory", "Base") != p_folderPath) - { - std::vector files; - std::string oldFolder = GetParameter("IndexDirectory", "Base"); - if (!oldFolder.empty() && *(oldFolder.rbegin()) != FolderSep) - oldFolder += FolderSep; - listdir((oldFolder + "*").c_str(), files); - for (auto file : files) - { - size_t firstSep = oldFolder.length(), lastSep = file.find_last_of(FolderSep); - std::string newFolder = - folderPath + ((lastSep > firstSep) ? file.substr(firstSep, lastSep - firstSep) : ""), - filename = file.substr(lastSep + 1); - if (!direxists(newFolder.c_str())) - mkdir(newFolder.c_str()); - SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "Copy file %s to %s...\n", file.c_str(), - (newFolder + FolderSep + filename).c_str()); - if (!copyfile(file.c_str(), (newFolder + FolderSep + filename).c_str())) - return ErrorCode::DiskIOFail; - } - SetParameter("IndexDirectory", p_folderPath, "Base"); - } - ErrorCode ret = ErrorCode::Success; { auto configFile = SPTAG::f_createIO(); @@ -457,6 +455,11 @@ ErrorCode VectorIndex::SaveIndex(const std::string &p_folderPath) handles.push_back(std::move(ptr)); } + if (m_pQuantizer) + { + ret = m_pQuantizer->SaveQuantizer(handles.back()); + } + size_t metaStart = GetIndexFiles()->size(); if (NeedRefine()) { @@ -469,13 +472,6 @@ ErrorCode VectorIndex::SaveIndex(const std::string &p_folderPath) if (ErrorCode::Success == ret) ret = SaveIndexData(handles); } - if (m_pMetadata != nullptr) - metaStart += 2; - - if (ErrorCode::Success == ret && m_pQuantizer) - { - ret = m_pQuantizer->SaveQuantizer(handles[metaStart]); - } return ret; } @@ -1010,7 +1006,7 @@ std::shared_ptr VectorIndex::Clone(std::string p_clone) else { std::string indexFolder = GetParameter("IndexDirectory", "Base"); - //ret = SaveIndex(indexFolder); + ret = SaveIndex(indexFolder); if (!copydirectory(indexFolder, p_clone)) { SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Failed to copy index directory contents to %s!\n", diff --git a/Test/src/SPFreshTest.cpp b/Test/src/SPFreshTest.cpp index 639f1c709..d410f5456 100644 --- a/Test/src/SPFreshTest.cpp +++ b/Test/src/SPFreshTest.cpp @@ -428,7 +428,7 @@ BOOST_AUTO_TEST_CASE(TestReopenIndexRecall) auto originalIndex = BuildIndex("original_index", vecset, metaset); BOOST_REQUIRE(originalIndex != nullptr); BOOST_REQUIRE(originalIndex->SaveIndex("original_index") == ErrorCode::Success); - float recall1 = Search(originalIndex, queryset, vecset, addvecset, K, truth, N); + float recall1 = Search(originalIndex, queryset, vecset, addvecset, K, truth, N); originalIndex = nullptr; std::shared_ptr loadedOnce; @@ -523,7 +523,7 @@ BOOST_AUTO_TEST_CASE(TestCloneRecall) BOOST_REQUIRE(originalIndex != nullptr); BOOST_REQUIRE(originalIndex->SaveIndex("original_index") == ErrorCode::Success); float originalRecall = Search(originalIndex, queryset, vecset, addvecset, K, truth, N); - + auto clonedIndex = originalIndex->Clone("cloned_index"); BOOST_REQUIRE(clonedIndex != nullptr); originalIndex.reset(); @@ -1216,68 +1216,4 @@ BOOST_AUTO_TEST_CASE(IterativeSearchPerf) std::filesystem::remove_all("original_index"); } -BOOST_AUTO_TEST_CASE(RefineTestIdx) -{ - using namespace SPFreshTest; - - constexpr int dimension = 1024; - - std::shared_ptr vecset = get_embeddings(0, 500, dimension, -1); - std::shared_ptr metaset = TestUtils::TestDataGenerator::GenerateMetadataSet(1000, 0); - - for (auto i = 0; i < 2; ++i) { - void* p = vecset->GetVector(i); - for (auto i = 0; i < dimension; ++i) { - std::cout << ((float*)p)[i] << " "; - } - std::cout << std::endl; - } - - auto originalIndex = BuildIndex("original_index", vecset, metaset, "COSINE"); - BOOST_REQUIRE(originalIndex != nullptr); - BOOST_REQUIRE(originalIndex->SaveIndex("original_index") == ErrorCode::Success); - originalIndex = nullptr; - - std::string prevPath = "original_index"; - for (int iter = 0; iter < 1; iter++) - { - std::string clone_path = "clone_index_" + std::to_string(iter); - std::shared_ptr prevIndex; - BOOST_REQUIRE(VectorIndex::LoadIndex(prevPath, prevIndex) == ErrorCode::Success); - BOOST_REQUIRE(prevIndex != nullptr); - auto t0 = std::chrono::high_resolution_clock::now(); - BOOST_REQUIRE(prevIndex->Check() == ErrorCode::Success); - std::cout << "Check time for iteration " << iter << ": " - << std::chrono::duration_cast(std::chrono::high_resolution_clock::now() - - t0) - .count() - << " ms" << std::endl; - - auto cloneIndex = prevIndex->Clone(clone_path); - auto *cloneIndexPtr = static_cast *>(cloneIndex.get()); - std::shared_ptr tmpvecs = get_embeddings(500, 1100, dimension, -1); - std::shared_ptr tmpmetas = TestUtils::TestDataGenerator::GenerateMetadataSet(1200, 1000); - auto t1 = std::chrono::high_resolution_clock::now(); - InsertVectors(cloneIndexPtr, 1, 1200, tmpvecs, tmpmetas); - std::cout << "Insert time for iteration " << iter << ": " - << std::chrono::duration_cast(std::chrono::high_resolution_clock::now() - - t1) - .count() - << " ms" << std::endl; - - for (auto i = 1000; i < 1900; ++i) - { - cloneIndexPtr->DeleteIndex(i); - } - - BOOST_REQUIRE(cloneIndex->SaveIndex(clone_path) == ErrorCode::Success); - cloneIndex = nullptr; - } - - for (int iter = 0; iter < 1; iter++) - { - std::filesystem::remove_all("clone_index_" + std::to_string(iter)); - } - // std::filesystem::remove_all("original_index"); -} BOOST_AUTO_TEST_SUITE_END() From 57bf5ed218fa2499b56bb45f2d9fadff262ff4dd Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Tue, 25 Nov 2025 13:49:32 -0800 Subject: [PATCH 03/38] fix errorcode --- AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h | 8 ++++++-- Test/src/SPFreshTest.cpp | 3 ++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h b/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h index 57b101557..e5603d8b1 100644 --- a/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h +++ b/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h @@ -552,7 +552,7 @@ namespace SPTAG::SPANN { index, (int)(m_postingSizes.GetSize(index) * m_vectorInfoSize), (int)(postingList.size()), (int)(ret == ErrorCode::Success)); PrintErrorInPosting(postingList, index); - finalcode = ret; + finalcode = ErrorCode::Fail; //return; } SizeType postVectorNum = (SizeType)(postingList.size() / m_vectorInfoSize); @@ -687,7 +687,11 @@ namespace SPTAG::SPANN { if ((ret=db->Get(headID, &postingList, MaxTimeout, &(p_exWorkSpace->m_diskRequests))) != ErrorCode::Success || !m_checkSum.ValidateChecksum(postingList.c_str(), (int)(postingList.size()), *m_checkSums[headID])) { - SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Split fail to get oversized postings: key=%d size=%d\n", headID, m_postingSizes.GetSize(headID)); + SPTAGLIB_LOG(Helper::LogLevel::LL_Error, + "Split fail to get oversized postings: key=%d required size=%d read size=%d checksum " + "issue=%d\n", + headID, (int)(m_postingSizes.GetSize(headID) * m_vectorInfoSize), + (int)(postingList.size()), (int)(ret == ErrorCode::Success)); return ret; } auto splitGetEnd = std::chrono::high_resolution_clock::now(); diff --git a/Test/src/SPFreshTest.cpp b/Test/src/SPFreshTest.cpp index d410f5456..8ac2c7b75 100644 --- a/Test/src/SPFreshTest.cpp +++ b/Test/src/SPFreshTest.cpp @@ -1098,6 +1098,7 @@ BOOST_AUTO_TEST_CASE(CacheTest) std::filesystem::remove_all("clone_index_" + std::to_string(iter)); } + /* std::cout << "=================Enable Cache===================" << std::endl; prevPath = "original_index"; for (int iter = 0; iter < iterations; iter++) @@ -1158,7 +1159,7 @@ BOOST_AUTO_TEST_CASE(CacheTest) { std::filesystem::remove_all("clone_index_" + std::to_string(iter)); } - + */ std::filesystem::remove_all("original_index"); } From f8ea350dfd63698aa6a1779d219a7cddfe1b238d Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Tue, 2 Dec 2025 20:42:54 -0800 Subject: [PATCH 04/38] Add checksum verification in merge --- AnnService/inc/Core/Common/CommonUtils.h | 24 +++ .../inc/Core/SPANN/ExtraDynamicSearcher.h | 19 +- .../inc/Core/SPANN/ExtraFileController.h | 202 +++++++++++------- .../inc/Core/SPANN/ExtraRocksDBController.h | 5 +- .../inc/Core/SPANN/ExtraSPDKController.h | 5 +- AnnService/inc/Helper/KeyValueIO.h | 5 +- AnnService/src/Core/SPANN/SPANNIndex.cpp | 4 + Test/src/KVTest.cpp | 3 +- 8 files changed, 173 insertions(+), 94 deletions(-) diff --git a/AnnService/inc/Core/Common/CommonUtils.h b/AnnService/inc/Core/Common/CommonUtils.h index 905de2cd7..697eb521f 100644 --- a/AnnService/inc/Core/Common/CommonUtils.h +++ b/AnnService/inc/Core/Common/CommonUtils.h @@ -100,6 +100,30 @@ namespace SPTAG } } } + + static void PrintPostingDiff(std::string &p1, std::string &p2, const char *pos) + { + if (p1.size() != p2.size()) + { + SPTAGLIB_LOG(Helper::LogLevel::LL_Error, + "Merge %s: p1 and p2 have different sizes: before=%u after=%u\n", pos, p1.size(), + p2.size()); + return; + } + std::string diff = ""; + for (size_t i = 0; i < p1.size(); i++) + { + if (p1[i] != p2[i]) + { + diff += "[" + std::to_string(i) + "]:" + std::to_string(int(p1[i])) + "^" + + std::to_string(int(p2[i])) + " "; + } + } + if (diff.size() != 0) + { + SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "%s: %s\n", pos, diff.substr(0, 1000).c_str()); + } + } }; } } diff --git a/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h b/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h index e5603d8b1..c78ed6cf1 100644 --- a/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h +++ b/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h @@ -1429,28 +1429,33 @@ namespace SPTAG::SPANN { { //std::shared_lock lock(m_rwLocks[headID]); //ROCKSDB std::unique_lock lock(m_rwLocks[headID]); //SPDK + ErrorCode ret; if (!p_index->ContainSample(headID)) { lock.unlock(); goto checkDeleted; } if (m_postingSizes.GetSize(headID) + appendNum > (m_postingSizeLimit + m_bufferSizeLimit)) { - SPTAGLIB_LOG(Helper::LogLevel::LL_Warning, "After appending, the number of vectors exceeds the postingsize + buffersize (%d + %d)! Do split now...\n", m_postingSizeLimit, m_bufferSizeLimit); - Split(p_exWorkSpace, p_index, headID, !m_opt->m_disableReassign, false, false); + SPTAGLIB_LOG(Helper::LogLevel::LL_Warning, "After appending, the number of vectors in %d exceeds the postingsize + buffersize (%d + %d)! Do split now...\n", headID, m_postingSizeLimit, m_bufferSizeLimit); + ret = Split(p_exWorkSpace, p_index, headID, !m_opt->m_disableReassign, false, false); + if (ret != ErrorCode::Success) + SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Split %d failed!\n", headID); lock.unlock(); goto checkDeleted; } - ErrorCode ret; auto appendIOBegin = std::chrono::high_resolution_clock::now(); - *m_checkSums[headID] = - m_checkSum.AppendChecksum(*m_checkSums[headID], appendPosting.c_str(), (int)(appendPosting.size())); - if ((ret=db->Merge(headID, appendPosting, MaxTimeout, &(p_exWorkSpace->m_diskRequests))) != ErrorCode::Success) { - SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Merge failed! Posting Size:%d, limit: %d\n", m_postingSizes.GetSize(headID), m_postingSizeLimit); + if ((ret = db->Merge(headID, appendPosting, MaxTimeout, &(p_exWorkSpace->m_diskRequests), [this, prefixChecksum = *m_checkSums[headID]] (const std::string& readPrefix) -> bool { + return this->m_checkSum.ValidateChecksum(readPrefix.c_str(), (int)(readPrefix.size()), prefixChecksum); + })) != ErrorCode::Success) + { + SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Merge failed for %d! Posting Size:%d, limit: %d\n", headID, m_postingSizes.GetSize(headID), m_postingSizeLimit); GetDBStats(); return ret; } auto appendIOEnd = std::chrono::high_resolution_clock::now(); appendIOSeconds = std::chrono::duration_cast(appendIOEnd - appendIOBegin).count(); + *m_checkSums[headID] = + m_checkSum.AppendChecksum(*m_checkSums[headID], appendPosting.c_str(), (int)(appendPosting.size())); m_postingSizes.IncSize(headID, appendNum); if (m_opt->m_consistencyCheck && (ret = db->Check(headID, m_postingSizes.GetSize(headID) * m_vectorInfoSize, nullptr)) != ErrorCode::Success) { diff --git a/AnnService/inc/Core/SPANN/ExtraFileController.h b/AnnService/inc/Core/SPANN/ExtraFileController.h index 70c1824e5..b026aeeb3 100644 --- a/AnnService/inc/Core/SPANN/ExtraFileController.h +++ b/AnnService/inc/Core/SPANN/ExtraFileController.h @@ -218,7 +218,7 @@ namespace SPTAG::SPANN { FileIO* fileIO; Helper::RequestQueue processIocp; std::vector reqs; - std::vector> pageBuffers; + Helper::PageBuffer pageBuffer; public: LRUCache(int64_t capacity, int64_t limit, FileIO* fileIO) { @@ -229,11 +229,10 @@ namespace SPTAG::SPANN { this->hits = 0; this->fileIO = fileIO; this->reqs.resize(limit); - this->pageBuffers.resize(limit); + this->pageBuffer.ReservePageBuffer(limit << PageSizeEx); for (int i = 0; i < limit; i++) { - this->pageBuffers[i].ReservePageBuffer(PageSize); auto& req = this->reqs[i]; - req.m_buffer = (char*)(this->pageBuffers[i].GetBuffer()); + req.m_buffer = (char *)(this->pageBuffer.GetBuffer() + (i << PageSizeEx)); req.m_extension = &processIocp; #ifdef _MSC_VER @@ -295,6 +294,7 @@ namespace SPTAG::SPANN { auto last = keys.back(); auto lastit = cache.find(last); if (!evict(last, lastit->second.first.data(), lastit->second.first.size(), lastit)) { + evict(key, it->second.first.data(), it->second.first.size(), it); return false; } } @@ -314,8 +314,7 @@ namespace SPTAG::SPANN { return false; } } - auto keys_it = keys.insert(keys.begin(), key); - cache.insert({key, {std::string((char*)value, put_size), keys_it}}); + cache.insert({key, {std::string((char *)value, put_size), keys.insert(keys.begin(), key)}}); size += put_size; return true; } @@ -329,22 +328,33 @@ namespace SPTAG::SPANN { return true; } - bool merge(SizeType key, void* value, int merge_size) { + bool merge(SizeType key, void *value, int merge_size, std::function checksum) + { // SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "LRUCache: merge size: %lld\n", merge_size); auto it = cache.find(key); if (it == cache.end()) { // SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "LRUCache: merge key not found\n"); + ErrorCode ret; std::string valstr; - if (fileIO->Get(key, &valstr, MaxTimeout, &reqs, false) != ErrorCode::Success) { - SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "LRUCache: merge key not found in file\n"); + if ((ret = fileIO->Get(key, &valstr, MaxTimeout, &reqs, false)) != ErrorCode::Success || !checksum(valstr)) { + SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "LRUCache: merge key not found in file or checksum issue = %d\n", (int)(ret == ErrorCode::Success)); return false; // If the key does not exist, return false } + valstr.append((char *)value, merge_size); + while (valstr.size() > (int)(capacity - size) && (!keys.empty())) + { + auto last = keys.back(); + auto lastit = cache.find(last); + if (!evict(last, lastit->second.first.data(), lastit->second.first.size(), lastit)) + { + return false; + } + } cache.insert({key, {valstr, keys.insert(keys.begin(), key)}}); size += valstr.size(); - it = cache.find(key); - } else { - hits++; + return true; } + hits++; if (merge_size + it->second.first.size() > limit) { evict(key, it->second.first.data(), it->second.first.size(), it); @@ -357,6 +367,7 @@ namespace SPTAG::SPANN { auto last = keys.back(); auto lastit = cache.find(last); if (!evict(last, lastit->second.first.data(), lastit->second.first.size(), lastit)) { + evict(key, it->second.first.data(), it->second.first.size(), it); return false; } } @@ -421,8 +432,9 @@ namespace SPTAG::SPANN { return caches[hash(key)]->del(key); } - bool merge(SizeType key, void* value, int merge_size) { - return caches[hash(key)]->merge(key, value, merge_size); + bool merge(SizeType key, void *value, int merge_size, std::function checksum) + { + return caches[hash(key)]->merge(key, value, merge_size, checksum); } bool flush() { @@ -732,6 +744,48 @@ namespace SPTAG::SPANN { { lock = &(m_pShardedLRUCache->getlock(key)); lock->lock(); + + if (m_pShardedLRUCache->put(key, (void *)(value.data()), (SPTAG::SizeType)(value.size()))) + { + if (At(key) == 0xffffffffffffffff) + { + uintptr_t tmpblocks = 0xffffffffffffffff; + if (m_buffer.unsafe_size() > m_bufferLimit) + { + while (!m_buffer.try_pop(tmpblocks)); + } + else + { + tmpblocks = (uintptr_t)(new AddressType[m_blockLimit]); + } + // The 0th element of the block address list represents the data size; set it to -1. + memset((AddressType *)tmpblocks, -1, sizeof(AddressType) * m_blockLimit); + At(key) = tmpblocks; + } + int64_t *postingSize = (int64_t *)At(key); + int oldblocks = (*postingSize < 0) ? 0 : ((*postingSize + PageSize - 1) >> PageSizeEx); + if (blocks - oldblocks > 0) + { + if (!m_pBlockController.GetBlocks(postingSize + oldblocks + 1, blocks - oldblocks)) + { + SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "[Put] Not enough blocks in the pool can be allocated!\n"); + if (lock) lock->unlock(); + return ErrorCode::DiskIOFail; + } + } + else if (blocks - oldblocks < 0) + { + if (!m_pBlockController.ReleaseBlocks(postingSize + blocks + 1, oldblocks - blocks)) + { + SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "[Put] Release blocks back to the pool failed!\n"); + if (lock) lock->unlock(); + return ErrorCode::DiskIOFail; + } + } + *postingSize = (int64_t)(value.size()); + lock->unlock(); + return ErrorCode::Success; + } } uintptr_t tmpblocks = 0xffffffffffffffff; // If this key has not been assigned mapping blocks yet, allocate a batch. @@ -759,16 +813,14 @@ namespace SPTAG::SPANN { return ErrorCode::DiskIOFail; } *postingSize = value.size(); - if (!useCache || m_pShardedLRUCache == nullptr || !m_pShardedLRUCache->put(key, (void*)(value.data()), (SPTAG::SizeType)(value.size()))) { - if (!m_pBlockController.WriteBlocks(postingSize + 1, blocks, value, timeout, reqs)) - { - m_pBlockController.ReleaseBlocks(postingSize + 1, blocks); - memset(postingSize + 1, -1, sizeof(AddressType) * blocks); - SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "[Put] Write new block failed!\n"); - if (lock) lock->unlock(); - return ErrorCode::DiskIOFail; - } - } + if (!m_pBlockController.WriteBlocks(postingSize + 1, blocks, value, timeout, reqs)) + { + m_pBlockController.ReleaseBlocks(postingSize + 1, blocks); + memset(postingSize + 1, -1, sizeof(AddressType) * blocks); + SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "[Put] Write new block failed!\n"); + if (lock) lock->unlock(); + return ErrorCode::DiskIOFail; + } At(key) = tmpblocks; } else { @@ -785,15 +837,13 @@ namespace SPTAG::SPANN { return ErrorCode::DiskIOFail; } *((int64_t*)partialtmpblocks) = value.size(); - if (!useCache || m_pShardedLRUCache == nullptr || !m_pShardedLRUCache->put(key, (void*)(value.data()), (SPTAG::SizeType)(value.size()))) { - if (!m_pBlockController.WriteBlocks((AddressType*)partialtmpblocks + 1, blocks, value, timeout, reqs)) - { - SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "[Put] Write new block failed!\n"); - m_pBlockController.ReleaseBlocks((AddressType*)partialtmpblocks + 1, blocks); - m_buffer.push(partialtmpblocks); - if (lock) lock->unlock(); - return ErrorCode::DiskIOFail; - } + if (!m_pBlockController.WriteBlocks((AddressType*)partialtmpblocks + 1, blocks, value, timeout, reqs)) + { + SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "[Put] Write new block failed!\n"); + m_pBlockController.ReleaseBlocks((AddressType*)partialtmpblocks + 1, blocks); + m_buffer.push(partialtmpblocks); + if (lock) lock->unlock(); + return ErrorCode::DiskIOFail; } // Release the original blocks @@ -859,23 +909,10 @@ namespace SPTAG::SPANN { return ErrorCode::Success; } - void PrintPostingDiff(std::string& p1, std::string& p2, const char* pos) { - if (p1.size() != p2.size()) { - SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Merge %s: p1 and p2 have different sizes: before=%u after=%u\n", pos, p1.size(), p2.size()); - return; - } - std::string diff = ""; - for (size_t i = 0; i < p1.size(); i+=4) { - if (p1[i] != p2[i]) { - diff += "[" + std::to_string(i) + "]:" + std::to_string(int(p1[i])) + "^" + std::to_string(int(p2[i])) + " "; - } - } - if (diff.size() != 0) { - SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Merge %s: %s\n", pos, diff.c_str()); - } - } - ErrorCode Merge(const SizeType key, const std::string& value, const std::chrono::microseconds& timeout, std::vector* reqs) { + ErrorCode Merge(const SizeType key, const std::string &value, const std::chrono::microseconds &timeout, + std::vector *reqs, std::function checksum) + { SizeType r = m_pBlockMapping.R(); if (key >= r) { @@ -912,10 +949,22 @@ namespace SPTAG::SPANN { if (lock) lock->unlock(); return ErrorCode::Posting_OverFlow; } + if (m_pShardedLRUCache && m_pShardedLRUCache->merge(key, (void *)(value.data()), value.size(), checksum)) + { + int oldblocks = ((*postingSize + PageSize - 1) >> PageSizeEx); + int allocblocks = newblocks - oldblocks; + if (allocblocks > 0 && !m_pBlockController.GetBlocks(postingSize + 1 + oldblocks, allocblocks)) + { + SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "[Merge] Not enough blocks in the pool can be allocated!\n"); + if (lock) lock->unlock(); + return ErrorCode::DiskIOFail; + } + *postingSize = newSize; + if (lock) lock->unlock(); + return ErrorCode::Success; + } - //std::string before; - //Get(key, &before, timeout, reqs); - + postingSize = (int64_t *)At(key); auto sizeInPage = (*postingSize) % PageSize; // Actual size of the last block int oldblocks = (*postingSize >> PageSizeEx); int allocblocks = newblocks - oldblocks; @@ -942,20 +991,18 @@ namespace SPTAG::SPANN { m_buffer.push(tmpblocks); if (lock) lock->unlock(); return ErrorCode::DiskIOFail; + } + if (!m_pBlockController.WriteBlocks((AddressType *)tmpblocks + 1 + oldblocks, allocblocks, newValue, + timeout, reqs)) + { + SPTAGLIB_LOG(Helper::LogLevel::LL_Error, + "[Merge] Write new block failed!\n"); + m_pBlockController.ReleaseBlocks((AddressType *)tmpblocks + 1 + oldblocks, allocblocks); + m_buffer.push(tmpblocks); + if (lock) lock->unlock(); + return ErrorCode::DiskIOFail; } - *((int64_t*)tmpblocks) = newSize; - if (m_pShardedLRUCache == nullptr || !m_pShardedLRUCache->merge(key, (void *)(value.data()), value.size())) { - if (!m_pBlockController.WriteBlocks((AddressType *)tmpblocks + 1 + oldblocks, allocblocks, newValue, - timeout, reqs)) - { - SPTAGLIB_LOG(Helper::LogLevel::LL_Error, - "[Merge] Write new block failed!\n"); - m_pBlockController.ReleaseBlocks((AddressType *)tmpblocks + 1 + oldblocks, allocblocks); - m_buffer.push(tmpblocks); - if (lock) lock->unlock(); - return ErrorCode::DiskIOFail; - } - } + *((int64_t *)tmpblocks) = newSize; // This is also to ensure checkpoint correctness, so we release the partially used block and allocate a new one. m_pBlockController.ReleaseBlocks(postingSize + 1 + oldblocks, 1); @@ -972,31 +1019,20 @@ namespace SPTAG::SPANN { if (lock) lock->unlock(); return ErrorCode::DiskIOFail; } - - if (m_pShardedLRUCache == nullptr || !m_pShardedLRUCache->merge(key, (void *)(value.data()), value.size())) { - if (!m_pBlockController.WriteBlocks(postingSize + 1 + oldblocks, allocblocks, value, timeout, reqs)) - { - SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "[Merge] Write new block failed!\n"); - m_pBlockController.ReleaseBlocks(postingSize + 1 + oldblocks, allocblocks); - if (lock) lock->unlock(); - return ErrorCode::DiskIOFail; - } + + if (!m_pBlockController.WriteBlocks(postingSize + 1 + oldblocks, allocblocks, value, timeout, reqs)) + { + SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "[Merge] Write new block failed!\n"); + m_pBlockController.ReleaseBlocks(postingSize + 1 + oldblocks, allocblocks); + if (lock) lock->unlock(); + return ErrorCode::DiskIOFail; } *postingSize = newSize; } - /* - std::string after; - Get(key, &after, timeout, reqs); - before += value; - PrintPostingDiff(before, after, "1"); - */ if (lock) lock->unlock(); return ErrorCode::Success; } - ErrorCode Merge(const std::string &key, const std::string& value, const std::chrono::microseconds& timeout, std::vector* reqs) { - return Merge(std::stoi(key), value, timeout, reqs); - } ErrorCode Delete(SizeType key) override { SizeType r = m_pBlockMapping.R(); diff --git a/AnnService/inc/Core/SPANN/ExtraRocksDBController.h b/AnnService/inc/Core/SPANN/ExtraRocksDBController.h index 732dda6f3..030a3a6d7 100644 --- a/AnnService/inc/Core/SPANN/ExtraRocksDBController.h +++ b/AnnService/inc/Core/SPANN/ExtraRocksDBController.h @@ -279,7 +279,10 @@ namespace SPTAG::SPANN return Put(k, value, timeout, reqs); } - ErrorCode Merge(const SizeType key, const std::string& value, const std::chrono::microseconds& timeout, std::vector* reqs) override { + ErrorCode Merge(const SizeType key, const std::string &value, const std::chrono::microseconds &timeout, + std::vector *reqs, + std::function checksum) override + { if (value.empty()) { SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Error! empty append posting!\n"); } diff --git a/AnnService/inc/Core/SPANN/ExtraSPDKController.h b/AnnService/inc/Core/SPANN/ExtraSPDKController.h index 396fb24be..739e3a958 100644 --- a/AnnService/inc/Core/SPANN/ExtraSPDKController.h +++ b/AnnService/inc/Core/SPANN/ExtraSPDKController.h @@ -348,7 +348,10 @@ namespace SPTAG::SPANN return ErrorCode::Success; } - ErrorCode Merge(SizeType key, const std::string& value, const std::chrono::microseconds& timeout, std::vector* reqs) override { + ErrorCode Merge(SizeType key, const std::string &value, const std::chrono::microseconds &timeout, + std::vector *reqs, + std::function checksum) override + { if (key >= m_pBlockMapping.R()) { SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Key range error: key: %d, mapping size: %d\n", key, m_pBlockMapping.R()); return ErrorCode::Fail; diff --git a/AnnService/inc/Helper/KeyValueIO.h b/AnnService/inc/Helper/KeyValueIO.h index 4bda94a91..2a4c4ea79 100644 --- a/AnnService/inc/Helper/KeyValueIO.h +++ b/AnnService/inc/Helper/KeyValueIO.h @@ -32,7 +32,10 @@ namespace SPTAG virtual ErrorCode Put(const SizeType key, const std::string& value, const std::chrono::microseconds& timeout, std::vector* reqs) = 0; - virtual ErrorCode Merge(const SizeType key, const std::string& value, const std::chrono::microseconds& timeout, std::vector* reqs) = 0; + virtual ErrorCode Merge(const SizeType key, const std::string &value, + const std::chrono::microseconds &timeout, + std::vector *reqs, + std::function checksum) = 0; virtual ErrorCode Delete(SizeType key) = 0; diff --git a/AnnService/src/Core/SPANN/SPANNIndex.cpp b/AnnService/src/Core/SPANN/SPANNIndex.cpp index 90340b8d4..8ce0d2f0b 100644 --- a/AnnService/src/Core/SPANN/SPANNIndex.cpp +++ b/AnnService/src/Core/SPANN/SPANNIndex.cpp @@ -1557,6 +1557,10 @@ template ErrorCode Index::Check() { std::atomic ret = ErrorCode::Success; + while (!AllFinished()) + { + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + } //if ((ret = m_index->Check()) != ErrorCode::Success) // return ret; diff --git a/Test/src/KVTest.cpp b/Test/src/KVTest.cpp index 9c49d4ae1..32fab3ac0 100644 --- a/Test/src/KVTest.cpp +++ b/Test/src/KVTest.cpp @@ -108,7 +108,8 @@ void Test(std::string path, std::string type, bool debug = false) { for (int j = 0; j < mergeIters; j++) { - db->Merge(i, std::to_string(i), MaxTimeout, &(workspace.m_diskRequests)); + db->Merge(i, std::to_string(i), MaxTimeout, &(workspace.m_diskRequests), + [](const std::string &prefix) -> bool { return true; }); } } t2 = std::chrono::high_resolution_clock::now(); From 574412e03fdcf03d09cafcc321404c815ddd4b3e Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Wed, 3 Dec 2025 20:39:19 -0800 Subject: [PATCH 05/38] add configuable merge during search and benchmark test --- .../inc/Core/SPANN/ExtraDynamicSearcher.h | 2 +- AnnService/inc/Core/SPANN/Options.h | 3 +- .../inc/Core/SPANN/ParameterDefinitionList.h | 1 + AnnService/src/KeyValueTest/main.cpp | 3 +- Test/inc/TestDataGenerator.h | 12 +- Test/src/SPFreshTest.cpp | 469 +++++++++++++++++- Test/src/TestDataGenerator.cpp | 74 ++- 7 files changed, 545 insertions(+), 19 deletions(-) diff --git a/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h b/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h index c78ed6cf1..854652380 100644 --- a/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h +++ b/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h @@ -1837,7 +1837,7 @@ namespace SPTAG::SPANN { queryResults.AddPoint(vectorID, distance2leaf); } auto compEnd = std::chrono::high_resolution_clock::now(); - //if (realNum <= m_mergeThreshold) MergeAsync(p_index.get(), curPostingID); // TODO: Control merge + if (m_opt->m_asyncMergeInSearch && realNum <= m_mergeThreshold) MergeAsync(p_index.get(), curPostingID); // TODO: Control merge compLatency += ((double)std::chrono::duration_cast(compEnd - compStart).count()); diff --git a/AnnService/inc/Core/SPANN/Options.h b/AnnService/inc/Core/SPANN/Options.h index 414111a1a..9246c492d 100644 --- a/AnnService/inc/Core/SPANN/Options.h +++ b/AnnService/inc/Core/SPANN/Options.h @@ -189,7 +189,8 @@ namespace SPTAG { bool m_checksumInRead; int m_cacheSize; int m_cacheShards; - + bool m_asyncMergeInSearch; + // Iterative int m_headBatch; int m_asyncAppendQueueSize; diff --git a/AnnService/inc/Core/SPANN/ParameterDefinitionList.h b/AnnService/inc/Core/SPANN/ParameterDefinitionList.h index 61c752e4c..32f01aff8 100644 --- a/AnnService/inc/Core/SPANN/ParameterDefinitionList.h +++ b/AnnService/inc/Core/SPANN/ParameterDefinitionList.h @@ -202,6 +202,7 @@ DefineSSDParameter(m_growthFileSize, int, 10, "GrowthFileSizeGB") DefineSSDParameter(m_growThreshold, float, 0.05, "GrowthThreshold") DefineSSDParameter(m_fDeletePercentageForRefine, float, 0.4F, "DeletePercentageForRefine") // Mutable DefineSSDParameter(m_oneClusterCutMax, bool, false, "OneClusterCutMax") // Mutable +DefineSSDParameter(m_asyncMergeInSearch, bool, true, "AsyncMergeInSearch") // Mutable DefineSSDParameter(m_consistencyCheck, bool, false, "ConsistencyCheck") // Mutable DefineSSDParameter(m_checksumCheck, bool, false, "ChecksumCheck") // Mutable DefineSSDParameter(m_checksumInRead, bool, false, "ChecksumInRead") // Mutable diff --git a/AnnService/src/KeyValueTest/main.cpp b/AnnService/src/KeyValueTest/main.cpp index 470dcc9bc..1fec3886b 100644 --- a/AnnService/src/KeyValueTest/main.cpp +++ b/AnnService/src/KeyValueTest/main.cpp @@ -397,7 +397,8 @@ int main(int argc, char *argv[]) { mergeValue += (char)(rand() % 256); } - fileIO.Merge(key, mergeValue, MaxTimeout, &(workspace.m_diskRequests)); + fileIO.Merge(key, mergeValue, MaxTimeout, &(workspace.m_diskRequests), + [](const std::string &) -> bool { return true; }); write_count++; read_count++; std::string readValue; diff --git a/Test/inc/TestDataGenerator.h b/Test/inc/TestDataGenerator.h index 883df381c..387aeaa36 100644 --- a/Test/inc/TestDataGenerator.h +++ b/Test/inc/TestDataGenerator.h @@ -15,7 +15,8 @@ namespace TestUtils { template class TestDataGenerator { public: - TestDataGenerator(int n, int q, int m, int k, std::string distMethod); + TestDataGenerator(int n, int q, int m, int k, std::string distMethod, int a = 0, bool isRandom = true, + std::string vectorPath = "", std::string queryPath = ""); void Run(std::shared_ptr& vecset, std::shared_ptr& metaset, @@ -29,15 +30,18 @@ namespace TestUtils { static std::shared_ptr GenerateMetadataSet(SPTAG::SizeType count, SPTAG::SizeType offsetStart); + static std::shared_ptr GenerateLoadVectorSet(SPTAG::SizeType count, SPTAG::DimensionType dim, + std::string path, SPTAG::SizeType start = 0); + void RunBatches(std::shared_ptr &vecset, std::shared_ptr &metaset, std::shared_ptr &addvecset, std::shared_ptr &addmetaset, std::shared_ptr &queryset, int base, int batchinsert, int batchdelete, int batches, std::shared_ptr &truths); private: - int m_n, m_q, m_m, m_k; + int m_n, m_a, m_q, m_m, m_k; std::string m_distMethod; - - bool FileExists(const std::string& filename); + bool m_isRandom; + std::string m_vectorPath, m_queryPath; std::shared_ptr LoadReader(const std::string& filename); diff --git a/Test/src/SPFreshTest.cpp b/Test/src/SPFreshTest.cpp index 8ac2c7b75..573f40239 100644 --- a/Test/src/SPFreshTest.cpp +++ b/Test/src/SPFreshTest.cpp @@ -95,6 +95,7 @@ std::shared_ptr BuildIndex(const std::string &outDirectory, std::sh ConsistencyCheck=true ChecksumCheck=true ChecksumInRead=false + AsyncMergeInSearch=true DeletePercentageForRefine=0.4 AsyncAppendQueueSize=0 AllowZeroReplica=false @@ -246,6 +247,385 @@ void InsertVectors(SPANN::Index *p_index, int insertThreads, int step std::this_thread::sleep_for(std::chrono::milliseconds(20)); } } + +template +void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, const std::string &truthPath, DistCalcMethod distMethod, + const std::string &indexPath, int dimension, int baseVectorCount, int insertVectorCount, int deleteVectorCount, + int batches, int topK, int numThreads, int numQueries) +{ + int oldM = M, oldK = K, oldN = N, oldQueries = queries; + N = baseVectorCount; + queries = numQueries; + M = dimension; + K = topK; + std::string dist = Helper::Convert::ConvertToString(distMethod); + int insertBatchSize = insertVectorCount / batches; + int deleteBatchSize = deleteVectorCount / batches; + + // Variables to collect JSON output data + std::ostringstream benchmark1Data, benchmark2Data, benchmark3Data, benchmark4Data, benchmark5Data, benchmark6Data; + + // Generate test data + std::shared_ptr vecset, addvecset, queryset, truth; + std::shared_ptr metaset, addmetaset; + TestUtils::TestDataGenerator generator(N, queries, M, K, dist, insertVectorCount, false, vectorPath, queryPath); + generator.RunBatches(vecset, metaset, addvecset, addmetaset, queryset, N, insertBatchSize, deleteBatchSize, + batches, truth); + + // Build initial index + BOOST_TEST_MESSAGE("\n=== Building Index ==="); + std::shared_ptr index = BuildIndex(indexPath, vecset, metaset, dist); + BOOST_REQUIRE(index != nullptr); + float recall = Search(index, queryset, vecset, addvecset, K, truth, N, 0); + BOOST_TEST_MESSAGE("Index built successfully with " << N << " vectors. Recall@" << K << "=" << recall << "."); + + // Benchmark 1: Insert performance + BOOST_TEST_MESSAGE("\n=== Benchmark 1: Insert Performance ==="); + { + auto start = std::chrono::high_resolution_clock::now(); + for (int iter = 0; iter < batches; iter++) + { + InsertVectors(static_cast *>(index.get()), numThreads, insertBatchSize, addvecset, + addmetaset, iter * insertBatchSize); + } + auto end = std::chrono::high_resolution_clock::now(); + + double seconds = std::chrono::duration_cast(end - start).count() / 1000000.0f; + double throughput = insertVectorCount / seconds; + + BOOST_TEST_MESSAGE(" Inserted: " << insertVectorCount << " vectors"); + BOOST_TEST_MESSAGE(" Time: " << seconds << " seconds"); + BOOST_TEST_MESSAGE(" Throughput: " << throughput << " vectors/sec"); + + // Collect JSON data for Benchmark 1 + benchmark1Data << std::fixed << std::setprecision(4); + benchmark1Data << "{\n"; + benchmark1Data << " \"inserted\": " << insertVectorCount << ",\n"; + benchmark1Data << " \"insert timeSeconds\": " << seconds << ",\n"; + benchmark1Data << " \"insert throughput\": " << throughput << "\n"; + + + std::vector threads; + threads.reserve(numThreads); + + std::atomic_size_t vectorsSent(0); + int totaldeleted = batches * deleteBatchSize; + auto func = [&]() { + size_t idx = 0; + while (true) + { + idx = vectorsSent.fetch_add(1); + if (idx < totaldeleted) + { + if ((idx & ((1 << 5) - 1)) == 0) + { + SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "Sent %.2lf%%...\n", idx * 100.0 / totaldeleted); + } + BOOST_REQUIRE(index->DeleteIndex(idx) == ErrorCode::Success); + } + else + { + return; + } + } + }; + + start = std::chrono::high_resolution_clock::now(); + for (int j = 0; j < numThreads; j++) + { + threads.emplace_back(func); + } + for (auto &thread : threads) + { + thread.join(); + } + end = std::chrono::high_resolution_clock::now(); + seconds = std::chrono::duration_cast(end - start).count() / 1000000.0f; + throughput = deleteVectorCount / seconds; + + benchmark1Data << " \"deleted\": " << deleteVectorCount << ",\n"; + benchmark1Data << " \"delete timeSeconds\": " << seconds << ",\n"; + benchmark1Data << " \"delete throughput\": " << throughput << "\n"; + benchmark1Data << " }"; + } + + // Benchmark 2: Query performance with detailed latency stats + BOOST_TEST_MESSAGE("\n=== Benchmark 2: Query Performance ==="); + { + std::vector latencies(numQueries); + std::atomic_size_t queriesSent(0); + std::vector results(numQueries); + + for (int i = 0; i < numQueries; i++) + { + results[i] = QueryResult((const T *)queryset->GetVector(i), K, false); + } + + std::vector threads; + threads.reserve(numThreads); + + auto batchStart = std::chrono::high_resolution_clock::now(); + + for (int i = 0; i < numThreads; i++) + { + threads.emplace_back([&]() { + size_t qid; + while ((qid = queriesSent.fetch_add(1)) < numQueries) + { + auto t1 = std::chrono::high_resolution_clock::now(); + index->SearchIndex(results[qid]); + auto t2 = std::chrono::high_resolution_clock::now(); + latencies[qid] = std::chrono::duration_cast(t2 - t1).count() / 1000.0f; + } + }); + } + + for (auto &thread : threads) + thread.join(); + + auto batchEnd = std::chrono::high_resolution_clock::now(); + float batchLatency = + std::chrono::duration_cast(batchEnd - batchStart).count() / 1000000.0f; + + // Calculate statistics + float mean = 0, minLat = (std::numeric_limits::max)(), maxLat = 0; + for (int i = 0; i < numQueries; i++) + { + mean += latencies[i]; + minLat = (std::min)(minLat, latencies[i]); + maxLat = (std::max)(maxLat, latencies[i]); + } + mean /= numQueries; + + std::sort(latencies.begin(), latencies.end()); + float p50 = latencies[static_cast(numQueries * 0.50)]; + float p90 = latencies[static_cast(numQueries * 0.90)]; + float p95 = latencies[static_cast(numQueries * 0.95)]; + float p99 = latencies[static_cast(numQueries * 0.99)]; + float qps = numQueries / batchLatency; + + BOOST_TEST_MESSAGE(" Queries: " << numQueries); + BOOST_TEST_MESSAGE(" Mean Latency: " << mean << " ms"); + BOOST_TEST_MESSAGE(" P50 Latency: " << p50 << " ms"); + BOOST_TEST_MESSAGE(" P90 Latency: " << p90 << " ms"); + BOOST_TEST_MESSAGE(" P95 Latency: " << p95 << " ms"); + BOOST_TEST_MESSAGE(" P99 Latency: " << p99 << " ms"); + BOOST_TEST_MESSAGE(" Min Latency: " << minLat << " ms"); + BOOST_TEST_MESSAGE(" Max Latency: " << maxLat << " ms"); + BOOST_TEST_MESSAGE(" QPS: " << qps); + + // Collect JSON data for Benchmark 2 + benchmark2Data << std::fixed << std::setprecision(4); + benchmark2Data << "{\n"; + benchmark2Data << " \"numQueries\": " << numQueries << ",\n"; + benchmark2Data << " \"meanLatency\": " << mean << ",\n"; + benchmark2Data << " \"p50\": " << p50 << ",\n"; + benchmark2Data << " \"p90\": " << p90 << ",\n"; + benchmark2Data << " \"p95\": " << p95 << ",\n"; + benchmark2Data << " \"p99\": " << p99 << ",\n"; + benchmark2Data << " \"minLatency\": " << minLat << ",\n"; + benchmark2Data << " \"maxLatency\": " << maxLat << ",\n"; + benchmark2Data << " \"qps\": " << qps << "\n"; + benchmark2Data << " }"; + } + + // Benchmark 3: Save index to disk + BOOST_TEST_MESSAGE("\n=== Benchmark 3: Save Index ==="); + { + auto start = std::chrono::high_resolution_clock::now(); + BOOST_REQUIRE(ErrorCode::Success == index->SaveIndex(indexPath + "_saved")); + auto end = std::chrono::high_resolution_clock::now(); + + double seconds = std::chrono::duration_cast(end - start).count() / 1000000.0f; + BOOST_TEST_MESSAGE(" Save Time: " << seconds << " seconds"); + BOOST_TEST_MESSAGE(" Save completed successfully"); + + // Collect JSON data for Benchmark 3 + benchmark3Data << std::fixed << std::setprecision(4); + benchmark3Data << "{\n"; + benchmark3Data << " \"timeSeconds\": " << seconds << "\n"; + benchmark3Data << " }"; + } + + BOOST_TEST_MESSAGE("Starting Benchmark 4..."); + + // Benchmark 4: Clean up memory and reload from disk + BOOST_TEST_MESSAGE("\n=== Benchmark 4: Memory Cleanup & Reload ==="); + + // Explicitly clean up memory + BOOST_TEST_MESSAGE("Releasing index from memory..."); + index = nullptr; + BOOST_TEST_MESSAGE("Memory cleanup complete"); + + // Reload index + BOOST_TEST_MESSAGE("Starting to reload index from: " << indexPath + "_saved"); + { + auto start = std::chrono::high_resolution_clock::now(); + ErrorCode loadResult = VectorIndex::LoadIndex(indexPath + "_saved", index); + BOOST_TEST_MESSAGE("LoadIndex returned with code: " << (int)loadResult); + BOOST_REQUIRE(loadResult == ErrorCode::Success); + auto end = std::chrono::high_resolution_clock::now(); + + BOOST_REQUIRE(index != nullptr); + + double seconds = std::chrono::duration_cast(end - start).count() / 1000000.0f; + int vectorCount = index->GetNumSamples(); + BOOST_TEST_MESSAGE(" Load Time: " << seconds << " seconds"); + BOOST_TEST_MESSAGE(" Index vectors after reload: " << vectorCount); + + // Collect JSON data for Benchmark 4 + benchmark4Data << std::fixed << std::setprecision(4); + benchmark4Data << "{\n"; + benchmark4Data << " \"timeSeconds\": " << seconds << ",\n"; + benchmark4Data << " \"vectorCount\": " << vectorCount << "\n"; + benchmark4Data << " }"; + } + + BOOST_TEST_MESSAGE("Benchmark 4 completed successfully"); + + std::vector results(numQueries); + // Benchmark 5: Query performance after reload + BOOST_TEST_MESSAGE("\n=== Benchmark 5: Query After Reload ==="); + { + std::vector latencies(numQueries); + std::atomic_size_t queriesSent(0); + + for (int i = 0; i < numQueries; i++) + { + results[i] = QueryResult((const T *)queryset->GetVector(i), K, false); + } + + std::vector threads; + threads.reserve(numThreads); + + auto batchStart = std::chrono::high_resolution_clock::now(); + + for (int i = 0; i < numThreads; i++) + { + threads.emplace_back([&]() { + size_t qid; + while ((qid = queriesSent.fetch_add(1)) < numQueries) + { + auto t1 = std::chrono::high_resolution_clock::now(); + index->SearchIndex(results[qid]); + auto t2 = std::chrono::high_resolution_clock::now(); + latencies[qid] = std::chrono::duration_cast(t2 - t1).count() / 1000.0f; + } + }); + } + + for (auto &thread : threads) + thread.join(); + + auto batchEnd = std::chrono::high_resolution_clock::now(); + float batchLatency = + std::chrono::duration_cast(batchEnd - batchStart).count() / 1000000.0f; + + float mean = 0; + for (int i = 0; i < numQueries; i++) + { + mean += latencies[i]; + } + mean /= numQueries; + + std::sort(latencies.begin(), latencies.end()); + float p95 = latencies[static_cast(numQueries * 0.95)]; + float p99 = latencies[static_cast(numQueries * 0.99)]; + float qps = numQueries / batchLatency; + + BOOST_TEST_MESSAGE(" Mean Latency: " << mean << " ms"); + BOOST_TEST_MESSAGE(" P95 Latency: " << p95 << " ms"); + BOOST_TEST_MESSAGE(" P99 Latency: " << p99 << " ms"); + BOOST_TEST_MESSAGE(" QPS: " << qps); + + // Collect JSON data for Benchmark 5 + benchmark5Data << std::fixed << std::setprecision(4); + benchmark5Data << "{\n"; + benchmark5Data << " \"meanLatency\": " << mean << ",\n"; + benchmark5Data << " \"p95\": " << p95 << ",\n"; + benchmark5Data << " \"p99\": " << p99 << ",\n"; + benchmark5Data << " \"qps\": " << qps << "\n"; + benchmark5Data << " }"; + } + + // Benchmark 6: Recall evaluation (if truth file provided) + BOOST_TEST_MESSAGE("Checking for truth file: " << truthPath); + float avgRecall = EvaluateRecall(results, index, queryset, truth, vecset, addvecset, N, K, batches); + BOOST_TEST_MESSAGE(" Recall@" << K << " = " << (avgRecall * 100.0f) << "%"); + BOOST_TEST_MESSAGE(" (Evaluated on " << numQueries << " queries against base vectors)"); + + // Collect JSON data for Benchmark 6 + benchmark6Data << std::fixed << std::setprecision(4); + benchmark6Data << "{\n"; + benchmark6Data << " \"recallAtK\": " << avgRecall << ",\n"; + benchmark6Data << " \"k\": " << K << ",\n"; + benchmark6Data << " \"numQueries\": " << numQueries << "\n"; + benchmark6Data << " }"; + + BOOST_TEST_MESSAGE("\n=== Benchmark Complete ==="); + + // Write JSON output + { + std::ofstream jsonFile("output.json"); + if (jsonFile.is_open()) + { + jsonFile << std::fixed << std::setprecision(4); + + // Get current timestamp + auto now = std::chrono::system_clock::now(); + auto time_t_now = std::chrono::system_clock::to_time_t(now); + std::tm tm_now; + localtime_s(&tm_now, &time_t_now); + + std::ostringstream timestampStream; + timestampStream << std::put_time(&tm_now, "%Y-%m-%dT%H:%M:%S"); + std::string timestamp = timestampStream.str(); + + jsonFile << "{\n"; + jsonFile << " \"timestamp\": \"" << timestamp << "\",\n"; + jsonFile << " \"config\": {\n"; + jsonFile << " \"vectorPath\": \"" << vectorPath << "\",\n"; + jsonFile << " \"queryPath\": \"" << queryPath << "\",\n"; + jsonFile << " \"truthPath\": \"" << truthPath << "\",\n"; + jsonFile << " \"indexPath\": \"" << indexPath << "\",\n"; + jsonFile << " \"ValueType\": \"" << Helper::Convert::ConvertToString(GetEnumValueType()) << "\",\n"; + jsonFile << " \"dimension\": " << dimension << ",\n"; + jsonFile << " \"baseVectorCount\": " << baseVectorCount << ",\n"; + jsonFile << " \"insertVectorCount\": " << insertVectorCount << ",\n"; + jsonFile << " \"DeleteVectorCount\": " << deleteVectorCount << ",\n"; + jsonFile << " \"BatchNum\": " << batches << ",\n"; + jsonFile << " \"topK\": " << topK << ",\n"; + jsonFile << " \"numQueries\": " << numQueries << ",\n"; + jsonFile << " \"numThreads\": " << numThreads << ",\n"; + jsonFile << " \"DistMethod\": \"" << Helper::Convert::ConvertToString(distMethod) << "\"\n"; + jsonFile << " },\n"; + jsonFile << " \"results\": {\n"; + jsonFile << " \"benchmark1_insert\": " << benchmark1Data.str() << ",\n"; + jsonFile << " \"benchmark2_query_before_save\": " << benchmark2Data.str() << ",\n"; + jsonFile << " \"benchmark3_save\": " << benchmark3Data.str() << ",\n"; + jsonFile << " \"benchmark4_reload\": " << benchmark4Data.str() << ",\n"; + jsonFile << " \"benchmark5_query_after_reload\": " << benchmark5Data.str(); + if (!benchmark6Data.str().empty()) + { + jsonFile << ",\n \"benchmark6_recall\": " << benchmark6Data.str() << "\n"; + } + else + { + jsonFile << "\n"; + } + jsonFile << " }\n"; + jsonFile << "}\n"; + + jsonFile.close(); + BOOST_TEST_MESSAGE("\n=== Benchmark results saved to output.json ==="); + } + else + { + BOOST_TEST_MESSAGE("\n=== Failed to create output.json ==="); + } + } +} + } // namespace SPFreshTest bool CompareFilesWithLogging(const std::filesystem::path &file1, const std::filesystem::path &file2) @@ -427,8 +807,8 @@ BOOST_AUTO_TEST_CASE(TestReopenIndexRecall) auto originalIndex = BuildIndex("original_index", vecset, metaset); BOOST_REQUIRE(originalIndex != nullptr); - BOOST_REQUIRE(originalIndex->SaveIndex("original_index") == ErrorCode::Success); - float recall1 = Search(originalIndex, queryset, vecset, addvecset, K, truth, N); + float recall1 = Search(originalIndex, queryset, vecset, addvecset, K, truth, N); + BOOST_REQUIRE(originalIndex->SaveIndex("original_index") == ErrorCode::Success); originalIndex = nullptr; std::shared_ptr loadedOnce; @@ -443,7 +823,7 @@ BOOST_AUTO_TEST_CASE(TestReopenIndexRecall) float recall2 = Search(loadedTwice, queryset, vecset, addvecset, K, truth, N); loadedTwice = nullptr; - BOOST_REQUIRE_MESSAGE(std::fabs(recall1 - recall2) < 1e-5, "Recall mismatch between original and reopened index"); + BOOST_REQUIRE_MESSAGE(std::fabs(recall1 - recall2) < 0.02, "Recall mismatch between original and reopened index"); std::filesystem::remove_all("original_index"); std::filesystem::remove_all("reopened_index"); @@ -535,7 +915,7 @@ BOOST_AUTO_TEST_CASE(TestCloneRecall) float clonedRecall = Search(loadedClonedIndex, queryset, vecset, addvecset, K, truth, N); loadedClonedIndex = nullptr; - BOOST_REQUIRE_MESSAGE(std::fabs(originalRecall - clonedRecall) < 1e-5, + BOOST_REQUIRE_MESSAGE(std::fabs(originalRecall - clonedRecall) < 0.02, "Recall mismatch between original and cloned index: " << "original=" << originalRecall << ", cloned=" << clonedRecall); @@ -1098,7 +1478,6 @@ BOOST_AUTO_TEST_CASE(CacheTest) std::filesystem::remove_all("clone_index_" + std::to_string(iter)); } - /* std::cout << "=================Enable Cache===================" << std::endl; prevPath = "original_index"; for (int iter = 0; iter < iterations; iter++) @@ -1159,7 +1538,6 @@ BOOST_AUTO_TEST_CASE(CacheTest) { std::filesystem::remove_all("clone_index_" + std::to_string(iter)); } - */ std::filesystem::remove_all("original_index"); } @@ -1217,4 +1595,83 @@ BOOST_AUTO_TEST_CASE(IterativeSearchPerf) std::filesystem::remove_all("original_index"); } +BOOST_AUTO_TEST_CASE(BenchmarkFromConfig) +{ + using namespace SPFreshTest; + + // Check if benchmark config is provided via environment variable + const char *configPath = std::getenv("BENCHMARK_CONFIG"); + if (configPath == nullptr) + { + BOOST_TEST_MESSAGE("Skipping benchmark test - BENCHMARK_CONFIG environment variable not set"); + return; + } + + BOOST_TEST_MESSAGE("Running benchmark with config: " << configPath); + + // Read benchmark configuration + Helper::IniReader iniReader; + if (ErrorCode::Success != iniReader.LoadIniFile(configPath)) + { + BOOST_FAIL("Failed to load benchmark config file: " << configPath); + return; + } + + // Parse config parameters + std::string vectorPath = iniReader.GetParameter("Benchmark", "VectorPath", std::string("")); + std::string queryPath = iniReader.GetParameter("Benchmark", "QueryPath", std::string("")); + std::string truthPath = iniReader.GetParameter("Benchmark", "TruthPath", std::string("")); + std::string indexPath = iniReader.GetParameter("Benchmark", "IndexPath", std::string("benchmark_index")); + + VectorValueType valueType = VectorValueType::Float; + std::string valueTypeStr = iniReader.GetParameter("Benchmark", "ValueType", std::string("Float")); + if (valueTypeStr == "Float") + valueType = VectorValueType::Float; + else if (valueTypeStr == "Int8") + valueType = VectorValueType::Int8; + else if (valueTypeStr == "UInt8") + valueType = VectorValueType::UInt8; + + int dimension = iniReader.GetParameter("Benchmark", "Dimension", 128); + int baseVectorCount = iniReader.GetParameter("Benchmark", "BaseVectorCount", 8000); + int insertVectorCount = iniReader.GetParameter("Benchmark", "InsertVectorCount", 2000); + int deleteVectorCount = iniReader.GetParameter("Benchmark", "DeleteVectorCount", 2000); + int batchNum = iniReader.GetParameter("Benchmark", "BatchNum", 100); + int topK = iniReader.GetParameter("Benchmark", "TopK", 10); + int numThreads = iniReader.GetParameter("Benchmark", "NumThreads", 32); + int numQueries = iniReader.GetParameter("Benchmark", "NumQueries", 1000); + DistCalcMethod distMethod = iniReader.GetParameter("Benchmark", "DistMethod", DistCalcMethod::L2); + + BOOST_TEST_MESSAGE("=== Benchmark Configuration ==="); + BOOST_TEST_MESSAGE("Vector Path: " << vectorPath); + BOOST_TEST_MESSAGE("Query Path: " << queryPath); + BOOST_TEST_MESSAGE("Base Vectors: " << baseVectorCount); + BOOST_TEST_MESSAGE("Insert Vectors: " << insertVectorCount); + BOOST_TEST_MESSAGE("Dimension: " << dimension); + BOOST_TEST_MESSAGE("Batch Number: " << batchNum); + BOOST_TEST_MESSAGE("Top-K: " << topK); + BOOST_TEST_MESSAGE("Threads: " << numThreads); + BOOST_TEST_MESSAGE("Queries: " << numQueries); + BOOST_TEST_MESSAGE("DistMethod: " << Helper::Convert::ConvertToString(distMethod)); + + // Dispatch to appropriate type + if (valueType == VectorValueType::Float) + { + RunBenchmark(vectorPath, queryPath, truthPath, distMethod, indexPath, dimension, baseVectorCount, insertVectorCount, 0, + batchNum, topK, numThreads, numQueries); + } + else if (valueType == VectorValueType::Int8) + { + RunBenchmark(vectorPath, queryPath, truthPath, distMethod, indexPath, dimension, baseVectorCount, + insertVectorCount, 0, batchNum, topK, numThreads, numQueries); + } + else if (valueType == VectorValueType::UInt8) + { + RunBenchmark(vectorPath, queryPath, truthPath, distMethod, indexPath, dimension, baseVectorCount, + insertVectorCount, 0, batchNum, topK, numThreads, numQueries); + } + + std::filesystem::remove_all(indexPath); + std::filesystem::remove_all(indexPath + "_saved"); +} BOOST_AUTO_TEST_SUITE_END() diff --git a/Test/src/TestDataGenerator.cpp b/Test/src/TestDataGenerator.cpp index 6e94425ac..7b3cdae88 100644 --- a/Test/src/TestDataGenerator.cpp +++ b/Test/src/TestDataGenerator.cpp @@ -9,8 +9,10 @@ namespace TestUtils { template -TestDataGenerator::TestDataGenerator(int n, int q, int m, int k, std::string distMethod) - : m_n(n), m_q(q), m_m(m), m_k(k), m_distMethod(std::move(distMethod)) +TestDataGenerator::TestDataGenerator(int n, int q, int m, int k, std::string distMethod, int a, bool isRandom, + std::string vectorPath, std::string queryPath) + : m_n(n), m_a((a == 0)? n : a), m_q(q), m_m(m), m_k(k), m_distMethod(std::move(distMethod)), m_isRandom(isRandom), + m_vectorPath(vectorPath), m_queryPath(queryPath) { } @@ -53,7 +55,14 @@ void TestDataGenerator::LoadOrGenerateBase(std::shared_ptr &vecset } else { - vecset = GenerateRandomVectorSet(m_n, m_m); + if (m_isRandom) + { + vecset = GenerateRandomVectorSet(m_n, m_m); + } + else + { + vecset = GenerateLoadVectorSet(m_n, m_m, m_vectorPath, 0); + } vecset->Save("perftest_vector.bin"); metaset = GenerateMetadataSet(m_n, 0); @@ -70,7 +79,14 @@ template void TestDataGenerator::LoadOrGenerateQuery(std::shared } else { - queryset = GenerateRandomVectorSet(m_q, m_m); + if (m_isRandom) + { + queryset = GenerateRandomVectorSet(m_q, m_m); + } + else + { + queryset = GenerateLoadVectorSet(m_q, m_m, m_queryPath, 0); + } queryset->Save("perftest_query.bin"); } } @@ -89,10 +105,17 @@ void TestDataGenerator::LoadOrGenerateAdd(std::shared_ptr &addvecs } else { - addvecset = GenerateRandomVectorSet(m_n, m_m); + if (m_isRandom) + { + addvecset = GenerateRandomVectorSet(m_a, m_m); + } + else + { + addvecset = GenerateLoadVectorSet(m_a, m_m, m_vectorPath, m_n); + } addvecset->Save("perftest_addvector.bin"); - addmetaset = GenerateMetadataSet(m_n, m_n); + addmetaset = GenerateMetadataSet(m_a, m_n); addmetaset->SaveMetadata("perftest_addmeta.bin", "perftest_addmetaidx.bin"); } } @@ -272,6 +295,45 @@ std::shared_ptr TestDataGenerator::GenerateMetadataSet(SizeType return std::make_shared(meta, metaoffset, count, count * 2, MaxSize, 10); } +template +std::shared_ptr TestDataGenerator::GenerateLoadVectorSet(SPTAG::SizeType count, + SPTAG::DimensionType dim, + std::string path, SPTAG::SizeType start) +{ + VectorFileType fileType = VectorFileType::DEFAULT; + if (path.find(".fvecs") != std::string::npos || path.find(".ivecs") != std::string::npos) + { + fileType = VectorFileType::XVEC; + } + auto vectorOptions = + std::shared_ptr(new Helper::ReaderOptions(GetEnumValueType(), dim, fileType)); + auto vectorReader = Helper::VectorSetReader::CreateInstance(vectorOptions); + + if (!fileexists(path.c_str()) || ErrorCode::Success != vectorReader->LoadFile(path)) + { + SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Cannot find or load %s. Using random generation!\n", path.c_str()); + return GenerateRandomVectorSet(count, dim); + } + + auto allVectors = vectorReader->GetVectorSet(); + int totalVectors = allVectors->Count(); + if (totalVectors - start < count) + { + SPTAGLIB_LOG(Helper::LogLevel::LL_Error, + "%s contains total %d vectors. Cannot get %d vectors start from %d. Using random generation!\n", path.c_str(), + totalVectors, count, start); + return GenerateRandomVectorSet(count, dim); + } + + SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "%s contains total %d vectors. Load %d vectors start from %d\n", + path.c_str(), totalVectors, count, start); + + + ByteArray baseData = ByteArray::Alloc(sizeof(T) * count * dim); + memcpy(baseData.Data(), (char *)(allVectors->GetData()) + sizeof(T) * start * dim, sizeof(T) * count * dim); + return std::make_shared(baseData, GetEnumValueType(), dim, count); +} + template std::shared_ptr TestDataGenerator::CombineVectorSets(std::shared_ptr base, std::shared_ptr add) From b79b98ac7032a1c09bb6cfdaf3b72643f0c671e8 Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Thu, 4 Dec 2025 13:45:58 -0800 Subject: [PATCH 06/38] fix linux localtime_s to localtime_r --- Test/src/SPFreshTest.cpp | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/Test/src/SPFreshTest.cpp b/Test/src/SPFreshTest.cpp index 573f40239..f88a15f85 100644 --- a/Test/src/SPFreshTest.cpp +++ b/Test/src/SPFreshTest.cpp @@ -572,10 +572,13 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c jsonFile << std::fixed << std::setprecision(4); // Get current timestamp - auto now = std::chrono::system_clock::now(); - auto time_t_now = std::chrono::system_clock::to_time_t(now); + auto time_t_now = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); std::tm tm_now; +#if defined(_MSC_VER) localtime_s(&tm_now, &time_t_now); +#else + localtime_r(&time_t_now, &tm_now); +#endif std::ostringstream timestampStream; timestampStream << std::put_time(&tm_now, "%Y-%m-%dT%H:%M:%S"); @@ -624,6 +627,11 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c BOOST_TEST_MESSAGE("\n=== Failed to create output.json ==="); } } + + M = oldM; + K = oldK; + N = oldN; + queries = oldQueries; } } // namespace SPFreshTest @@ -1038,7 +1046,6 @@ BOOST_AUTO_TEST_CASE(IndexPersistenceAndInsertMultipleThreads) // Cleanup std::filesystem::remove_all("insert_test_index_multi"); std::filesystem::remove_all("insert_cloned_index_multi"); - std::filesystem::remove_all("insert_final_index_multi"); } BOOST_AUTO_TEST_CASE(IndexSaveDuringQuery) @@ -1657,18 +1664,18 @@ BOOST_AUTO_TEST_CASE(BenchmarkFromConfig) // Dispatch to appropriate type if (valueType == VectorValueType::Float) { - RunBenchmark(vectorPath, queryPath, truthPath, distMethod, indexPath, dimension, baseVectorCount, insertVectorCount, 0, - batchNum, topK, numThreads, numQueries); + RunBenchmark(vectorPath, queryPath, truthPath, distMethod, indexPath, dimension, baseVectorCount, + insertVectorCount, deleteVectorCount, batchNum, topK, numThreads, numQueries); } else if (valueType == VectorValueType::Int8) { RunBenchmark(vectorPath, queryPath, truthPath, distMethod, indexPath, dimension, baseVectorCount, - insertVectorCount, 0, batchNum, topK, numThreads, numQueries); + insertVectorCount, deleteVectorCount, batchNum, topK, numThreads, numQueries); } else if (valueType == VectorValueType::UInt8) { RunBenchmark(vectorPath, queryPath, truthPath, distMethod, indexPath, dimension, baseVectorCount, - insertVectorCount, 0, batchNum, topK, numThreads, numQueries); + insertVectorCount, deleteVectorCount, batchNum, topK, numThreads, numQueries); } std::filesystem::remove_all(indexPath); From a26f2edfc2f18757cc0eb1c4c288b8a75e54f3ac Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Wed, 10 Dec 2025 16:24:25 -0800 Subject: [PATCH 07/38] fix the benchmark to avoid create workspace after reload during search --- Test/src/SPFreshTest.cpp | 410 ++++++++++++++++++--------------------- 1 file changed, 191 insertions(+), 219 deletions(-) diff --git a/Test/src/SPFreshTest.cpp b/Test/src/SPFreshTest.cpp index f88a15f85..ae72746f4 100644 --- a/Test/src/SPFreshTest.cpp +++ b/Test/src/SPFreshTest.cpp @@ -34,7 +34,7 @@ int queries = 10; template std::shared_ptr BuildIndex(const std::string &outDirectory, std::shared_ptr vecset, - std::shared_ptr metaset, const std::string &distMethod = "L2") + std::shared_ptr metaset, const std::string &distMethod = "L2", int searchthread = 2) { auto vecIndex = VectorIndex::CreateInstance(IndexAlgoType::SPANN, GetEnumValueType()); @@ -76,7 +76,8 @@ std::shared_ptr BuildIndex(const std::string &outDirectory, std::sh SpdkBatchSize=64 ExcludeHead=false ResultNum=10 - SearchThreadNum=2 + SearchThreadNum=)" + std::to_string(searchthread) + + R"( Update=true SteadyState=true InsertThreadNum=1 @@ -91,11 +92,11 @@ std::shared_ptr BuildIndex(const std::string &outDirectory, std::sh BufferLength=6 InPlace=true StartFileSizeGB=1 - OneClusterCutMax=true - ConsistencyCheck=true - ChecksumCheck=true + OneClusterCutMax=false + ConsistencyCheck=false + ChecksumCheck=false ChecksumInRead=false - AsyncMergeInSearch=true + AsyncMergeInSearch=false DeletePercentageForRefine=0.4 AsyncAppendQueueSize=0 AllowZeroReplica=false @@ -248,6 +249,105 @@ void InsertVectors(SPANN::Index *p_index, int insertThreads, int step } } + +template +void BenchmarkQueryPerformance(std::shared_ptr &index, std::shared_ptr &queryset, + std::shared_ptr &truth, std::shared_ptr &vecset, + std::shared_ptr &addvecset, const std::string &truthPath, + SizeType baseVectorCount, int topK, int numThreads, int numQueries, int batches, + std::ostringstream &benchmarkData) +{ + // Benchmark: Query performance with detailed latency stats + std::vector latencies(numQueries); + std::atomic_size_t queriesSent(0); + std::vector results(numQueries); + + for (int i = 0; i < numQueries; i++) + { + results[i] = QueryResult((const T *)queryset->GetVector(i), topK, false); + } + + std::vector threads; + threads.reserve(numThreads); + + auto batchStart = std::chrono::high_resolution_clock::now(); + + for (int i = 0; i < numThreads; i++) + { + threads.emplace_back([&]() { + size_t qid; + while ((qid = queriesSent.fetch_add(1)) < numQueries) + { + auto t1 = std::chrono::high_resolution_clock::now(); + index->SearchIndex(results[qid]); + auto t2 = std::chrono::high_resolution_clock::now(); + latencies[qid] = std::chrono::duration_cast(t2 - t1).count() / 1000.0f; + } + }); + } + + for (auto &thread : threads) + thread.join(); + + auto batchEnd = std::chrono::high_resolution_clock::now(); + float batchLatency = + std::chrono::duration_cast(batchEnd - batchStart).count() / 1000000.0f; + + // Calculate statistics + float mean = 0, minLat = (std::numeric_limits::max)(), maxLat = 0; + for (int i = 0; i < numQueries; i++) + { + mean += latencies[i]; + minLat = (std::min)(minLat, latencies[i]); + maxLat = (std::max)(maxLat, latencies[i]); + } + mean /= numQueries; + + std::sort(latencies.begin(), latencies.end()); + float p50 = latencies[static_cast(numQueries * 0.50)]; + float p90 = latencies[static_cast(numQueries * 0.90)]; + float p95 = latencies[static_cast(numQueries * 0.95)]; + float p99 = latencies[static_cast(numQueries * 0.99)]; + float qps = numQueries / batchLatency; + + BOOST_TEST_MESSAGE(" Queries: " << numQueries); + BOOST_TEST_MESSAGE(" Mean Latency: " << mean << " ms"); + BOOST_TEST_MESSAGE(" P50 Latency: " << p50 << " ms"); + BOOST_TEST_MESSAGE(" P90 Latency: " << p90 << " ms"); + BOOST_TEST_MESSAGE(" P95 Latency: " << p95 << " ms"); + BOOST_TEST_MESSAGE(" P99 Latency: " << p99 << " ms"); + BOOST_TEST_MESSAGE(" Min Latency: " << minLat << " ms"); + BOOST_TEST_MESSAGE(" Max Latency: " << maxLat << " ms"); + BOOST_TEST_MESSAGE(" QPS: " << qps); + + // Collect JSON data for Benchmark + benchmarkData << std::fixed << std::setprecision(4); + benchmarkData << "{\n"; + benchmarkData << " \"numQueries\": " << numQueries << ",\n"; + benchmarkData << " \"meanLatency\": " << mean << ",\n"; + benchmarkData << " \"p50\": " << p50 << ",\n"; + benchmarkData << " \"p90\": " << p90 << ",\n"; + benchmarkData << " \"p95\": " << p95 << ",\n"; + benchmarkData << " \"p99\": " << p99 << ",\n"; + benchmarkData << " \"minLatency\": " << minLat << ",\n"; + benchmarkData << " \"maxLatency\": " << maxLat << ",\n"; + benchmarkData << " \"qps\": " << qps << ",\n"; + + + // Recall evaluation (if truth file provided) + BOOST_TEST_MESSAGE("Checking for truth file: " << truthPath); + float avgRecall = EvaluateRecall(results, index, queryset, truth, vecset, addvecset, baseVectorCount, topK, batches); + BOOST_TEST_MESSAGE(" Recall@" << topK << " = " << (avgRecall * 100.0f) << "%"); + BOOST_TEST_MESSAGE(" (Evaluated on " << numQueries << " queries against base vectors)"); + benchmarkData << std::fixed << std::setprecision(4); + benchmarkData << " \"recall\": {\n"; + benchmarkData << " \"recallAtK\": " << avgRecall << ",\n"; + benchmarkData << " \"k\": " << topK << ",\n"; + benchmarkData << " \"numQueries\": " << numQueries << "\n"; + benchmarkData << " }\n"; + benchmarkData << " }"; +} + template void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, const std::string &truthPath, DistCalcMethod distMethod, const std::string &indexPath, int dimension, int baseVectorCount, int insertVectorCount, int deleteVectorCount, @@ -263,7 +363,8 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c int deleteBatchSize = deleteVectorCount / batches; // Variables to collect JSON output data - std::ostringstream benchmark1Data, benchmark2Data, benchmark3Data, benchmark4Data, benchmark5Data, benchmark6Data; + std::ostringstream tmpbenchmark, benchmark0Data, benchmark1Data, benchmark2Data, benchmark3Data, benchmark4Data, benchmark5Data, + benchmark6Data; // Generate test data std::shared_ptr vecset, addvecset, queryset, truth; @@ -274,159 +375,97 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c // Build initial index BOOST_TEST_MESSAGE("\n=== Building Index ==="); - std::shared_ptr index = BuildIndex(indexPath, vecset, metaset, dist); + std::shared_ptr index = BuildIndex(indexPath, vecset, metaset, dist, numThreads); BOOST_REQUIRE(index != nullptr); - float recall = Search(index, queryset, vecset, addvecset, K, truth, N, 0); - BOOST_TEST_MESSAGE("Index built successfully with " << N << " vectors. Recall@" << K << "=" << recall << "."); + + BOOST_TEST_MESSAGE("Index built successfully with " << baseVectorCount << " vectors"); + + // Benchmark 0: Query performance before insertions + BOOST_TEST_MESSAGE("\n=== Benchmark 0: Query Before Insertions ==="); + BenchmarkQueryPerformance(index, queryset, truth, vecset, addvecset, truthPath, baseVectorCount, topK, + numThreads, numQueries, batches, benchmark0Data); // Benchmark 1: Insert performance - BOOST_TEST_MESSAGE("\n=== Benchmark 1: Insert Performance ==="); + if (insertVectorCount > 0) { - auto start = std::chrono::high_resolution_clock::now(); - for (int iter = 0; iter < batches; iter++) + BOOST_TEST_MESSAGE("\n=== Benchmark 1: Insert Performance ==="); { - InsertVectors(static_cast *>(index.get()), numThreads, insertBatchSize, addvecset, - addmetaset, iter * insertBatchSize); - } - auto end = std::chrono::high_resolution_clock::now(); - - double seconds = std::chrono::duration_cast(end - start).count() / 1000000.0f; - double throughput = insertVectorCount / seconds; - - BOOST_TEST_MESSAGE(" Inserted: " << insertVectorCount << " vectors"); - BOOST_TEST_MESSAGE(" Time: " << seconds << " seconds"); - BOOST_TEST_MESSAGE(" Throughput: " << throughput << " vectors/sec"); - - // Collect JSON data for Benchmark 1 - benchmark1Data << std::fixed << std::setprecision(4); - benchmark1Data << "{\n"; - benchmark1Data << " \"inserted\": " << insertVectorCount << ",\n"; - benchmark1Data << " \"insert timeSeconds\": " << seconds << ",\n"; - benchmark1Data << " \"insert throughput\": " << throughput << "\n"; - - - std::vector threads; - threads.reserve(numThreads); - - std::atomic_size_t vectorsSent(0); - int totaldeleted = batches * deleteBatchSize; - auto func = [&]() { - size_t idx = 0; - while (true) + auto start = std::chrono::high_resolution_clock::now(); + for (int iter = 0; iter < batches; iter++) { - idx = vectorsSent.fetch_add(1); - if (idx < totaldeleted) - { - if ((idx & ((1 << 5) - 1)) == 0) - { - SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "Sent %.2lf%%...\n", idx * 100.0 / totaldeleted); - } - BOOST_REQUIRE(index->DeleteIndex(idx) == ErrorCode::Success); - } - else - { - return; - } + InsertVectors(static_cast *>(index.get()), numThreads, insertBatchSize, addvecset, + addmetaset, iter * insertBatchSize); } - }; - - start = std::chrono::high_resolution_clock::now(); - for (int j = 0; j < numThreads; j++) - { - threads.emplace_back(func); - } - for (auto &thread : threads) - { - thread.join(); - } - end = std::chrono::high_resolution_clock::now(); - seconds = std::chrono::duration_cast(end - start).count() / 1000000.0f; - throughput = deleteVectorCount / seconds; + auto end = std::chrono::high_resolution_clock::now(); - benchmark1Data << " \"deleted\": " << deleteVectorCount << ",\n"; - benchmark1Data << " \"delete timeSeconds\": " << seconds << ",\n"; - benchmark1Data << " \"delete throughput\": " << throughput << "\n"; - benchmark1Data << " }"; - } + double seconds = std::chrono::duration_cast(end - start).count() / 1000000.0f; + double throughput = insertVectorCount / seconds; - // Benchmark 2: Query performance with detailed latency stats - BOOST_TEST_MESSAGE("\n=== Benchmark 2: Query Performance ==="); - { - std::vector latencies(numQueries); - std::atomic_size_t queriesSent(0); - std::vector results(numQueries); + BOOST_TEST_MESSAGE(" Inserted: " << insertVectorCount << " vectors"); + BOOST_TEST_MESSAGE(" Time: " << seconds << " seconds"); + BOOST_TEST_MESSAGE(" Throughput: " << throughput << " vectors/sec"); - for (int i = 0; i < numQueries; i++) - { - results[i] = QueryResult((const T *)queryset->GetVector(i), K, false); + // Collect JSON data for Benchmark 1 + benchmark1Data << std::fixed << std::setprecision(4); + benchmark1Data << "{\n"; + benchmark1Data << " \"inserted\": " << insertVectorCount << ",\n"; + benchmark1Data << " \"insert timeSeconds\": " << seconds << ",\n"; + benchmark1Data << " \"insert throughput\": " << throughput << "\n"; } - std::vector threads; - threads.reserve(numThreads); + if (deleteVectorCount > 0) + { - auto batchStart = std::chrono::high_resolution_clock::now(); + std::vector threads; + threads.reserve(numThreads); - for (int i = 0; i < numThreads; i++) - { - threads.emplace_back([&]() { - size_t qid; - while ((qid = queriesSent.fetch_add(1)) < numQueries) + std::atomic_size_t vectorsSent(0); + int totaldeleted = batches * deleteBatchSize; + auto func = [&]() { + size_t idx = 0; + while (true) { - auto t1 = std::chrono::high_resolution_clock::now(); - index->SearchIndex(results[qid]); - auto t2 = std::chrono::high_resolution_clock::now(); - latencies[qid] = std::chrono::duration_cast(t2 - t1).count() / 1000.0f; + idx = vectorsSent.fetch_add(1); + if (idx < totaldeleted) + { + if ((idx & ((1 << 5) - 1)) == 0) + { + SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "Sent %.2lf%%...\n", + idx * 100.0 / totaldeleted); + } + BOOST_REQUIRE(index->DeleteIndex(idx) == ErrorCode::Success); + } + else + { + return; + } } - }); - } + }; - for (auto &thread : threads) - thread.join(); - - auto batchEnd = std::chrono::high_resolution_clock::now(); - float batchLatency = - std::chrono::duration_cast(batchEnd - batchStart).count() / 1000000.0f; - - // Calculate statistics - float mean = 0, minLat = (std::numeric_limits::max)(), maxLat = 0; - for (int i = 0; i < numQueries; i++) - { - mean += latencies[i]; - minLat = (std::min)(minLat, latencies[i]); - maxLat = (std::max)(maxLat, latencies[i]); + auto start = std::chrono::high_resolution_clock::now(); + for (int j = 0; j < numThreads; j++) + { + threads.emplace_back(func); + } + for (auto &thread : threads) + { + thread.join(); + } + auto end = std::chrono::high_resolution_clock::now(); + double seconds = std::chrono::duration_cast(end - start).count() / 1000000.0f; + double throughput = deleteVectorCount / seconds; + + benchmark1Data << " \"deleted\": " << deleteVectorCount << ",\n"; + benchmark1Data << " \"delete timeSeconds\": " << seconds << ",\n"; + benchmark1Data << " \"delete throughput\": " << throughput << "\n"; + } - mean /= numQueries; - - std::sort(latencies.begin(), latencies.end()); - float p50 = latencies[static_cast(numQueries * 0.50)]; - float p90 = latencies[static_cast(numQueries * 0.90)]; - float p95 = latencies[static_cast(numQueries * 0.95)]; - float p99 = latencies[static_cast(numQueries * 0.99)]; - float qps = numQueries / batchLatency; - - BOOST_TEST_MESSAGE(" Queries: " << numQueries); - BOOST_TEST_MESSAGE(" Mean Latency: " << mean << " ms"); - BOOST_TEST_MESSAGE(" P50 Latency: " << p50 << " ms"); - BOOST_TEST_MESSAGE(" P90 Latency: " << p90 << " ms"); - BOOST_TEST_MESSAGE(" P95 Latency: " << p95 << " ms"); - BOOST_TEST_MESSAGE(" P99 Latency: " << p99 << " ms"); - BOOST_TEST_MESSAGE(" Min Latency: " << minLat << " ms"); - BOOST_TEST_MESSAGE(" Max Latency: " << maxLat << " ms"); - BOOST_TEST_MESSAGE(" QPS: " << qps); - - // Collect JSON data for Benchmark 2 - benchmark2Data << std::fixed << std::setprecision(4); - benchmark2Data << "{\n"; - benchmark2Data << " \"numQueries\": " << numQueries << ",\n"; - benchmark2Data << " \"meanLatency\": " << mean << ",\n"; - benchmark2Data << " \"p50\": " << p50 << ",\n"; - benchmark2Data << " \"p90\": " << p90 << ",\n"; - benchmark2Data << " \"p95\": " << p95 << ",\n"; - benchmark2Data << " \"p99\": " << p99 << ",\n"; - benchmark2Data << " \"minLatency\": " << minLat << ",\n"; - benchmark2Data << " \"maxLatency\": " << maxLat << ",\n"; - benchmark2Data << " \"qps\": " << qps << "\n"; - benchmark2Data << " }"; + benchmark1Data << " }"; + + // Benchmark 2: Query performance with detailed latency stats + BOOST_TEST_MESSAGE("\n=== Benchmark 2: Query After Insertions ==="); + BenchmarkQueryPerformance(index, queryset, truth, vecset, addvecset, truthPath, baseVectorCount, topK, + numThreads, numQueries, batches, benchmark2Data); } // Benchmark 3: Save index to disk @@ -483,84 +522,13 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c BOOST_TEST_MESSAGE("Benchmark 4 completed successfully"); - std::vector results(numQueries); // Benchmark 5: Query performance after reload BOOST_TEST_MESSAGE("\n=== Benchmark 5: Query After Reload ==="); - { - std::vector latencies(numQueries); - std::atomic_size_t queriesSent(0); - - for (int i = 0; i < numQueries; i++) - { - results[i] = QueryResult((const T *)queryset->GetVector(i), K, false); - } - - std::vector threads; - threads.reserve(numThreads); - - auto batchStart = std::chrono::high_resolution_clock::now(); - - for (int i = 0; i < numThreads; i++) - { - threads.emplace_back([&]() { - size_t qid; - while ((qid = queriesSent.fetch_add(1)) < numQueries) - { - auto t1 = std::chrono::high_resolution_clock::now(); - index->SearchIndex(results[qid]); - auto t2 = std::chrono::high_resolution_clock::now(); - latencies[qid] = std::chrono::duration_cast(t2 - t1).count() / 1000.0f; - } - }); - } + BenchmarkQueryPerformance(index, queryset, truth, vecset, addvecset, truthPath, baseVectorCount, topK, + numThreads, numQueries, batches, tmpbenchmark); - for (auto &thread : threads) - thread.join(); - - auto batchEnd = std::chrono::high_resolution_clock::now(); - float batchLatency = - std::chrono::duration_cast(batchEnd - batchStart).count() / 1000000.0f; - - float mean = 0; - for (int i = 0; i < numQueries; i++) - { - mean += latencies[i]; - } - mean /= numQueries; - - std::sort(latencies.begin(), latencies.end()); - float p95 = latencies[static_cast(numQueries * 0.95)]; - float p99 = latencies[static_cast(numQueries * 0.99)]; - float qps = numQueries / batchLatency; - - BOOST_TEST_MESSAGE(" Mean Latency: " << mean << " ms"); - BOOST_TEST_MESSAGE(" P95 Latency: " << p95 << " ms"); - BOOST_TEST_MESSAGE(" P99 Latency: " << p99 << " ms"); - BOOST_TEST_MESSAGE(" QPS: " << qps); - - // Collect JSON data for Benchmark 5 - benchmark5Data << std::fixed << std::setprecision(4); - benchmark5Data << "{\n"; - benchmark5Data << " \"meanLatency\": " << mean << ",\n"; - benchmark5Data << " \"p95\": " << p95 << ",\n"; - benchmark5Data << " \"p99\": " << p99 << ",\n"; - benchmark5Data << " \"qps\": " << qps << "\n"; - benchmark5Data << " }"; - } - - // Benchmark 6: Recall evaluation (if truth file provided) - BOOST_TEST_MESSAGE("Checking for truth file: " << truthPath); - float avgRecall = EvaluateRecall(results, index, queryset, truth, vecset, addvecset, N, K, batches); - BOOST_TEST_MESSAGE(" Recall@" << K << " = " << (avgRecall * 100.0f) << "%"); - BOOST_TEST_MESSAGE(" (Evaluated on " << numQueries << " queries against base vectors)"); - - // Collect JSON data for Benchmark 6 - benchmark6Data << std::fixed << std::setprecision(4); - benchmark6Data << "{\n"; - benchmark6Data << " \"recallAtK\": " << avgRecall << ",\n"; - benchmark6Data << " \"k\": " << K << ",\n"; - benchmark6Data << " \"numQueries\": " << numQueries << "\n"; - benchmark6Data << " }"; + BenchmarkQueryPerformance(index, queryset, truth, vecset, addvecset, truthPath, baseVectorCount, topK, + numThreads, numQueries, batches, benchmark5Data); BOOST_TEST_MESSAGE("\n=== Benchmark Complete ==="); @@ -603,8 +571,12 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c jsonFile << " \"DistMethod\": \"" << Helper::Convert::ConvertToString(distMethod) << "\"\n"; jsonFile << " },\n"; jsonFile << " \"results\": {\n"; - jsonFile << " \"benchmark1_insert\": " << benchmark1Data.str() << ",\n"; - jsonFile << " \"benchmark2_query_before_save\": " << benchmark2Data.str() << ",\n"; + jsonFile << " \"benchmark0_query_before_insert\": " << benchmark0Data.str() << ",\n"; + if (!benchmark1Data.str().empty()) + { + jsonFile << " \"benchmark1_insert\": " << benchmark1Data.str() << ",\n"; + jsonFile << " \"benchmark2_query_before_save\": " << benchmark2Data.str() << ",\n"; + } jsonFile << " \"benchmark3_save\": " << benchmark3Data.str() << ",\n"; jsonFile << " \"benchmark4_reload\": " << benchmark4Data.str() << ",\n"; jsonFile << " \"benchmark5_query_after_reload\": " << benchmark5Data.str(); From da41c2b9b7382930952326ff0e7abc17cee1e6bd Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Wed, 10 Dec 2025 16:57:14 -0800 Subject: [PATCH 08/38] add output file to the benchmark --- Test/src/SPFreshTest.cpp | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/Test/src/SPFreshTest.cpp b/Test/src/SPFreshTest.cpp index ae72746f4..fd3637c68 100644 --- a/Test/src/SPFreshTest.cpp +++ b/Test/src/SPFreshTest.cpp @@ -350,8 +350,8 @@ void BenchmarkQueryPerformance(std::shared_ptr &index, std::shared_ template void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, const std::string &truthPath, DistCalcMethod distMethod, - const std::string &indexPath, int dimension, int baseVectorCount, int insertVectorCount, int deleteVectorCount, - int batches, int topK, int numThreads, int numQueries) + const std::string &indexPath, int dimension, int baseVectorCount, int insertVectorCount, int deleteVectorCount, int batches, int topK, int numThreads, int numQueries, + const std::string &outputFile = "output.json") { int oldM = M, oldK = K, oldN = N, oldQueries = queries; N = baseVectorCount; @@ -534,7 +534,7 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c // Write JSON output { - std::ofstream jsonFile("output.json"); + std::ofstream jsonFile(outputFile); if (jsonFile.is_open()) { jsonFile << std::fixed << std::setprecision(4); @@ -1633,21 +1633,28 @@ BOOST_AUTO_TEST_CASE(BenchmarkFromConfig) BOOST_TEST_MESSAGE("Queries: " << numQueries); BOOST_TEST_MESSAGE("DistMethod: " << Helper::Convert::ConvertToString(distMethod)); + // Get output file path from environment variable or use default + const char *outputPath = std::getenv("BENCHMARK_OUTPUT"); + std::string outputFile = outputPath ? std::string(outputPath) : "output.json"; + BOOST_TEST_MESSAGE("Output File: " << outputFile); + // Dispatch to appropriate type if (valueType == VectorValueType::Float) { RunBenchmark(vectorPath, queryPath, truthPath, distMethod, indexPath, dimension, baseVectorCount, - insertVectorCount, deleteVectorCount, batchNum, topK, numThreads, numQueries); + insertVectorCount, deleteVectorCount, batchNum, topK, numThreads, numQueries, outputFile); } else if (valueType == VectorValueType::Int8) { RunBenchmark(vectorPath, queryPath, truthPath, distMethod, indexPath, dimension, baseVectorCount, - insertVectorCount, deleteVectorCount, batchNum, topK, numThreads, numQueries); + insertVectorCount, deleteVectorCount, batchNum, topK, numThreads, numQueries, + outputFile); } else if (valueType == VectorValueType::UInt8) { RunBenchmark(vectorPath, queryPath, truthPath, distMethod, indexPath, dimension, baseVectorCount, - insertVectorCount, deleteVectorCount, batchNum, topK, numThreads, numQueries); + insertVectorCount, deleteVectorCount, batchNum, topK, numThreads, numQueries, + outputFile); } std::filesystem::remove_all(indexPath); From c1036bc9404f99a313d3de5588b8453d873d9749 Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Thu, 11 Dec 2025 14:04:45 -0800 Subject: [PATCH 09/38] fix the BatchNum --- Test/src/SPFreshTest.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Test/src/SPFreshTest.cpp b/Test/src/SPFreshTest.cpp index fd3637c68..398973f64 100644 --- a/Test/src/SPFreshTest.cpp +++ b/Test/src/SPFreshTest.cpp @@ -359,8 +359,8 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c M = dimension; K = topK; std::string dist = Helper::Convert::ConvertToString(distMethod); - int insertBatchSize = insertVectorCount / batches; - int deleteBatchSize = deleteVectorCount / batches; + int insertBatchSize = insertVectorCount / max(batches, 1); + int deleteBatchSize = deleteVectorCount / max(batches, 1); // Variables to collect JSON output data std::ostringstream tmpbenchmark, benchmark0Data, benchmark1Data, benchmark2Data, benchmark3Data, benchmark4Data, benchmark5Data, @@ -383,7 +383,7 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c // Benchmark 0: Query performance before insertions BOOST_TEST_MESSAGE("\n=== Benchmark 0: Query Before Insertions ==="); BenchmarkQueryPerformance(index, queryset, truth, vecset, addvecset, truthPath, baseVectorCount, topK, - numThreads, numQueries, batches, benchmark0Data); + numThreads, numQueries, 0, benchmark0Data); // Benchmark 1: Insert performance if (insertVectorCount > 0) From 20fa8475966ea0ad6a13fbc34dc0f08cdb60c7bc Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Thu, 11 Dec 2025 16:18:08 -0800 Subject: [PATCH 10/38] fix benchmark to include multiple batch perf --- Test/src/SPFreshTest.cpp | 285 +++++++++++++++++++-------------------- 1 file changed, 136 insertions(+), 149 deletions(-) diff --git a/Test/src/SPFreshTest.cpp b/Test/src/SPFreshTest.cpp index 398973f64..6884d6ee5 100644 --- a/Test/src/SPFreshTest.cpp +++ b/Test/src/SPFreshTest.cpp @@ -255,7 +255,7 @@ void BenchmarkQueryPerformance(std::shared_ptr &index, std::shared_ std::shared_ptr &truth, std::shared_ptr &vecset, std::shared_ptr &addvecset, const std::string &truthPath, SizeType baseVectorCount, int topK, int numThreads, int numQueries, int batches, - std::ostringstream &benchmarkData) + std::ostringstream &benchmarkData, std::string prefix = "") { // Benchmark: Query performance with detailed latency stats std::vector latencies(numQueries); @@ -322,16 +322,16 @@ void BenchmarkQueryPerformance(std::shared_ptr &index, std::shared_ // Collect JSON data for Benchmark benchmarkData << std::fixed << std::setprecision(4); - benchmarkData << "{\n"; - benchmarkData << " \"numQueries\": " << numQueries << ",\n"; - benchmarkData << " \"meanLatency\": " << mean << ",\n"; - benchmarkData << " \"p50\": " << p50 << ",\n"; - benchmarkData << " \"p90\": " << p90 << ",\n"; - benchmarkData << " \"p95\": " << p95 << ",\n"; - benchmarkData << " \"p99\": " << p99 << ",\n"; - benchmarkData << " \"minLatency\": " << minLat << ",\n"; - benchmarkData << " \"maxLatency\": " << maxLat << ",\n"; - benchmarkData << " \"qps\": " << qps << ",\n"; + benchmarkData << prefix << "{\n"; + benchmarkData << prefix << " \"numQueries\": " << numQueries << ",\n"; + benchmarkData << prefix << " \"meanLatency\": " << mean << ",\n"; + benchmarkData << prefix << " \"p50\": " << p50 << ",\n"; + benchmarkData << prefix << " \"p90\": " << p90 << ",\n"; + benchmarkData << prefix << " \"p95\": " << p95 << ",\n"; + benchmarkData << prefix << " \"p99\": " << p99 << ",\n"; + benchmarkData << prefix << " \"minLatency\": " << minLat << ",\n"; + benchmarkData << prefix << " \"maxLatency\": " << maxLat << ",\n"; + benchmarkData << prefix << " \"qps\": " << qps << ",\n"; // Recall evaluation (if truth file provided) @@ -340,12 +340,12 @@ void BenchmarkQueryPerformance(std::shared_ptr &index, std::shared_ BOOST_TEST_MESSAGE(" Recall@" << topK << " = " << (avgRecall * 100.0f) << "%"); BOOST_TEST_MESSAGE(" (Evaluated on " << numQueries << " queries against base vectors)"); benchmarkData << std::fixed << std::setprecision(4); - benchmarkData << " \"recall\": {\n"; - benchmarkData << " \"recallAtK\": " << avgRecall << ",\n"; - benchmarkData << " \"k\": " << topK << ",\n"; - benchmarkData << " \"numQueries\": " << numQueries << "\n"; - benchmarkData << " }\n"; - benchmarkData << " }"; + benchmarkData << prefix << " \"recall\": {\n"; + benchmarkData << prefix << " \"recallAtK\": " << avgRecall << ",\n"; + benchmarkData << prefix << " \"k\": " << topK << ",\n"; + benchmarkData << prefix << " \"numQueries\": " << numQueries << "\n"; + benchmarkData << prefix << " }\n"; + benchmarkData << prefix << " }"; } template @@ -363,8 +363,7 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c int deleteBatchSize = deleteVectorCount / max(batches, 1); // Variables to collect JSON output data - std::ostringstream tmpbenchmark, benchmark0Data, benchmark1Data, benchmark2Data, benchmark3Data, benchmark4Data, benchmark5Data, - benchmark6Data; + std::ostringstream tmpbenchmark, benchmark0Data, benchmark1Data; // Generate test data std::shared_ptr vecset, addvecset, queryset, truth; @@ -375,6 +374,7 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c // Build initial index BOOST_TEST_MESSAGE("\n=== Building Index ==="); + std::filesystem::remove_all(indexPath); std::shared_ptr index = BuildIndex(indexPath, vecset, metaset, dist, numThreads); BOOST_REQUIRE(index != nullptr); @@ -382,153 +382,152 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c // Benchmark 0: Query performance before insertions BOOST_TEST_MESSAGE("\n=== Benchmark 0: Query Before Insertions ==="); + BenchmarkQueryPerformance(index, queryset, truth, vecset, addvecset, truthPath, baseVectorCount, topK, + numThreads, numQueries, 0, tmpbenchmark); BenchmarkQueryPerformance(index, queryset, truth, vecset, addvecset, truthPath, baseVectorCount, topK, numThreads, numQueries, 0, benchmark0Data); + BOOST_REQUIRE(index->SaveIndex(indexPath) == ErrorCode::Success); + index = nullptr; + + for (int iter = 0; iter < batches; iter++) + { + if (direxists((indexPath + "_" + std::to_string(iter)).c_str())) + { + std::filesystem::remove_all(indexPath + "_" + std::to_string(iter)); + } + } // Benchmark 1: Insert performance - if (insertVectorCount > 0) + if (insertBatchSize > 0) { BOOST_TEST_MESSAGE("\n=== Benchmark 1: Insert Performance ==="); { - auto start = std::chrono::high_resolution_clock::now(); + benchmark1Data << std::fixed << std::setprecision(4); + benchmark1Data << "{\n"; + std::string prevPath = indexPath; for (int iter = 0; iter < batches; iter++) { - InsertVectors(static_cast *>(index.get()), numThreads, insertBatchSize, addvecset, + benchmark1Data << " \"batch_" << iter + 1 << "\": {\n"; + + std::string clonePath = indexPath + "_" + std::to_string(iter); + std::shared_ptr prevIndex, clonedIndex; + auto start = std::chrono::high_resolution_clock::now(); + BOOST_REQUIRE(VectorIndex::LoadIndex(prevPath, prevIndex) == ErrorCode::Success); + auto end = std::chrono::high_resolution_clock::now(); + BOOST_REQUIRE(prevIndex != nullptr); + BOOST_REQUIRE(prevIndex->Check() == ErrorCode::Success); + + double seconds = + std::chrono::duration_cast(end - start).count() / 1000000.0f; + int vectorCount = prevIndex->GetNumSamples(); + BOOST_TEST_MESSAGE(" Load Time: " << seconds << " seconds"); + BOOST_TEST_MESSAGE(" Index vectors after reload: " << vectorCount); + + // Collect JSON data for Benchmark 4 + benchmark1Data << " \"Load timeSeconds\": " << seconds << ",\n"; + benchmark1Data << " \"Load vectorCount\": " << vectorCount << ",\n"; + + auto cloneIndex = prevIndex->Clone(clonePath); + prevIndex = nullptr; + + start = std::chrono::high_resolution_clock::now(); + InsertVectors(static_cast *>(cloneIndex.get()), numThreads, insertBatchSize, + addvecset, addmetaset, iter * insertBatchSize); - } - auto end = std::chrono::high_resolution_clock::now(); + end = std::chrono::high_resolution_clock::now(); - double seconds = std::chrono::duration_cast(end - start).count() / 1000000.0f; - double throughput = insertVectorCount / seconds; + seconds = + std::chrono::duration_cast(end - start).count() / 1000000.0f; + double throughput = insertBatchSize / seconds; - BOOST_TEST_MESSAGE(" Inserted: " << insertVectorCount << " vectors"); - BOOST_TEST_MESSAGE(" Time: " << seconds << " seconds"); - BOOST_TEST_MESSAGE(" Throughput: " << throughput << " vectors/sec"); + BOOST_TEST_MESSAGE(" Inserted: " << insertBatchSize << " vectors"); + BOOST_TEST_MESSAGE(" Time: " << seconds << " seconds"); + BOOST_TEST_MESSAGE(" Throughput: " << throughput << " vectors/sec"); - // Collect JSON data for Benchmark 1 - benchmark1Data << std::fixed << std::setprecision(4); - benchmark1Data << "{\n"; - benchmark1Data << " \"inserted\": " << insertVectorCount << ",\n"; - benchmark1Data << " \"insert timeSeconds\": " << seconds << ",\n"; - benchmark1Data << " \"insert throughput\": " << throughput << "\n"; - } - - if (deleteVectorCount > 0) - { - - std::vector threads; - threads.reserve(numThreads); + // Collect JSON data for Benchmark 1 + benchmark1Data << " \"inserted\": " << insertBatchSize << ",\n"; + benchmark1Data << " \"insert timeSeconds\": " << seconds << ",\n"; + benchmark1Data << " \"insert throughput\": " << throughput << ",\n"; - std::atomic_size_t vectorsSent(0); - int totaldeleted = batches * deleteBatchSize; - auto func = [&]() { - size_t idx = 0; - while (true) + if (deleteBatchSize > 0) { - idx = vectorsSent.fetch_add(1); - if (idx < totaldeleted) - { - if ((idx & ((1 << 5) - 1)) == 0) + std::vector threads; + threads.reserve(numThreads); + + int startidx = iter * deleteBatchSize; + std::atomic_size_t vectorsSent(startidx); + int totaldeleted = startidx + deleteBatchSize; + auto func = [&]() { + size_t idx = startidx; + while (true) { - SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "Sent %.2lf%%...\n", - idx * 100.0 / totaldeleted); + idx = vectorsSent.fetch_add(1); + if (idx < totaldeleted) + { + if ((idx & ((1 << 5) - 1)) == 0) + { + SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "Sent %.2lf%%...\n", + (idx - startidx) * 100.0 / deleteBatchSize); + } + BOOST_REQUIRE(cloneIndex->DeleteIndex(idx) == ErrorCode::Success); + } + else + { + return; + } } - BOOST_REQUIRE(index->DeleteIndex(idx) == ErrorCode::Success); + }; + + start = std::chrono::high_resolution_clock::now(); + for (int j = 0; j < numThreads; j++) + { + threads.emplace_back(func); } - else + for (auto &thread : threads) { - return; + thread.join(); } + end = std::chrono::high_resolution_clock::now(); + double seconds = + std::chrono::duration_cast(end - start).count() / 1000000.0f; + double throughput = deleteBatchSize / seconds; + + benchmark1Data << " \"deleted\": " << deleteVectorCount << ",\n"; + benchmark1Data << " \"delete timeSeconds\": " << seconds << ",\n"; + benchmark1Data << " \"delete throughput\": " << throughput << ",\n"; } - }; - auto start = std::chrono::high_resolution_clock::now(); - for (int j = 0; j < numThreads; j++) - { - threads.emplace_back(func); - } - for (auto &thread : threads) - { - thread.join(); - } - auto end = std::chrono::high_resolution_clock::now(); - double seconds = std::chrono::duration_cast(end - start).count() / 1000000.0f; - double throughput = deleteVectorCount / seconds; - - benchmark1Data << " \"deleted\": " << deleteVectorCount << ",\n"; - benchmark1Data << " \"delete timeSeconds\": " << seconds << ",\n"; - benchmark1Data << " \"delete throughput\": " << throughput << "\n"; - - } - benchmark1Data << " }"; - - // Benchmark 2: Query performance with detailed latency stats - BOOST_TEST_MESSAGE("\n=== Benchmark 2: Query After Insertions ==="); - BenchmarkQueryPerformance(index, queryset, truth, vecset, addvecset, truthPath, baseVectorCount, topK, - numThreads, numQueries, batches, benchmark2Data); - } + BOOST_TEST_MESSAGE("\n=== Benchmark 2: Query After Insertions and Deletions ==="); + benchmark1Data << " \"search\":"; + BenchmarkQueryPerformance(cloneIndex, queryset, truth, vecset, addvecset, truthPath, baseVectorCount, + topK, numThreads, numQueries, iter + 1, benchmark1Data, " "); + benchmark1Data << ",\n"; - // Benchmark 3: Save index to disk - BOOST_TEST_MESSAGE("\n=== Benchmark 3: Save Index ==="); - { - auto start = std::chrono::high_resolution_clock::now(); - BOOST_REQUIRE(ErrorCode::Success == index->SaveIndex(indexPath + "_saved")); - auto end = std::chrono::high_resolution_clock::now(); - - double seconds = std::chrono::duration_cast(end - start).count() / 1000000.0f; - BOOST_TEST_MESSAGE(" Save Time: " << seconds << " seconds"); - BOOST_TEST_MESSAGE(" Save completed successfully"); - - // Collect JSON data for Benchmark 3 - benchmark3Data << std::fixed << std::setprecision(4); - benchmark3Data << "{\n"; - benchmark3Data << " \"timeSeconds\": " << seconds << "\n"; - benchmark3Data << " }"; - } + start = std::chrono::high_resolution_clock::now(); + BOOST_REQUIRE(cloneIndex->SaveIndex(clonePath) == ErrorCode::Success); + end = std::chrono::high_resolution_clock::now(); - BOOST_TEST_MESSAGE("Starting Benchmark 4..."); + seconds = std::chrono::duration_cast(end - start).count() / 1000000.0f; + BOOST_TEST_MESSAGE(" Save Time: " << seconds << " seconds"); + BOOST_TEST_MESSAGE(" Save completed successfully"); - // Benchmark 4: Clean up memory and reload from disk - BOOST_TEST_MESSAGE("\n=== Benchmark 4: Memory Cleanup & Reload ==="); + // Collect JSON data for Benchmark 3 + benchmark1Data << " \"save timeSeconds\": " << seconds << "\n"; - // Explicitly clean up memory - BOOST_TEST_MESSAGE("Releasing index from memory..."); - index = nullptr; - BOOST_TEST_MESSAGE("Memory cleanup complete"); + if (iter != batches - 1) + benchmark1Data << " },\n"; + else + benchmark1Data << " }\n"; - // Reload index - BOOST_TEST_MESSAGE("Starting to reload index from: " << indexPath + "_saved"); - { - auto start = std::chrono::high_resolution_clock::now(); - ErrorCode loadResult = VectorIndex::LoadIndex(indexPath + "_saved", index); - BOOST_TEST_MESSAGE("LoadIndex returned with code: " << (int)loadResult); - BOOST_REQUIRE(loadResult == ErrorCode::Success); - auto end = std::chrono::high_resolution_clock::now(); - - BOOST_REQUIRE(index != nullptr); - - double seconds = std::chrono::duration_cast(end - start).count() / 1000000.0f; - int vectorCount = index->GetNumSamples(); - BOOST_TEST_MESSAGE(" Load Time: " << seconds << " seconds"); - BOOST_TEST_MESSAGE(" Index vectors after reload: " << vectorCount); - - // Collect JSON data for Benchmark 4 - benchmark4Data << std::fixed << std::setprecision(4); - benchmark4Data << "{\n"; - benchmark4Data << " \"timeSeconds\": " << seconds << ",\n"; - benchmark4Data << " \"vectorCount\": " << vectorCount << "\n"; - benchmark4Data << " }"; - } - BOOST_TEST_MESSAGE("Benchmark 4 completed successfully"); - // Benchmark 5: Query performance after reload - BOOST_TEST_MESSAGE("\n=== Benchmark 5: Query After Reload ==="); - BenchmarkQueryPerformance(index, queryset, truth, vecset, addvecset, truthPath, baseVectorCount, topK, - numThreads, numQueries, batches, tmpbenchmark); + cloneIndex = nullptr; + prevPath = clonePath; + } + } + benchmark1Data << " }"; + } - BenchmarkQueryPerformance(index, queryset, truth, vecset, addvecset, truthPath, baseVectorCount, topK, - numThreads, numQueries, batches, benchmark5Data); BOOST_TEST_MESSAGE("\n=== Benchmark Complete ==="); @@ -574,19 +573,7 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c jsonFile << " \"benchmark0_query_before_insert\": " << benchmark0Data.str() << ",\n"; if (!benchmark1Data.str().empty()) { - jsonFile << " \"benchmark1_insert\": " << benchmark1Data.str() << ",\n"; - jsonFile << " \"benchmark2_query_before_save\": " << benchmark2Data.str() << ",\n"; - } - jsonFile << " \"benchmark3_save\": " << benchmark3Data.str() << ",\n"; - jsonFile << " \"benchmark4_reload\": " << benchmark4Data.str() << ",\n"; - jsonFile << " \"benchmark5_query_after_reload\": " << benchmark5Data.str(); - if (!benchmark6Data.str().empty()) - { - jsonFile << ",\n \"benchmark6_recall\": " << benchmark6Data.str() << "\n"; - } - else - { - jsonFile << "\n"; + jsonFile << " \"benchmark1_insert\": " << benchmark1Data.str() << "\n"; } jsonFile << " }\n"; jsonFile << "}\n"; From fa5fb2913151111665444daf81f6c7a3a0292c7c Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Thu, 11 Dec 2025 18:00:02 -0800 Subject: [PATCH 11/38] write the result in time --- Test/src/SPFreshTest.cpp | 148 ++++++++++++++++++--------------------- 1 file changed, 67 insertions(+), 81 deletions(-) diff --git a/Test/src/SPFreshTest.cpp b/Test/src/SPFreshTest.cpp index 6884d6ee5..6e52f655c 100644 --- a/Test/src/SPFreshTest.cpp +++ b/Test/src/SPFreshTest.cpp @@ -255,7 +255,7 @@ void BenchmarkQueryPerformance(std::shared_ptr &index, std::shared_ std::shared_ptr &truth, std::shared_ptr &vecset, std::shared_ptr &addvecset, const std::string &truthPath, SizeType baseVectorCount, int topK, int numThreads, int numQueries, int batches, - std::ostringstream &benchmarkData, std::string prefix = "") + std::ostream &benchmarkData, std::string prefix = "") { // Benchmark: Query performance with detailed latency stats std::vector latencies(numQueries); @@ -363,7 +363,7 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c int deleteBatchSize = deleteVectorCount / max(batches, 1); // Variables to collect JSON output data - std::ostringstream tmpbenchmark, benchmark0Data, benchmark1Data; + std::ostringstream tmpbenchmark; // Generate test data std::shared_ptr vecset, addvecset, queryset, truth; @@ -372,6 +372,45 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c generator.RunBatches(vecset, metaset, addvecset, addmetaset, queryset, N, insertBatchSize, deleteBatchSize, batches, truth); + + std::ofstream jsonFile(outputFile); + BOOST_REQUIRE(jsonFile.is_open()); + + jsonFile << std::fixed << std::setprecision(4); + + // Get current timestamp + auto time_t_now = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); + std::tm tm_now; +#if defined(_MSC_VER) + localtime_s(&tm_now, &time_t_now); +#else + localtime_r(&time_t_now, &tm_now); +#endif + + std::ostringstream timestampStream; + timestampStream << std::put_time(&tm_now, "%Y-%m-%dT%H:%M:%S"); + std::string timestamp = timestampStream.str(); + + jsonFile << "{\n"; + jsonFile << " \"timestamp\": \"" << timestamp << "\",\n"; + jsonFile << " \"config\": {\n"; + jsonFile << " \"vectorPath\": \"" << vectorPath << "\",\n"; + jsonFile << " \"queryPath\": \"" << queryPath << "\",\n"; + jsonFile << " \"truthPath\": \"" << truthPath << "\",\n"; + jsonFile << " \"indexPath\": \"" << indexPath << "\",\n"; + jsonFile << " \"ValueType\": \"" << Helper::Convert::ConvertToString(GetEnumValueType()) << "\",\n"; + jsonFile << " \"dimension\": " << dimension << ",\n"; + jsonFile << " \"baseVectorCount\": " << baseVectorCount << ",\n"; + jsonFile << " \"insertVectorCount\": " << insertVectorCount << ",\n"; + jsonFile << " \"DeleteVectorCount\": " << deleteVectorCount << ",\n"; + jsonFile << " \"BatchNum\": " << batches << ",\n"; + jsonFile << " \"topK\": " << topK << ",\n"; + jsonFile << " \"numQueries\": " << numQueries << ",\n"; + jsonFile << " \"numThreads\": " << numThreads << ",\n"; + jsonFile << " \"DistMethod\": \"" << Helper::Convert::ConvertToString(distMethod) << "\"\n"; + jsonFile << " },\n"; + jsonFile << " \"results\": {\n"; + // Build initial index BOOST_TEST_MESSAGE("\n=== Building Index ==="); std::filesystem::remove_all(indexPath); @@ -384,8 +423,12 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c BOOST_TEST_MESSAGE("\n=== Benchmark 0: Query Before Insertions ==="); BenchmarkQueryPerformance(index, queryset, truth, vecset, addvecset, truthPath, baseVectorCount, topK, numThreads, numQueries, 0, tmpbenchmark); + jsonFile << " \"benchmark0_query_before_insert\": "; BenchmarkQueryPerformance(index, queryset, truth, vecset, addvecset, truthPath, baseVectorCount, topK, - numThreads, numQueries, 0, benchmark0Data); + numThreads, numQueries, 0, jsonFile); + jsonFile << ",\n"; + jsonFile.flush(); + BOOST_REQUIRE(index->SaveIndex(indexPath) == ErrorCode::Success); index = nullptr; @@ -402,12 +445,11 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c { BOOST_TEST_MESSAGE("\n=== Benchmark 1: Insert Performance ==="); { - benchmark1Data << std::fixed << std::setprecision(4); - benchmark1Data << "{\n"; + jsonFile << " \"benchmark1_insert\": {\n"; std::string prevPath = indexPath; for (int iter = 0; iter < batches; iter++) { - benchmark1Data << " \"batch_" << iter + 1 << "\": {\n"; + jsonFile << " \"batch_" << iter + 1 << "\": {\n"; std::string clonePath = indexPath + "_" + std::to_string(iter); std::shared_ptr prevIndex, clonedIndex; @@ -424,8 +466,8 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c BOOST_TEST_MESSAGE(" Index vectors after reload: " << vectorCount); // Collect JSON data for Benchmark 4 - benchmark1Data << " \"Load timeSeconds\": " << seconds << ",\n"; - benchmark1Data << " \"Load vectorCount\": " << vectorCount << ",\n"; + jsonFile << " \"Load timeSeconds\": " << seconds << ",\n"; + jsonFile << " \"Load vectorCount\": " << vectorCount << ",\n"; auto cloneIndex = prevIndex->Clone(clonePath); prevIndex = nullptr; @@ -445,9 +487,9 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c BOOST_TEST_MESSAGE(" Throughput: " << throughput << " vectors/sec"); // Collect JSON data for Benchmark 1 - benchmark1Data << " \"inserted\": " << insertBatchSize << ",\n"; - benchmark1Data << " \"insert timeSeconds\": " << seconds << ",\n"; - benchmark1Data << " \"insert throughput\": " << throughput << ",\n"; + jsonFile << " \"inserted\": " << insertBatchSize << ",\n"; + jsonFile << " \"insert timeSeconds\": " << seconds << ",\n"; + jsonFile << " \"insert throughput\": " << throughput << ",\n"; if (deleteBatchSize > 0) { @@ -492,16 +534,16 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c std::chrono::duration_cast(end - start).count() / 1000000.0f; double throughput = deleteBatchSize / seconds; - benchmark1Data << " \"deleted\": " << deleteVectorCount << ",\n"; - benchmark1Data << " \"delete timeSeconds\": " << seconds << ",\n"; - benchmark1Data << " \"delete throughput\": " << throughput << ",\n"; + jsonFile << " \"deleted\": " << deleteVectorCount << ",\n"; + jsonFile << " \"delete timeSeconds\": " << seconds << ",\n"; + jsonFile << " \"delete throughput\": " << throughput << ",\n"; } BOOST_TEST_MESSAGE("\n=== Benchmark 2: Query After Insertions and Deletions ==="); - benchmark1Data << " \"search\":"; + jsonFile << " \"search\":"; BenchmarkQueryPerformance(cloneIndex, queryset, truth, vecset, addvecset, truthPath, baseVectorCount, - topK, numThreads, numQueries, iter + 1, benchmark1Data, " "); - benchmark1Data << ",\n"; + topK, numThreads, numQueries, iter + 1, jsonFile, " "); + jsonFile << ",\n"; start = std::chrono::high_resolution_clock::now(); BOOST_REQUIRE(cloneIndex->SaveIndex(clonePath) == ErrorCode::Success); @@ -512,80 +554,24 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c BOOST_TEST_MESSAGE(" Save completed successfully"); // Collect JSON data for Benchmark 3 - benchmark1Data << " \"save timeSeconds\": " << seconds << "\n"; + jsonFile << " \"save timeSeconds\": " << seconds << "\n"; if (iter != batches - 1) - benchmark1Data << " },\n"; + jsonFile << " },\n"; else - benchmark1Data << " }\n"; - - + jsonFile << " }\n"; cloneIndex = nullptr; prevPath = clonePath; + jsonFile.flush(); } } - benchmark1Data << " }"; + jsonFile << " }\n"; } - - BOOST_TEST_MESSAGE("\n=== Benchmark Complete ==="); - - // Write JSON output - { - std::ofstream jsonFile(outputFile); - if (jsonFile.is_open()) - { - jsonFile << std::fixed << std::setprecision(4); - - // Get current timestamp - auto time_t_now = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); - std::tm tm_now; -#if defined(_MSC_VER) - localtime_s(&tm_now, &time_t_now); -#else - localtime_r(&time_t_now, &tm_now); -#endif - - std::ostringstream timestampStream; - timestampStream << std::put_time(&tm_now, "%Y-%m-%dT%H:%M:%S"); - std::string timestamp = timestampStream.str(); - - jsonFile << "{\n"; - jsonFile << " \"timestamp\": \"" << timestamp << "\",\n"; - jsonFile << " \"config\": {\n"; - jsonFile << " \"vectorPath\": \"" << vectorPath << "\",\n"; - jsonFile << " \"queryPath\": \"" << queryPath << "\",\n"; - jsonFile << " \"truthPath\": \"" << truthPath << "\",\n"; - jsonFile << " \"indexPath\": \"" << indexPath << "\",\n"; - jsonFile << " \"ValueType\": \"" << Helper::Convert::ConvertToString(GetEnumValueType()) << "\",\n"; - jsonFile << " \"dimension\": " << dimension << ",\n"; - jsonFile << " \"baseVectorCount\": " << baseVectorCount << ",\n"; - jsonFile << " \"insertVectorCount\": " << insertVectorCount << ",\n"; - jsonFile << " \"DeleteVectorCount\": " << deleteVectorCount << ",\n"; - jsonFile << " \"BatchNum\": " << batches << ",\n"; - jsonFile << " \"topK\": " << topK << ",\n"; - jsonFile << " \"numQueries\": " << numQueries << ",\n"; - jsonFile << " \"numThreads\": " << numThreads << ",\n"; - jsonFile << " \"DistMethod\": \"" << Helper::Convert::ConvertToString(distMethod) << "\"\n"; - jsonFile << " },\n"; - jsonFile << " \"results\": {\n"; - jsonFile << " \"benchmark0_query_before_insert\": " << benchmark0Data.str() << ",\n"; - if (!benchmark1Data.str().empty()) - { - jsonFile << " \"benchmark1_insert\": " << benchmark1Data.str() << "\n"; - } - jsonFile << " }\n"; - jsonFile << "}\n"; - - jsonFile.close(); - BOOST_TEST_MESSAGE("\n=== Benchmark results saved to output.json ==="); - } - else - { - BOOST_TEST_MESSAGE("\n=== Failed to create output.json ==="); - } - } + jsonFile << " }\n"; + jsonFile << "}\n"; + jsonFile.close(); M = oldM; K = oldK; From bcafeb7fa6183282458f3559d1864dd826fab9c3 Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Fri, 12 Dec 2025 12:30:12 -0800 Subject: [PATCH 12/38] let benchmark fit for larger dataset --- AnnService/inc/Core/MetadataSet.h | 4 +- AnnService/src/Core/MetadataSet.cpp | 26 +++- Test/inc/TestDataGenerator.h | 13 ++ Test/src/SPFreshTest.cpp | 194 ++++++++++++++++++++++++---- Test/src/TestDataGenerator.cpp | 174 +++++++++++++++++++++++-- 5 files changed, 361 insertions(+), 50 deletions(-) diff --git a/AnnService/inc/Core/MetadataSet.h b/AnnService/inc/Core/MetadataSet.h index d16c037ff..0325bda31 100644 --- a/AnnService/inc/Core/MetadataSet.h +++ b/AnnService/inc/Core/MetadataSet.h @@ -93,7 +93,7 @@ class MemMetadataSet : public MetadataSet std::uint64_t p_blockSize, std::uint64_t p_capacity, std::uint64_t p_metaSize); MemMetadataSet(const std::string& p_metafile, const std::string& p_metaindexfile, - std::uint64_t p_blockSize, std::uint64_t p_capacity, std::uint64_t p_metaSize); + std::uint64_t p_blockSize, std::uint64_t p_capacity, std::uint64_t p_metaSize, int start = 0, int count = -1); MemMetadataSet(std::shared_ptr p_metain, std::shared_ptr p_metaindexin, std::uint64_t p_blockSize, std::uint64_t p_capacity, std::uint64_t p_metaSize); @@ -118,7 +118,7 @@ class MemMetadataSet : public MetadataSet private: ErrorCode Init(std::shared_ptr p_metain, std::shared_ptr p_metaindexin, - std::uint64_t p_blockSize, std::uint64_t p_capacity, std::uint64_t p_metaSize); + std::uint64_t p_blockSize, std::uint64_t p_capacity, std::uint64_t p_metaSize, int start = 0, int count = -1); std::shared_ptr m_lock; diff --git a/AnnService/src/Core/MetadataSet.cpp b/AnnService/src/Core/MetadataSet.cpp index bcef57bb5..0c724eb60 100644 --- a/AnnService/src/Core/MetadataSet.cpp +++ b/AnnService/src/Core/MetadataSet.cpp @@ -276,23 +276,34 @@ MemMetadataSet::MemMetadataSet(std::uint64_t p_blockSize, std::uint64_t p_capaci } ErrorCode MemMetadataSet::Init(std::shared_ptr p_metain, std::shared_ptr p_metaindexin, - std::uint64_t p_blockSize, std::uint64_t p_capacity, std::uint64_t p_metaSize) + std::uint64_t p_blockSize, std::uint64_t p_capacity, std::uint64_t p_metaSize, int start, int count) { IOBINARY(p_metaindexin, ReadBinary, sizeof(m_count), (char *)&m_count); + if (start > m_count) start = m_count; + if (count < 0 || count > m_count - start) count = m_count - start; + + std::uint64_t offset = 0; m_pOffsets.reset(new MetadataOffsets, std::default_delete()); auto &m_offsets = *static_cast(m_pOffsets.get()); m_offsets.reserve(p_blockSize, p_capacity); { std::vector tmp(m_count + 1, 0); IOBINARY(p_metaindexin, ReadBinary, sizeof(std::uint64_t) * (m_count + 1), (char *)tmp.data()); - m_offsets.assign(tmp.data(), tmp.data() + tmp.size()); + offset = tmp[start]; + if (offset > 0) + { + for (int i = start; i <= start + count; i++) + tmp[i] -= offset; + } + m_offsets.assign(tmp.data() + start, tmp.data() + start + count + 1); } - m_metadataHolder = ByteArray::Alloc(m_offsets[m_count]); - IOBINARY(p_metain, ReadBinary, m_metadataHolder.Length(), (char *)m_metadataHolder.Data()); + m_metadataHolder = ByteArray::Alloc(m_offsets[count]); + IOBINARY(p_metain, ReadBinary, m_metadataHolder.Length(), (char *)m_metadataHolder.Data(), offset); m_newdata.reserve(p_blockSize * p_metaSize); m_lock.reset(new std::shared_timed_mutex, std::default_delete()); - SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "Load MetaIndex(%d) Meta(%llu)\n", m_count, m_offsets[m_count]); + m_count = count; + SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "Load MetaIndex(%d) Meta(%llu) Offset(%llu)\n", m_count, m_offsets[m_count], offset); return ErrorCode::Success; } @@ -307,7 +318,8 @@ MemMetadataSet::MemMetadataSet(std::shared_ptr p_metain, std::sh } MemMetadataSet::MemMetadataSet(const std::string &p_metafile, const std::string &p_metaindexfile, - std::uint64_t p_blockSize, std::uint64_t p_capacity, std::uint64_t p_metaSize) + std::uint64_t p_blockSize, std::uint64_t p_capacity, std::uint64_t p_metaSize, + int start, int count) { std::shared_ptr ptrMeta = f_createIO(), ptrMetaIndex = f_createIO(); if (ptrMeta == nullptr || ptrMetaIndex == nullptr || @@ -318,7 +330,7 @@ MemMetadataSet::MemMetadataSet(const std::string &p_metafile, const std::string p_metaindexfile.c_str()); throw std::runtime_error("Cannot open MemMetadataSet files"); } - if (Init(ptrMeta, ptrMetaIndex, p_blockSize, p_capacity, p_metaSize) != ErrorCode::Success) + if (Init(ptrMeta, ptrMetaIndex, p_blockSize, p_capacity, p_metaSize, start, count) != ErrorCode::Success) { SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "ERROR: Cannot read MemMetadataSet!\n"); throw std::runtime_error("Cannot read MemMetadataSet"); diff --git a/Test/inc/TestDataGenerator.h b/Test/inc/TestDataGenerator.h index 387aeaa36..bafca1b1b 100644 --- a/Test/inc/TestDataGenerator.h +++ b/Test/inc/TestDataGenerator.h @@ -37,6 +37,12 @@ namespace TestUtils { std::shared_ptr &addvecset, std::shared_ptr &addmetaset, std::shared_ptr &queryset, int base, int batchinsert, int batchdelete, int batches, std::shared_ptr &truths); + + void RunLargeBatches(std::string &vecset, std::string &metaset, std::string &metaidx, + std::string &addset, std::string &addmetaset, std::string &addmetaidx, + std::string &queryset, int bash, int batchinsert, int batchdelete, int batches, + std::string &truth); + private: int m_n, m_a, m_q, m_m, m_k; std::string m_distMethod; @@ -64,6 +70,13 @@ namespace TestUtils { std::shared_ptr CombineVectorSets(std::shared_ptr base, std::shared_ptr additional); + + void GenerateVectorSet(std::string &vecset, std::string &metaset, std::string &metaidx, std::string &vecPath, + SPTAG::SizeType start, int count); + + void GenerateBatchTruth(const std::string &filename, std::string &pvecset, std::string &paddset, std::string &pqueryset, + int base, int batchinsert, int batchdelete, int batches, + bool normalize); }; } // namespace TestUtils \ No newline at end of file diff --git a/Test/src/SPFreshTest.cpp b/Test/src/SPFreshTest.cpp index 6e52f655c..b331dd8b1 100644 --- a/Test/src/SPFreshTest.cpp +++ b/Test/src/SPFreshTest.cpp @@ -126,6 +126,103 @@ std::shared_ptr BuildIndex(const std::string &outDirectory, std::sh return vecIndex; } +template +std::shared_ptr BuildLargeIndex(const std::string &outDirectory, std::string &pvecset, + std::string& pmetaset, std::string& pmetaidx, const std::string &distMethod = "L2", + int searchthread = 2) +{ + auto vecIndex = VectorIndex::CreateInstance(IndexAlgoType::SPANN, GetEnumValueType()); + + std::string configuration = R"( + [Base] + DistCalcMethod=L2 + IndexAlgoType=BKT + VectorPath=)" + pvecset + R"( + ValueType=)" + Helper::Convert::ConvertToString(GetEnumValueType()) + + R"( + Dim=)" + std::to_string(M) + + R"( + IndexDirectory=)" + outDirectory + + R"( + + [SelectHead] + isExecute=true + NumberOfThreads=16 + SelectThreshold=0 + SplitFactor=0 + SplitThreshold=0 + Ratio=0.2 + + [BuildHead] + isExecute=true + NumberOfThreads=16 + + [BuildSSDIndex] + isExecute=true + BuildSsdIndex=true + InternalResultNum=64 + SearchInternalResultNum=64 + NumberOfThreads=16 + PostingPageLimit=)" + std::to_string(4 * sizeof(T)) + + R"( + SearchPostingPageLimit=)" + + std::to_string(4 * sizeof(T)) + R"( + TmpDir=tmpdir + Storage=FILEIO + SpdkBatchSize=64 + ExcludeHead=false + ResultNum=10 + SearchThreadNum=)" + std::to_string(searchthread) + + R"( + Update=true + SteadyState=true + InsertThreadNum=1 + AppendThreadNum=1 + ReassignThreadNum=0 + DisableReassign=false + ReassignK=64 + LatencyLimit=50.0 + SearchDuringUpdate=true + MergeThreshold=10 + Sampling=4 + BufferLength=6 + InPlace=true + StartFileSizeGB=1 + OneClusterCutMax=false + ConsistencyCheck=false + ChecksumCheck=false + ChecksumInRead=false + AsyncMergeInSearch=false + DeletePercentageForRefine=0.4 + AsyncAppendQueueSize=0 + AllowZeroReplica=false + )"; + + std::shared_ptr buffer(new Helper::SimpleBufferIO()); + Helper::IniReader reader; + if (!buffer->Initialize(configuration.data(), std::ios::in, configuration.size())) + return nullptr; + if (ErrorCode::Success != reader.LoadIni(buffer)) + return nullptr; + + std::string sections[] = {"Base", "SelectHead", "BuildHead", "BuildSSDIndex"}; + for (const auto &sec : sections) + { + auto params = reader.GetParameters(sec.c_str()); + for (const auto &[key, val] : params) + { + vecIndex->SetParameter(key.c_str(), val.c_str(), sec.c_str()); + } + } + + auto buildStatus = vecIndex->BuildIndex(); + if (buildStatus != ErrorCode::Success) + return nullptr; + + vecIndex->SetMetadata(new SPTAG::FileMetadataSet(pmetaset, pmetaidx)); + return vecIndex; +} + template std::vector SearchOnly(std::shared_ptr &vecIndex, std::shared_ptr &queryset, int k) { @@ -149,7 +246,7 @@ std::vector SearchOnly(std::shared_ptr &vecIndex, std: template float EvaluateRecall(const std::vector &res, std::shared_ptr &vecIndex, std::shared_ptr &queryset, std::shared_ptr &truth, - std::shared_ptr &baseVec, std::shared_ptr &addVec, SizeType baseCount, int k, int batch) + std::shared_ptr &baseVec, std::shared_ptr &addVec, SizeType baseCount, int k, int batch, int totalbatches = -1) { if (!truth) { @@ -160,18 +257,28 @@ float EvaluateRecall(const std::vector &res, std::shared_ptr(truth->Dimension())); float totalRecall = 0.0f; float eps = 1e-4f; - + int distbase = (totalbatches + 1) * queryset->Count(); for (SizeType i = 0; i < queryset->Count(); ++i) { const SizeType *truthNN = reinterpret_cast(truth->GetVector(i + batch * queryset->Count())); + float *truthD = nullptr; + if (truth->Count() == 2 * distbase) + { + truthD = reinterpret_cast(truth->GetVector(distbase + i + batch * queryset->Count())); + } for (int j = 0; j < recallK; ++j) { SizeType truthVid = truthNN[j]; - float truthDist = - (truthVid < baseCount) + float truthDist = MaxDist; + if (baseVec != nullptr && addVec != nullptr) + truthDist = (truthVid < baseCount) ? vecIndex->ComputeDistance(queryset->GetVector(i), baseVec->GetVector(truthVid)) : vecIndex->ComputeDistance(queryset->GetVector(i), addVec->GetVector(truthVid - baseCount)); - + else if (truthD) + { + truthDist = truthD[j]; + } + for (int l = 0; l < k; ++l) { const auto result = res[i].GetResult(l); @@ -252,9 +359,8 @@ void InsertVectors(SPANN::Index *p_index, int insertThreads, int step template void BenchmarkQueryPerformance(std::shared_ptr &index, std::shared_ptr &queryset, - std::shared_ptr &truth, std::shared_ptr &vecset, - std::shared_ptr &addvecset, const std::string &truthPath, - SizeType baseVectorCount, int topK, int numThreads, int numQueries, int batches, + std::shared_ptr &truth, const std::string &truthPath, + SizeType baseVectorCount, int topK, int numThreads, int numQueries, int batches, int totalbatches, std::ostream &benchmarkData, std::string prefix = "") { // Benchmark: Query performance with detailed latency stats @@ -336,7 +442,8 @@ void BenchmarkQueryPerformance(std::shared_ptr &index, std::shared_ // Recall evaluation (if truth file provided) BOOST_TEST_MESSAGE("Checking for truth file: " << truthPath); - float avgRecall = EvaluateRecall(results, index, queryset, truth, vecset, addvecset, baseVectorCount, topK, batches); + std::shared_ptr pvecset, paddvecset; + float avgRecall = EvaluateRecall(results, index, queryset, truth, pvecset, paddvecset, baseVectorCount, topK, batches, totalbatches); BOOST_TEST_MESSAGE(" Recall@" << topK << " = " << (avgRecall * 100.0f) << "%"); BOOST_TEST_MESSAGE(" (Evaluated on " << numQueries << " queries against base vectors)"); benchmarkData << std::fixed << std::setprecision(4); @@ -366,11 +473,10 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c std::ostringstream tmpbenchmark; // Generate test data - std::shared_ptr vecset, addvecset, queryset, truth; - std::shared_ptr metaset, addmetaset; + std::string pvecset, paddset, pqueryset, ptruth, pmeta, pmetaidx, paddmeta, paddmetaidx; TestUtils::TestDataGenerator generator(N, queries, M, K, dist, insertVectorCount, false, vectorPath, queryPath); - generator.RunBatches(vecset, metaset, addvecset, addmetaset, queryset, N, insertBatchSize, deleteBatchSize, - batches, truth); + generator.RunLargeBatches(pvecset, pmeta, pmetaidx, paddset, paddmeta, paddmetaidx, pqueryset, N, insertBatchSize, deleteBatchSize, + batches, ptruth); std::ofstream jsonFile(outputFile); @@ -414,18 +520,46 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c // Build initial index BOOST_TEST_MESSAGE("\n=== Building Index ==="); std::filesystem::remove_all(indexPath); - std::shared_ptr index = BuildIndex(indexPath, vecset, metaset, dist, numThreads); + std::shared_ptr index = BuildLargeIndex(indexPath, pvecset, pmeta, pmetaidx, dist, numThreads); BOOST_REQUIRE(index != nullptr); BOOST_TEST_MESSAGE("Index built successfully with " << baseVectorCount << " vectors"); + auto vectorOptions = std::shared_ptr( + new Helper::ReaderOptions(GetEnumValueType(), M, VectorFileType::DEFAULT)); + + auto queryReader = Helper::VectorSetReader::CreateInstance(vectorOptions); + if (!fileexists(pqueryset.c_str()) || ErrorCode::Success != queryReader->LoadFile(pqueryset)) + { + SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Cannot find or load %s. Using random generation!\n", + pqueryset.c_str()); + return; + } + auto queryset = queryReader->GetVectorSet(); + + auto addReader = Helper::VectorSetReader::CreateInstance(vectorOptions); + if (!fileexists(paddset.c_str()) || ErrorCode::Success != addReader->LoadFile(paddset)) + { + SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Cannot find or load %s. Using random generation!\n", paddset.c_str()); + return; + } + + auto opts = std::make_shared(GetEnumValueType(), K, VectorFileType::DEFAULT); + auto reader = Helper::VectorSetReader::CreateInstance(opts); + if (ErrorCode::Success != reader->LoadFile(ptruth)) + { + SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Failed to read file %s\n", ptruth.c_str()); + return; + } + auto truth = reader->GetVectorSet(); + // Benchmark 0: Query performance before insertions BOOST_TEST_MESSAGE("\n=== Benchmark 0: Query Before Insertions ==="); - BenchmarkQueryPerformance(index, queryset, truth, vecset, addvecset, truthPath, baseVectorCount, topK, - numThreads, numQueries, 0, tmpbenchmark); + BenchmarkQueryPerformance(index, queryset, truth,truthPath, baseVectorCount, topK, + numThreads, numQueries, 0, batches, tmpbenchmark); jsonFile << " \"benchmark0_query_before_insert\": "; - BenchmarkQueryPerformance(index, queryset, truth, vecset, addvecset, truthPath, baseVectorCount, topK, - numThreads, numQueries, 0, jsonFile); + BenchmarkQueryPerformance(index, queryset, truth, truthPath, baseVectorCount, topK, + numThreads, numQueries, 0, batches, jsonFile); jsonFile << ",\n"; jsonFile.flush(); @@ -452,7 +586,7 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c jsonFile << " \"batch_" << iter + 1 << "\": {\n"; std::string clonePath = indexPath + "_" + std::to_string(iter); - std::shared_ptr prevIndex, clonedIndex; + std::shared_ptr prevIndex, cloneIndex; auto start = std::chrono::high_resolution_clock::now(); BOOST_REQUIRE(VectorIndex::LoadIndex(prevPath, prevIndex) == ErrorCode::Success); auto end = std::chrono::high_resolution_clock::now(); @@ -469,15 +603,19 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c jsonFile << " \"Load timeSeconds\": " << seconds << ",\n"; jsonFile << " \"Load vectorCount\": " << vectorCount << ",\n"; - auto cloneIndex = prevIndex->Clone(clonePath); + cloneIndex = prevIndex->Clone(clonePath); prevIndex = nullptr; - start = std::chrono::high_resolution_clock::now(); - InsertVectors(static_cast *>(cloneIndex.get()), numThreads, insertBatchSize, - addvecset, - addmetaset, iter * insertBatchSize); - end = std::chrono::high_resolution_clock::now(); - + int insertStart = iter * insertBatchSize; + { + std::shared_ptr addset = addReader->GetVectorSet(insertStart, insertStart + insertBatchSize); + std::shared_ptr addmetaset(new MemMetadataSet( + paddmeta, paddmetaidx, cloneIndex->m_iDataBlockSize, cloneIndex->m_iDataCapacity, 10, insertStart, insertBatchSize)); + start = std::chrono::high_resolution_clock::now(); + InsertVectors(static_cast *>(cloneIndex.get()), numThreads, insertBatchSize, + addset, addmetaset, 0); + end = std::chrono::high_resolution_clock::now(); + } seconds = std::chrono::duration_cast(end - start).count() / 1000000.0f; double throughput = insertBatchSize / seconds; @@ -541,8 +679,8 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c BOOST_TEST_MESSAGE("\n=== Benchmark 2: Query After Insertions and Deletions ==="); jsonFile << " \"search\":"; - BenchmarkQueryPerformance(cloneIndex, queryset, truth, vecset, addvecset, truthPath, baseVectorCount, - topK, numThreads, numQueries, iter + 1, jsonFile, " "); + BenchmarkQueryPerformance(cloneIndex, queryset, truth, truthPath, baseVectorCount, + topK, numThreads, numQueries, iter + 1, batches, jsonFile, " "); jsonFile << ",\n"; start = std::chrono::high_resolution_clock::now(); diff --git a/Test/src/TestDataGenerator.cpp b/Test/src/TestDataGenerator.cpp index 7b3cdae88..ecfafbe54 100644 --- a/Test/src/TestDataGenerator.cpp +++ b/Test/src/TestDataGenerator.cpp @@ -29,6 +29,7 @@ void TestDataGenerator::Run(std::shared_ptr &vecset, std::shared_p LoadOrGenerateTruth("perftest_addtruth." + m_distMethod, CombineVectorSets(vecset, addvecset), queryset, addtruth, true); } + template void TestDataGenerator::RunBatches(std::shared_ptr& vecset, std::shared_ptr& metaset, @@ -42,7 +43,161 @@ void TestDataGenerator::RunBatches(std::shared_ptr& vecset, LoadOrGenerateBatchTruth("perftest_batchtruth." + m_distMethod, CombineVectorSets(vecset, addvecset), queryset, truths, base, batchinsert, batchdelete, batches, true); } - + +template +void TestDataGenerator::RunLargeBatches(std::string &vecset, std::string &metaset, std::string &metaidx, + std::string &addset, std::string &addmetaset, std::string &addmetaidx, + std::string &queryset, int base, int batchinsert, int batchdelete, + int batches, std::string &truth) +{ + vecset = "perftest_vector.bin"; + metaset = "perftest_meta.bin"; + metaidx = "perftest_metaidx.bin"; + addset = "perftest_addvector.bin"; + addmetaset = "perftest_addmeta.bin"; + addmetaidx = "perftest_addmetaidx.bin"; + queryset = "perftest_query.bin"; + truth = "perftest_batchtruth." + m_distMethod; + std::string empty; + + GenerateVectorSet(vecset, metaset, metaidx, m_vectorPath, 0, m_n); + GenerateVectorSet(queryset, empty, empty, m_queryPath, 0, m_q); + GenerateVectorSet(addset, addmetaset, addmetaidx, m_vectorPath, m_n, m_a); + GenerateBatchTruth(truth, vecset, addset, queryset, base, batchinsert, batchdelete, batches, true); +} + +template +void TestDataGenerator::GenerateVectorSet(std::string & pvecset, std::string & pmetaset, std::string & pmetaidx, std::string& pvecPath, SPTAG::SizeType start, int count) +{ + if (!fileexists(pvecset.c_str())) + { + std::shared_ptr vecset; + if (m_isRandom) + { + vecset = GenerateRandomVectorSet(count, m_m); + } + else + { + vecset = GenerateLoadVectorSet(count, m_m, pvecPath, start); + } + vecset->Save(pvecset); + } + + if (pmetaset.empty() || pmetaidx.empty()) + return; + + if (!fileexists(pmetaset.c_str()) || !fileexists(pmetaidx.c_str())) + { + auto metaset = GenerateMetadataSet(count, start); + metaset->SaveMetadata(pmetaset, pmetaidx); + } +} + +template +void TestDataGenerator::GenerateBatchTruth(const std::string &filename, std::string &pvecset, std::string &paddset, std::string &pqueryset, int base, + int batchinsert, int batchdelete, int batches, bool normalize) +{ + if (fileexists(filename.c_str())) + return; + + auto vectorOptions = std::shared_ptr(new Helper::ReaderOptions(GetEnumValueType(), m_m, VectorFileType::DEFAULT)); + auto vectorReader = Helper::VectorSetReader::CreateInstance(vectorOptions); + if (!fileexists(pvecset.c_str()) || ErrorCode::Success != vectorReader->LoadFile(pvecset)) + { + SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Cannot find or load %s. Using random generation!\n", pvecset.c_str()); + return; + } + auto vecset = vectorReader->GetVectorSet(); + + auto queryReader = Helper::VectorSetReader::CreateInstance(vectorOptions); + if (!fileexists(pqueryset.c_str()) || ErrorCode::Success != queryReader->LoadFile(pqueryset)) + { + SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Cannot find or load %s. Using random generation!\n", + pqueryset.c_str()); + return; + } + auto queryset = queryReader->GetVectorSet(); + + auto addReader = Helper::VectorSetReader::CreateInstance(vectorOptions); + if (!fileexists(paddset.c_str()) || ErrorCode::Success != addReader->LoadFile(paddset)) + { + SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Cannot find or load %s. Using random generation!\n", + paddset.c_str()); + return; + } + auto addset = addReader->GetVectorSet(); + + + DistCalcMethod distMethod; + Helper::Convert::ConvertStringTo(m_distMethod.c_str(), distMethod); + if (normalize && distMethod == DistCalcMethod::Cosine) + { + COMMON::Utils::BatchNormalize((T *)vecset->GetData(), vecset->Count(), vecset->Dimension(), + COMMON::Utils::GetBase(), 5); + COMMON::Utils::BatchNormalize((T *)addset->GetData(), addset->Count(), addset->Dimension(), + COMMON::Utils::GetBase(), 5); + } + + ByteArray tru = ByteArray::Alloc(2 * sizeof(float) * (batches + 1) * queryset->Count() * m_k); + int distbase = (batches + 1) * queryset->Count() * m_k; + int start = 0; + int end = base; + int maxthreads = std::thread::hardware_concurrency(); + for (int iter = 0; iter < batches + 1; iter++) + { + std::vector mythreads; + mythreads.reserve(maxthreads); + std::atomic_size_t sent(0); + for (int tid = 0; tid < maxthreads; tid++) + { + mythreads.emplace_back([&, tid]() { + size_t i = 0; + while (true) + { + i = sent.fetch_add(1); + if (i < queryset->Count()) + { + SizeType *neighbors = ((SizeType *)tru.Data()) + iter * (queryset->Count() * m_k) + i * m_k; + float *dists = ((float *)tru.Data()) + distbase + iter * (queryset->Count() * m_k) + i * m_k; + COMMON::QueryResultSet res((const T *)queryset->GetVector(i), m_k); + for (SizeType j = start; j < end; ++j) + { + float dist = MaxDist; + if (j < vecset->Count()) + dist = COMMON::DistanceUtils::ComputeDistance(res.GetTarget(), + reinterpret_cast(vecset->GetVector(j)), m_m, distMethod); + else + dist = COMMON::DistanceUtils::ComputeDistance(res.GetTarget(), + reinterpret_cast(addset->GetVector(j - vecset->Count())), m_m, distMethod); + + res.AddPoint(j, dist); + } + res.SortResult(); + for (int j = 0; j < m_k; ++j) + { + neighbors[j] = res.GetResult(j)->VID; + dists[j] = res.GetResult(j)->Dist; + } + } + else + { + return; + } + } + }); + } + for (auto &t : mythreads) + { + t.join(); + } + mythreads.clear(); + start += batchdelete; + end += batchinsert; + } + auto truths = std::make_shared(tru, GetEnumValueType(), m_k, 2 * (batches + 1) * queryset->Count()); + truths->Save(filename); +} + template void TestDataGenerator::LoadOrGenerateBase(std::shared_ptr &vecset, std::shared_ptr &metaset) { @@ -315,23 +470,16 @@ std::shared_ptr TestDataGenerator::GenerateLoadVectorSet(SP return GenerateRandomVectorSet(count, dim); } - auto allVectors = vectorReader->GetVectorSet(); - int totalVectors = allVectors->Count(); - if (totalVectors - start < count) + auto allVectors = vectorReader->GetVectorSet(start, start + count); + if (allVectors->Count() < count) { SPTAGLIB_LOG(Helper::LogLevel::LL_Error, - "%s contains total %d vectors. Cannot get %d vectors start from %d. Using random generation!\n", path.c_str(), - totalVectors, count, start); + "Cannot get %d vectors start from %d. Using random generation!\n", count, start); return GenerateRandomVectorSet(count, dim); } - SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "%s contains total %d vectors. Load %d vectors start from %d\n", - path.c_str(), totalVectors, count, start); - - - ByteArray baseData = ByteArray::Alloc(sizeof(T) * count * dim); - memcpy(baseData.Data(), (char *)(allVectors->GetData()) + sizeof(T) * start * dim, sizeof(T) * count * dim); - return std::make_shared(baseData, GetEnumValueType(), dim, count); + SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "Load %d vectors start from %d\n", count, start); + return allVectors; } template From 3107cf60734bb9634e4730ccce20831f0cea7f79 Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Fri, 12 Dec 2025 13:02:20 -0800 Subject: [PATCH 13/38] ensure metadata size --- Test/src/TestDataGenerator.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Test/src/TestDataGenerator.cpp b/Test/src/TestDataGenerator.cpp index ecfafbe54..0290fd13d 100644 --- a/Test/src/TestDataGenerator.cpp +++ b/Test/src/TestDataGenerator.cpp @@ -436,7 +436,7 @@ std::shared_ptr TestDataGenerator::GenerateRandomVectorSet(SizeTyp template std::shared_ptr TestDataGenerator::GenerateMetadataSet(SizeType count, SizeType offsetBase) { - ByteArray meta = ByteArray::Alloc(count * 6); + ByteArray meta = ByteArray::Alloc(count * 10); ByteArray metaoffset = ByteArray::Alloc((count + 1) * sizeof(std::uint64_t)); std::uint64_t offset = 0; for (SizeType i = 0; i < count; i++) From b87551c34c72e12b4b0a1ab16db565c5f2d85f8e Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Fri, 12 Dec 2025 14:42:03 -0800 Subject: [PATCH 14/38] add build time --- Test/src/SPFreshTest.cpp | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/Test/src/SPFreshTest.cpp b/Test/src/SPFreshTest.cpp index b331dd8b1..8be600ae1 100644 --- a/Test/src/SPFreshTest.cpp +++ b/Test/src/SPFreshTest.cpp @@ -132,7 +132,7 @@ std::shared_ptr BuildLargeIndex(const std::string &outDirectory, st int searchthread = 2) { auto vecIndex = VectorIndex::CreateInstance(IndexAlgoType::SPANN, GetEnumValueType()); - + int maxthreads = std::thread::hardware_concurrency(); std::string configuration = R"( [Base] DistCalcMethod=L2 @@ -147,7 +147,7 @@ std::shared_ptr BuildLargeIndex(const std::string &outDirectory, st [SelectHead] isExecute=true - NumberOfThreads=16 + NumberOfThreads=)" + std::to_string(maxthreads) + R"( SelectThreshold=0 SplitFactor=0 SplitThreshold=0 @@ -155,15 +155,15 @@ std::shared_ptr BuildLargeIndex(const std::string &outDirectory, st [BuildHead] isExecute=true - NumberOfThreads=16 + NumberOfThreads=)" + std::to_string(maxthreads) + R"( [BuildSSDIndex] isExecute=true BuildSsdIndex=true InternalResultNum=64 SearchInternalResultNum=64 - NumberOfThreads=16 - PostingPageLimit=)" + std::to_string(4 * sizeof(T)) + + NumberOfThreads=)" + std::to_string(maxthreads) + R"( + PostingPageLimit=)" + std::to_string(4 * sizeof(T)) + R"( SearchPostingPageLimit=)" + std::to_string(4 * sizeof(T)) + R"( @@ -520,9 +520,13 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c // Build initial index BOOST_TEST_MESSAGE("\n=== Building Index ==="); std::filesystem::remove_all(indexPath); + auto buildstart = std::chrono::high_resolution_clock::now(); std::shared_ptr index = BuildLargeIndex(indexPath, pvecset, pmeta, pmetaidx, dist, numThreads); BOOST_REQUIRE(index != nullptr); - + auto buildend = std::chrono::high_resolution_clock::now(); + double buildseconds = + std::chrono::duration_cast(buildend - buildstart).count() / 1000000.0f; + jsonFile << " \"build timeSeconds\": " << buildseconds << ",\n"; BOOST_TEST_MESSAGE("Index built successfully with " << baseVectorCount << " vectors"); auto vectorOptions = std::shared_ptr( @@ -609,8 +613,8 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c int insertStart = iter * insertBatchSize; { std::shared_ptr addset = addReader->GetVectorSet(insertStart, insertStart + insertBatchSize); - std::shared_ptr addmetaset(new MemMetadataSet( - paddmeta, paddmetaidx, cloneIndex->m_iDataBlockSize, cloneIndex->m_iDataCapacity, 10, insertStart, insertBatchSize)); + std::shared_ptr addmetaset(new MemMetadataSet(paddmeta, paddmetaidx, cloneIndex->m_iDataBlockSize, + cloneIndex->m_iDataCapacity, 10, insertStart, insertBatchSize), std::default_delete()); start = std::chrono::high_resolution_clock::now(); InsertVectors(static_cast *>(cloneIndex.get()), numThreads, insertBatchSize, addset, addmetaset, 0); @@ -701,7 +705,7 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c cloneIndex = nullptr; prevPath = clonePath; - jsonFile.flush(); + jsonFile.flush(); } } jsonFile << " }\n"; From 40a89d697336665f6b787f11dc15de96137ac955 Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Fri, 12 Dec 2025 15:48:50 -0800 Subject: [PATCH 15/38] add rebuild and resume configuration --- Test/src/SPFreshTest.cpp | 65 +++++++++++++++++++++++----------------- 1 file changed, 38 insertions(+), 27 deletions(-) diff --git a/Test/src/SPFreshTest.cpp b/Test/src/SPFreshTest.cpp index 8be600ae1..870cac0ad 100644 --- a/Test/src/SPFreshTest.cpp +++ b/Test/src/SPFreshTest.cpp @@ -456,9 +456,10 @@ void BenchmarkQueryPerformance(std::shared_ptr &index, std::shared_ } template -void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, const std::string &truthPath, DistCalcMethod distMethod, - const std::string &indexPath, int dimension, int baseVectorCount, int insertVectorCount, int deleteVectorCount, int batches, int topK, int numThreads, int numQueries, - const std::string &outputFile = "output.json") +void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, const std::string &truthPath, + DistCalcMethod distMethod, const std::string &indexPath, int dimension, int baseVectorCount, + int insertVectorCount, int deleteVectorCount, int batches, int topK, int numThreads, int numQueries, + const std::string &outputFile = "output.json", const bool rebuild = true, const int resume = -1) { int oldM = M, oldK = K, oldN = N, oldQueries = queries; N = baseVectorCount; @@ -475,9 +476,8 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c // Generate test data std::string pvecset, paddset, pqueryset, ptruth, pmeta, pmetaidx, paddmeta, paddmetaidx; TestUtils::TestDataGenerator generator(N, queries, M, K, dist, insertVectorCount, false, vectorPath, queryPath); - generator.RunLargeBatches(pvecset, pmeta, pmetaidx, paddset, paddmeta, paddmetaidx, pqueryset, N, insertBatchSize, deleteBatchSize, - batches, ptruth); - + generator.RunLargeBatches(pvecset, pmeta, pmetaidx, paddset, paddmeta, paddmetaidx, pqueryset, N, insertBatchSize, + deleteBatchSize, batches, ptruth); std::ofstream jsonFile(outputFile); BOOST_REQUIRE(jsonFile.is_open()); @@ -517,17 +517,25 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c jsonFile << " },\n"; jsonFile << " \"results\": {\n"; + std::shared_ptr index; // Build initial index BOOST_TEST_MESSAGE("\n=== Building Index ==="); - std::filesystem::remove_all(indexPath); - auto buildstart = std::chrono::high_resolution_clock::now(); - std::shared_ptr index = BuildLargeIndex(indexPath, pvecset, pmeta, pmetaidx, dist, numThreads); - BOOST_REQUIRE(index != nullptr); - auto buildend = std::chrono::high_resolution_clock::now(); - double buildseconds = - std::chrono::duration_cast(buildend - buildstart).count() / 1000000.0f; - jsonFile << " \"build timeSeconds\": " << buildseconds << ",\n"; - BOOST_TEST_MESSAGE("Index built successfully with " << baseVectorCount << " vectors"); + if (rebuild || !direxists(indexPath.c_str())) { + std::filesystem::remove_all(indexPath); + auto buildstart = std::chrono::high_resolution_clock::now(); + index = BuildLargeIndex(indexPath, pvecset, pmeta, pmetaidx, dist, numThreads); + BOOST_REQUIRE(index != nullptr); + auto buildend = std::chrono::high_resolution_clock::now(); + double buildseconds = + std::chrono::duration_cast(buildend - buildstart).count() / 1000000.0f; + jsonFile << " \"build timeSeconds\": " << buildseconds << ",\n"; + BOOST_TEST_MESSAGE("Index built successfully with " << baseVectorCount << " vectors"); + } + else + { + BOOST_REQUIRE(VectorIndex::LoadIndex(indexPath, index) == ErrorCode::Success); + BOOST_REQUIRE(index != nullptr); + } auto vectorOptions = std::shared_ptr( new Helper::ReaderOptions(GetEnumValueType(), M, VectorFileType::DEFAULT)); @@ -570,13 +578,6 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c BOOST_REQUIRE(index->SaveIndex(indexPath) == ErrorCode::Success); index = nullptr; - for (int iter = 0; iter < batches; iter++) - { - if (direxists((indexPath + "_" + std::to_string(iter)).c_str())) - { - std::filesystem::remove_all(indexPath + "_" + std::to_string(iter)); - } - } // Benchmark 1: Insert performance if (insertBatchSize > 0) @@ -585,7 +586,11 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c { jsonFile << " \"benchmark1_insert\": {\n"; std::string prevPath = indexPath; - for (int iter = 0; iter < batches; iter++) + if (resume >= 0) + { + prevPath = indexPath + "_" + std::to_string(resume); + } + for (int iter = resume + 1; iter < batches; iter++) { jsonFile << " \"batch_" << iter + 1 << "\": {\n"; @@ -705,7 +710,10 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c cloneIndex = nullptr; prevPath = clonePath; - jsonFile.flush(); + jsonFile.flush(); + + if (iter > 0) + std::filesystem::remove_all(indexPath + "_" + std::to_string(iter - 1)); } } jsonFile << " }\n"; @@ -1735,6 +1743,8 @@ BOOST_AUTO_TEST_CASE(BenchmarkFromConfig) int numThreads = iniReader.GetParameter("Benchmark", "NumThreads", 32); int numQueries = iniReader.GetParameter("Benchmark", "NumQueries", 1000); DistCalcMethod distMethod = iniReader.GetParameter("Benchmark", "DistMethod", DistCalcMethod::L2); + bool rebuild = iniReader.GetParameter("Benchmark", "Rebuild", true); + int resume = iniReader.GetParameter("Benchmark", "Resume", -1); BOOST_TEST_MESSAGE("=== Benchmark Configuration ==="); BOOST_TEST_MESSAGE("Vector Path: " << vectorPath); @@ -1757,19 +1767,20 @@ BOOST_AUTO_TEST_CASE(BenchmarkFromConfig) if (valueType == VectorValueType::Float) { RunBenchmark(vectorPath, queryPath, truthPath, distMethod, indexPath, dimension, baseVectorCount, - insertVectorCount, deleteVectorCount, batchNum, topK, numThreads, numQueries, outputFile); + insertVectorCount, deleteVectorCount, batchNum, topK, numThreads, numQueries, outputFile, + rebuild, resume); } else if (valueType == VectorValueType::Int8) { RunBenchmark(vectorPath, queryPath, truthPath, distMethod, indexPath, dimension, baseVectorCount, insertVectorCount, deleteVectorCount, batchNum, topK, numThreads, numQueries, - outputFile); + outputFile, rebuild, resume); } else if (valueType == VectorValueType::UInt8) { RunBenchmark(vectorPath, queryPath, truthPath, distMethod, indexPath, dimension, baseVectorCount, insertVectorCount, deleteVectorCount, batchNum, topK, numThreads, numQueries, - outputFile); + outputFile, rebuild, resume); } std::filesystem::remove_all(indexPath); From b78aa907c99f6264b6813c26f8e0b522661fe449 Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Fri, 12 Dec 2025 15:59:17 -0800 Subject: [PATCH 16/38] change printstep --- Test/src/SPFreshTest.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Test/src/SPFreshTest.cpp b/Test/src/SPFreshTest.cpp index 870cac0ad..23c4bcc3e 100644 --- a/Test/src/SPFreshTest.cpp +++ b/Test/src/SPFreshTest.cpp @@ -93,7 +93,7 @@ std::shared_ptr BuildIndex(const std::string &outDirectory, std::sh InPlace=true StartFileSizeGB=1 OneClusterCutMax=false - ConsistencyCheck=false + ConsistencyCheck=true ChecksumCheck=false ChecksumInRead=false AsyncMergeInSearch=false @@ -316,6 +316,7 @@ void InsertVectors(SPANN::Index *p_index, int insertThreads, int step std::vector threads; + int printstep = step / 50; std::atomic_size_t vectorsSent(start); auto func = [&]() { size_t index = start; @@ -324,7 +325,7 @@ void InsertVectors(SPANN::Index *p_index, int insertThreads, int step index = vectorsSent.fetch_add(1); if (index < start + step) { - if ((index & ((1 << 5) - 1)) == 0) + if ((index & (printstep - 1)) == 0) { SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "Sent %.2lf%%...\n", index * 100.0 / step); } @@ -646,6 +647,7 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c int startidx = iter * deleteBatchSize; std::atomic_size_t vectorsSent(startidx); int totaldeleted = startidx + deleteBatchSize; + int printstep = deleteBatchSize / 50; auto func = [&]() { size_t idx = startidx; while (true) @@ -653,7 +655,7 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c idx = vectorsSent.fetch_add(1); if (idx < totaldeleted) { - if ((idx & ((1 << 5) - 1)) == 0) + if ((idx & (printstep - 1)) == 0) { SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "Sent %.2lf%%...\n", (idx - startidx) * 100.0 / deleteBatchSize); From d29ce7332ccbd105a4d2a7f6a34946abba7f6718 Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Fri, 12 Dec 2025 16:05:15 -0800 Subject: [PATCH 17/38] fix print step --- Test/src/SPFreshTest.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Test/src/SPFreshTest.cpp b/Test/src/SPFreshTest.cpp index 23c4bcc3e..9865fe83c 100644 --- a/Test/src/SPFreshTest.cpp +++ b/Test/src/SPFreshTest.cpp @@ -325,7 +325,7 @@ void InsertVectors(SPANN::Index *p_index, int insertThreads, int step index = vectorsSent.fetch_add(1); if (index < start + step) { - if ((index & (printstep - 1)) == 0) + if ((index % (printstep - 1)) == 0) { SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "Sent %.2lf%%...\n", index * 100.0 / step); } @@ -655,7 +655,7 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c idx = vectorsSent.fetch_add(1); if (idx < totaldeleted) { - if ((idx & (printstep - 1)) == 0) + if ((idx % (printstep - 1)) == 0) { SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "Sent %.2lf%%...\n", (idx - startidx) * 100.0 / deleteBatchSize); From c6e0dfc73fbdde13ea4564f2b824c80e0ae0949b Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Mon, 15 Dec 2025 14:49:23 -0800 Subject: [PATCH 18/38] add more append threads --- Test/inc/Test.h | 1 + Test/src/SPFreshTest.cpp | 6 +++--- Test/src/TestDataGenerator.cpp | 1 + 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Test/inc/Test.h b/Test/inc/Test.h index da6c096ba..2e3c93003 100644 --- a/Test/inc/Test.h +++ b/Test/inc/Test.h @@ -3,5 +3,6 @@ #pragma once +#define BOOST_TEST_LOG_LEVEL test_suite #include #include diff --git a/Test/src/SPFreshTest.cpp b/Test/src/SPFreshTest.cpp index 9865fe83c..eebef9e83 100644 --- a/Test/src/SPFreshTest.cpp +++ b/Test/src/SPFreshTest.cpp @@ -177,7 +177,7 @@ std::shared_ptr BuildLargeIndex(const std::string &outDirectory, st Update=true SteadyState=true InsertThreadNum=1 - AppendThreadNum=1 + AppendThreadNum=4 ReassignThreadNum=0 DisableReassign=false ReassignK=64 @@ -333,8 +333,8 @@ void InsertVectors(SPANN::Index *p_index, int insertThreads, int step std::uint64_t *offsets = new std::uint64_t[2]{0, p_meta.Length()}; std::shared_ptr meta(new MemMetadataSet( p_meta, ByteArray((std::uint8_t *)offsets, 2 * sizeof(std::uint64_t), true), 1)); - auto status = p_index->AddIndex(addset->GetVector((SizeType)index), 1, p_opts.m_dim, meta, true); - BOOST_REQUIRE(status == ErrorCode::Success); + BOOST_REQUIRE(p_index->AddIndex(addset->GetVector((SizeType)index), 1, p_opts.m_dim, meta, true) == + ErrorCode::Success); } else { diff --git a/Test/src/TestDataGenerator.cpp b/Test/src/TestDataGenerator.cpp index 0290fd13d..3ac1bf682 100644 --- a/Test/src/TestDataGenerator.cpp +++ b/Test/src/TestDataGenerator.cpp @@ -145,6 +145,7 @@ void TestDataGenerator::GenerateBatchTruth(const std::string &filename, std:: int maxthreads = std::thread::hardware_concurrency(); for (int iter = 0; iter < batches + 1; iter++) { + SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "Generating groundtruth for batch %d\n", iter); std::vector mythreads; mythreads.reserve(maxthreads); std::atomic_size_t sent(0); From e06a423a38f3a5b42bcb44bf3efd6cd02da2356d Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Mon, 15 Dec 2025 16:13:57 -0800 Subject: [PATCH 19/38] add more log and change Multiget to use buffer --- .../inc/Core/SPANN/ExtraDynamicSearcher.h | 18 ++++++++++-------- AnnService/inc/Helper/ThreadPool.h | 6 +++++- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h b/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h index 854652380..e79dc6b7b 100644 --- a/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h +++ b/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h @@ -1267,8 +1267,6 @@ namespace SPTAG::SPANN { std::vector HeadPrevTopK; newHeadsDist.clear(); newHeadsDist.resize(0); - postingLists.clear(); - postingLists.resize(0); COMMON::QueryResultSet nearbyHeads(headVector, m_opt->m_reassignK); p_index->SearchIndex(nearbyHeads); BasicResult* queryResults = nearbyHeads.GetResults(); @@ -1284,7 +1282,10 @@ namespace SPTAG::SPANN { } auto reassignScanIOBegin = std::chrono::high_resolution_clock::now(); ErrorCode ret; - if ((ret=db->MultiGet(HeadPrevTopK, &postingLists, m_hardLatencyLimit, &(p_exWorkSpace->m_diskRequests))) != ErrorCode::Success || !ValidatePostings(HeadPrevTopK, postingLists)) { + if ((ret = db->MultiGet(HeadPrevTopK, p_exWorkSpace->m_pageBuffers, m_hardLatencyLimit, + &(p_exWorkSpace->m_diskRequests))) != ErrorCode::Success || + !ValidatePostings(HeadPrevTopK, p_exWorkSpace->m_pageBuffers)) + { SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "ReAssign can't get all the near postings\n"); return ret; } @@ -1293,15 +1294,16 @@ namespace SPTAG::SPANN { auto elapsedMSeconds = std::chrono::duration_cast(reassignScanIOEnd - reassignScanIOBegin).count(); m_stat.m_reassignScanIOCost += elapsedMSeconds; - for (int i = 0; i < postingLists.size(); i++) { - auto& postingList = postingLists[i]; - size_t postVectorNum = postingList.size() / m_vectorInfoSize; - auto* postingP = reinterpret_cast(postingList.data()); + for (int i = 0; i < HeadPrevTopK.size(); i++) + { + auto &buffer = (p_exWorkSpace->m_pageBuffers[i]); + size_t postVectorNum = (int)(buffer.GetAvailableSize() / m_vectorInfoSize); + auto *postingP = buffer.GetBuffer(); for (int j = 0; j < postVectorNum; j++) { uint8_t* vectorId = postingP + j * m_vectorInfoSize; SizeType vid = *(reinterpret_cast(vectorId)); // SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "%d: VID: %d, Head: %d, size:%d/%d\n", i, vid, HeadPrevTopK[i], postingLists.size(), HeadPrevTopK.size()); - uint8_t version = *(reinterpret_cast(vectorId + sizeof(int))); + uint8_t version = *(reinterpret_cast(vectorId + sizeof(SizeType))); ValueType* vector = reinterpret_cast(vectorId + m_metaDataSize); if (reAssignVectorsTopK.find(vid) == reAssignVectorsTopK.end() && !m_versionMap->Deleted(vid) && m_versionMap->GetVersion(vid) == version) { m_stat.m_reAssignScanNum++; diff --git a/AnnService/inc/Helper/ThreadPool.h b/AnnService/inc/Helper/ThreadPool.h index 96d781833..e80330f8d 100644 --- a/AnnService/inc/Helper/ThreadPool.h +++ b/AnnService/inc/Helper/ThreadPool.h @@ -104,7 +104,11 @@ namespace SPTAG inline uint32_t runningJobs() { return currentJobs; } - inline bool allClear() { return currentJobs == 0 && jobsize() == 0; } + inline bool allClear() { + SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "currentJobs: %u, jobsize: %zu\n", currentJobs.load(), + jobsize()); + return currentJobs == 0 && jobsize() == 0; + } protected: std::atomic_uint32_t currentJobs{ 0 }; From 2e28ed2014156cbf5418c6d17e005e4baf201f83 Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Mon, 15 Dec 2025 17:11:31 -0800 Subject: [PATCH 20/38] clean some logs --- AnnService/inc/Helper/ThreadPool.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/AnnService/inc/Helper/ThreadPool.h b/AnnService/inc/Helper/ThreadPool.h index e80330f8d..6aee44b30 100644 --- a/AnnService/inc/Helper/ThreadPool.h +++ b/AnnService/inc/Helper/ThreadPool.h @@ -105,9 +105,10 @@ namespace SPTAG inline uint32_t runningJobs() { return currentJobs; } inline bool allClear() { - SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "currentJobs: %u, jobsize: %zu\n", currentJobs.load(), - jobsize()); - return currentJobs == 0 && jobsize() == 0; + size_t totaljobs = jobsize(); + if (totaljobs % 10000 == 0) + SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "jobsize: %zu\n", totaljobs); + return currentJobs == 0 && totaljobs == 0; } protected: From 1e08a007dd01a22ed0cee30a936370faed8740c3 Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Tue, 16 Dec 2025 19:10:03 -0800 Subject: [PATCH 21/38] ensure workspace posting size. --- AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h | 2 +- Test/src/SPFreshTest.cpp | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h b/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h index e79dc6b7b..706247ae4 100644 --- a/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h +++ b/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h @@ -1361,7 +1361,7 @@ namespace SPTAG::SPANN { p_exWorkSpace->Clear(m_opt->m_searchInternalResultNum, (max(m_opt->m_postingPageLimit, m_opt->m_searchPostingPageLimit) + m_opt->m_bufferLength) << PageSizeEx, true, m_opt->m_enableDataCompression); } else { - p_exWorkSpace->Initialize(m_opt->m_maxCheck, m_opt->m_hashExp, m_opt->m_searchInternalResultNum, (max(m_opt->m_postingPageLimit, m_opt->m_searchPostingPageLimit) + m_opt->m_bufferLength) << PageSizeEx, true, m_opt->m_enableDataCompression); + p_exWorkSpace->Initialize(m_opt->m_maxCheck, m_opt->m_hashExp, max(m_opt->m_searchInternalResultNum, m_opt->m_reassignK), (max(m_opt->m_postingPageLimit, m_opt->m_searchPostingPageLimit) + m_opt->m_bufferLength) << PageSizeEx, true, m_opt->m_enableDataCompression); int wid = 0; if (m_freeWorkSpaceIds == nullptr || !m_freeWorkSpaceIds->try_pop(wid)) { diff --git a/Test/src/SPFreshTest.cpp b/Test/src/SPFreshTest.cpp index eebef9e83..ed7fec898 100644 --- a/Test/src/SPFreshTest.cpp +++ b/Test/src/SPFreshTest.cpp @@ -129,7 +129,7 @@ std::shared_ptr BuildIndex(const std::string &outDirectory, std::sh template std::shared_ptr BuildLargeIndex(const std::string &outDirectory, std::string &pvecset, std::string& pmetaset, std::string& pmetaidx, const std::string &distMethod = "L2", - int searchthread = 2) + int searchthread = 2, int insertthread = 2) { auto vecIndex = VectorIndex::CreateInstance(IndexAlgoType::SPANN, GetEnumValueType()); int maxthreads = std::thread::hardware_concurrency(); @@ -155,6 +155,7 @@ std::shared_ptr BuildLargeIndex(const std::string &outDirectory, st [BuildHead] isExecute=true + AddCountForRebuild=10000 NumberOfThreads=)" + std::to_string(maxthreads) + R"( [BuildSSDIndex] @@ -172,12 +173,11 @@ std::shared_ptr BuildLargeIndex(const std::string &outDirectory, st SpdkBatchSize=64 ExcludeHead=false ResultNum=10 - SearchThreadNum=)" + std::to_string(searchthread) + - R"( + SearchThreadNum=)" + std::to_string(searchthread) + R"( Update=true SteadyState=true InsertThreadNum=1 - AppendThreadNum=4 + AppendThreadNum=)" + std::to_string(insertthread) + R"( ReassignThreadNum=0 DisableReassign=false ReassignK=64 @@ -524,7 +524,7 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c if (rebuild || !direxists(indexPath.c_str())) { std::filesystem::remove_all(indexPath); auto buildstart = std::chrono::high_resolution_clock::now(); - index = BuildLargeIndex(indexPath, pvecset, pmeta, pmetaidx, dist, numThreads); + index = BuildLargeIndex(indexPath, pvecset, pmeta, pmetaidx, dist, numThreads, numThreads); BOOST_REQUIRE(index != nullptr); auto buildend = std::chrono::high_resolution_clock::now(); double buildseconds = @@ -690,6 +690,8 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c BOOST_TEST_MESSAGE("\n=== Benchmark 2: Query After Insertions and Deletions ==="); jsonFile << " \"search\":"; + BenchmarkQueryPerformance(cloneIndex, queryset, truth, truthPath, baseVectorCount, topK, numThreads, + numQueries, iter + 1, batches, tmpbenchmark, " "); BenchmarkQueryPerformance(cloneIndex, queryset, truth, truthPath, baseVectorCount, topK, numThreads, numQueries, iter + 1, batches, jsonFile, " "); jsonFile << ",\n"; From d3f2a7642540cdadbaa5a92f604d4dd678d968da Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Wed, 17 Dec 2025 13:34:43 -0800 Subject: [PATCH 22/38] add Get to PageBuffer API --- .../inc/Core/SPANN/ExtraDynamicSearcher.h | 6 +- .../inc/Core/SPANN/ExtraFileController.h | 64 +++++++++++++++++-- .../inc/Core/SPANN/ExtraRocksDBController.h | 2 +- .../inc/Core/SPANN/ExtraSPDKController.h | 2 +- AnnService/inc/Helper/KeyValueIO.h | 6 +- .../src/Core/SPANN/ExtraFileController.cpp | 46 +++++++++++++ AnnService/src/KeyValueTest/main.cpp | 2 +- Test/src/KVTest.cpp | 2 +- 8 files changed, 116 insertions(+), 14 deletions(-) diff --git a/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h b/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h index 706247ae4..caeba6c4e 100644 --- a/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h +++ b/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h @@ -1446,8 +1446,10 @@ namespace SPTAG::SPANN { } auto appendIOBegin = std::chrono::high_resolution_clock::now(); - if ((ret = db->Merge(headID, appendPosting, MaxTimeout, &(p_exWorkSpace->m_diskRequests), [this, prefixChecksum = *m_checkSums[headID]] (const std::string& readPrefix) -> bool { - return this->m_checkSum.ValidateChecksum(readPrefix.c_str(), (int)(readPrefix.size()), prefixChecksum); + if ((ret = db->Merge( + headID, appendPosting, MaxTimeout, &(p_exWorkSpace->m_diskRequests), + [this, prefixChecksum = *m_checkSums[headID]](const void *val, const int size) -> bool { + return this->m_checkSum.ValidateChecksum((const char*)val, size, prefixChecksum); })) != ErrorCode::Success) { SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Merge failed for %d! Posting Size:%d, limit: %d\n", headID, m_postingSizes.GetSize(headID), m_postingSizeLimit); diff --git a/AnnService/inc/Core/SPANN/ExtraFileController.h b/AnnService/inc/Core/SPANN/ExtraFileController.h index b026aeeb3..7c46e41bc 100644 --- a/AnnService/inc/Core/SPANN/ExtraFileController.h +++ b/AnnService/inc/Core/SPANN/ExtraFileController.h @@ -75,6 +75,8 @@ namespace SPTAG::SPANN { bool ReadBlocks(AddressType* p_data, std::string* p_value, const std::chrono::microseconds &timeout, std::vector* reqs); + bool ReadBlocks(AddressType *p_data, Helper::PageBuffer &p_value, const std::chrono::microseconds &timeout, std::vector *reqs); + bool ReadBlocks(const std::vector& p_data, std::vector* p_value, const std::chrono::microseconds &timeout, std::vector* reqs); bool ReadBlocks(const std::vector& p_data, std::vector>& p_value, const std::chrono::microseconds& timeout, std::vector* reqs); @@ -328,19 +330,21 @@ namespace SPTAG::SPANN { return true; } - bool merge(SizeType key, void *value, int merge_size, std::function checksum) + bool merge(SizeType key, void *value, int merge_size, std::function checksum) { // SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "LRUCache: merge size: %lld\n", merge_size); auto it = cache.find(key); if (it == cache.end()) { // SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "LRUCache: merge key not found\n"); ErrorCode ret; - std::string valstr; - if ((ret = fileIO->Get(key, &valstr, MaxTimeout, &reqs, false)) != ErrorCode::Success || !checksum(valstr)) { + if ((ret = fileIO->Get(key, pageBuffer, MaxTimeout, &reqs, false)) != ErrorCode::Success || + !checksum(pageBuffer.GetBuffer(), pageBuffer.GetAvailableSize())) { SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "LRUCache: merge key not found in file or checksum issue = %d\n", (int)(ret == ErrorCode::Success)); return false; // If the key does not exist, return false } - valstr.append((char *)value, merge_size); + std::string valstr(pageBuffer.GetAvailableSize() + merge_size, '\0'); + memcpy(valstr.data(), pageBuffer.GetBuffer(), pageBuffer.GetAvailableSize()); + memcpy(valstr.data() + pageBuffer.GetAvailableSize(), value, merge_size); while (valstr.size() > (int)(capacity - size) && (!keys.empty())) { auto last = keys.back(); @@ -432,7 +436,8 @@ namespace SPTAG::SPANN { return caches[hash(key)]->del(key); } - bool merge(SizeType key, void *value, int merge_size, std::function checksum) + bool merge(SizeType key, void *value, int merge_size, + std::function checksum) { return caches[hash(key)]->merge(key, value, merge_size, checksum); } @@ -622,6 +627,52 @@ namespace SPTAG::SPANN { return Get(std::stoi(key), value, timeout, reqs, true); } + ErrorCode Get(const SizeType key, Helper::PageBuffer &value, + const std::chrono::microseconds &timeout, std::vector *reqs, bool useCache = true) override + { + auto get_begin_time = std::chrono::high_resolution_clock::now(); + SizeType r = m_pBlockMapping.R(); + + if (key >= r) + { + SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Key OverFlow! Key:%d R:%d\n", key, r); + return ErrorCode::Key_OverFlow; + } + AddressType *addr = (AddressType *)(At(key)); + if (((uintptr_t)addr) == 0xffffffffffffffff) + { + SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Key NotFound! Key:%d\n", key); + return ErrorCode::Key_NotFound; + } + + int size = (int)(addr[0]); + if (size < 0) + return ErrorCode::Posting_SizeError; + + if (useCache && m_pShardedLRUCache) + { + value.ReservePageBuffer(size); + if (m_pShardedLRUCache->get(key, value.GetBuffer(), size)) + { + value.SetAvailableSize(size); + return ErrorCode::Success; + } + } + + // if (m_pBlockController.ReadBlocks((AddressType*)At(key), value)) { + // return ErrorCode::Success; + // } + auto begin_time = std::chrono::high_resolution_clock::now(); + auto result = m_pBlockController.ReadBlocks((AddressType *)At(key), value, timeout, reqs); + auto end_time = std::chrono::high_resolution_clock::now(); + read_time_vec += std::chrono::duration_cast(end_time - begin_time).count(); + get_times_vec++; + auto get_end_time = std::chrono::high_resolution_clock::now(); + get_time_vec += + std::chrono::duration_cast(get_end_time - get_begin_time).count(); + return result ? ErrorCode::Success : ErrorCode::Fail; + } + ErrorCode MultiGet(const std::vector& keys, std::vector>& values, const std::chrono::microseconds &timeout, std::vector* reqs) override { std::vector blocks; @@ -911,7 +962,8 @@ namespace SPTAG::SPANN { ErrorCode Merge(const SizeType key, const std::string &value, const std::chrono::microseconds &timeout, - std::vector *reqs, std::function checksum) + std::vector *reqs, + std::function checksum) { SizeType r = m_pBlockMapping.R(); if (key >= r) diff --git a/AnnService/inc/Core/SPANN/ExtraRocksDBController.h b/AnnService/inc/Core/SPANN/ExtraRocksDBController.h index 030a3a6d7..8ecc4ff82 100644 --- a/AnnService/inc/Core/SPANN/ExtraRocksDBController.h +++ b/AnnService/inc/Core/SPANN/ExtraRocksDBController.h @@ -281,7 +281,7 @@ namespace SPTAG::SPANN ErrorCode Merge(const SizeType key, const std::string &value, const std::chrono::microseconds &timeout, std::vector *reqs, - std::function checksum) override + std::function checksum) override { if (value.empty()) { SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Error! empty append posting!\n"); diff --git a/AnnService/inc/Core/SPANN/ExtraSPDKController.h b/AnnService/inc/Core/SPANN/ExtraSPDKController.h index 739e3a958..1954588d9 100644 --- a/AnnService/inc/Core/SPANN/ExtraSPDKController.h +++ b/AnnService/inc/Core/SPANN/ExtraSPDKController.h @@ -350,7 +350,7 @@ namespace SPTAG::SPANN ErrorCode Merge(SizeType key, const std::string &value, const std::chrono::microseconds &timeout, std::vector *reqs, - std::function checksum) override + std::function checksum) override { if (key >= m_pBlockMapping.R()) { SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Key range error: key: %d, mapping size: %d\n", key, m_pBlockMapping.R()); diff --git a/AnnService/inc/Helper/KeyValueIO.h b/AnnService/inc/Helper/KeyValueIO.h index 2a4c4ea79..21cb7193e 100644 --- a/AnnService/inc/Helper/KeyValueIO.h +++ b/AnnService/inc/Helper/KeyValueIO.h @@ -22,7 +22,9 @@ namespace SPTAG virtual ErrorCode Get(const SizeType key, std::string* value, const std::chrono::microseconds& timeout, std::vector* reqs) = 0; - virtual ErrorCode MultiGet(const std::vector& keys, std::vector>& values, const std::chrono::microseconds& timeout, std::vector* reqs) = 0; + virtual ErrorCode Get(const SizeType key, Helper::PageBuffer &value, const std::chrono::microseconds &timeout, std::vector *reqs, bool useCache = true) { return ErrorCode::Undefined; } + + virtual ErrorCode MultiGet(const std::vector& keys, std::vector>& values, const std::chrono::microseconds& timeout, std::vector* reqs) { return ErrorCode::Undefined; } virtual ErrorCode MultiGet(const std::vector& keys, std::vector* values, const std::chrono::microseconds& timeout, std::vector* reqs) = 0; @@ -35,7 +37,7 @@ namespace SPTAG virtual ErrorCode Merge(const SizeType key, const std::string &value, const std::chrono::microseconds &timeout, std::vector *reqs, - std::function checksum) = 0; + std::function checksum) = 0; virtual ErrorCode Delete(SizeType key) = 0; diff --git a/AnnService/src/Core/SPANN/ExtraFileController.cpp b/AnnService/src/Core/SPANN/ExtraFileController.cpp index ea69c0c38..0c794f3bb 100644 --- a/AnnService/src/Core/SPANN/ExtraFileController.cpp +++ b/AnnService/src/Core/SPANN/ExtraFileController.cpp @@ -228,6 +228,52 @@ bool FileIO::BlockController::ReadBlocks(AddressType *p_data, std::string *p_val return true; } +bool FileIO::BlockController::ReadBlocks( + AddressType *p_data, Helper::PageBuffer &p_value, const std::chrono::microseconds &timeout, + std::vector *reqs) +{ + if ((uintptr_t)p_data == 0xffffffffffffffff) + { + p_value.SetAvailableSize(0); + return true; + } + + const int64_t postingSize = (int64_t)(p_data[0]); + auto blockNum = (postingSize + PageSize - 1) >> PageSizeEx; + if (blockNum > reqs->size()) + { + SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "FileIO::BlockController::ReadBlocks: %d > %u\n", (int)blockNum, + reqs->size()); + p_value.SetAvailableSize(0); + return false; + } + + p_value.SetAvailableSize(postingSize); + AddressType currOffset = 0; + AddressType dataIdx = 1; + for (int i = 0; i < blockNum; i++) + { + Helper::AsyncReadRequest &curr = reqs->at(i); + curr.m_readSize = (postingSize - currOffset) < PageSize ? (postingSize - currOffset) : PageSize; + curr.m_offset = p_data[dataIdx] * PageSize; + currOffset += PageSize; + dataIdx++; + } + + std::uint32_t totalReads = m_fileHandle->BatchReadFile(reqs->data(), blockNum, timeout, m_batchSize); + read_submit_vec += blockNum; + read_complete_vec += totalReads; + + if (totalReads < blockNum) + { + SPTAGLIB_LOG(Helper::LogLevel::LL_Warning, "FileIO::BlockController::ReadBlocks: %u < %u\n", totalReads, + blockNum); + m_batchReadTimeouts++; + return false; + } + return true; +} + bool FileIO::BlockController::ReadBlocks(const std::vector &p_data, std::vector *p_values, const std::chrono::microseconds &timeout, std::vector *reqs) diff --git a/AnnService/src/KeyValueTest/main.cpp b/AnnService/src/KeyValueTest/main.cpp index 1fec3886b..c88de7f2a 100644 --- a/AnnService/src/KeyValueTest/main.cpp +++ b/AnnService/src/KeyValueTest/main.cpp @@ -398,7 +398,7 @@ int main(int argc, char *argv[]) mergeValue += (char)(rand() % 256); } fileIO.Merge(key, mergeValue, MaxTimeout, &(workspace.m_diskRequests), - [](const std::string &) -> bool { return true; }); + [](const void* val, const int size) -> bool { return true; }); write_count++; read_count++; std::string readValue; diff --git a/Test/src/KVTest.cpp b/Test/src/KVTest.cpp index 32fab3ac0..ebb2b3609 100644 --- a/Test/src/KVTest.cpp +++ b/Test/src/KVTest.cpp @@ -109,7 +109,7 @@ void Test(std::string path, std::string type, bool debug = false) for (int j = 0; j < mergeIters; j++) { db->Merge(i, std::to_string(i), MaxTimeout, &(workspace.m_diskRequests), - [](const std::string &prefix) -> bool { return true; }); + [](const void* val, const int size) -> bool { return true; }); } } t2 = std::chrono::high_resolution_clock::now(); From 1fe1bb32c4c2913569828da3fe8d6425ee3a38ca Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Thu, 18 Dec 2025 16:13:23 -0800 Subject: [PATCH 23/38] fix cache issue --- .../inc/Core/SPANN/ExtraDynamicSearcher.h | 1 - .../inc/Core/SPANN/ExtraFileController.h | 185 ++++++++---------- AnnService/inc/Core/SPANN/Options.h | 1 + .../inc/Core/SPANN/ParameterDefinitionList.h | 3 +- Test/src/SPFreshTest.cpp | 35 ++-- 5 files changed, 106 insertions(+), 119 deletions(-) diff --git a/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h b/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h index caeba6c4e..6109ab4a1 100644 --- a/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h +++ b/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h @@ -2225,7 +2225,6 @@ namespace SPTAG::SPANN { std::vector replicaCountDist(m_opt->m_replicaCount + 1, 0); for (int i = 0; i < replicaCount.size(); ++i) { - if (headVectorIDS.count(i) > 0) continue; ++replicaCountDist[replicaCount[i]]; } diff --git a/AnnService/inc/Core/SPANN/ExtraFileController.h b/AnnService/inc/Core/SPANN/ExtraFileController.h index 7c46e41bc..236dc4ae5 100644 --- a/AnnService/inc/Core/SPANN/ExtraFileController.h +++ b/AnnService/inc/Core/SPANN/ExtraFileController.h @@ -209,12 +209,18 @@ namespace SPTAG::SPANN { } }; + struct CacheEntry + { + std::string value; + std::list::iterator iter; + }; + class LRUCache { int64_t capacity; int64_t limit; int64_t size; std::list keys; // Page Address - std::unordered_map::iterator>> cache; // Page Address -> Page Address in Cache + std::unordered_map cache; // Page Address -> Page Address in Cache int64_t queries; std::atomic hits; FileIO* fileIO; @@ -232,7 +238,7 @@ namespace SPTAG::SPANN { this->fileIO = fileIO; this->reqs.resize(limit); this->pageBuffer.ReservePageBuffer(limit << PageSizeEx); - for (int i = 0; i < limit; i++) { + for (uint64_t i = 0; i < limit; i++) { auto& req = this->reqs[i]; req.m_buffer = (char *)(this->pageBuffer.GetBuffer() + (i << PageSizeEx)); req.m_extension = &processIocp; @@ -250,7 +256,8 @@ namespace SPTAG::SPANN { ~LRUCache() {} - bool evict(SizeType key, void* value, int vsize, std::unordered_map::iterator>>::iterator& it) { + bool evict(SizeType key, void *value, int vsize, std::unordered_map::iterator &it) + { if (value != nullptr) { std::string valstr((char*)value, vsize); if (fileIO->Put(key, valstr, MaxTimeout, &reqs, false) != ErrorCode::Success) { @@ -259,24 +266,41 @@ namespace SPTAG::SPANN { } } - size -= it->second.first.size(); - keys.erase(it->second.second); + size -= it->second.value.size(); + keys.erase(it->second.iter); cache.erase(it); return true; } - bool get(SizeType key, void* value, int& get_size) { + bool get(SizeType key, Helper::PageBuffer &buffer) + { queries++; auto it = cache.find(key); - if (it == cache.end()) { - return false; // If the key does not exist, return -1 + if (it == cache.end()) + { + return false; } - if (get_size > it->second.first.size()) { - SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Cache get error: key %d required size %d, real size = %d\n", key, get_size, (int)(it->second.first.size())); - get_size = (int)(it->second.first.size()); + + size_t data_size = it->second.value.size(); + buffer.ReservePageBuffer(data_size); + memcpy(buffer.GetBuffer(), it->second.value.data(), data_size); + buffer.SetAvailableSize(data_size); + hits++; + return true; + } + + bool get(SizeType key, std::string &value) + { + queries++; + auto it = cache.find(key); + if (it == cache.end()) + { + return false; } - // Update access order, move the key to the head of the linked list - memcpy((char*)value, it->second.first.data(), get_size); + + size_t data_size = it->second.value.size(); + value.resize(data_size); + memcpy(value.data(), it->second.value.data(), data_size); hits++; return true; } @@ -285,23 +309,23 @@ namespace SPTAG::SPANN { auto it = cache.find(key); if (it != cache.end()) { if (put_size > limit) { - evict(key, it->second.first.data(), it->second.first.size(), it); + evict(key, it->second.value.data(), it->second.value.size(), it); return false; } - keys.splice(keys.begin(), keys, it->second.second); - it->second.second = keys.begin(); + keys.splice(keys.begin(), keys, it->second.iter); + it->second.iter = keys.begin(); - auto delta_size = put_size - (int)(it->second.first.size()); + auto delta_size = put_size - (int)(it->second.value.size()); while ((int)(capacity - size) < delta_size && (keys.size() > 1)) { auto last = keys.back(); auto lastit = cache.find(last); - if (!evict(last, lastit->second.first.data(), lastit->second.first.size(), lastit)) { - evict(key, it->second.first.data(), it->second.first.size(), it); + if (!evict(last, lastit->second.value.data(), lastit->second.value.size(), lastit)) { + evict(key, it->second.value.data(), it->second.value.size(), it); return false; } } - it->second.first.resize(put_size); - memcpy(it->second.first.data(), (char*)value, put_size); + it->second.value.resize(put_size); + memcpy(it->second.value.data(), (char*)value, put_size); size += delta_size; hits++; return true; @@ -312,11 +336,12 @@ namespace SPTAG::SPANN { while (put_size > (int)(capacity - size) && (!keys.empty())) { auto last = keys.back(); auto lastit = cache.find(last); - if (!evict(last, lastit->second.first.data(), lastit->second.first.size(), lastit)) { + if (!evict(last, lastit->second.value.data(), lastit->second.value.size(), lastit)) { return false; } } - cache.insert({key, {std::string((char *)value, put_size), keys.insert(keys.begin(), key)}}); + auto keys_it = keys.insert(keys.begin(), key); + cache.insert({key, {std::string((char *)value, put_size), keys_it}}); size += put_size; return true; } @@ -339,7 +364,7 @@ namespace SPTAG::SPANN { ErrorCode ret; if ((ret = fileIO->Get(key, pageBuffer, MaxTimeout, &reqs, false)) != ErrorCode::Success || !checksum(pageBuffer.GetBuffer(), pageBuffer.GetAvailableSize())) { - SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "LRUCache: merge key not found in file or checksum issue = %d\n", (int)(ret == ErrorCode::Success)); + SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "LRUCache: merge key %d not found in file or checksum issue = %d\n", key, (int)(ret == ErrorCode::Success)); return false; // If the key does not exist, return false } std::string valstr(pageBuffer.GetAvailableSize() + merge_size, '\0'); @@ -349,33 +374,34 @@ namespace SPTAG::SPANN { { auto last = keys.back(); auto lastit = cache.find(last); - if (!evict(last, lastit->second.first.data(), lastit->second.first.size(), lastit)) + if (!evict(last, lastit->second.value.data(), lastit->second.value.size(), lastit)) { return false; } } - cache.insert({key, {valstr, keys.insert(keys.begin(), key)}}); + auto keys_it = keys.insert(keys.begin(), key); + cache.insert({key, {valstr, keys_it}}); size += valstr.size(); return true; } hits++; - if (merge_size + it->second.first.size() > limit) { - evict(key, it->second.first.data(), it->second.first.size(), it); + if (merge_size + it->second.value.size() > limit) { + evict(key, it->second.value.data(), it->second.value.size(), it); // SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "LRUCache: merge size exceeded\n"); return false; } - keys.splice(keys.begin(), keys, it->second.second); - it->second.second = keys.begin(); + keys.splice(keys.begin(), keys, it->second.iter); + it->second.iter = keys.begin(); while((int)(capacity - size) < merge_size && (keys.size() > 1)) { auto last = keys.back(); auto lastit = cache.find(last); - if (!evict(last, lastit->second.first.data(), lastit->second.first.size(), lastit)) { - evict(key, it->second.first.data(), it->second.first.size(), it); + if (!evict(last, lastit->second.value.data(), lastit->second.value.size(), lastit)) { + evict(key, it->second.value.data(), it->second.value.size(), it); return false; } } - it->second.first.append((char*)value, merge_size); + it->second.value.append((char*)value, merge_size); size += merge_size; // SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "LRUCache: merge success\n"); return true; @@ -387,8 +413,8 @@ namespace SPTAG::SPANN { bool flush() { for (auto it = cache.begin(); it != cache.end(); it++) { - if (fileIO->Put(it->first, it->second.first, MaxTimeout, &reqs, false) != ErrorCode::Success) { - SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "LRUCache: evict key:%d value size:%d to file failed\n", it->first, (int)(it->second.first.size())); + if (fileIO->Put(it->first, it->second.value, MaxTimeout, &reqs, false) != ErrorCode::Success) { + SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "LRUCache: evict key:%d value size:%d to file failed\n", it->first, (int)(it->second.value.size())); return false; } } @@ -422,10 +448,18 @@ namespace SPTAG::SPANN { } } - bool get(SizeType key, void* value, int& get_size) { + bool get(SizeType key, Helper::PageBuffer &buffer) + { SizeType cid = hash(key); std::shared_lock lock(m_rwMutexs[cid]); - return caches[cid]->get(key, value, get_size); + return caches[cid]->get(key, buffer); + } + + bool get(SizeType key, std::string &value) + { + SizeType cid = hash(key); + std::shared_lock lock(m_rwMutexs[cid]); + return caches[cid]->get(key, value); } bool put(SizeType key, void* value, int put_size) { @@ -582,9 +616,7 @@ namespace SPTAG::SPANN { } ErrorCode Get(const SizeType key, std::string* value, const std::chrono::microseconds &timeout, std::vector* reqs, bool useCache) { - auto get_begin_time = std::chrono::high_resolution_clock::now(); SizeType r = m_pBlockMapping.R(); - if (key >= r) { SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Key OverFlow! Key:%d R:%d\n", key, r); return ErrorCode::Key_OverFlow; @@ -599,23 +631,12 @@ namespace SPTAG::SPANN { if (size < 0) return ErrorCode::Posting_SizeError; if (useCache && m_pShardedLRUCache) { - value->resize(size); - if (m_pShardedLRUCache->get(key, value->data(), size)) { - value->resize(size); + if (m_pShardedLRUCache->get(key, *value)) { return ErrorCode::Success; } } - // if (m_pBlockController.ReadBlocks((AddressType*)At(key), value)) { - // return ErrorCode::Success; - // } - auto begin_time = std::chrono::high_resolution_clock::now(); auto result = m_pBlockController.ReadBlocks((AddressType*)At(key), value, timeout, reqs); - auto end_time = std::chrono::high_resolution_clock::now(); - read_time_vec += std::chrono::duration_cast(end_time - begin_time).count(); - get_times_vec++; - auto get_end_time = std::chrono::high_resolution_clock::now(); - get_time_vec += std::chrono::duration_cast(get_end_time - get_begin_time).count(); return result ? ErrorCode::Success : ErrorCode::Fail; } @@ -630,9 +651,7 @@ namespace SPTAG::SPANN { ErrorCode Get(const SizeType key, Helper::PageBuffer &value, const std::chrono::microseconds &timeout, std::vector *reqs, bool useCache = true) override { - auto get_begin_time = std::chrono::high_resolution_clock::now(); SizeType r = m_pBlockMapping.R(); - if (key >= r) { SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Key OverFlow! Key:%d R:%d\n", key, r); @@ -646,30 +665,17 @@ namespace SPTAG::SPANN { } int size = (int)(addr[0]); - if (size < 0) - return ErrorCode::Posting_SizeError; + if (size < 0) return ErrorCode::Posting_SizeError; if (useCache && m_pShardedLRUCache) { - value.ReservePageBuffer(size); - if (m_pShardedLRUCache->get(key, value.GetBuffer(), size)) + if (m_pShardedLRUCache->get(key, value)) { - value.SetAvailableSize(size); return ErrorCode::Success; } } - // if (m_pBlockController.ReadBlocks((AddressType*)At(key), value)) { - // return ErrorCode::Success; - // } - auto begin_time = std::chrono::high_resolution_clock::now(); auto result = m_pBlockController.ReadBlocks((AddressType *)At(key), value, timeout, reqs); - auto end_time = std::chrono::high_resolution_clock::now(); - read_time_vec += std::chrono::duration_cast(end_time - begin_time).count(); - get_times_vec++; - auto get_end_time = std::chrono::high_resolution_clock::now(); - get_time_vec += - std::chrono::duration_cast(get_end_time - get_begin_time).count(); return result ? ErrorCode::Success : ErrorCode::Fail; } @@ -681,19 +687,12 @@ namespace SPTAG::SPANN { int i = 0; for (SizeType key : keys) { r = m_pBlockMapping.R(); - if (key < r) { - AddressType* addr = (AddressType*)(At(key)); - if (m_pShardedLRUCache && ((uintptr_t)addr) != 0xffffffffffffffff && addr[0] >= 0) { - int size = (int)(addr[0]); - values[i].ReservePageBuffer(size); - if (m_pShardedLRUCache->get(key, values[i].GetBuffer(), size)) { - values[i].SetAvailableSize(size); - blocks.push_back(nullptr); - } - else { - blocks.push_back(addr); - } + if (key < r) { + if (m_pShardedLRUCache && m_pShardedLRUCache->get(key, values[i])) + { + blocks.push_back(nullptr); } else { + AddressType* addr = (AddressType*)(At(key)); blocks.push_back(addr); } } @@ -702,7 +701,6 @@ namespace SPTAG::SPANN { } i++; } - // if (m_pBlockController.ReadBlocks(blocks, values, timeout)) return ErrorCode::Success; auto result = m_pBlockController.ReadBlocks(blocks, values, timeout, reqs); return result ? ErrorCode::Success : ErrorCode::Fail; } @@ -718,19 +716,11 @@ namespace SPTAG::SPANN { for (SizeType key : keys) { r = m_pBlockMapping.R(); if (key < r) { - AddressType* addr = (AddressType*)(At(key)); - if (m_pShardedLRUCache && ((uintptr_t)addr) != 0xffffffffffffffff && addr[0] >= 0) { - int size = (int)(addr[0]); - (*values)[i].resize(size); - if (m_pShardedLRUCache->get(key, (*values)[i].data(), size)) { - (*values)[i].resize(size); - blocks.push_back(nullptr); - } - else { - blocks.push_back(addr); - } + if (m_pShardedLRUCache && m_pShardedLRUCache->get(key, (*values)[i])) { + blocks.push_back(nullptr); } else { + AddressType* addr = (AddressType*)(At(key)); blocks.push_back(addr); } } @@ -739,7 +729,6 @@ namespace SPTAG::SPANN { } i++; } - // if (m_pBlockController.ReadBlocks(blocks, values, timeout)) return ErrorCode::Success; auto result = m_pBlockController.ReadBlocks(blocks, values, timeout, reqs); return result ? ErrorCode::Success : ErrorCode::Fail; } @@ -1139,10 +1128,6 @@ namespace SPTAG::SPANN { int remainGB = ((long long)(remainBlocks + reserveBlocks) >> (30 - PageSizeEx)); // int remainGB = remainBlocks >> 20 << 2; SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "Total %d blocks, Remain %d blocks, Reserve %d blocks, totally %d GB\n", totalBlocks, remainBlocks, reserveBlocks, remainGB); - double average_read_time = (double)read_time_vec / get_times_vec; - double average_get_time = (double)get_time_vec / get_times_vec; - SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "Get times: %llu, get time: %llu us, read time: %llu us\n", get_times_vec.load(), get_time_vec.load(), read_time_vec.load()); - SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "Average read time: %lf us, average get time: %lf us\n", average_read_time, average_get_time); if (m_pShardedLRUCache) { auto cache_stat = m_pShardedLRUCache->get_stat(); SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "Cache queries: %lld, Cache hits: %lld, Hit rates: %lf\n", cache_stat.first, cache_stat.second, cache_stat.second == 0 ? 0 : (double)cache_stat.second / cache_stat.first); @@ -1227,10 +1212,6 @@ namespace SPTAG::SPANN { } private: - std::atomic read_time_vec = 0; - std::atomic get_time_vec = 0; - std::atomic get_times_vec = 0; - std::string m_mappingPath; SizeType m_blockLimit; COMMON::Dataset m_pBlockMapping; @@ -1240,7 +1221,7 @@ namespace SPTAG::SPANN { std::shared_ptr m_compactionThreadPool; BlockController m_pBlockController; - ShardedLRUCache *m_pShardedLRUCache; + ShardedLRUCache *m_pShardedLRUCache{nullptr}; bool m_shutdownCalled; std::shared_mutex m_updateMutex; diff --git a/AnnService/inc/Core/SPANN/Options.h b/AnnService/inc/Core/SPANN/Options.h index 9246c492d..f49621230 100644 --- a/AnnService/inc/Core/SPANN/Options.h +++ b/AnnService/inc/Core/SPANN/Options.h @@ -190,6 +190,7 @@ namespace SPTAG { int m_cacheSize; int m_cacheShards; bool m_asyncMergeInSearch; + bool m_centeringToZero; // Iterative int m_headBatch; diff --git a/AnnService/inc/Core/SPANN/ParameterDefinitionList.h b/AnnService/inc/Core/SPANN/ParameterDefinitionList.h index 32f01aff8..0a88e3f1d 100644 --- a/AnnService/inc/Core/SPANN/ParameterDefinitionList.h +++ b/AnnService/inc/Core/SPANN/ParameterDefinitionList.h @@ -210,7 +210,8 @@ DefineSSDParameter(m_cacheSize, int, 0, "CacheSizeGB") // Mutable DefineSSDParameter(m_cacheShards, int, 1, "CacheShards") // Mutable DefineSSDParameter(m_asyncAppendQueueSize, int, 0, "AsyncAppendQueueSize") // Mutable DefineSSDParameter(m_allowZeroReplica, bool, false, "AllowZeroReplica") - +DefineSSDParameter(m_centeringToZero, bool, false, "CenteringToZero") + // Iterative DefineSSDParameter(m_headBatch, int, 32, "IterativeSearchHeadBatch") // Mutable diff --git a/Test/src/SPFreshTest.cpp b/Test/src/SPFreshTest.cpp index ed7fec898..32e737cd2 100644 --- a/Test/src/SPFreshTest.cpp +++ b/Test/src/SPFreshTest.cpp @@ -37,7 +37,7 @@ std::shared_ptr BuildIndex(const std::string &outDirectory, std::sh std::shared_ptr metaset, const std::string &distMethod = "L2", int searchthread = 2) { auto vecIndex = VectorIndex::CreateInstance(IndexAlgoType::SPANN, GetEnumValueType()); - + int maxthreads = std::thread::hardware_concurrency(); std::string configuration = R"( [Base] DistCalcMethod=L2 @@ -51,7 +51,7 @@ std::shared_ptr BuildIndex(const std::string &outDirectory, std::sh [SelectHead] isExecute=true - NumberOfThreads=16 + NumberOfThreads=)" + std::to_string(maxthreads) + R"( SelectThreshold=0 SplitFactor=0 SplitThreshold=0 @@ -59,25 +59,22 @@ std::shared_ptr BuildIndex(const std::string &outDirectory, std::sh [BuildHead] isExecute=true - NumberOfThreads=16 + NumberOfThreads=)" + std::to_string(maxthreads) + R"( [BuildSSDIndex] isExecute=true BuildSsdIndex=true InternalResultNum=64 SearchInternalResultNum=64 - NumberOfThreads=16 - PostingPageLimit=)" + std::to_string(4 * sizeof(T)) + - R"( - SearchPostingPageLimit=)" + - std::to_string(4 * sizeof(T)) + R"( + NumberOfThreads=)" + std::to_string(maxthreads) + R"( + PostingPageLimit=)" + std::to_string(4 * sizeof(T)) + R"( + SearchPostingPageLimit=)" + std::to_string(4 * sizeof(T)) + R"( TmpDir=tmpdir Storage=FILEIO SpdkBatchSize=64 ExcludeHead=false ResultNum=10 - SearchThreadNum=)" + std::to_string(searchthread) + - R"( + SearchThreadNum=)" + std::to_string(searchthread) + R"( Update=true SteadyState=true InsertThreadNum=1 @@ -89,12 +86,12 @@ std::shared_ptr BuildIndex(const std::string &outDirectory, std::sh SearchDuringUpdate=true MergeThreshold=10 Sampling=4 - BufferLength=6 + BufferLength=)" + std::to_string(6 * sizeof(T)) + R"( InPlace=true StartFileSizeGB=1 OneClusterCutMax=false ConsistencyCheck=true - ChecksumCheck=false + ChecksumCheck=true ChecksumInRead=false AsyncMergeInSearch=false DeletePercentageForRefine=0.4 @@ -185,7 +182,7 @@ std::shared_ptr BuildLargeIndex(const std::string &outDirectory, st SearchDuringUpdate=true MergeThreshold=10 Sampling=4 - BufferLength=6 + BufferLength=)" + std::to_string(6 * sizeof(T)) + R"( InPlace=true StartFileSizeGB=1 OneClusterCutMax=false @@ -613,8 +610,17 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c jsonFile << " \"Load timeSeconds\": " << seconds << ",\n"; jsonFile << " \"Load vectorCount\": " << vectorCount << ",\n"; + start = std::chrono::high_resolution_clock::now(); cloneIndex = prevIndex->Clone(clonePath); + end = std::chrono::high_resolution_clock::now(); + seconds = std::chrono::duration_cast(end - start).count() / 1000000.0f; + jsonFile << " \"Clone timeSeconds\": " << seconds << ",\n"; + prevIndex = nullptr; + ErrorCode cloneret = cloneIndex->Check(); + BOOST_REQUIRE(cloneret == ErrorCode::Success); + SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "Cloned index from %s to %s, check:%d, time: %f seconds\n", + prevPath.c_str(), clonePath.c_str(), (int)(cloneret == ErrorCode::Success), seconds); int insertStart = iter * insertBatchSize; { @@ -1787,7 +1793,6 @@ BOOST_AUTO_TEST_CASE(BenchmarkFromConfig) outputFile, rebuild, resume); } - std::filesystem::remove_all(indexPath); - std::filesystem::remove_all(indexPath + "_saved"); + //std::filesystem::remove_all(indexPath); } BOOST_AUTO_TEST_SUITE_END() From 89cac4b38dd77b4d5d1a4cd97d03747d080c1bb9 Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Tue, 30 Dec 2025 15:51:33 -0800 Subject: [PATCH 24/38] fix unsafe_delete --- .../inc/Core/SPANN/ExtraDynamicSearcher.h | 81 +++++++++++-------- Test/src/SPFreshTest.cpp | 14 ++-- 2 files changed, 55 insertions(+), 40 deletions(-) diff --git a/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h b/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h index 6109ab4a1..8185b3b1f 100644 --- a/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h +++ b/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h @@ -200,6 +200,7 @@ namespace SPTAG::SPANN { std::unordered_setm_splitList; Helper::Concurrent::ConcurrentMap m_mergeList; + std::shared_timed_mutex m_mergeListLock; public: ExtraDynamicSearcher(SPANN::Options& p_opt) { @@ -579,9 +580,6 @@ namespace SPTAG::SPANN { if (vectorCount <= m_mergeThreshold) mergelist.insert(p_headmapping->at(index)); postingList.resize(vectorCount * m_vectorInfoSize); - new_postingSizes.UpdateSize(p_headmapping->at(index), vectorCount); - *new_checkSums[p_headmapping->at(index)] = - m_checkSum.CalcChecksum(postingList.c_str(), (int)(postingList.size())); if ((ret = db->Put(p_headmapping->at(index), postingList, MaxTimeout, &(workSpace.m_diskRequests))) != ErrorCode::Success) @@ -591,6 +589,9 @@ namespace SPTAG::SPANN { finalcode = ret; return; } + new_postingSizes.UpdateSize(p_headmapping->at(index), vectorCount); + *new_checkSums[p_headmapping->at(index)] = + m_checkSum.CalcChecksum(postingList.c_str(), (int)(postingList.size())); if (m_opt->m_consistencyCheck && (ret = db->Check(p_headmapping->at(index), new_postingSizes.GetSize(p_headmapping->at(index)) * m_vectorInfoSize, nullptr)) != ErrorCode::Success) { SPTAGLIB_LOG(Helper::LogLevel::LL_Error, @@ -706,8 +707,8 @@ namespace SPTAG::SPANN { //COMMON::Dataset smallSample(0, m_opt->m_dim, p_index->m_iDataBlockSize, p_index->m_iDataCapacity); // smallSample[i] -> VID //std::vector localIndicesInsert(postVectorNum); // smallSample[i] = j <-> localindices[j] = i //std::vector localIndicesInsertVersion(postVectorNum); - std::vector localIndices(postVectorNum); - int index = 0; + std::vector localIndices; + localIndices.reserve(postVectorNum); uint8_t* vectorId = postingP; for (int j = 0; j < postVectorNum; j++, vectorId += m_vectorInfoSize) { @@ -720,8 +721,7 @@ namespace SPTAG::SPANN { //localIndicesInsert[index] = VID; //localIndicesInsertVersion[index] = version; //smallSample.AddBatch(1, (ValueType*)(vectorId + m_metaDataSize)); - localIndices[index] = j; - index++; + localIndices.push_back(j); } // double gcEndTime = sw.getElapsedMs(); // m_splitGcCost += gcEndTime; @@ -731,19 +731,19 @@ namespace SPTAG::SPANN { //SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "DEBUG: in place or not prereassign & index < m_postingSizeLimit. GC begin...\n"); char* ptr = (char*)(postingList.c_str()); - for (int j = 0; j < index; j++, ptr += m_vectorInfoSize) + for (int j = 0; j < localIndices.size(); j++, ptr += m_vectorInfoSize) { if (j == localIndices[j]) continue; memcpy(ptr, postingList.c_str() + localIndices[j] * m_vectorInfoSize, m_vectorInfoSize); //Serialize(ptr, localIndicesInsert[j], localIndicesInsertVersion[j], smallSample[j]); } postingList.resize(index * m_vectorInfoSize); - m_postingSizes.UpdateSize(headID, index); - *m_checkSums[headID] = m_checkSum.CalcChecksum(postingList.c_str(), (int)(postingList.size())); if ((ret=db->Put(headID, postingList, MaxTimeout, &(p_exWorkSpace->m_diskRequests))) != ErrorCode::Success) { SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Split Fail to write back postings\n"); return ret; } + m_postingSizes.UpdateSize(headID, index); + *m_checkSums[headID] = m_checkSum.CalcChecksum(postingList.c_str(), (int)(postingList.size())); if (m_opt->m_consistencyCheck && (ret = db->Check(headID, m_postingSizes.GetSize(headID) * m_vectorInfoSize, nullptr)) != ErrorCode::Success) { SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Split: Check failed after Put %d\n", headID); @@ -761,8 +761,6 @@ namespace SPTAG::SPANN { //SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "GC triggered: %d, new length: %d\n", headID, index); return ErrorCode::Success; } - //SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "Resize\n"); - localIndices.resize(index); auto clusterBegin = std::chrono::high_resolution_clock::now(); // k = 2, maybe we can change the split number, now it is fixed @@ -789,13 +787,14 @@ namespace SPTAG::SPANN { //Serialize(ptr, localIndicesInsert[j], localIndicesInsertVersion[j], smallSample[j]); } SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "Cluserting Failed (The same vector), Cluster total dist:%f Only Keep %d vectors.\n", totaldist, cut); - - m_postingSizes.UpdateSize(headID, cut); - *m_checkSums[headID] = m_checkSum.CalcChecksum(newpostingList.c_str(), (int)(newpostingList.size())); + if ((ret=db->Put(headID, newpostingList, MaxTimeout, &(p_exWorkSpace->m_diskRequests))) != ErrorCode::Success) { SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Split fail to override postings cut to limit\n"); return ret; } + m_postingSizes.UpdateSize(headID, cut); + *m_checkSums[headID] = + m_checkSum.CalcChecksum(newpostingList.c_str(), (int)(newpostingList.size())); if (m_opt->m_consistencyCheck && (ret = db->Check(headID, m_postingSizes.GetSize(headID) * m_vectorInfoSize, nullptr)) != ErrorCode::Success) { SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Split: Consolidate Check failed after Put %d\n", headID); @@ -825,14 +824,14 @@ namespace SPTAG::SPANN { newHeadsID.push_back(headID); newHeadVID = headID; theSameHead = true; - m_postingSizes.UpdateSize(newHeadVID, args.counts[k]); - *m_checkSums[newHeadVID] = - m_checkSum.CalcChecksum(newPostingLists[k].c_str(), (int)(newPostingLists[k].size())); auto splitPutBegin = std::chrono::high_resolution_clock::now(); if (!preReassign && (ret=db->Put(newHeadVID, newPostingLists[k], MaxTimeout, &(p_exWorkSpace->m_diskRequests))) != ErrorCode::Success) { SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Fail to override postings\n"); return ret; } + m_postingSizes.UpdateSize(newHeadVID, args.counts[k]); + *m_checkSums[newHeadVID] = + m_checkSum.CalcChecksum(newPostingLists[k].c_str(), (int)(newPostingLists[k].size())); if (m_opt->m_consistencyCheck && (ret = db->Check(newHeadVID, m_postingSizes.GetSize(newHeadVID) * m_vectorInfoSize, nullptr)) != ErrorCode::Success) { SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Split: Cluster Write Check failed after Put %d\n", newHeadVID); @@ -997,7 +996,10 @@ namespace SPTAG::SPANN { m_mergeLock.unlock(); return ret; } - m_mergeList.unsafe_erase(headID); + { + std::unique_lock lock(m_mergeListLock); + m_mergeList.unsafe_erase(headID); + } m_mergeLock.unlock(); return ErrorCode::Success; } @@ -1010,7 +1012,12 @@ namespace SPTAG::SPANN { { BasicResult* queryResult = queryResults.GetResult(i); int nextLength = m_postingSizes.GetSize(queryResult->VID); - if (currentLength + nextLength < m_postingSizeLimit && m_mergeList.find(queryResult->VID) == m_mergeList.end()) + bool listContains = false; + { + std::shared_lock anotherLock(m_mergeListLock); + listContains = m_mergeList.contains(queryResult->VID); + } + if (currentLength + nextLength < m_postingSizeLimit && !listContains) { { std::unique_lock anotherLock(m_rwLocks[queryResult->VID], std::defer_lock); @@ -1031,11 +1038,6 @@ namespace SPTAG::SPANN { } postingP = reinterpret_cast(nextPostingList.data()); postVectorNum = nextPostingList.size() / m_vectorInfoSize; - if (currentLength + postVectorNum > m_postingSizeLimit) - { - continue; - } - nextLength = 0; vectorId = postingP; for (int j = 0; j < postVectorNum; j++, vectorId += m_vectorInfoSize) @@ -1144,7 +1146,7 @@ namespace SPTAG::SPANN { if (m_opt->m_excludehead) { SizeType vid = (SizeType)(*(m_vectorTranslateMap->At(deletedHead))); - if (!m_versionMap->Deleted(vid)) + if (vid != MaxSize && !m_versionMap->Deleted(vid)) { std::shared_ptr vectorinfo = std::make_shared(m_vectorInfoSize, ' '); @@ -1155,7 +1157,10 @@ namespace SPTAG::SPANN { } } - m_mergeList.unsafe_erase(headID); + { + std::unique_lock lock(m_mergeListLock); + m_mergeList.unsafe_erase(headID); + } m_stat.m_mergeNum++; return ErrorCode::Success; @@ -1176,7 +1181,10 @@ namespace SPTAG::SPANN { SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Merge: Check failed after put original posting %d\n", headID); return ret; } - m_mergeList.unsafe_erase(headID); + { + std::unique_lock lock(m_mergeListLock); + m_mergeList.unsafe_erase(headID); + } m_mergeLock.unlock(); } return ErrorCode::Success; @@ -1208,11 +1216,16 @@ namespace SPTAG::SPANN { inline void MergeAsync(VectorIndex* p_index, SizeType headID, std::function p_callback = nullptr) { - if (m_mergeList.find(headID) != m_mergeList.end()) { - return; - } Helper::Concurrent::ConcurrentMap::value_type workPair(headID, headID); - m_mergeList.insert(workPair); + { + std::shared_lock lock(m_mergeListLock); + auto res = m_mergeList.insert(workPair); + if (!res.second) + { + // Already in queue + return; + } + } auto* curJob = new MergeAsyncJob(p_index, this, headID, m_opt->m_disableReassign, p_callback); m_splitThreadPool->add(curJob); @@ -1232,7 +1245,7 @@ namespace SPTAG::SPANN { if (m_opt->m_excludehead && !theSameHead) { SizeType vid = (SizeType)(*(m_vectorTranslateMap->At(headID))); - if (!m_versionMap->Deleted(vid)) + if (vid != MaxSize && !m_versionMap->Deleted(vid)) { std::shared_ptr vectorinfo = std::make_shared(m_vectorInfoSize, ' '); Serialize(vectorinfo->data(), vid, m_versionMap->GetVersion(vid), headVector); @@ -2370,7 +2383,6 @@ namespace SPTAG::SPANN { Serialize(ptr, fullID, version, p_fullVectors->GetVector(fullID)); ptr += m_vectorInfoSize; } - *m_checkSums[index] = m_checkSum.CalcChecksum(postinglist.c_str(), (int)(postinglist.size())); ErrorCode tmp; if ((tmp = db->Put(index, postinglist, MaxTimeout, &(workSpace.m_diskRequests))) != ErrorCode::Success) @@ -2379,6 +2391,7 @@ namespace SPTAG::SPANN { ret = tmp; return; } + *m_checkSums[index] = m_checkSum.CalcChecksum(postinglist.c_str(), (int)(postinglist.size())); if (m_opt->m_consistencyCheck && (tmp = db->Check(index, m_postingSizes.GetSize(index) * m_vectorInfoSize, nullptr)) != ErrorCode::Success) { diff --git a/Test/src/SPFreshTest.cpp b/Test/src/SPFreshTest.cpp index 32e737cd2..f98424b70 100644 --- a/Test/src/SPFreshTest.cpp +++ b/Test/src/SPFreshTest.cpp @@ -38,6 +38,7 @@ std::shared_ptr BuildIndex(const std::string &outDirectory, std::sh { auto vecIndex = VectorIndex::CreateInstance(IndexAlgoType::SPANN, GetEnumValueType()); int maxthreads = std::thread::hardware_concurrency(); + int postingLimit = 4 * sizeof(T); std::string configuration = R"( [Base] DistCalcMethod=L2 @@ -67,8 +68,8 @@ std::shared_ptr BuildIndex(const std::string &outDirectory, std::sh InternalResultNum=64 SearchInternalResultNum=64 NumberOfThreads=)" + std::to_string(maxthreads) + R"( - PostingPageLimit=)" + std::to_string(4 * sizeof(T)) + R"( - SearchPostingPageLimit=)" + std::to_string(4 * sizeof(T)) + R"( + PostingPageLimit=)" + std::to_string(postingLimit) + R"( + SearchPostingPageLimit=)" + std::to_string(postingLimit) + R"( TmpDir=tmpdir Storage=FILEIO SpdkBatchSize=64 @@ -86,7 +87,7 @@ std::shared_ptr BuildIndex(const std::string &outDirectory, std::sh SearchDuringUpdate=true MergeThreshold=10 Sampling=4 - BufferLength=)" + std::to_string(6 * sizeof(T)) + R"( + BufferLength=)" + std::to_string(postingLimit) + R"( InPlace=true StartFileSizeGB=1 OneClusterCutMax=false @@ -130,6 +131,7 @@ std::shared_ptr BuildLargeIndex(const std::string &outDirectory, st { auto vecIndex = VectorIndex::CreateInstance(IndexAlgoType::SPANN, GetEnumValueType()); int maxthreads = std::thread::hardware_concurrency(); + int postingLimit = 4 * sizeof(T); std::string configuration = R"( [Base] DistCalcMethod=L2 @@ -161,10 +163,10 @@ std::shared_ptr BuildLargeIndex(const std::string &outDirectory, st InternalResultNum=64 SearchInternalResultNum=64 NumberOfThreads=)" + std::to_string(maxthreads) + R"( - PostingPageLimit=)" + std::to_string(4 * sizeof(T)) + + PostingPageLimit=)" + std::to_string(postingLimit) + R"( SearchPostingPageLimit=)" + - std::to_string(4 * sizeof(T)) + R"( + std::to_string(postingLimit) + R"( TmpDir=tmpdir Storage=FILEIO SpdkBatchSize=64 @@ -182,7 +184,7 @@ std::shared_ptr BuildLargeIndex(const std::string &outDirectory, st SearchDuringUpdate=true MergeThreshold=10 Sampling=4 - BufferLength=)" + std::to_string(6 * sizeof(T)) + R"( + BufferLength=)" + std::to_string(postingLimit) + R"( InPlace=true StartFileSizeGB=1 OneClusterCutMax=false From 2bc00cabb51556a8167bbb1380520223e6a9b505 Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Tue, 30 Dec 2025 15:58:11 -0800 Subject: [PATCH 25/38] fix unsafe delete --- AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h b/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h index 8185b3b1f..88b355d5f 100644 --- a/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h +++ b/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h @@ -726,7 +726,7 @@ namespace SPTAG::SPANN { // double gcEndTime = sw.getElapsedMs(); // m_splitGcCost += gcEndTime; - if (!preReassign && index < m_postingSizeLimit) + if (!preReassign && localIndices.size() < m_postingSizeLimit) { //SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "DEBUG: in place or not prereassign & index < m_postingSizeLimit. GC begin...\n"); @@ -737,12 +737,12 @@ namespace SPTAG::SPANN { memcpy(ptr, postingList.c_str() + localIndices[j] * m_vectorInfoSize, m_vectorInfoSize); //Serialize(ptr, localIndicesInsert[j], localIndicesInsertVersion[j], smallSample[j]); } - postingList.resize(index * m_vectorInfoSize); + postingList.resize(localIndices.size() * m_vectorInfoSize); if ((ret=db->Put(headID, postingList, MaxTimeout, &(p_exWorkSpace->m_diskRequests))) != ErrorCode::Success) { SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Split Fail to write back postings\n"); return ret; } - m_postingSizes.UpdateSize(headID, index); + m_postingSizes.UpdateSize(headID, localIndices.size()); *m_checkSums[headID] = m_checkSum.CalcChecksum(postingList.c_str(), (int)(postingList.size())); if (m_opt->m_consistencyCheck && (ret = db->Check(headID, m_postingSizes.GetSize(headID) * m_vectorInfoSize, nullptr)) != ErrorCode::Success) { @@ -1015,7 +1015,7 @@ namespace SPTAG::SPANN { bool listContains = false; { std::shared_lock anotherLock(m_mergeListLock); - listContains = m_mergeList.contains(queryResult->VID); + listContains = (m_mergeList.find(queryResult->VID) != m_mergeList.end()); } if (currentLength + nextLength < m_postingSizeLimit && !listContains) { From 130dcfb7a912997130e87a981c2285462c32b284 Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Tue, 30 Dec 2025 19:52:28 -0800 Subject: [PATCH 26/38] avoid dataset incblock leak --- AnnService/inc/Core/Common/Dataset.h | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/AnnService/inc/Core/Common/Dataset.h b/AnnService/inc/Core/Common/Dataset.h index 86dc8bcd7..9cd1388c9 100644 --- a/AnnService/inc/Core/Common/Dataset.h +++ b/AnnService/inc/Core/Common/Dataset.h @@ -260,12 +260,15 @@ namespace SPTAG { if (data != nullptr) { if (ownData) ALIGN_FREE(data); - for (char* ptr : *incBlocks) ALIGN_FREE(ptr); - incBlocks->clear(); } - rows = rows_; + + + for (char *ptr : *incBlocks) + ALIGN_FREE(ptr); + incBlocks->clear(); incRows = 0; + if (rowEnd_ >= colStart_) cols = rowEnd_; else cols = cols_ * sizeof(T); data = (char*)data_; From f771a7cdab9dbc6ba32af033590ef69507553dad Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Fri, 2 Jan 2026 15:22:05 -0800 Subject: [PATCH 27/38] Add split VID check --- .../inc/Core/SPANN/ExtraDynamicSearcher.h | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h b/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h index 88b355d5f..5d23318be 100644 --- a/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h +++ b/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h @@ -681,6 +681,8 @@ namespace SPTAG::SPANN { std::unique_lock lock(m_rwLocks[headID], std::defer_lock); if (requirelock) lock.lock(); + int retry = 0; + Retry: if (!p_index->ContainSample(headID)) return ErrorCode::Success; std::string postingList; @@ -715,6 +717,21 @@ namespace SPTAG::SPANN { //LOG(Helper::LogLevel::LL_Info, "vector index/total:id: %d/%d:%d\n", j, m_postingSizes[headID].load(), *(reinterpret_cast(vectorId))); uint8_t version = *(vectorId + sizeof(int)); int VID = *((int*)(vectorId)); + if (VID < 0 || VID >= m_versionMap->Count()) + { + if (retry < 3) + { + retry++; + goto Retry; + } + else + { + SPTAGLIB_LOG(Helper::LogLevel::LL_Error, + "Split fail: Get posting fail after 3 times retries."); + return ErrorCode::DiskIOFail; + } + } + //if (VID >= m_versionMap->Count()) SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "DEBUG: vector ID:%d total size:%d\n", VID, m_versionMap->Count()); if (m_versionMap->Deleted(VID) || m_versionMap->GetVersion(VID) != version) continue; From 6714e5ad017711794e5becd4d3004334e657eb74 Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Fri, 2 Jan 2026 16:06:40 -0800 Subject: [PATCH 28/38] fix dataset --- AnnService/inc/Core/Common/Dataset.h | 10 ++++++---- AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/AnnService/inc/Core/Common/Dataset.h b/AnnService/inc/Core/Common/Dataset.h index 9cd1388c9..dcdebbb04 100644 --- a/AnnService/inc/Core/Common/Dataset.h +++ b/AnnService/inc/Core/Common/Dataset.h @@ -263,10 +263,12 @@ namespace SPTAG } rows = rows_; - - for (char *ptr : *incBlocks) - ALIGN_FREE(ptr); - incBlocks->clear(); + if (incBlocks != nullptr) + { + for (char *ptr : *incBlocks) + ALIGN_FREE(ptr); + incBlocks->clear(); + } incRows = 0; if (rowEnd_ >= colStart_) cols = rowEnd_; diff --git a/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h b/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h index 5d23318be..36ee7a6ba 100644 --- a/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h +++ b/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h @@ -727,7 +727,7 @@ namespace SPTAG::SPANN { else { SPTAGLIB_LOG(Helper::LogLevel::LL_Error, - "Split fail: Get posting fail after 3 times retries."); + "Split fail: Get posting %d fail after 3 times retries.\n", headID); return ErrorCode::DiskIOFail; } } From 2baa3581c57b23ac8811ac2f5c092dedc7476eb6 Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Fri, 9 Jan 2026 15:31:00 -0800 Subject: [PATCH 29/38] merge changes in internal version --- .vscode/c_cpp_properties.json | 21 ++ .vscode/launch.json | 219 ++++++++++++++++++ .vscode/settings.json | 103 ++++++++ .vscode/tasks.json | 27 +++ AnnService/CoreLibrary.vcxproj | 2 +- AnnService/CoreLibrary.vcxproj.filters | 2 +- AnnService/inc/Core/BKT/Index.h | 8 +- AnnService/inc/Core/Common/Labelset.h | 6 +- AnnService/inc/Core/KDT/Index.h | 6 +- .../inc/Core/SPANN/ExtraDynamicSearcher.h | 2 +- .../inc/Core/SPANN/ExtraFileController.h | 171 ++++++++------ .../inc/Core/SPANN/ExtraStaticSearcher.h | 2 +- AnnService/inc/Core/SPANN/IExtraSearcher.h | 2 +- AnnService/inc/Core/SPANN/Index.h | 2 +- AnnService/inc/Helper/KeyValueIO.h | 7 +- AnnService/src/Core/BKT/BKTIndex.cpp | 22 +- AnnService/src/Core/KDT/KDTIndex.cpp | 22 +- AnnService/src/Core/SPANN/SPANNIndex.cpp | 2 +- 18 files changed, 519 insertions(+), 107 deletions(-) create mode 100644 .vscode/c_cpp_properties.json create mode 100644 .vscode/launch.json create mode 100644 .vscode/settings.json create mode 100644 .vscode/tasks.json diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json new file mode 100644 index 000000000..8be2bc624 --- /dev/null +++ b/.vscode/c_cpp_properties.json @@ -0,0 +1,21 @@ +{ + "configurations": [ + { + "name": "Linux", + "includePath": [ + "${workspaceFolder}/**", + "${workspaceFolder}/AnnService/inc", + "${workspaceFolder}/Test/inc", + "${workspaceFolder}/ThirdParty/**", + "/usr/include/**" + ], + "defines": [], + "compilerPath": "/usr/bin/g++", + "cStandard": "c17", + "cppStandard": "c++17", + "intelliSenseMode": "gcc-x64", + "compileCommands": "${workspaceFolder}/build/compile_commands.json" + } + ], + "version": 4 +} \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 000000000..bed3879ae --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,219 @@ +{ + "version": "2.0.0", + "configurations": [ + { + "name": "(gdb) Launch SPFresh Test", + "type": "cppdbg", + "request": "launch", + "preLaunchTask": "Build Debug", + "stopAtEntry": false, + "program": "${workspaceFolder}/Debug/SPTAGTest", + "args": [ "--run_test=SPFreshTest/CacheTest" ], + "cwd": "${workspaceFolder}/Debug", + "environment": [ + { + "name": "UDF_RUNTIME_DIR", + "value": "${workspaceFolder}" + }, + { + "name": "STARROCKS_HOME", + "value": "${workspaceFolder}" + }, + { + "name": "PYTHON_INSTALL_DIR", + "value": "${workspaceFolder}/thirdparty/installed/python" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/lib/jvm/java-1.11.0-openjdk-amd64/lib/server:${workspaceFolder}/Debug" + }, + { + "name": "LD_PRELOAD", + "value": "/usr/lib/gcc/x86_64-linux-gnu/11/libasan.so" + }, + { + "name": "PCI_ALLOWED", + "value": "1462:00:00.0" + }, + { + "name": "SPFRESH_SPDK_USE_SSD_IMPL", + "value": "1" + }, + { + "name": "SPFRESH_SPDK_CONF", + "value": "./bdev.json" + }, + { + "name": "SPFRESH_SPDK_BDEV", + "value": "Nvme0n1" + }, + { + "name": "BENCHMARK_CONFIG", + "value": "benchmark.ini" + } + ], + "externalConsole": false, + "MIMode": "gdb", + "miDebuggerPath": "/usr/bin/gdb", + "additionalSOLibSearchPath": "${workspaceFolder}/Debug", + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + }, + { + "description": "Set Disassembly Flavor to Intel", + "text": "-gdb-set disassembly-flavor intel", + "ignoreFailures": true + }, + { + "description": "Skip standard library files", + "text": "-gdb-set skip-solib-deps on", + "ignoreFailures": true + } + ] + }, + { + "name": "(gdb) Launch All Tests", + "type": "cppdbg", + "request": "launch", + "stopAtEntry": false, + "program": "${workspaceFolder}/Debug/SPTAGTest", + "args": [ "--run_test=SPFreshTest" ], + "cwd": "${workspaceFolder}/Debug", + "environment": [ + { + "name": "UDF_RUNTIME_DIR", + "value": "${workspaceFolder}" + }, + { + "name": "STARROCKS_HOME", + "value": "${workspaceFolder}" + }, + { + "name": "PYTHON_INSTALL_DIR", + "value": "${workspaceFolder}/thirdparty/installed/python" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/lib/jvm/java-1.11.0-openjdk-amd64/lib/server:${workspaceFolder}/Debug" + }, + { + "name": "LD_PRELOAD", + "value": "/usr/lib/gcc/x86_64-linux-gnu/11/libasan.so" + }, + { + "name": "PCI_ALLOWED", + "value": "1462:00:00.0" + }, + { + "name": "SPFRESH_SPDK_USE_SSD_IMPL", + "value": "1" + }, + { + "name": "SPFRESH_SPDK_CONF", + "value": "./bdev.json" + }, + { + "name": "SPFRESH_SPDK_BDEV", + "value": "Nvme0n1" + } + ], + "externalConsole": false, + "MIMode": "gdb", + "miDebuggerPath": "/usr/bin/gdb", + "additionalSOLibSearchPath": "${workspaceFolder}/Debug", + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + }, + { + "description": "Set Disassembly Flavor to Intel", + "text": "-gdb-set disassembly-flavor intel", + "ignoreFailures": true + }, + { + "description": "Skip standard library files", + "text": "-gdb-set skip-solib-deps on", + "ignoreFailures": true + } + ] + }, + { + "name": "(gdb) Launch SPFresh Benchmark", + "type": "cppdbg", + "request": "launch", + "preLaunchTask": "Build Debug", + "stopAtEntry": false, + "program": "${workspaceFolder}/Debug/SPTAGTest", + "args": [ "--run_test=SPFreshTest/BenchmarkFromConfig" ], + "cwd": "${workspaceFolder}/Debug", + "environment": [ + { + "name": "UDF_RUNTIME_DIR", + "value": "${workspaceFolder}" + }, + { + "name": "STARROCKS_HOME", + "value": "${workspaceFolder}" + }, + { + "name": "PYTHON_INSTALL_DIR", + "value": "${workspaceFolder}/thirdparty/installed/python" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/lib/jvm/java-1.11.0-openjdk-amd64/lib/server:${workspaceFolder}/Debug" + }, + { + "name": "LD_PRELOAD", + "value": "/usr/lib/gcc/x86_64-linux-gnu/11/libasan.so" + }, + { + "name": "PCI_ALLOWED", + "value": "1462:00:00.0" + }, + { + "name": "SPFRESH_SPDK_USE_SSD_IMPL", + "value": "1" + }, + { + "name": "SPFRESH_SPDK_CONF", + "value": "./bdev.json" + }, + { + "name": "SPFRESH_SPDK_BDEV", + "value": "Nvme0n1" + }, + { + "name": "BENCHMARK_CONFIG", + "value": "/home/superbench/SPTAG/Debug/benchmark.ini" + } + ], + "externalConsole": false, + "MIMode": "gdb", + "miDebuggerPath": "/usr/bin/gdb", + "additionalSOLibSearchPath": "${workspaceFolder}/Debug", + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + }, + { + "description": "Set Disassembly Flavor to Intel", + "text": "-gdb-set disassembly-flavor intel", + "ignoreFailures": true + }, + { + "description": "Skip standard library files", + "text": "-gdb-set skip-solib-deps on", + "ignoreFailures": true + } + ] + } + ] +} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 000000000..42f8caebc --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,103 @@ +{ + "debug.allowBreakpointsEverywhere": true, + "debug.disassemblyView.showSourceCode": true, + "C_Cpp.debugging.engineLogging": false, + "C_Cpp.loggingLevel": "Information", + "files.associations": { + "array": "cpp", + "bitset": "cpp", + "string_view": "cpp", + "initializer_list": "cpp", + "ranges": "cpp", + "span": "cpp", + "regex": "cpp", + "utility": "cpp", + "valarray": "cpp", + "barrier": "cpp", + "__hash_table": "cpp", + "__split_buffer": "cpp", + "__tree": "cpp", + "deque": "cpp", + "iterator": "cpp", + "list": "cpp", + "map": "cpp", + "queue": "cpp", + "random": "cpp", + "set": "cpp", + "stack": "cpp", + "string": "cpp", + "unordered_map": "cpp", + "unordered_set": "cpp", + "vector": "cpp", + "atomic": "cpp", + "cctype": "cpp", + "clocale": "cpp", + "cmath": "cpp", + "csetjmp": "cpp", + "csignal": "cpp", + "cstdarg": "cpp", + "cstddef": "cpp", + "cstdio": "cpp", + "cstdlib": "cpp", + "cstring": "cpp", + "ctime": "cpp", + "cwchar": "cpp", + "cwctype": "cpp", + "any": "cpp", + "strstream": "cpp", + "bit": "cpp", + "cfenv": "cpp", + "charconv": "cpp", + "chrono": "cpp", + "cinttypes": "cpp", + "codecvt": "cpp", + "compare": "cpp", + "complex": "cpp", + "concepts": "cpp", + "condition_variable": "cpp", + "coroutine": "cpp", + "cstdint": "cpp", + "cuchar": "cpp", + "forward_list": "cpp", + "exception": "cpp", + "algorithm": "cpp", + "functional": "cpp", + "memory": "cpp", + "memory_resource": "cpp", + "numeric": "cpp", + "optional": "cpp", + "ratio": "cpp", + "source_location": "cpp", + "system_error": "cpp", + "tuple": "cpp", + "type_traits": "cpp", + "fstream": "cpp", + "future": "cpp", + "iomanip": "cpp", + "iosfwd": "cpp", + "iostream": "cpp", + "istream": "cpp", + "latch": "cpp", + "limits": "cpp", + "mutex": "cpp", + "new": "cpp", + "numbers": "cpp", + "ostream": "cpp", + "scoped_allocator": "cpp", + "semaphore": "cpp", + "shared_mutex": "cpp", + "sstream": "cpp", + "stdexcept": "cpp", + "stop_token": "cpp", + "streambuf": "cpp", + "syncstream": "cpp", + "thread": "cpp", + "typeindex": "cpp", + "typeinfo": "cpp", + "variant": "cpp", + "expected": "cpp", + "spanstream": "cpp", + "stacktrace": "cpp", + "__nullptr": "cpp" + } +} \ No newline at end of file diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 000000000..cbd8b4c74 --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,27 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "Build Debug", + "type": "shell", + "command": "make", + "args": ["-j4"], + "group": { + "kind": "build", + "isDefault": true + }, + "options": { + "cwd": "${workspaceFolder}/build" + }, + "problemMatcher": ["$gcc"], + "presentation": { + "echo": true, + "reveal": "always", + "focus": false, + "panel": "shared", + "showReuseMessage": true, + "clear": false + } + } + ] +} \ No newline at end of file diff --git a/AnnService/CoreLibrary.vcxproj b/AnnService/CoreLibrary.vcxproj index 721d74d4b..5a66222f8 100644 --- a/AnnService/CoreLibrary.vcxproj +++ b/AnnService/CoreLibrary.vcxproj @@ -94,7 +94,7 @@ - + diff --git a/AnnService/CoreLibrary.vcxproj.filters b/AnnService/CoreLibrary.vcxproj.filters index 3f7b3fa11..63b5e7997 100644 --- a/AnnService/CoreLibrary.vcxproj.filters +++ b/AnnService/CoreLibrary.vcxproj.filters @@ -160,7 +160,7 @@ Header Files\Helper - + Header Files\Core\Common diff --git a/AnnService/inc/Core/BKT/Index.h b/AnnService/inc/Core/BKT/Index.h index dd9734bed..8d5c41ccc 100644 --- a/AnnService/inc/Core/BKT/Index.h +++ b/AnnService/inc/Core/BKT/Index.h @@ -15,7 +15,7 @@ #include "inc/Core/Common/WorkSpacePool.h" #include "inc/Core/Common/RelativeNeighborhoodGraph.h" #include "inc/Core/Common/BKTree.h" -#include "inc/Core/Common/Labelset.h" +#include "inc/Core/Common/LabelSet.h" #include "inc/Helper/SimpleIniReader.h" #include "inc/Helper/StringConvert.h" #include "inc/Helper/ThreadPool.h" @@ -94,7 +94,7 @@ namespace SPTAG float m_fDeletePercentageForRefine; std::mutex m_dataAddLock; // protect data and graph std::shared_timed_mutex m_dataDeleteLock; - COMMON::Labelset m_deletedID; + COMMON::LabelSet m_deletedID; Helper::ThreadPool m_threadPool; int m_iNumberOfThreads; @@ -230,12 +230,12 @@ namespace SPTAG int SearchIndexIterative(COMMON::QueryResultSet& p_query, COMMON::WorkSpace& p_space, bool p_isFirst, int batch, bool p_searchDeleted, bool p_searchDuplicated) const; - template &, SizeType, float), bool (*checkFilter)(const std::shared_ptr &, SizeType, std::function)> + template &, SizeType, float), bool (*checkFilter)(const std::shared_ptr &, SizeType, std::function)> int SearchIterative(COMMON::QueryResultSet& p_query, COMMON::WorkSpace& p_space, bool p_isFirst, int batch) const; void SearchIndex(COMMON::QueryResultSet &p_query, COMMON::WorkSpace &p_space, bool p_searchDeleted, bool p_searchDuplicated, std::function filterFunc = nullptr) const; - template &, SizeType, float), bool(*checkFilter)(const std::shared_ptr&, SizeType, std::function)> + template &, SizeType, float), bool(*checkFilter)(const std::shared_ptr&, SizeType, std::function)> void Search(COMMON::QueryResultSet& p_query, COMMON::WorkSpace& p_space, std::function filterFunc) const; }; } // namespace BKT diff --git a/AnnService/inc/Core/Common/Labelset.h b/AnnService/inc/Core/Common/Labelset.h index c34f563de..65b5ba7f3 100644 --- a/AnnService/inc/Core/Common/Labelset.h +++ b/AnnService/inc/Core/Common/Labelset.h @@ -11,7 +11,7 @@ namespace SPTAG { namespace COMMON { - class Labelset + class LabelSet { public: enum class InvalidIDBehavior @@ -23,10 +23,10 @@ namespace SPTAG private: std::atomic m_inserted; Dataset m_data; - InvalidIDBehavior m_invalidIDBehaviorSetting; + InvalidIDBehavior m_invalidIDBehaviorSetting{}; public: - Labelset() + LabelSet() { m_inserted = 0; m_data.SetName("DeleteID"); diff --git a/AnnService/inc/Core/KDT/Index.h b/AnnService/inc/Core/KDT/Index.h index 9d11dac43..064a7ac16 100644 --- a/AnnService/inc/Core/KDT/Index.h +++ b/AnnService/inc/Core/KDT/Index.h @@ -15,7 +15,7 @@ #include "inc/Core/Common/WorkSpacePool.h" #include "inc/Core/Common/RelativeNeighborhoodGraph.h" #include "inc/Core/Common/KDTree.h" -#include "inc/Core/Common/Labelset.h" +#include "inc/Core/Common/LabelSet.h" #include "inc/Helper/SimpleIniReader.h" #include "inc/Helper/StringConvert.h" #include "inc/Helper/ThreadPool.h" @@ -71,7 +71,7 @@ namespace SPTAG float m_fDeletePercentageForRefine; std::mutex m_dataAddLock; // protect data and graph std::shared_timed_mutex m_dataDeleteLock; - COMMON::Labelset m_deletedID; + COMMON::LabelSet m_deletedID; Helper::ThreadPool m_threadPool; int m_iNumberOfThreads; @@ -204,7 +204,7 @@ namespace SPTAG private: template void SearchIndex(COMMON::QueryResultSet &p_query, COMMON::WorkSpace &p_space, bool p_searchDeleted) const; - template + template void Search(COMMON::QueryResultSet& p_query, COMMON::WorkSpace& p_space) const; }; } // namespace KDT diff --git a/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h b/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h index 36ee7a6ba..b737bccb5 100644 --- a/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h +++ b/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h @@ -2529,7 +2529,7 @@ namespace SPTAG::SPANN { return (postingID < m_postingSizes.GetPostingNum()) && (m_postingSizes.GetSize(postingID) > 0); } - virtual ErrorCode CheckPosting(SizeType postingID, std::vector *visited = nullptr, + virtual ErrorCode CheckPosting(SizeType postingID, std::vector *visited = nullptr, ExtraWorkSpace *p_exWorkSpace = nullptr) override { if (postingID < 0 || postingID >= m_postingSizes.GetPostingNum()) diff --git a/AnnService/inc/Core/SPANN/ExtraFileController.h b/AnnService/inc/Core/SPANN/ExtraFileController.h index 236dc4ae5..f4b8bf243 100644 --- a/AnnService/inc/Core/SPANN/ExtraFileController.h +++ b/AnnService/inc/Core/SPANN/ExtraFileController.h @@ -4,7 +4,7 @@ #include "inc/Helper/KeyValueIO.h" #include "inc/Core/Common/Dataset.h" -#include "inc/Core/Common/Labelset.h" +#include "inc/Core/Common/LabelSet.h" #include "inc/Core/Common/FineGrainedLock.h" #include "inc/Core/VectorIndex.h" #include "inc/Helper/ThreadPool.h" @@ -27,7 +27,7 @@ namespace SPTAG::SPANN { Helper::Concurrent::ConcurrentQueue m_blockAddresses; Helper::Concurrent::ConcurrentQueue m_blockAddresses_reserve; - COMMON::Labelset m_available; + COMMON::LabelSet m_available; std::atomic read_complete_vec = 0; std::atomic read_submit_vec = 0; @@ -105,15 +105,18 @@ namespace SPTAG::SPANN { SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "FileIO::BlockController::Checkpoint - Reload reserved blocks...\n"); AddressType currBlockAddress = 0; - while (m_blockAddresses_reserve.try_pop(currBlockAddress)) - { + int reloadCount = 0; + + while (m_blockAddresses_reserve.try_pop(currBlockAddress)) { m_blockAddresses.push(currBlockAddress); + ++reloadCount; } AddressType blocks = RemainBlocks(); AddressType totalBlocks = m_totalAllocatedBlocks.load(); + SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "FileIO::BlockController::Checkpoint - Reloaded blocks: %d\n", reloadCount); SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "FileIO::BlockController::Checkpoint - Total allocated blocks: %llu\n", static_cast(totalBlocks)); - SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "FileIO::BlockController::Checkpoint - Remaining free blocks: %llu\n", blocks); + SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "FileIO::BlockController::Checkpoint - Remaining free blocks: %llu\n", static_cast(blocks)); SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "FileIO::BlockController::Checkpoint - Saving to file: %s\n", filename.c_str()); auto ptr = f_createIO(); @@ -126,26 +129,15 @@ namespace SPTAG::SPANN { for (auto it = m_blockAddresses.unsafe_begin(); it != m_blockAddresses.unsafe_end(); it++) { IOBINARY(ptr, WriteBinary, sizeof(AddressType), reinterpret_cast(&(*it))); } - /* - int i = 0; - for (auto it = m_blockAddresses.unsafe_begin(); it != m_blockAddresses.unsafe_end(); it++) { - std::cout << *it << " "; - i++; - if (i == 10) break; - } - std::cout << std::endl; - */ - SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "Save Finish!\n"); + + SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "FileIO::BlockController::Checkpoint - Save Finish!\n"); return ErrorCode::Success; } ErrorCode LoadBlockPool(std::string prefix, AddressType startNumBlocks, bool allowInit, int blockSize, int blockCapacity) { std::string blockfile = prefix + "_blockpool"; if (allowInit && !fileexists(blockfile.c_str())) { - SPTAGLIB_LOG(Helper::LogLevel::LL_Info, - "FileIO::BlockController::LoadBlockPool: initializing fresh pool (no existing file " - "found: %s)\n", - blockfile.c_str()); + SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "FileIO::BlockController::LoadBlockPool: initializing fresh pool (no existing file found: %s)\n", blockfile.c_str()); m_available.Initialize(startNumBlocks, blockSize, blockCapacity); for(AddressType i = 0; i < startNumBlocks; i++) { m_blockAddresses.push(i); @@ -208,7 +200,15 @@ namespace SPTAG::SPANN { return ErrorCode::Success; } }; - + + struct CacheCounters + { + int64_t query{}; + int64_t query_hits{}; + int64_t put{}; + int64_t evict{}; + }; + struct CacheEntry { std::string value; @@ -221,20 +221,20 @@ namespace SPTAG::SPANN { int64_t size; std::list keys; // Page Address std::unordered_map cache; // Page Address -> Page Address in Cache - int64_t queries; - std::atomic hits; FileIO* fileIO; Helper::RequestQueue processIocp; std::vector reqs; Helper::PageBuffer pageBuffer; + std::atomic query_counter{}; + std::atomic query_hits_counter{}; + std::atomic put_counter{}; + std::atomic evict_counter{}; public: LRUCache(int64_t capacity, int64_t limit, FileIO* fileIO) { this->capacity = capacity; this->limit = min(capacity, (limit << PageSizeEx)); this->size = 0; - this->queries = 0; - this->hits = 0; this->fileIO = fileIO; this->reqs.resize(limit); this->pageBuffer.ReservePageBuffer(limit << PageSizeEx); @@ -256,9 +256,9 @@ namespace SPTAG::SPANN { ~LRUCache() {} - bool evict(SizeType key, void *value, int vsize, std::unordered_map::iterator &it) - { + bool evict(SizeType key, void* value, int vsize, std::unordered_map::iterator& it) { if (value != nullptr) { + ++evict_counter; std::string valstr((char*)value, vsize); if (fileIO->Put(key, valstr, MaxTimeout, &reqs, false) != ErrorCode::Success) { SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "LRUCache: evict key:%d value size:%d to file failed\n", key, vsize); @@ -272,12 +272,10 @@ namespace SPTAG::SPANN { return true; } - bool get(SizeType key, Helper::PageBuffer &buffer) - { - queries++; + bool get(SizeType key, Helper::PageBuffer& buffer) { + ++query_counter; auto it = cache.find(key); - if (it == cache.end()) - { + if (it == cache.end()) { return false; } @@ -285,27 +283,26 @@ namespace SPTAG::SPANN { buffer.ReservePageBuffer(data_size); memcpy(buffer.GetBuffer(), it->second.value.data(), data_size); buffer.SetAvailableSize(data_size); - hits++; + ++query_hits_counter; return true; } - bool get(SizeType key, std::string &value) - { - queries++; + bool get(SizeType key, std::string& value) { + ++query_counter; auto it = cache.find(key); - if (it == cache.end()) - { + if (it == cache.end()) { return false; } size_t data_size = it->second.value.size(); value.resize(data_size); memcpy(value.data(), it->second.value.data(), data_size); - hits++; + ++query_hits_counter; return true; } bool put(SizeType key, void* value, int put_size) { + ++put_counter; auto it = cache.find(key); if (it != cache.end()) { if (put_size > limit) { @@ -315,7 +312,7 @@ namespace SPTAG::SPANN { keys.splice(keys.begin(), keys, it->second.iter); it->second.iter = keys.begin(); - auto delta_size = put_size - (int)(it->second.value.size()); + int delta_size = put_size - (int)(it->second.value.size()); while ((int)(capacity - size) < delta_size && (keys.size() > 1)) { auto last = keys.back(); auto lastit = cache.find(last); @@ -327,7 +324,6 @@ namespace SPTAG::SPANN { it->second.value.resize(put_size); memcpy(it->second.value.data(), (char*)value, put_size); size += delta_size; - hits++; return true; } if (put_size > limit) { @@ -357,6 +353,7 @@ namespace SPTAG::SPANN { bool merge(SizeType key, void *value, int merge_size, std::function checksum) { + ++put_counter; // SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "LRUCache: merge size: %lld\n", merge_size); auto it = cache.find(key); if (it == cache.end()) { @@ -384,7 +381,6 @@ namespace SPTAG::SPANN { size += valstr.size(); return true; } - hits++; if (merge_size + it->second.value.size() > limit) { evict(key, it->second.value.data(), it->second.value.size(), it); @@ -406,11 +402,11 @@ namespace SPTAG::SPANN { // SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "LRUCache: merge success\n"); return true; } - - std::pair get_stat() { - return {queries, hits.load()}; - } + CacheCounters get_stat() { + return CacheCounters{query_counter.load(), query_hits_counter.load(), put_counter.load(), evict_counter.load()}; + } + bool flush() { for (auto it = cache.begin(); it != cache.end(); it++) { if (fileIO->Put(it->first, it->second.value, MaxTimeout, &reqs, false) != ErrorCode::Success) { @@ -423,13 +419,18 @@ namespace SPTAG::SPANN { size = 0; return true; } + + int64_t GetApproximateMemoryUsage() const + { + return static_cast(size); + } }; class ShardedLRUCache { int shards; std::vector caches; std::unique_ptr m_rwMutexs; - + public: ShardedLRUCache(int shards, int64_t capacity, int64_t limit, FileIO* fileIO) : shards(shards) { caches.resize(shards); @@ -478,6 +479,7 @@ namespace SPTAG::SPANN { bool flush() { for (int i = 0; i < shards; i++) { + std::unique_lock lock(m_rwMutexs[i]); if (!caches[i]->flush()) return false; } return true; @@ -488,19 +490,31 @@ namespace SPTAG::SPANN { return m_rwMutexs[hash(key)]; } - SizeType hash(SizeType key) const - { + SizeType hash(SizeType key) const { return key % shards; } - - std::pair get_stat() { - int64_t queries = 0, hits = 0; + + CacheCounters get_stat() { + CacheCounters result; for (int i = 0; i < shards; i++) { auto stat = caches[i]->get_stat(); - queries += stat.first; - hits += stat.second; + result.query += stat.query; + result.query_hits += stat.query_hits; + result.put += stat.put; + result.evict += stat.evict; } - return {queries, hits}; + + return result; + } + + int64_t GetApproximateMemoryUsage() const + { + int64_t result = 0; + for (int i = 0; i < shards; i++) + { + result += caches[i]->GetApproximateMemoryUsage(); + } + return result; } }; @@ -682,19 +696,18 @@ namespace SPTAG::SPANN { ErrorCode MultiGet(const std::vector& keys, std::vector>& values, const std::chrono::microseconds &timeout, std::vector* reqs) override { std::vector blocks; - std::set lock_keys; SizeType r; int i = 0; for (SizeType key : keys) { - r = m_pBlockMapping.R(); - if (key < r) { + r = m_pBlockMapping.R(); + if (key < r) { if (m_pShardedLRUCache && m_pShardedLRUCache->get(key, values[i])) { blocks.push_back(nullptr); } else { AddressType* addr = (AddressType*)(At(key)); blocks.push_back(addr); - } + } } else { SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Fail to read key:%d total key number:%d\n", key, r); @@ -709,7 +722,6 @@ namespace SPTAG::SPANN { ErrorCode MultiGet(const std::vector& keys, std::vector* values, const std::chrono::microseconds& timeout, std::vector* reqs) { std::vector blocks; - std::set lock_keys; SizeType r; values->resize(keys.size()); int i = 0; @@ -905,7 +917,7 @@ namespace SPTAG::SPANN { return Put(std::stoi(key), value, timeout, reqs, true); } - ErrorCode Check(const SizeType key, int size, std::vector *visited) override + ErrorCode Check(const SizeType key, int size, std::vector *visited) override { SizeType r = m_pBlockMapping.R(); @@ -932,24 +944,39 @@ namespace SPTAG::SPANN { (int)(postingSize[i]), m_pBlockController.TotalBlocks()); return ErrorCode::Block_IDError; } - - if (visited == nullptr) - continue; + if (visited == nullptr) continue; - if (visited->at(postingSize[i])) + if (postingSize[i] >= visited->size()) + { + SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "[Check] Key %d failed: BLOCK %lld exceed total block size %zu!\n", key, + postingSize[i], visited->size()); + continue; + } + + if (visited->at(postingSize[i]) > 0) { - SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "[Check] Block %lld double used!\n", postingSize[i]); + SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "[Check] Key %d failed: Block %lld double used!\n", key, postingSize[i]); return ErrorCode::Block_IDError; } else { - visited->at(postingSize[i]) = true; + InterlockedExchange8((char*)(&(visited->at(postingSize[i]))), 1); } } return ErrorCode::Success; } + int64_t GetApproximateMemoryUsage() const override + { + int64_t result = m_pBlockMapping.BufferSize(); + if (m_pShardedLRUCache) + { + result += m_pShardedLRUCache->GetApproximateMemoryUsage(); + } + return result; + } + ErrorCode Merge(const SizeType key, const std::string &value, const std::chrono::microseconds &timeout, std::vector *reqs, std::function checksum) @@ -1130,7 +1157,17 @@ namespace SPTAG::SPANN { SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "Total %d blocks, Remain %d blocks, Reserve %d blocks, totally %d GB\n", totalBlocks, remainBlocks, reserveBlocks, remainGB); if (m_pShardedLRUCache) { auto cache_stat = m_pShardedLRUCache->get_stat(); - SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "Cache queries: %lld, Cache hits: %lld, Hit rates: %lf\n", cache_stat.first, cache_stat.second, cache_stat.second == 0 ? 0 : (double)cache_stat.second / cache_stat.first); + if (cache_stat.query + cache_stat.put) + { + SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "LRUCache query: %lld/%lld (%f), put: %lld/%lld (%f)\n", + cache_stat.query_hits, + cache_stat.query, + cache_stat.query ? cache_stat.query_hits / (float) cache_stat.query : 0.0, + cache_stat.evict, + cache_stat.put, + cache_stat.put ? cache_stat.evict / (float) cache_stat.put : 0.0 + ); + } } m_pBlockController.IOStatistics(); } diff --git a/AnnService/inc/Core/SPANN/ExtraStaticSearcher.h b/AnnService/inc/Core/SPANN/ExtraStaticSearcher.h index 2149e0add..2d21f2ac4 100644 --- a/AnnService/inc/Core/SPANN/ExtraStaticSearcher.h +++ b/AnnService/inc/Core/SPANN/ExtraStaticSearcher.h @@ -1056,7 +1056,7 @@ namespace SPTAG return m_listInfos[postingID].listEleCount != 0; } - virtual ErrorCode CheckPosting(SizeType postingID, std::vector *visited = nullptr, + virtual ErrorCode CheckPosting(SizeType postingID, std::vector *visited = nullptr, ExtraWorkSpace *p_exWorkSpace = nullptr) override { if (postingID < 0 || postingID >= m_totalListCount) diff --git a/AnnService/inc/Core/SPANN/IExtraSearcher.h b/AnnService/inc/Core/SPANN/IExtraSearcher.h index 6e8aad0fa..be59225c6 100644 --- a/AnnService/inc/Core/SPANN/IExtraSearcher.h +++ b/AnnService/inc/Core/SPANN/IExtraSearcher.h @@ -330,7 +330,7 @@ namespace SPTAG { virtual void ForceCompaction() { return; } virtual bool CheckValidPosting(SizeType postingID) = 0; - virtual ErrorCode CheckPosting(SizeType postingiD, std::vector *visited = nullptr, + virtual ErrorCode CheckPosting(SizeType postingiD, std::vector *visited = nullptr, ExtraWorkSpace *p_exWorkSpace = nullptr) = 0; virtual SizeType SearchVector(ExtraWorkSpace* p_exWorkSpace, std::shared_ptr& p_vectorSet, std::shared_ptr p_index, int testNum = 64, SizeType VID = -1) { return -1; } diff --git a/AnnService/inc/Core/SPANN/Index.h b/AnnService/inc/Core/SPANN/Index.h index 54b33c2b4..0479d6c85 100644 --- a/AnnService/inc/Core/SPANN/Index.h +++ b/AnnService/inc/Core/SPANN/Index.h @@ -17,7 +17,7 @@ #include "inc/Core/Common/VersionLabel.h" #include "inc/Core/Common/PostingSizeRecord.h" -#include "inc/Core/Common/Labelset.h" +#include "inc/Core/Common/LabelSet.h" #include "inc/Helper/SimpleIniReader.h" #include "inc/Helper/StringConvert.h" #include "inc/Helper/ThreadPool.h" diff --git a/AnnService/inc/Helper/KeyValueIO.h b/AnnService/inc/Helper/KeyValueIO.h index 21cb7193e..69e029bfd 100644 --- a/AnnService/inc/Helper/KeyValueIO.h +++ b/AnnService/inc/Helper/KeyValueIO.h @@ -51,11 +51,16 @@ namespace SPTAG virtual bool Available() { return false; } - virtual ErrorCode Check(const SizeType key, int size, std::vector *visited) + virtual ErrorCode Check(const SizeType key, int size, std::vector *visited) { return ErrorCode::Undefined; } + virtual int64_t GetApproximateMemoryUsage() const + { + return 0; + } + virtual ErrorCode Checkpoint(std::string prefix) {return ErrorCode::Undefined;} virtual ErrorCode StartToScan(SizeType& key, std::string* value) {return ErrorCode::Undefined;} diff --git a/AnnService/src/Core/BKT/BKTIndex.cpp b/AnnService/src/Core/BKT/BKTIndex.cpp index 603272a30..77ecce4fc 100644 --- a/AnnService/src/Core/BKT/BKTIndex.cpp +++ b/AnnService/src/Core/BKT/BKTIndex.cpp @@ -69,9 +69,9 @@ template ErrorCode Index::LoadIndexDataFromMemory(const std::vec return ErrorCode::FailedParseValue; if (p_indexBlobs.size() <= 3) m_deletedID.Initialize(m_pSamples.R(), m_iDataBlockSize, m_iDataCapacity, - COMMON::Labelset::InvalidIDBehavior::AlwaysContains); + COMMON::LabelSet::InvalidIDBehavior::AlwaysContains); else if (m_deletedID.Load((char *)p_indexBlobs[3].Data(), m_iDataBlockSize, m_iDataCapacity, - COMMON::Labelset::InvalidIDBehavior::AlwaysContains) != ErrorCode::Success) + COMMON::LabelSet::InvalidIDBehavior::AlwaysContains) != ErrorCode::Success) return ErrorCode::FailedParseValue; if (m_pSamples.R() != m_pGraph.R() || m_pSamples.R() != m_deletedID.R()) @@ -104,9 +104,9 @@ ErrorCode Index::LoadIndexData(const std::vector -template &, SizeType, float), bool (*checkFilter)(const std::shared_ptr &, SizeType, std::function)> void Index::Search(COMMON::QueryResultSet &p_query, COMMON::WorkSpace &p_space, @@ -394,7 +394,7 @@ void Index::Search(COMMON::QueryResultSet &p_query, COMMON::WorkSpace &p_s } template -template &, SizeType, float), bool (*checkFilter)(const std::shared_ptr &, SizeType, std::function)> int Index::SearchIterative(COMMON::QueryResultSet &p_query, COMMON::WorkSpace &p_space, bool p_isFirst, @@ -485,7 +485,7 @@ template bool AlwaysTrue(Args...) return true; } -bool CheckIfNotDeleted(const COMMON::Labelset &deletedIDs, SizeType node) +bool CheckIfNotDeleted(const COMMON::LabelSet &deletedIDs, SizeType node) { return !deletedIDs.Contains(node); } @@ -828,7 +828,7 @@ ErrorCode Index::BuildIndex(const void *p_data, SizeType p_vectorNum, Dimensi m_pSamples.Initialize(p_vectorNum, p_dimension, m_iDataBlockSize, m_iDataCapacity, (T *)p_data, p_shareOwnership); m_deletedID.Initialize(p_vectorNum, m_iDataBlockSize, m_iDataCapacity, - COMMON::Labelset::InvalidIDBehavior::AlwaysContains); + COMMON::LabelSet::InvalidIDBehavior::AlwaysContains); if (DistCalcMethod::Cosine == m_iDistCalcMethod && !p_normalized) { @@ -906,7 +906,7 @@ template ErrorCode Index::RefineIndex(std::shared_ptrm_deletedID.Initialize(newR, m_iDataBlockSize, m_iDataCapacity, - COMMON::Labelset::InvalidIDBehavior::AlwaysContains); + COMMON::LabelSet::InvalidIDBehavior::AlwaysContains); COMMON::BKTree *newtree = &(ptr->m_pTrees); (*newtree).BuildTrees(ptr->m_pSamples, ptr->m_iDistCalcMethod, m_iNumberOfThreads); m_pGraph.RefineGraph(this, indices, reverseIndices, nullptr, &(ptr->m_pGraph), &(ptr->m_pTrees.GetSampleMap())); @@ -985,9 +985,9 @@ ErrorCode Index::RefineIndex(const std::vector ErrorCode Index::LoadIndexDataFromMemory(const std::vec return ErrorCode::FailedParseValue; if (p_indexBlobs.size() <= 3) m_deletedID.Initialize(m_pSamples.R(), m_iDataBlockSize, m_iDataCapacity, - COMMON::Labelset::InvalidIDBehavior::AlwaysContains); + COMMON::LabelSet::InvalidIDBehavior::AlwaysContains); else if (m_deletedID.Load((char *)p_indexBlobs[3].Data(), m_iDataBlockSize, m_iDataCapacity, - COMMON::Labelset::InvalidIDBehavior::AlwaysContains) != ErrorCode::Success) + COMMON::LabelSet::InvalidIDBehavior::AlwaysContains) != ErrorCode::Success) return ErrorCode::FailedParseValue; if (m_pSamples.R() != m_pGraph.R() || m_pSamples.R() != m_deletedID.R()) @@ -103,9 +103,9 @@ ErrorCode Index::LoadIndexData(const std::vector -template +template void Index::Search(COMMON::QueryResultSet &p_query, COMMON::WorkSpace &p_space) const { std::shared_lock lock(*(m_pTrees.m_lock)); @@ -272,12 +272,12 @@ void Index::Search(COMMON::QueryResultSet &p_query, COMMON::WorkSpace &p_s namespace StaticDispatch { -bool AlwaysTrue(const COMMON::Labelset &deletedIDs, SizeType node) +bool AlwaysTrue(const COMMON::LabelSet &deletedIDs, SizeType node) { return true; } -bool CheckIfNotDeleted(const COMMON::Labelset &deletedIDs, SizeType node) +bool CheckIfNotDeleted(const COMMON::LabelSet &deletedIDs, SizeType node) { return !deletedIDs.Contains(node); } @@ -490,7 +490,7 @@ ErrorCode Index::BuildIndex(const void *p_data, SizeType p_vectorNum, Dimensi m_pSamples.Initialize(p_vectorNum, p_dimension, m_iDataBlockSize, m_iDataCapacity, (T *)p_data, p_shareOwnership); m_deletedID.Initialize(p_vectorNum, m_iDataBlockSize, m_iDataCapacity, - COMMON::Labelset::InvalidIDBehavior::AlwaysContains); + COMMON::LabelSet::InvalidIDBehavior::AlwaysContains); if (DistCalcMethod::Cosine == m_iDistCalcMethod && !p_normalized) { @@ -567,7 +567,7 @@ template ErrorCode Index::RefineIndex(std::shared_ptrm_deletedID.Initialize(newR, m_iDataBlockSize, m_iDataCapacity, - COMMON::Labelset::InvalidIDBehavior::AlwaysContains); + COMMON::LabelSet::InvalidIDBehavior::AlwaysContains); COMMON::KDTree *newtree = &(ptr->m_pTrees); (*newtree).BuildTrees(ptr->m_pSamples, m_iNumberOfThreads); @@ -666,9 +666,9 @@ ErrorCode Index::RefineIndex(const std::vector::Check() std::vector mythreads; mythreads.reserve(m_options.m_iSSDNumberOfThreads); std::atomic_size_t sent(0); - std::vector checked(m_extraSearcher->GetNumBlocks(), false); + std::vector checked(m_extraSearcher->GetNumBlocks(), false); for (int tid = 0; tid < m_options.m_iSSDNumberOfThreads; tid++) { mythreads.emplace_back([&, tid]() { From 9f2ed71264f0ea2cd7c69d14b8b104113929e641 Mon Sep 17 00:00:00 2001 From: Zhou Lin Date: Tue, 27 Jan 2026 06:26:38 +0000 Subject: [PATCH 30/38] Fix labelset compiling issue --- .../Core/Common/{Labelset.h => LabelSet.h} | 0 AnnService/inc/Core/Common/WorkSpace.h | 27 +++++++++++++++++++ AnnService/inc/Core/SPANN/Index.h | 1 + 3 files changed, 28 insertions(+) rename AnnService/inc/Core/Common/{Labelset.h => LabelSet.h} (100%) diff --git a/AnnService/inc/Core/Common/Labelset.h b/AnnService/inc/Core/Common/LabelSet.h similarity index 100% rename from AnnService/inc/Core/Common/Labelset.h rename to AnnService/inc/Core/Common/LabelSet.h diff --git a/AnnService/inc/Core/Common/WorkSpace.h b/AnnService/inc/Core/Common/WorkSpace.h index 0eca10ef8..50e52794f 100644 --- a/AnnService/inc/Core/Common/WorkSpace.h +++ b/AnnService/inc/Core/Common/WorkSpace.h @@ -19,6 +19,8 @@ namespace SPTAG class IWorkSpaceFactory { public: + using Ptr = std::unique_ptr; + virtual std::unique_ptr GetWorkSpace() = 0; virtual void ReturnWorkSpace(std::unique_ptr ws) = 0; }; @@ -38,7 +40,32 @@ namespace SPTAG { m_workspace = std::move(ws); } + }; + + template + class SharedPoolWorkSpaceFactory : public IWorkSpaceFactory { + public: + virtual std::unique_ptr GetWorkSpace() override + { + std::unique_ptr ws; + std::lock_guard lock(m_mutex); + if (!m_pool.empty()) { + ws = std::move(m_pool.back()); + m_pool.pop_back(); + } + return ws; + } + + void ReturnWorkSpace(std::unique_ptr ws) override { + if (ws) { + std::lock_guard lock(m_mutex); + m_pool.emplace_back(std::move(ws)); + } + } + private: + std::mutex m_mutex; + std::vector> m_pool; }; class OptHashPosVector diff --git a/AnnService/inc/Core/SPANN/Index.h b/AnnService/inc/Core/SPANN/Index.h index 0479d6c85..446cb784c 100644 --- a/AnnService/inc/Core/SPANN/Index.h +++ b/AnnService/inc/Core/SPANN/Index.h @@ -72,6 +72,7 @@ namespace SPTAG Index() { m_workSpaceFactory = std::make_unique>(); + //m_workSpaceFactory = std::make_unique>(); m_fComputeDistance = std::function(COMMON::DistanceCalcSelector(m_options.m_distCalcMethod)); m_iBaseSquare = (m_options.m_distCalcMethod == DistCalcMethod::Cosine) ? COMMON::Utils::GetBase() * COMMON::Utils::GetBase() : 1; } From 133dfbf29f7bcbda639cb6abff7209c195ed4f40 Mon Sep 17 00:00:00 2001 From: MaggieQi Date: Tue, 27 Jan 2026 14:42:04 +0800 Subject: [PATCH 31/38] test quantization in SPFresh (#440) * test quantization in SPFresh * enable BKT in quantization testcase --------- Co-authored-by: qiazh --- AnnService/inc/Core/Common/LabelSet.h | 2 +- CMakeLists.txt | 3 +- Dockerfile | 4 +- Test/inc/TestDataGenerator.h | 6 +- Test/src/SPFreshTest.cpp | 310 ++++++++++++++++++++++++-- Test/src/TestDataGenerator.cpp | 7 +- benchmark.ini | 19 ++ 7 files changed, 321 insertions(+), 30 deletions(-) create mode 100644 benchmark.ini diff --git a/AnnService/inc/Core/Common/LabelSet.h b/AnnService/inc/Core/Common/LabelSet.h index 65b5ba7f3..df03c19d4 100644 --- a/AnnService/inc/Core/Common/LabelSet.h +++ b/AnnService/inc/Core/Common/LabelSet.h @@ -162,4 +162,4 @@ namespace SPTAG } } -#endif // _SPTAG_COMMON_LABELSET_H_ \ No newline at end of file +#endif // _SPTAG_COMMON_LABELSET_H_ diff --git a/CMakeLists.txt b/CMakeLists.txt index a49d6dc8b..2aab627c9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,7 +26,8 @@ if(${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU") message(FATAL_ERROR "GCC version must be at least 5.0!") endif() set (CMAKE_CXX_FLAGS "-Wall -Wunreachable-code -Wno-reorder -Wno-pessimizing-move -Wno-delete-non-virtual-dtor -Wno-sign-compare -Wno-unknown-pragmas -Wcast-align -lm -lrt -std=c++17 -fopenmp") - set (CMAKE_CXX_FLAGS_RELEASE "-DNDEBUG -O3 -march=native") + set (CMAKE_CXX_FLAGS_RELEASE "-DNDEBUG -O3 -mno-avx -mno-avx2") + set (CMAKE_C_FLAGS_RELEASE "-DNDEBUG -O3 -mno-avx -mno-avx2") set (CMAKE_CXX_FLAGS_DEBUG "-g -DDEBUG") if(USE_ASAN) diff --git a/Dockerfile b/Dockerfile index 1ac755f5e..6ad8fb079 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,9 +4,9 @@ WORKDIR /app ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get -y install wget build-essential swig cmake git libnuma-dev python3.8-dev python3-distutils gcc-8 g++-8 \ - libboost-filesystem-dev libboost-test-dev libboost-serialization-dev libboost-regex-dev libboost-serialization-dev libboost-regex-dev libboost-thread-dev libboost-system-dev + libboost-filesystem-dev libboost-test-dev libboost-serialization-dev libboost-regex-dev libboost-serialization-dev libboost-regex-dev libboost-thread-dev libboost-system-dev libtbb-dev -RUN wget https://bootstrap.pypa.io/get-pip.py && python3.8 get-pip.py && python3.8 -m pip install numpy +RUN wget https://bootstrap.pypa.io/pip/3.8/get-pip.py && python3.8 get-pip.py && python3.8 -m pip install numpy ENV PYTHONPATH=/app/Release diff --git a/Test/inc/TestDataGenerator.h b/Test/inc/TestDataGenerator.h index bafca1b1b..924a53b60 100644 --- a/Test/inc/TestDataGenerator.h +++ b/Test/inc/TestDataGenerator.h @@ -39,9 +39,9 @@ namespace TestUtils { std::shared_ptr &truths); void RunLargeBatches(std::string &vecset, std::string &metaset, std::string &metaidx, - std::string &addset, std::string &addmetaset, std::string &addmetaidx, - std::string &queryset, int bash, int batchinsert, int batchdelete, int batches, - std::string &truth); + std::string &addset, std::string &addmetaset, std::string &addmetaidx, + std::string &queryset, int bash, int batchinsert, int batchdelete, int batches, + std::string &truth, bool generateTruth = true); private: int m_n, m_a, m_q, m_m, m_k; diff --git a/Test/src/SPFreshTest.cpp b/Test/src/SPFreshTest.cpp index f98424b70..61232c1f8 100644 --- a/Test/src/SPFreshTest.cpp +++ b/Test/src/SPFreshTest.cpp @@ -7,19 +7,24 @@ #include "inc/Core/SPANN/Index.h" #include "inc/Core/SPANN/SPANNResultIterator.h" #include "inc/Core/VectorIndex.h" +#include "inc/Core/Common/IQuantizer.h" +#include "inc/Core/Common/PQQuantizer.h" #include "inc/Helper/DiskIO.h" #include "inc/Helper/SimpleIniReader.h" #include "inc/Helper/VectorSetReader.h" #include "inc/Helper/StringConvert.h" +#include "inc/Quantizer/Training.h" #include "inc/Test.h" #include "inc/TestDataGenerator.h" #include #include #include +#include #include #include #include +#include #include #include @@ -32,12 +37,126 @@ DimensionType M = 100; int K = 10; int queries = 10; +std::shared_ptr ConvertToFloatVectorSet(const std::shared_ptr& src) +{ + if (!src) + return nullptr; + + if (src->GetValueType() == VectorValueType::Float) + return src; + + SizeType count = src->Count(); + DimensionType dim = src->Dimension(); + ByteArray bytes = ByteArray::Alloc(sizeof(float) * (size_t)count * (size_t)dim); + float* out = reinterpret_cast(bytes.Data()); + + switch (src->GetValueType()) + { + case VectorValueType::Int8: + { + auto* in = reinterpret_cast(src->GetData()); + for (size_t i = 0; i < (size_t)count * (size_t)dim; ++i) + out[i] = static_cast(in[i]); + break; + } + case VectorValueType::UInt8: + { + auto* in = reinterpret_cast(src->GetData()); + for (size_t i = 0; i < (size_t)count * (size_t)dim; ++i) + out[i] = static_cast(in[i]); + break; + } + case VectorValueType::Int16: + { + auto* in = reinterpret_cast(src->GetData()); + for (size_t i = 0; i < (size_t)count * (size_t)dim; ++i) + out[i] = static_cast(in[i]); + break; + } + default: + return nullptr; + } + + return std::make_shared(bytes, VectorValueType::Float, dim, count); +} + + void RemoveStaticSsdIndexIfDynamic(const std::shared_ptr& index, const std::string& indexPath) + { + if (!index) + return; + + std::string storage = index->GetParameter("Storage", "BuildSSDIndex"); + if (storage == "STATIC") + return; + + std::string ssdIndex = index->GetParameter("SSDIndex", "Base"); + if (ssdIndex.empty()) + return; + + std::filesystem::path ssdIndexPath = std::filesystem::path(indexPath) / ssdIndex; + if (std::filesystem::exists(ssdIndexPath)) + { + std::filesystem::remove(ssdIndexPath); + } + } + +std::shared_ptr SliceQuantizedVectorSet(const ByteArray& data, SizeType start, SizeType count, DimensionType dim) +{ + size_t offset = (size_t)start * (size_t)dim; + size_t length = (size_t)count * (size_t)dim; + ByteArray view(data.Data() + offset, length, false); + return std::make_shared(view, VectorValueType::UInt8, dim, count); +} + +std::shared_ptr EnsurePQQuantizer(std::shared_ptr index, + const std::string& quantizerFile, + const std::shared_ptr& trainVectors, + DimensionType quantizedDim, + int threadNum) +{ + if (!index || !trainVectors) + return nullptr; + + std::shared_ptr quantizer; + if (index->LoadQuantizer(quantizerFile) == ErrorCode::Success) + { + quantizer = index->GetQuantizer(); + return quantizer; + } + + if (quantizedDim <= 0 || (trainVectors->Dimension() % quantizedDim) != 0) + return nullptr; + + auto options = std::make_shared( + trainVectors->Count(), false, 0.0f, QuantizerType::PQQuantizer, quantizerFile, quantizedDim, "", ""); + options->m_dimension = trainVectors->Dimension(); + options->m_threadNum = threadNum; + options->m_inputValueType = VectorValueType::Float; + options->m_trainingSamples = trainVectors->Count(); + + ByteArray pq_vector_array = ByteArray::Alloc(sizeof(std::uint8_t) * (size_t)quantizedDim * (size_t)trainVectors->Count()); + auto pq_vectors = std::make_shared(pq_vector_array, VectorValueType::UInt8, quantizedDim, trainVectors->Count()); + + auto codebooks = TrainPQQuantizer(options, trainVectors, pq_vectors); + if (!codebooks) + return nullptr; + + quantizer = std::make_shared>( + quantizedDim, 256, trainVectors->Dimension() / quantizedDim, false, std::move(codebooks)); + + auto fp = SPTAG::f_createIO(); + if (fp != nullptr && fp->Initialize(quantizerFile.c_str(), std::ios::binary | std::ios::out)) + quantizer->SaveQuantizer(fp); + + return quantizer; +} + template std::shared_ptr BuildIndex(const std::string &outDirectory, std::shared_ptr vecset, std::shared_ptr metaset, const std::string &distMethod = "L2", int searchthread = 2) { auto vecIndex = VectorIndex::CreateInstance(IndexAlgoType::SPANN, GetEnumValueType()); - int maxthreads = std::thread::hardware_concurrency(); + int maxthreads = std::max(1, searchthread); int postingLimit = 4 * sizeof(T); std::string configuration = R"( [Base] @@ -53,6 +172,7 @@ std::shared_ptr BuildIndex(const std::string &outDirectory, std::sh [SelectHead] isExecute=true NumberOfThreads=)" + std::to_string(maxthreads) + R"( + SelectHeadType=Random SelectThreshold=0 SplitFactor=0 SplitThreshold=0 @@ -130,7 +250,7 @@ std::shared_ptr BuildLargeIndex(const std::string &outDirectory, st int searchthread = 2, int insertthread = 2) { auto vecIndex = VectorIndex::CreateInstance(IndexAlgoType::SPANN, GetEnumValueType()); - int maxthreads = std::thread::hardware_concurrency(); + int maxthreads = std::max(1, searchthread); int postingLimit = 4 * sizeof(T); std::string configuration = R"( [Base] @@ -147,6 +267,7 @@ std::shared_ptr BuildLargeIndex(const std::string &outDirectory, st [SelectHead] isExecute=true NumberOfThreads=)" + std::to_string(maxthreads) + R"( + SelectHeadType=Random SelectThreshold=0 SplitFactor=0 SplitThreshold=0 @@ -332,8 +453,20 @@ void InsertVectors(SPANN::Index *p_index, int insertThreads, int step std::uint64_t *offsets = new std::uint64_t[2]{0, p_meta.Length()}; std::shared_ptr meta(new MemMetadataSet( p_meta, ByteArray((std::uint8_t *)offsets, 2 * sizeof(std::uint64_t), true), 1)); - BOOST_REQUIRE(p_index->AddIndex(addset->GetVector((SizeType)index), 1, p_opts.m_dim, meta, true) == - ErrorCode::Success); + // For quantized index, pass GetFeatureDim() which returns reconstruct dimension + DimensionType dim = p_index->GetFeatureDim(); + ErrorCode ret = p_index->AddIndex(addset->GetVector((SizeType)index), 1, dim, meta, true); + if (ret != ErrorCode::Success) + { + SPTAGLIB_LOG(Helper::LogLevel::LL_Error, + "AddIndex failed. VID:%zu Dim:%d IndexDim:%d Storage:%s Error:%d\n", + index, + dim, + p_index->GetFeatureDim(), + p_index->GetParameter("Storage", "BuildSSDIndex").c_str(), + static_cast(ret)); + } + BOOST_REQUIRE(ret == ErrorCode::Success); } else { @@ -441,6 +574,14 @@ void BenchmarkQueryPerformance(std::shared_ptr &index, std::shared_ // Recall evaluation (if truth file provided) + if (!truth || truthPath.empty() || truthPath == "none") + { + BOOST_TEST_MESSAGE(" Recall evaluation skipped (no truth data)"); + benchmarkData << prefix << " \"recall\": null\n"; + benchmarkData << prefix << " }"; + return; + } + BOOST_TEST_MESSAGE("Checking for truth file: " << truthPath); std::shared_ptr pvecset, paddvecset; float avgRecall = EvaluateRecall(results, index, queryset, truth, pvecset, paddvecset, baseVectorCount, topK, batches, totalbatches); @@ -459,7 +600,8 @@ template void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, const std::string &truthPath, DistCalcMethod distMethod, const std::string &indexPath, int dimension, int baseVectorCount, int insertVectorCount, int deleteVectorCount, int batches, int topK, int numThreads, int numQueries, - const std::string &outputFile = "output.json", const bool rebuild = true, const int resume = -1) + const std::string &outputFile = "output.json", const bool rebuild = true, const int resume = -1, + const std::string &quantizerFilePath = std::string(""), int quantizedDim = 0) { int oldM = M, oldK = K, oldN = N, oldQueries = queries; N = baseVectorCount; @@ -474,10 +616,12 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c std::ostringstream tmpbenchmark; // Generate test data + bool generateTruth = !(truthPath.empty() || truthPath == "none"); + bool enableQuantization = !quantizerFilePath.empty(); std::string pvecset, paddset, pqueryset, ptruth, pmeta, pmetaidx, paddmeta, paddmetaidx; TestUtils::TestDataGenerator generator(N, queries, M, K, dist, insertVectorCount, false, vectorPath, queryPath); generator.RunLargeBatches(pvecset, pmeta, pmetaidx, paddset, paddmeta, paddmetaidx, pqueryset, N, insertBatchSize, - deleteBatchSize, batches, ptruth); + deleteBatchSize, batches, ptruth, generateTruth); std::ofstream jsonFile(outputFile); BOOST_REQUIRE(jsonFile.is_open()); @@ -518,13 +662,83 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c jsonFile << " \"results\": {\n"; std::shared_ptr index; + std::shared_ptr quantizer; + DimensionType quantizedDimUsed = 0; + ByteArray quantizedAddBytes; // Build initial index BOOST_TEST_MESSAGE("\n=== Building Index ==="); if (rebuild || !direxists(indexPath.c_str())) { std::filesystem::remove_all(indexPath); auto buildstart = std::chrono::high_resolution_clock::now(); - index = BuildLargeIndex(indexPath, pvecset, pmeta, pmetaidx, dist, numThreads, numThreads); - BOOST_REQUIRE(index != nullptr); + + if (enableQuantization) + { + auto vectorOptions = std::shared_ptr( + new Helper::ReaderOptions(GetEnumValueType(), M, VectorFileType::DEFAULT)); + auto baseReader = Helper::VectorSetReader::CreateInstance(vectorOptions); + BOOST_REQUIRE(ErrorCode::Success == baseReader->LoadFile(pvecset)); + auto baseVectorsRaw = baseReader->GetVectorSet(); + + auto baseVectorsFloat = ConvertToFloatVectorSet(baseVectorsRaw); + BOOST_REQUIRE(baseVectorsFloat != nullptr); + + if (quantizedDim <= 0) + quantizedDim = dimension / 2; + BOOST_REQUIRE(quantizedDim > 0 && (dimension % quantizedDim) == 0); + + index = VectorIndex::CreateInstance(IndexAlgoType::SPANN, VectorValueType::UInt8); + BOOST_REQUIRE(index != nullptr); + index->SetParameter("IndexAlgoType", "BKT", "Base"); + index->SetParameter("IndexDirectory", indexPath.c_str(), "Base"); + index->SetParameter("DistCalcMethod", Helper::Convert::ConvertToString(distMethod).c_str(), "Base"); + index->SetParameter("QuantizerFilePath", quantizerFilePath.c_str(), "Base"); + + index->SetParameter("isExecute", "true", "SelectHead"); + index->SetParameter("NumberOfThreads", std::to_string(numThreads).c_str(), "SelectHead"); + index->SetParameter("SelectHeadType", "Random", "SelectHead"); + index->SetParameter("Ratio", "0.2", "SelectHead"); + + index->SetParameter("isExecute", "true", "BuildHead"); + index->SetParameter("NumberOfThreads", std::to_string(numThreads).c_str(), "BuildHead"); + + index->SetParameter("isExecute", "true", "BuildSSDIndex"); + index->SetParameter("BuildSsdIndex", "true", "BuildSSDIndex"); + index->SetParameter("NumberOfThreads", std::to_string(numThreads).c_str(), "BuildSSDIndex"); + index->SetParameter("InternalResultNum", "64", "BuildSSDIndex"); + index->SetParameter("SearchInternalResultNum", "64", "BuildSSDIndex"); + index->SetParameter("Storage", "FILEIO", "BuildSSDIndex"); + index->SetParameter("Update", "true", "BuildSSDIndex"); + index->SetParameter("SteadyState", "true", "BuildSSDIndex"); + index->SetParameter("InsertThreadNum", "1", "BuildSSDIndex"); + index->SetParameter("AppendThreadNum", "1", "BuildSSDIndex"); + index->SetParameter("ReassignThreadNum", "0", "BuildSSDIndex"); + index->SetParameter("DisableReassign", "false", "BuildSSDIndex"); + index->SetParameter("ReassignK", "64", "BuildSSDIndex"); + index->SetParameter("LatencyLimit", "50.0", "BuildSSDIndex"); + index->SetParameter("SearchDuringUpdate", "true", "BuildSSDIndex"); + index->SetParameter("MergeThreshold", "10", "BuildSSDIndex"); + index->SetParameter("Sampling", "4", "BuildSSDIndex"); + + quantizer = EnsurePQQuantizer(index, quantizerFilePath, baseVectorsFloat, (DimensionType)quantizedDim, numThreads); + BOOST_REQUIRE(quantizer != nullptr); + index->SetQuantizer(quantizer); + index->SetQuantizerADC(false); + quantizedDimUsed = static_cast(quantizer->GetNumSubvectors()); + index->SetParameter("Dim", std::to_string(quantizedDimUsed).c_str(), "Base"); + + ByteArray quantizedBaseBytes = ByteArray::Alloc((size_t)baseVectorCount * (size_t)quantizedDimUsed); + BOOST_REQUIRE(index->QuantizeVector(baseVectorsFloat->GetData(), baseVectorCount, quantizedBaseBytes) == ErrorCode::Success); + auto quantizedBase = std::make_shared(quantizedBaseBytes, VectorValueType::UInt8, quantizedDimUsed, baseVectorCount); + + std::shared_ptr metaset(new MemMetadataSet(pmeta, pmetaidx, baseVectorCount * 2, MaxSize, 10)); + BOOST_REQUIRE(index->BuildIndex(quantizedBase, metaset, true, false, false) == ErrorCode::Success); + } + else + { + index = BuildLargeIndex(indexPath, pvecset, pmeta, pmetaidx, dist, numThreads, numThreads); + BOOST_REQUIRE(index != nullptr); + } + auto buildend = std::chrono::high_resolution_clock::now(); double buildseconds = std::chrono::duration_cast(buildend - buildstart).count() / 1000000.0f; @@ -556,14 +770,39 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c return; } - auto opts = std::make_shared(GetEnumValueType(), K, VectorFileType::DEFAULT); - auto reader = Helper::VectorSetReader::CreateInstance(opts); - if (ErrorCode::Success != reader->LoadFile(ptruth)) + if (enableQuantization) { - SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Failed to read file %s\n", ptruth.c_str()); - return; + if (!quantizer) + { + quantizer = index->GetQuantizer(); + } + BOOST_REQUIRE(quantizer != nullptr); + quantizedDimUsed = static_cast(quantizer->GetNumSubvectors()); + + auto queryFloat = ConvertToFloatVectorSet(queryset); + BOOST_REQUIRE(queryFloat != nullptr); + ByteArray quantizedQueryBytes = ByteArray::Alloc((size_t)numQueries * (size_t)quantizedDimUsed); + BOOST_REQUIRE(index->QuantizeVector(queryFloat->GetData(), numQueries, quantizedQueryBytes) == ErrorCode::Success); + queryset = std::make_shared(quantizedQueryBytes, VectorValueType::UInt8, quantizedDimUsed, numQueries); + + auto addFloat = ConvertToFloatVectorSet(addReader->GetVectorSet()); + BOOST_REQUIRE(addFloat != nullptr); + quantizedAddBytes = ByteArray::Alloc((size_t)insertVectorCount * (size_t)quantizedDimUsed); + BOOST_REQUIRE(index->QuantizeVector(addFloat->GetData(), insertVectorCount, quantizedAddBytes) == ErrorCode::Success); + } + + std::shared_ptr truth; + if (generateTruth) + { + auto opts = std::make_shared(GetEnumValueType(), K, VectorFileType::DEFAULT); + auto reader = Helper::VectorSetReader::CreateInstance(opts); + if (ErrorCode::Success != reader->LoadFile(ptruth)) + { + SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Failed to read file %s\n", ptruth.c_str()); + return; + } + truth = reader->GetVectorSet(); } - auto truth = reader->GetVectorSet(); // Benchmark 0: Query performance before insertions BOOST_TEST_MESSAGE("\n=== Benchmark 0: Query Before Insertions ==="); @@ -576,6 +815,7 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c jsonFile.flush(); BOOST_REQUIRE(index->SaveIndex(indexPath) == ErrorCode::Success); + RemoveStaticSsdIndexIfDynamic(index, indexPath); index = nullptr; @@ -595,6 +835,10 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c jsonFile << " \"batch_" << iter + 1 << "\": {\n"; std::string clonePath = indexPath + "_" + std::to_string(iter); + if (std::filesystem::exists(clonePath)) + { + std::filesystem::remove_all(clonePath); + } std::shared_ptr prevIndex, cloneIndex; auto start = std::chrono::high_resolution_clock::now(); BOOST_REQUIRE(VectorIndex::LoadIndex(prevPath, prevIndex) == ErrorCode::Success); @@ -619,6 +863,14 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c jsonFile << " \"Clone timeSeconds\": " << seconds << ",\n"; prevIndex = nullptr; + + // If using quantization, update dimension after clone + if (enableQuantization) + { + cloneIndex->SetParameter("Dim", std::to_string(quantizedDimUsed)); + SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "Set cloned index Dim to %d for quantization\n", quantizedDimUsed); + } + ErrorCode cloneret = cloneIndex->Check(); BOOST_REQUIRE(cloneret == ErrorCode::Success); SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "Cloned index from %s to %s, check:%d, time: %f seconds\n", @@ -626,7 +878,15 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c int insertStart = iter * insertBatchSize; { - std::shared_ptr addset = addReader->GetVectorSet(insertStart, insertStart + insertBatchSize); + std::shared_ptr addset; + if (enableQuantization) + { + addset = SliceQuantizedVectorSet(quantizedAddBytes, insertStart, insertBatchSize, quantizedDimUsed); + } + else + { + addset = addReader->GetVectorSet(insertStart, insertStart + insertBatchSize); + } std::shared_ptr addmetaset(new MemMetadataSet(paddmeta, paddmetaidx, cloneIndex->m_iDataBlockSize, cloneIndex->m_iDataCapacity, 10, insertStart, insertBatchSize), std::default_delete()); start = std::chrono::high_resolution_clock::now(); @@ -707,6 +967,7 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c start = std::chrono::high_resolution_clock::now(); BOOST_REQUIRE(cloneIndex->SaveIndex(clonePath) == ErrorCode::Success); end = std::chrono::high_resolution_clock::now(); + RemoveStaticSsdIndexIfDynamic(cloneIndex, clonePath); seconds = std::chrono::duration_cast(end - start).count() / 1000000.0f; BOOST_TEST_MESSAGE(" Save Time: " << seconds << " seconds"); @@ -1736,6 +1997,8 @@ BOOST_AUTO_TEST_CASE(BenchmarkFromConfig) std::string queryPath = iniReader.GetParameter("Benchmark", "QueryPath", std::string("")); std::string truthPath = iniReader.GetParameter("Benchmark", "TruthPath", std::string("")); std::string indexPath = iniReader.GetParameter("Benchmark", "IndexPath", std::string("benchmark_index")); + std::string quantizerFilePath = iniReader.GetParameter("Benchmark", "QuantizerFilePath", std::string("")); + int quantizedDim = iniReader.GetParameter("Benchmark", "QuantizedDim", 0); VectorValueType valueType = VectorValueType::Float; std::string valueTypeStr = iniReader.GetParameter("Benchmark", "ValueType", std::string("Float")); @@ -1769,6 +2032,11 @@ BOOST_AUTO_TEST_CASE(BenchmarkFromConfig) BOOST_TEST_MESSAGE("Threads: " << numThreads); BOOST_TEST_MESSAGE("Queries: " << numQueries); BOOST_TEST_MESSAGE("DistMethod: " << Helper::Convert::ConvertToString(distMethod)); + if (!quantizerFilePath.empty()) + { + BOOST_TEST_MESSAGE("QuantizerFilePath: " << quantizerFilePath); + BOOST_TEST_MESSAGE("QuantizedDim: " << quantizedDim); + } // Get output file path from environment variable or use default const char *outputPath = std::getenv("BENCHMARK_OUTPUT"); @@ -1779,20 +2047,20 @@ BOOST_AUTO_TEST_CASE(BenchmarkFromConfig) if (valueType == VectorValueType::Float) { RunBenchmark(vectorPath, queryPath, truthPath, distMethod, indexPath, dimension, baseVectorCount, - insertVectorCount, deleteVectorCount, batchNum, topK, numThreads, numQueries, outputFile, - rebuild, resume); + insertVectorCount, deleteVectorCount, batchNum, topK, numThreads, numQueries, outputFile, + rebuild, resume, quantizerFilePath, quantizedDim); } else if (valueType == VectorValueType::Int8) { RunBenchmark(vectorPath, queryPath, truthPath, distMethod, indexPath, dimension, baseVectorCount, - insertVectorCount, deleteVectorCount, batchNum, topK, numThreads, numQueries, - outputFile, rebuild, resume); + insertVectorCount, deleteVectorCount, batchNum, topK, numThreads, numQueries, + outputFile, rebuild, resume, quantizerFilePath, quantizedDim); } else if (valueType == VectorValueType::UInt8) { RunBenchmark(vectorPath, queryPath, truthPath, distMethod, indexPath, dimension, baseVectorCount, - insertVectorCount, deleteVectorCount, batchNum, topK, numThreads, numQueries, - outputFile, rebuild, resume); + insertVectorCount, deleteVectorCount, batchNum, topK, numThreads, numQueries, + outputFile, rebuild, resume, quantizerFilePath, quantizedDim); } //std::filesystem::remove_all(indexPath); diff --git a/Test/src/TestDataGenerator.cpp b/Test/src/TestDataGenerator.cpp index 3ac1bf682..f3a57cb11 100644 --- a/Test/src/TestDataGenerator.cpp +++ b/Test/src/TestDataGenerator.cpp @@ -48,7 +48,7 @@ template void TestDataGenerator::RunLargeBatches(std::string &vecset, std::string &metaset, std::string &metaidx, std::string &addset, std::string &addmetaset, std::string &addmetaidx, std::string &queryset, int base, int batchinsert, int batchdelete, - int batches, std::string &truth) + int batches, std::string &truth, bool generateTruth) { vecset = "perftest_vector.bin"; metaset = "perftest_meta.bin"; @@ -63,7 +63,10 @@ void TestDataGenerator::RunLargeBatches(std::string &vecset, std::string &met GenerateVectorSet(vecset, metaset, metaidx, m_vectorPath, 0, m_n); GenerateVectorSet(queryset, empty, empty, m_queryPath, 0, m_q); GenerateVectorSet(addset, addmetaset, addmetaidx, m_vectorPath, m_n, m_a); - GenerateBatchTruth(truth, vecset, addset, queryset, base, batchinsert, batchdelete, batches, true); + if (generateTruth) + { + GenerateBatchTruth(truth, vecset, addset, queryset, base, batchinsert, batchdelete, batches, true); + } } template diff --git a/benchmark.ini b/benchmark.ini new file mode 100644 index 000000000..e2b400767 --- /dev/null +++ b/benchmark.ini @@ -0,0 +1,19 @@ +[Benchmark] +VectorPath=sift1b/base.100M.u8bin +QueryPath=sift1b/query.public.10K.u8bin +TruthPath=none +IndexPath=proidx/spann_index +ValueType=UInt8 +Dimension=128 +BaseVectorCount=10000 +InsertVectorCount=10000 +DeleteVectorCount=0 +BatchNum=10 +TopK=5 +NumThreads=8 +NumQueries=100 +DistMethod=L2 +Rebuild=true +Resume=-1 +QuantizerFilePath=quantizer.bin +QuantizedDim=64 From be5422f1a27662ceb18447cc759cd5b1834206c7 Mon Sep 17 00:00:00 2001 From: Zhou Lin Date: Tue, 27 Jan 2026 10:28:38 +0000 Subject: [PATCH 32/38] fix SPFresh Dynamic Quantize vector --- .vscode/launch.json | 2 +- .../inc/Core/SPANN/ExtraDynamicSearcher.h | 69 ++++++++++++++++--- Test/src/SPFreshTest.cpp | 35 +++++----- 3 files changed, 79 insertions(+), 27 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index bed3879ae..0cb6b4ec2 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -190,7 +190,7 @@ }, { "name": "BENCHMARK_CONFIG", - "value": "/home/superbench/SPTAG/Debug/benchmark.ini" + "value": "benchmark.ini" } ], "externalConsole": false, diff --git a/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h b/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h index b737bccb5..1401626d9 100644 --- a/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h +++ b/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h @@ -262,9 +262,20 @@ namespace SPTAG::SPANN { //headCandidates: search data structrue for "vid" vector //headID: the head vector that stands for vid - bool IsAssumptionBroken(VectorIndex* p_index, SizeType headID, QueryResult& headCandidates, SizeType vid) + bool IsAssumptionBroken(VectorIndex* p_index, SizeType headID, ValueType* vector, SizeType vid) { + COMMON::QueryResultSet headCandidates(vector, m_opt->m_reassignK); + void* rec_query = nullptr; + if (p_index->m_pQuantizer) { + rec_query = ALIGN_ALLOC(p_index->m_pQuantizer->ReconstructSize()); + p_index->m_pQuantizer->ReconstructVector((const uint8_t*)headCandidates.GetTarget(), rec_query); + headCandidates.SetTarget((ValueType*)rec_query, p_index->m_pQuantizer); + } p_index->SearchIndex(headCandidates); + if (rec_query) + { + ALIGN_FREE(rec_query); + } int replicaCount = 0; BasicResult* queryResults = headCandidates.GetResults(); std::vector selections(static_cast(m_opt->m_replicaCount)); @@ -314,8 +325,8 @@ namespace SPTAG::SPANN { avgDist += dist; distanceSet.push_back(dist); if (m_versionMap->Deleted(vid) || m_versionMap->GetVersion(vid) != version) continue; - COMMON::QueryResultSet headCandidates(reinterpret_cast(vectorId + m_metaDataSize), 64); - if (brokenID.find(vid) == brokenID.end() && IsAssumptionBroken(p_index, headID, headCandidates, vid)) { + + if (brokenID.find(vid) == brokenID.end() && IsAssumptionBroken(p_index, headID, reinterpret_cast(vectorId + m_metaDataSize), vid)) { /* float_t headDist = p_index->ComputeDistance(headCandidates.GetTarget(), p_index->GetSample(SplitHead)); float_t newHeadDist_1 = p_index->ComputeDistance(headCandidates.GetTarget(), p_index->GetSample(newHeads[0])); @@ -368,9 +379,19 @@ namespace SPTAG::SPANN { //"headID" is the head vector before split void QuantifySplitCaseB(ExtraWorkSpace* p_exWorkSpace, VectorIndex* p_index, SizeType headID, std::vector& newHeads, SizeType SplitHead, int split_order, int assumptionBrokenNum_top0, std::set& brokenID) { - COMMON::QueryResultSet nearbyHeads(reinterpret_cast(p_index->GetSample(headID)), 64); - std::vector postingLists; + COMMON::QueryResultSet nearbyHeads((ValueType*)(p_index->GetSample(headID)), m_opt->m_reassignK); + void* rec_query = nullptr; + if (p_index->m_pQuantizer) { + rec_query = ALIGN_ALLOC(p_index->m_pQuantizer->ReconstructSize()); + p_index->m_pQuantizer->ReconstructVector((const uint8_t*)nearbyHeads.GetTarget(), rec_query); + nearbyHeads.SetTarget((ValueType*)rec_query, p_index->m_pQuantizer); + } p_index->SearchIndex(nearbyHeads); + if (rec_query) + { + ALIGN_FREE(rec_query); + } + std::vector postingLists; std::string postingList; BasicResult* queryResults = nearbyHeads.GetResults(); int topk = 8; @@ -1021,9 +1042,18 @@ namespace SPTAG::SPANN { return ErrorCode::Success; } - QueryResult queryResults(p_index->GetSample(headID), m_opt->m_internalResultNum, false); + COMMON::QueryResultSet queryResults((ValueType*)(p_index->GetSample(headID)), m_opt->m_internalResultNum); + void* rec_query = nullptr; + if (p_index->m_pQuantizer) { + rec_query = ALIGN_ALLOC(p_index->m_pQuantizer->ReconstructSize()); + p_index->m_pQuantizer->ReconstructVector((const uint8_t*)queryResults.GetTarget(), rec_query); + queryResults.SetTarget((ValueType*)rec_query, p_index->m_pQuantizer); + } p_index->SearchIndex(queryResults); - + if (rec_query) + { + ALIGN_FREE(rec_query); + } std::string nextPostingList; for (int i = 1; i < queryResults.GetResultNum(); ++i) { @@ -1297,8 +1327,18 @@ namespace SPTAG::SPANN { std::vector HeadPrevTopK; newHeadsDist.clear(); newHeadsDist.resize(0); - COMMON::QueryResultSet nearbyHeads(headVector, m_opt->m_reassignK); + COMMON::QueryResultSet nearbyHeads((ValueType*)headVector, m_opt->m_reassignK); + void* rec_query = nullptr; + if (p_index->m_pQuantizer) { + rec_query = ALIGN_ALLOC(p_index->m_pQuantizer->ReconstructSize()); + p_index->m_pQuantizer->ReconstructVector((const uint8_t*)nearbyHeads.GetTarget(), rec_query); + nearbyHeads.SetTarget((ValueType*)rec_query, p_index->m_pQuantizer); + } p_index->SearchIndex(nearbyHeads); + if (rec_query) + { + ALIGN_FREE(rec_query); + } BasicResult* queryResults = nearbyHeads.GetResults(); for (int i = 0; i < nearbyHeads.GetResultNum(); i++) { auto vid = queryResults[i].VID; @@ -1351,9 +1391,18 @@ namespace SPTAG::SPANN { bool RNGSelection(std::vector& selections, ValueType* queryVector, VectorIndex* p_index, SizeType p_fullID, int& replicaCount, int checkHeadID = -1) { - QueryResult queryResults(queryVector, m_opt->m_internalResultNum, false); + COMMON::QueryResultSet queryResults(queryVector, m_opt->m_internalResultNum); + void* rec_query = nullptr; + if (p_index->m_pQuantizer) { + rec_query = ALIGN_ALLOC(p_index->m_pQuantizer->ReconstructSize()); + p_index->m_pQuantizer->ReconstructVector((const uint8_t*)queryResults.GetTarget(), rec_query); + queryResults.SetTarget((ValueType*)rec_query, p_index->m_pQuantizer); + } p_index->SearchIndex(queryResults); - + if (rec_query) + { + ALIGN_FREE(rec_query); + } replicaCount = 0; for (int i = 0; i < queryResults.GetResultNum() && replicaCount < m_opt->m_replicaCount; ++i) { diff --git a/Test/src/SPFreshTest.cpp b/Test/src/SPFreshTest.cpp index 61232c1f8..bc4412b10 100644 --- a/Test/src/SPFreshTest.cpp +++ b/Test/src/SPFreshTest.cpp @@ -172,7 +172,7 @@ std::shared_ptr BuildIndex(const std::string &outDirectory, std::sh [SelectHead] isExecute=true NumberOfThreads=)" + std::to_string(maxthreads) + R"( - SelectHeadType=Random + SelectHeadType=BKT SelectThreshold=0 SplitFactor=0 SplitThreshold=0 @@ -210,7 +210,7 @@ std::shared_ptr BuildIndex(const std::string &outDirectory, std::sh BufferLength=)" + std::to_string(postingLimit) + R"( InPlace=true StartFileSizeGB=1 - OneClusterCutMax=false + OneClusterCutMax=true ConsistencyCheck=true ChecksumCheck=true ChecksumInRead=false @@ -267,7 +267,7 @@ std::shared_ptr BuildLargeIndex(const std::string &outDirectory, st [SelectHead] isExecute=true NumberOfThreads=)" + std::to_string(maxthreads) + R"( - SelectHeadType=Random + SelectHeadType=BKT SelectThreshold=0 SplitFactor=0 SplitThreshold=0 @@ -308,7 +308,7 @@ std::shared_ptr BuildLargeIndex(const std::string &outDirectory, st BufferLength=)" + std::to_string(postingLimit) + R"( InPlace=true StartFileSizeGB=1 - OneClusterCutMax=false + OneClusterCutMax=true ConsistencyCheck=false ChecksumCheck=false ChecksumInRead=false @@ -366,7 +366,7 @@ std::vector SearchOnly(std::shared_ptr &vecIndex, std: template float EvaluateRecall(const std::vector &res, std::shared_ptr &vecIndex, std::shared_ptr &queryset, std::shared_ptr &truth, - std::shared_ptr &baseVec, std::shared_ptr &addVec, SizeType baseCount, int k, int batch, int totalbatches = -1) + std::shared_ptr &baseVec, std::shared_ptr &addVec, SizeType baseCount, int recallK, int k, int batch, int totalbatches = -1) { if (!truth) { @@ -374,7 +374,7 @@ float EvaluateRecall(const std::vector &res, std::shared_ptr(truth->Dimension())); + recallK = min(recallK, static_cast(truth->Dimension())); float totalRecall = 0.0f; float eps = 1e-4f; int distbase = (totalbatches + 1) * queryset->Count(); @@ -413,7 +413,7 @@ float EvaluateRecall(const std::vector &res, std::shared_ptrCount() * recallK); - SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "Recall %d@%d = %.4f\n", k, recallK, avgRecall); + SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "Recall %d@%d = %.4f\n", recallK, k, avgRecall); return avgRecall; } @@ -423,7 +423,7 @@ float Search(std::shared_ptr &vecIndex, std::shared_ptr std::shared_ptr &truth, SizeType baseCount, int batch = 0) { auto results = SearchOnly(vecIndex, queryset, k); - return EvaluateRecall(results, vecIndex, queryset, truth, baseVec, addVec, baseCount, k, batch); + return EvaluateRecall(results, vecIndex, queryset, truth, baseVec, addVec, baseCount, k, k, batch); } template @@ -493,7 +493,7 @@ void InsertVectors(SPANN::Index *p_index, int insertThreads, int step template void BenchmarkQueryPerformance(std::shared_ptr &index, std::shared_ptr &queryset, std::shared_ptr &truth, const std::string &truthPath, - SizeType baseVectorCount, int topK, int numThreads, int numQueries, int batches, int totalbatches, + SizeType baseVectorCount, int topK, int searchK, int numThreads, int numQueries, int batches, int totalbatches, std::ostream &benchmarkData, std::string prefix = "") { // Benchmark: Query performance with detailed latency stats @@ -503,7 +503,7 @@ void BenchmarkQueryPerformance(std::shared_ptr &index, std::shared_ for (int i = 0; i < numQueries; i++) { - results[i] = QueryResult((const T *)queryset->GetVector(i), topK, false); + results[i] = QueryResult((const T *)queryset->GetVector(i), searchK, false); } std::vector threads; @@ -584,8 +584,8 @@ void BenchmarkQueryPerformance(std::shared_ptr &index, std::shared_ BOOST_TEST_MESSAGE("Checking for truth file: " << truthPath); std::shared_ptr pvecset, paddvecset; - float avgRecall = EvaluateRecall(results, index, queryset, truth, pvecset, paddvecset, baseVectorCount, topK, batches, totalbatches); - BOOST_TEST_MESSAGE(" Recall@" << topK << " = " << (avgRecall * 100.0f) << "%"); + float avgRecall = EvaluateRecall(results, index, queryset, truth, pvecset, paddvecset, baseVectorCount, topK, searchK, batches, totalbatches); + BOOST_TEST_MESSAGE(" Recall" << topK << "@" << searchK << " = " << (avgRecall * 100.0f) << "%"); BOOST_TEST_MESSAGE(" (Evaluated on " << numQueries << " queries against base vectors)"); benchmarkData << std::fixed << std::setprecision(4); benchmarkData << prefix << " \"recall\": {\n"; @@ -648,6 +648,7 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c jsonFile << " \"queryPath\": \"" << queryPath << "\",\n"; jsonFile << " \"truthPath\": \"" << truthPath << "\",\n"; jsonFile << " \"indexPath\": \"" << indexPath << "\",\n"; + jsonFile << " \"quantizerPath\": \"" << quantizerFilePath << "\",\n"; jsonFile << " \"ValueType\": \"" << Helper::Convert::ConvertToString(GetEnumValueType()) << "\",\n"; jsonFile << " \"dimension\": " << dimension << ",\n"; jsonFile << " \"baseVectorCount\": " << baseVectorCount << ",\n"; @@ -661,6 +662,7 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c jsonFile << " },\n"; jsonFile << " \"results\": {\n"; + int SearchK = enableQuantization? topK * 4 : topK; std::shared_ptr index; std::shared_ptr quantizer; DimensionType quantizedDimUsed = 0; @@ -718,6 +720,7 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c index->SetParameter("SearchDuringUpdate", "true", "BuildSSDIndex"); index->SetParameter("MergeThreshold", "10", "BuildSSDIndex"); index->SetParameter("Sampling", "4", "BuildSSDIndex"); + index->SetParameter("OneClusterCutMax", "true", "BuildSSDIndex"); quantizer = EnsurePQQuantizer(index, quantizerFilePath, baseVectorsFloat, (DimensionType)quantizedDim, numThreads); BOOST_REQUIRE(quantizer != nullptr); @@ -806,10 +809,10 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c // Benchmark 0: Query performance before insertions BOOST_TEST_MESSAGE("\n=== Benchmark 0: Query Before Insertions ==="); - BenchmarkQueryPerformance(index, queryset, truth,truthPath, baseVectorCount, topK, + BenchmarkQueryPerformance(index, queryset, truth,truthPath, baseVectorCount, topK, SearchK, numThreads, numQueries, 0, batches, tmpbenchmark); jsonFile << " \"benchmark0_query_before_insert\": "; - BenchmarkQueryPerformance(index, queryset, truth, truthPath, baseVectorCount, topK, + BenchmarkQueryPerformance(index, queryset, truth, truthPath, baseVectorCount, topK, SearchK, numThreads, numQueries, 0, batches, jsonFile); jsonFile << ",\n"; jsonFile.flush(); @@ -958,10 +961,10 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c BOOST_TEST_MESSAGE("\n=== Benchmark 2: Query After Insertions and Deletions ==="); jsonFile << " \"search\":"; - BenchmarkQueryPerformance(cloneIndex, queryset, truth, truthPath, baseVectorCount, topK, numThreads, + BenchmarkQueryPerformance(cloneIndex, queryset, truth, truthPath, baseVectorCount, topK, SearchK, numThreads, numQueries, iter + 1, batches, tmpbenchmark, " "); BenchmarkQueryPerformance(cloneIndex, queryset, truth, truthPath, baseVectorCount, - topK, numThreads, numQueries, iter + 1, batches, jsonFile, " "); + topK, SearchK, numThreads, numQueries, iter + 1, batches, jsonFile, " "); jsonFile << ",\n"; start = std::chrono::high_resolution_clock::now(); From 90993f426f6501d5769245fba8647a550efb2ff2 Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Wed, 28 Jan 2026 05:51:12 +0000 Subject: [PATCH 33/38] add some fixes to dynamic cases --- .../inc/Core/SPANN/ExtraDynamicSearcher.h | 2 +- AnnService/src/Core/SPANN/SPANNIndex.cpp | 1 + Test/src/SPFreshTest.cpp | 182 +++++++++--------- 3 files changed, 94 insertions(+), 91 deletions(-) diff --git a/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h b/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h index 1401626d9..2167cf007 100644 --- a/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h +++ b/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h @@ -802,7 +802,7 @@ namespace SPTAG::SPANN { auto clusterBegin = std::chrono::high_resolution_clock::now(); // k = 2, maybe we can change the split number, now it is fixed - SPTAG::COMMON::KmeansArgs args(2, smallSample.C(), (SizeType)localIndices.size(), 1, p_index->GetDistCalcMethod()); + SPTAG::COMMON::KmeansArgs args(2, smallSample.C(), (SizeType)localIndices.size(), 1, p_index->GetDistCalcMethod(), p_index->m_pQuantizer); std::shuffle(localIndices.begin(), localIndices.end(), std::mt19937(std::random_device()())); int numClusters = SPTAG::COMMON::KmeansClustering(smallSample, localIndices, 0, (SizeType)localIndices.size(), args, 1000, 100.0F, false, nullptr); diff --git a/AnnService/src/Core/SPANN/SPANNIndex.cpp b/AnnService/src/Core/SPANN/SPANNIndex.cpp index 260c08e8d..0c34a17b8 100644 --- a/AnnService/src/Core/SPANN/SPANNIndex.cpp +++ b/AnnService/src/Core/SPANN/SPANNIndex.cpp @@ -938,6 +938,7 @@ bool Index::SelectHeadInternal(std::shared_ptr &p_re bkt->m_iSamples = m_options.m_iSamples; bkt->m_iTreeNumber = m_options.m_iTreeNumber; bkt->m_fBalanceFactor = m_options.m_fBalanceFactor; + bkt->m_pQuantizer = m_pQuantizer; SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "Start invoking BuildTrees.\n"); SPTAGLIB_LOG( Helper::LogLevel::LL_Info, diff --git a/Test/src/SPFreshTest.cpp b/Test/src/SPFreshTest.cpp index bc4412b10..1bb513dc3 100644 --- a/Test/src/SPFreshTest.cpp +++ b/Test/src/SPFreshTest.cpp @@ -108,20 +108,23 @@ std::shared_ptr SliceQuantizedVectorSet(const ByteArray& data, SizeTy return std::make_shared(view, VectorValueType::UInt8, dim, count); } -std::shared_ptr EnsurePQQuantizer(std::shared_ptr index, - const std::string& quantizerFile, +std::shared_ptr EnsurePQQuantizer(const std::string& quantizerFile, const std::shared_ptr& trainVectors, DimensionType quantizedDim, int threadNum) { - if (!index || !trainVectors) + if (!trainVectors) return nullptr; std::shared_ptr quantizer; - if (index->LoadQuantizer(quantizerFile) == ErrorCode::Success) - { - quantizer = index->GetQuantizer(); - return quantizer; + if (fileexists(quantizerFile.c_str())) { + auto ptr = SPTAG::f_createIO(); + if (ptr->Initialize(quantizerFile.c_str(), std::ios::binary | std::ios::in)) + { + quantizer = COMMON::IQuantizer::LoadIQuantizer(ptr); + BOOST_REQUIRE(quantizer != nullptr); + return quantizer; + } } if (quantizedDim <= 0 || (trainVectors->Dimension() % quantizedDim) != 0) @@ -156,11 +159,11 @@ std::shared_ptr BuildIndex(const std::string &outDirectory, std::sh std::shared_ptr metaset, const std::string &distMethod = "L2", int searchthread = 2) { auto vecIndex = VectorIndex::CreateInstance(IndexAlgoType::SPANN, GetEnumValueType()); - int maxthreads = std::max(1, searchthread); + int maxthreads = std::thread::hardware_concurrency(); int postingLimit = 4 * sizeof(T); std::string configuration = R"( [Base] - DistCalcMethod=L2 + DistCalcMethod=)" + distMethod + R"( IndexAlgoType=BKT ValueType=)" + Helper::Convert::ConvertToString(GetEnumValueType()) + R"( @@ -247,14 +250,14 @@ std::shared_ptr BuildIndex(const std::string &outDirectory, std::sh template std::shared_ptr BuildLargeIndex(const std::string &outDirectory, std::string &pvecset, std::string& pmetaset, std::string& pmetaidx, const std::string &distMethod = "L2", - int searchthread = 2, int insertthread = 2) + int searchthread = 2, int insertthread = 2, std::shared_ptr quantizer = nullptr, std::string quantizerFilePath = "quantizer.bin") { auto vecIndex = VectorIndex::CreateInstance(IndexAlgoType::SPANN, GetEnumValueType()); - int maxthreads = std::max(1, searchthread); + int maxthreads = std::thread::hardware_concurrency(); int postingLimit = 4 * sizeof(T); std::string configuration = R"( [Base] - DistCalcMethod=L2 + DistCalcMethod=)" + distMethod + R"( IndexAlgoType=BKT VectorPath=)" + pvecset + R"( ValueType=)" + Helper::Convert::ConvertToString(GetEnumValueType()) + @@ -335,6 +338,13 @@ std::shared_ptr BuildLargeIndex(const std::string &outDirectory, st } } + if (quantizer) + { + vecIndex->SetParameter("QuantizerFilePath", quantizerFilePath.c_str(), "Base"); + vecIndex->SetQuantizer(quantizer); + vecIndex->SetQuantizerADC(false); + vecIndex->SetParameter("Dim", std::to_string(quantizer->GetNumSubvectors()).c_str(), "Base"); + } auto buildStatus = vecIndex->BuildIndex(); if (buildStatus != ErrorCode::Success) return nullptr; @@ -596,6 +606,43 @@ void BenchmarkQueryPerformance(std::shared_ptr &index, std::shared_ benchmarkData << prefix << " }"; } +ErrorCode QuantizeVectors(const std::shared_ptr& quantizer, + const std::shared_ptr& srcVectors, + ByteArray& quantizedBytes) +{ + if (!quantizer || !srcVectors) + return ErrorCode::Fail; + + int maxthreads = std::thread::hardware_concurrency(); + std::vector threads; + threads.reserve(maxthreads); + std::atomic_size_t vectorsSent(0); + auto func = [&]() { + size_t index = 0; + while (true) + { + index = vectorsSent.fetch_add(1); + if (index < srcVectors->Count()) + { + quantizer->QuantizeVector(srcVectors->GetVector(index), quantizedBytes.Data() + index * (size_t)(quantizer->GetNumSubvectors()), false); + } + else + { + return; + } + } + }; + for (int j = 0; j < maxthreads; j++) + { + threads.emplace_back(func); + } + for (auto &thread : threads) + { + thread.join(); + } + return ErrorCode::Success; +} + template void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, const std::string &truthPath, DistCalcMethod distMethod, const std::string &indexPath, int dimension, int baseVectorCount, @@ -665,8 +712,7 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c int SearchK = enableQuantization? topK * 4 : topK; std::shared_ptr index; std::shared_ptr quantizer; - DimensionType quantizedDimUsed = 0; - ByteArray quantizedAddBytes; + // Build initial index BOOST_TEST_MESSAGE("\n=== Building Index ==="); if (rebuild || !direxists(indexPath.c_str())) { @@ -684,57 +730,23 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c auto baseVectorsFloat = ConvertToFloatVectorSet(baseVectorsRaw); BOOST_REQUIRE(baseVectorsFloat != nullptr); - if (quantizedDim <= 0) - quantizedDim = dimension / 2; + if (quantizedDim <= 0) quantizedDim = dimension / 2; BOOST_REQUIRE(quantizedDim > 0 && (dimension % quantizedDim) == 0); - index = VectorIndex::CreateInstance(IndexAlgoType::SPANN, VectorValueType::UInt8); - BOOST_REQUIRE(index != nullptr); - index->SetParameter("IndexAlgoType", "BKT", "Base"); - index->SetParameter("IndexDirectory", indexPath.c_str(), "Base"); - index->SetParameter("DistCalcMethod", Helper::Convert::ConvertToString(distMethod).c_str(), "Base"); - index->SetParameter("QuantizerFilePath", quantizerFilePath.c_str(), "Base"); - - index->SetParameter("isExecute", "true", "SelectHead"); - index->SetParameter("NumberOfThreads", std::to_string(numThreads).c_str(), "SelectHead"); - index->SetParameter("SelectHeadType", "Random", "SelectHead"); - index->SetParameter("Ratio", "0.2", "SelectHead"); - - index->SetParameter("isExecute", "true", "BuildHead"); - index->SetParameter("NumberOfThreads", std::to_string(numThreads).c_str(), "BuildHead"); - - index->SetParameter("isExecute", "true", "BuildSSDIndex"); - index->SetParameter("BuildSsdIndex", "true", "BuildSSDIndex"); - index->SetParameter("NumberOfThreads", std::to_string(numThreads).c_str(), "BuildSSDIndex"); - index->SetParameter("InternalResultNum", "64", "BuildSSDIndex"); - index->SetParameter("SearchInternalResultNum", "64", "BuildSSDIndex"); - index->SetParameter("Storage", "FILEIO", "BuildSSDIndex"); - index->SetParameter("Update", "true", "BuildSSDIndex"); - index->SetParameter("SteadyState", "true", "BuildSSDIndex"); - index->SetParameter("InsertThreadNum", "1", "BuildSSDIndex"); - index->SetParameter("AppendThreadNum", "1", "BuildSSDIndex"); - index->SetParameter("ReassignThreadNum", "0", "BuildSSDIndex"); - index->SetParameter("DisableReassign", "false", "BuildSSDIndex"); - index->SetParameter("ReassignK", "64", "BuildSSDIndex"); - index->SetParameter("LatencyLimit", "50.0", "BuildSSDIndex"); - index->SetParameter("SearchDuringUpdate", "true", "BuildSSDIndex"); - index->SetParameter("MergeThreshold", "10", "BuildSSDIndex"); - index->SetParameter("Sampling", "4", "BuildSSDIndex"); - index->SetParameter("OneClusterCutMax", "true", "BuildSSDIndex"); - - quantizer = EnsurePQQuantizer(index, quantizerFilePath, baseVectorsFloat, (DimensionType)quantizedDim, numThreads); + quantizer = EnsurePQQuantizer(quantizerFilePath, baseVectorsFloat, (DimensionType)quantizedDim, numThreads); BOOST_REQUIRE(quantizer != nullptr); - index->SetQuantizer(quantizer); - index->SetQuantizerADC(false); - quantizedDimUsed = static_cast(quantizer->GetNumSubvectors()); - index->SetParameter("Dim", std::to_string(quantizedDimUsed).c_str(), "Base"); - ByteArray quantizedBaseBytes = ByteArray::Alloc((size_t)baseVectorCount * (size_t)quantizedDimUsed); - BOOST_REQUIRE(index->QuantizeVector(baseVectorsFloat->GetData(), baseVectorCount, quantizedBaseBytes) == ErrorCode::Success); - auto quantizedBase = std::make_shared(quantizedBaseBytes, VectorValueType::UInt8, quantizedDimUsed, baseVectorCount); + std::string pquanvecset = "perftest_quanvectors.bin"; + { + ByteArray quantizedBaseBytes = ByteArray::Alloc((size_t)baseVectorCount * (size_t)quantizer->GetNumSubvectors()); + BOOST_REQUIRE(QuantizeVectors(quantizer, baseVectorsFloat, quantizedBaseBytes) == ErrorCode::Success); + auto quantizedBase = std::make_shared(quantizedBaseBytes, VectorValueType::UInt8, quantizer->GetNumSubvectors(), baseVectorCount); + quantizedBase->Save(pquanvecset); + } - std::shared_ptr metaset(new MemMetadataSet(pmeta, pmetaidx, baseVectorCount * 2, MaxSize, 10)); - BOOST_REQUIRE(index->BuildIndex(quantizedBase, metaset, true, false, false) == ErrorCode::Success); + index = BuildLargeIndex(indexPath, pquanvecset, pmeta, pmetaidx, dist, numThreads, numThreads, quantizer); + BOOST_REQUIRE(index != nullptr); + index->SetQuantizerADC(false); } else { @@ -765,14 +777,6 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c return; } auto queryset = queryReader->GetVectorSet(); - - auto addReader = Helper::VectorSetReader::CreateInstance(vectorOptions); - if (!fileexists(paddset.c_str()) || ErrorCode::Success != addReader->LoadFile(paddset)) - { - SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Cannot find or load %s. Using random generation!\n", paddset.c_str()); - return; - } - if (enableQuantization) { if (!quantizer) @@ -780,18 +784,14 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c quantizer = index->GetQuantizer(); } BOOST_REQUIRE(quantizer != nullptr); - quantizedDimUsed = static_cast(quantizer->GetNumSubvectors()); - - auto queryFloat = ConvertToFloatVectorSet(queryset); - BOOST_REQUIRE(queryFloat != nullptr); - ByteArray quantizedQueryBytes = ByteArray::Alloc((size_t)numQueries * (size_t)quantizedDimUsed); - BOOST_REQUIRE(index->QuantizeVector(queryFloat->GetData(), numQueries, quantizedQueryBytes) == ErrorCode::Success); - queryset = std::make_shared(quantizedQueryBytes, VectorValueType::UInt8, quantizedDimUsed, numQueries); - - auto addFloat = ConvertToFloatVectorSet(addReader->GetVectorSet()); - BOOST_REQUIRE(addFloat != nullptr); - quantizedAddBytes = ByteArray::Alloc((size_t)insertVectorCount * (size_t)quantizedDimUsed); - BOOST_REQUIRE(index->QuantizeVector(addFloat->GetData(), insertVectorCount, quantizedAddBytes) == ErrorCode::Success); + queryset = ConvertToFloatVectorSet(queryset); + } + + auto addReader = Helper::VectorSetReader::CreateInstance(vectorOptions); + if (!fileexists(paddset.c_str()) || ErrorCode::Success != addReader->LoadFile(paddset)) + { + SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Cannot find or load %s. Using random generation!\n", paddset.c_str()); + return; } std::shared_ptr truth; @@ -870,8 +870,7 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c // If using quantization, update dimension after clone if (enableQuantization) { - cloneIndex->SetParameter("Dim", std::to_string(quantizedDimUsed)); - SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "Set cloned index Dim to %d for quantization\n", quantizedDimUsed); + cloneIndex->SetParameter("Dim", std::to_string(quantizer->GetNumSubvectors()).c_str(), "Base"); } ErrorCode cloneret = cloneIndex->Check(); @@ -881,14 +880,17 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c int insertStart = iter * insertBatchSize; { - std::shared_ptr addset; - if (enableQuantization) - { - addset = SliceQuantizedVectorSet(quantizedAddBytes, insertStart, insertBatchSize, quantizedDimUsed); - } - else - { - addset = addReader->GetVectorSet(insertStart, insertStart + insertBatchSize); + std::shared_ptr addset = addReader->GetVectorSet(insertStart, insertStart + insertBatchSize); + ByteArray quantizedAddBytes; + if (enableQuantization) { + auto addFloat = ConvertToFloatVectorSet(addset); + BOOST_REQUIRE(addFloat != nullptr); + quantizedAddBytes = ByteArray::Alloc((size_t)addFloat->Count() * (size_t)(quantizer->GetNumSubvectors())); + BOOST_REQUIRE(QuantizeVectors(quantizer, addFloat, quantizedAddBytes) == ErrorCode::Success); + addset = std::make_shared(quantizedAddBytes, + VectorValueType::UInt8, + quantizer->GetNumSubvectors(), + addFloat->Count()); } std::shared_ptr addmetaset(new MemMetadataSet(paddmeta, paddmetaidx, cloneIndex->m_iDataBlockSize, cloneIndex->m_iDataCapacity, 10, insertStart, insertBatchSize), std::default_delete()); From 66d754add55afc3572580e61d044546f45364259 Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Wed, 28 Jan 2026 08:06:46 +0000 Subject: [PATCH 34/38] fix BKT reconstruct crashes --- AnnService/inc/Core/Common/BKTree.h | 19 ++++++++++--------- Test/src/SPFreshTest.cpp | 2 +- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/AnnService/inc/Core/Common/BKTree.h b/AnnService/inc/Core/Common/BKTree.h index 00019d97c..9addfb035 100644 --- a/AnnService/inc/Core/Common/BKTree.h +++ b/AnnService/inc/Core/Common/BKTree.h @@ -40,9 +40,10 @@ namespace SPTAG int _TH; DistCalcMethod _M; T* centers; - T* newTCenters; + T* newTCenters; SizeType* counts; - float* newCenters; + char* reconstructVectors; + float* newCenters; SizeType* newCounts; int* label; SizeType* clusterIdx; @@ -52,10 +53,11 @@ namespace SPTAG std::function fComputeDistance; const std::shared_ptr& m_pQuantizer; - KmeansArgs(int k, DimensionType dim, SizeType datasize, int threadnum, DistCalcMethod distMethod, const std::shared_ptr& quantizer = nullptr) : _K(k), _DK(k), _D(dim), _RD(dim), _TH(threadnum), _M(distMethod), m_pQuantizer(quantizer) { + KmeansArgs(int k, DimensionType dim, SizeType datasize, int threadnum, DistCalcMethod distMethod, const std::shared_ptr& quantizer = nullptr) : _K(k), _DK(k), _D(dim), _RD(dim), _TH(threadnum), _M(distMethod), m_pQuantizer(quantizer), reconstructVectors(nullptr) { if (m_pQuantizer) { _RD = m_pQuantizer->ReconstructDim(); fComputeDistance = m_pQuantizer->DistanceCalcSelector(distMethod); + reconstructVectors = new char [_TH * m_pQuantizer->ReconstructSize()]; } else { fComputeDistance = COMMON::DistanceCalcSelector(distMethod); @@ -75,8 +77,9 @@ namespace SPTAG ~KmeansArgs() { ALIGN_FREE(centers); - ALIGN_FREE(newTCenters); + ALIGN_FREE(newTCenters); delete[] counts; + if (reconstructVectors) delete[] reconstructVectors; delete[] newCenters; delete[] newCounts; delete[] label; @@ -239,7 +242,7 @@ namespace SPTAG float idist = 0; R *reconstructVector = nullptr; if (args.m_pQuantizer) - reconstructVector = (R *)ALIGN_ALLOC(args.m_pQuantizer->ReconstructSize()); + reconstructVector = (R *)(args.reconstructVectors + tid * args.m_pQuantizer->ReconstructSize()); for (SizeType i = istart; i < iend; i++) { @@ -248,7 +251,7 @@ namespace SPTAG for (int k = 0; k < args._DK; k++) { float dist = args.fComputeDistance(data[indices[i]], args.centers + k * args._D, args._D) + - lambda * args.counts[k]; + lambda * args.counts[k]; if (dist > -MaxDist && dist < smallestDist) { clusterid = k; @@ -264,7 +267,7 @@ namespace SPTAG if (args.m_pQuantizer) { args.m_pQuantizer->ReconstructVector((const uint8_t *)data[indices[i]], - reconstructVector); + reconstructVector); } else { @@ -289,8 +292,6 @@ namespace SPTAG } } } - if (args.m_pQuantizer) - ALIGN_FREE(reconstructVector); COMMON::Utils::atomic_float_add(&currDist, idist); }); } diff --git a/Test/src/SPFreshTest.cpp b/Test/src/SPFreshTest.cpp index 1bb513dc3..a830432ff 100644 --- a/Test/src/SPFreshTest.cpp +++ b/Test/src/SPFreshTest.cpp @@ -270,7 +270,7 @@ std::shared_ptr BuildLargeIndex(const std::string &outDirectory, st [SelectHead] isExecute=true NumberOfThreads=)" + std::to_string(maxthreads) + R"( - SelectHeadType=BKT + SelectHeadType=Random SelectThreshold=0 SplitFactor=0 SplitThreshold=0 From 6c8c4ee50e4fec707674cb3bcda6c16cc2a9a54c Mon Sep 17 00:00:00 2001 From: Zhou Lin Date: Wed, 28 Jan 2026 10:00:45 +0000 Subject: [PATCH 35/38] remove OPQ training dependance --- Tools/OPQ/OPQ_gpu_train_infer.py | 173 +++++++++++++++++++++---------- 1 file changed, 119 insertions(+), 54 deletions(-) diff --git a/Tools/OPQ/OPQ_gpu_train_infer.py b/Tools/OPQ/OPQ_gpu_train_infer.py index 082b272d3..5a8fce7ca 100644 --- a/Tools/OPQ/OPQ_gpu_train_infer.py +++ b/Tools/OPQ/OPQ_gpu_train_infer.py @@ -1,6 +1,10 @@ import numpy as np +import math +import tqdm +import time from struct import pack, unpack, calcsize from struct import pack, unpack, calcsize +from typing import Dict, List import heapq import argparse import copy @@ -26,7 +30,7 @@ def get_config(): parser.add_argument('--data_format', type = str, default = "DEFAULT", help='data format') parser.add_argument('--task', type = int, default = 0, help='task id') parser.add_argument('-log_dir', type = str, default = "", help='debug log dir in cosmos') - + parser.add_argument('--T', type = int, default = 32, help="thread number") parser.add_argument('--train_samples', type = int, default = 1000000, help='OPQ, PQ training samples') parser.add_argument('--quan_type', type = str, default = 'none', help='quantizer type') @@ -63,7 +67,7 @@ def __init__(self, filename, featuredim, batchsize, normalize, datatype, targett def norm(self, data): square = np.sqrt(np.sum(np.square(data), axis=1)) - data[square < 1e-6] = 1e-6 / math.sqrt(float(self.featuredim)) + data[square < 1e-6] = 1e-6 / math.sqrt(float(self.featuredim)) square[square < 1e-6] = 1e-6 data = data / square.reshape([-1, 1]) return data @@ -96,7 +100,7 @@ def readbatch(self): print ('Load batch query size:%r' % (i)) if self.normalize != 0: return i, self.norm(self.query[0:i]) return i, self.query[0:i] - + def readallbatches(self): numQuerys = self.query.shape[0] data = [] @@ -156,7 +160,7 @@ def gpusearch(args): fout = open('truth.txt.%d' % batch, 'w') foutd = open('dist.bin.%d' % batch, 'wb') - + foutd.write(pack('i', RQ)) foutd.write(pack('i', args.k)) @@ -170,7 +174,7 @@ def gpusearch(args): fout.close() foutd.close() - + if args.B <= 0 or args.B >= totaldata: args.B = totaldata truth = [[] for j in range(RQ)] @@ -206,34 +210,102 @@ def gpusearch(args): fout.close() foutd.close() +def search(faiss_index, + query_embeddings: np.ndarray, + topk: int = 1000, + nprobe: int = None, + batch_size: int = 64): + import faiss + if nprobe is not None: + if isinstance(faiss_index, faiss.IndexPreTransform): + ivf_index = faiss.downcast_index(faiss_index.index) + ivf_index.nprobe = nprobe + else: + faiss_index.nprobe = nprobe + + start_time = time.time() + if batch_size: + batch_num = math.ceil(len(query_embeddings) / batch_size) + all_scores = [] + all_search_results = [] + for step in tqdm(range(batch_num)): + start = batch_size * step + end = min(batch_size * (step + 1), len(query_embeddings)) + batch_emb = np.array(query_embeddings[start:end]) + score, batch_results = faiss_index.search(batch_emb, topk) + all_search_results.extend([list(x) for x in batch_results]) + all_scores.extend([list(x) for x in score]) + else: + all_scores, all_search_results = faiss_index.search(query_embeddings, topk) + search_time = time.time() - start_time + print( + f'number of query:{len(query_embeddings)}, searching time per query: {search_time / len(query_embeddings)}') + return all_scores, all_search_results + +def evaluate(retrieve_results: List[List[int]], + ground_truths: Dict[int, List[int]], + MRR_cutoffs: List[int] = [10], + Recall_cutoffs: List[int] = [5, 10, 50], + qids: List[int] = None): + """ + calculate MRR and Recall + """ + MRR = [0.0] * len(MRR_cutoffs) + Recall = [0.0] * len(Recall_cutoffs) + ranking = [] + if qids is None: + qids = list(range(len(retrieve_results))) + for qid, candidate_pid in zip(qids, retrieve_results): + if qid in ground_truths: + target_pid = ground_truths[qid] + ranking.append(-1) + + for i in range(0, max(MRR_cutoffs)): + if candidate_pid[i] in target_pid: + ranking.pop() + ranking.append(i + 1) + for inx, cutoff in enumerate(MRR_cutoffs): + if i <= cutoff - 1: + MRR[inx] += 1 / (i + 1) + break + + for i, k in enumerate(Recall_cutoffs): + Recall[i] += (len(set.intersection(set(target_pid), set(candidate_pid[:k]))) / len(set(target_pid))) + + if len(ranking) == 0: + raise IOError("No matching QIDs found. Are you sure you are scoring the evaluation set?") + + print(f"{len(ranking)} matching queries found") + MRR = [x / len(ranking) for x in MRR] + for i, k in enumerate(MRR_cutoffs): + print(f'MRR@{k}:{MRR[i]}') + + Recall = [x / len(ranking) for x in Recall] + for i, k in enumerate(Recall_cutoffs): + print(f'Recall@{k}:{Recall[i]}') + + return MRR, Recall + def train_pq(args): import faiss - from LibVQ.base_index import FaissIndex output_dir = args.output_dir datareader = DataReader(args.data_file, args.dim, args.train_samples, args.data_normalize, args.data_type, args.target_type) - + print ('train PQ...') - - index_method = 'pq' - ivf_centers_num = -1 + subvector_num = args.quan_dim subvector_bits = 8 numData, data = datareader.readbatch() - + faiss.omp_set_num_threads(args.T) - index = FaissIndex(index_method=index_method, - emb_size=len(data[0]), - ivf_centers_num=ivf_centers_num, - subvector_num=subvector_num, - subvector_bits=subvector_bits, - dist_mode='l2') + faiss_index = faiss.index_factory(len(data[0]), f"PQ{subvector_num}x{subvector_bits}", faiss.METRIC_L2) print('Training the index with doc embeddings') - index.fit(data) - + faiss_index.train(data) + rtype = np.uint8(0) if args.data_type == 'uint8': rtype = np.uint8(1) @@ -241,8 +313,7 @@ def train_pq(args): rtype = np.uint8(2) elif args.data_type == 'float32': rtype = np.uint8(3) - - faiss_index = index.index + ivf_index = faiss.downcast_index(faiss_index) centroid_embedings = faiss.vector_to_array(ivf_index.pq.centroids) codebooks = centroid_embedings.reshape(ivf_index.pq.M, ivf_index.pq.ksub, ivf_index.pq.dsub) @@ -276,8 +347,8 @@ def train_pq(args): writeitems = 0 while numData > 0: - if args.quan_test > 0: index.add(data) - + if args.quan_test > 0: faiss_index.add(data) + codes = ivf_index.pq.compute_codes(data) print ('codes shape:') @@ -289,10 +360,10 @@ def train_pq(args): if len(args.output_rec_vector_file) > 0: reconstructed = ivf_index.pq.decode(codes).astype(args.data_type) frec.write(reconstructed.tobytes()) - + writeitems += numData numData, data = datareader.readbatch() - + datareader.close() if len(args.output_quan_vector_file) > 0: @@ -330,37 +401,32 @@ def train_pq(args): f.close() # Test the performance - index.test(query, qid2ground_truths, topk=args.k, batch_size=64, - MRR_cutoffs=[5, 10], Recall_cutoffs=[5, 10, 20, 40]) + scores, retrieve_results = search(faiss_index, query, topk=args.k, nprobe = 1, batch_size=64) + evaluate(retrieve_results, qid2ground_truths, MRR_cutoffs=[5, 10], Recall_cutoffs=[5, 10, 20, 40], qids=None) def train_opq(args): import faiss - from LibVQ.base_index import FaissIndex output_dir = args.output_dir datareader = DataReader(args.data_file, args.dim, args.train_samples, args.data_normalize, args.data_type, args.target_type) - + print ('train OPQ...') - + index_method = 'opq' ivf_centers_num = -1 subvector_num = args.quan_dim subvector_bits = 8 numData, data = datareader.readbatch() - + data = data.astype(np.float32) + args.data_type = 'float32' faiss.omp_set_num_threads(args.T) - index = FaissIndex(index_method=index_method, - emb_size=len(data[0]), - ivf_centers_num=ivf_centers_num, - subvector_num=subvector_num, - subvector_bits=subvector_bits, - dist_mode='l2') + faiss_index = faiss.index_factory(len(data[0]), f"OPQ{subvector_num},PQ{subvector_num}x{subvector_bits}", faiss.METRIC_L2) print('Training the index with doc embeddings') - index.fit(data) - + faiss_index.train(data) + rtype = np.uint8(0) if args.data_type == 'uint8': rtype = np.uint8(1) @@ -368,8 +434,7 @@ def train_opq(args): rtype = np.uint8(2) elif args.data_type == 'float32': rtype = np.uint8(3) - - faiss_index = index.index + if isinstance(faiss_index, faiss.IndexPreTransform): vt = faiss.downcast_VectorTransform(faiss_index.chain.at(0)) assert isinstance(vt, faiss.LinearTransform) @@ -395,9 +460,8 @@ def train_opq(args): f.write(codebooks.tobytes()) f.write(rotate_matrix.tobytes()) - if args.quan_test == 0 and len(args.output_quan_vector_file) == 0 and len(args.output_rec_vector_file) == 0: + if args.quan_test == 0 and len(args.output_quan_vector_file) == 0 and len(args.output_rec_vector_file) == 0: os.rename(args.output_truth, os.path.join(output_dir, 'truth.txt' + '.' + str(args.task))) - ret = subprocess.run(['ZipKDTree.exe', output_dir, args.output_truth]) print (ret) return @@ -413,8 +477,8 @@ def train_opq(args): writeitems = 0 while numData > 0: - if args.quan_test > 0: index.add(data) - + if args.quan_test > 0: faiss_index.add(data) + rdata = np.matmul(data, rotate.T) codes = ivf_index.pq.compute_codes(rdata) @@ -427,9 +491,10 @@ def train_opq(args): Y = ivf_index.pq.decode(codes) reconstructed = np.matmul(Y, rotate).astype(args.data_type) frec.write(reconstructed.tobytes()) - + writeitems += numData numData, data = datareader.readbatch() + data = data.astype(np.float32) datareader.close() @@ -469,8 +534,8 @@ def train_opq(args): f.close() # Test the performance - index.test(query, qid2ground_truths, topk=args.k, batch_size=64, - MRR_cutoffs=[5, 10], Recall_cutoffs=[5, 10, 20, 40]) + scores, retrieve_results = search(faiss_index, query, topk=args.k, nprobe = 1, batch_size=64) + evaluate(retrieve_results, qid2ground_truths, MRR_cutoffs=[5, 10], Recall_cutoffs=[5, 10, 20, 40], qids=None) def quan_reconstruct_vectors(args): import faiss @@ -508,7 +573,7 @@ def quan_reconstruct_vectors(args): if pqtype == 2: f.write(rotate_matrix.tobytes()) f.close() - if len(args.output_quan_vector_file) == 0 and len(args.output_rec_vector_file) == 0: + if len(args.output_quan_vector_file) == 0 and len(args.output_rec_vector_file) == 0: os.rename(args.output_truth, os.path.join(output_dir, 'truth.txt' + '.' + str(args.task))) ret = subprocess.run(['ZipKDTree.exe', output_dir, args.output_truth]) print (ret) @@ -566,7 +631,7 @@ def fourcc(x): writeitems = 0 while numData > 0: print (data[0]) - if pqtype == 2: + if pqtype == 2: data = np.matmul(data, rotate_matrix) print ('rotate:') print (data[0]) @@ -587,7 +652,7 @@ def fourcc(x): print ('rotateback:') print (recY[0]) frec.write(recY.tobytes()) - + writeitems += numData numData, data = datareader.readbatch() @@ -635,7 +700,7 @@ def fourcc(x): ret = subprocess.run([target, args.data_file, os.path.join(args.output_dir, 'vectors.bin.%d' % args.task), os.path.join(args.output_dir, 'meta.bin.%d' % args.task), os.path.join(args.output_dir, 'metaindex.bin.%d' % args.task), str(args.dim), casttype, '0']) args.data_file = os.path.join(args.output_dir, 'vectors.bin.%d' % args.task) print(ret) - + if args.query_file[-4:] != '.bin': casttype = 'BYTE' if args.data_type == 'uint8': casttype = 'UBYTE' @@ -645,8 +710,8 @@ def fourcc(x): args.query_file = args.query_file + '_queryVector.bin' print(ret) - gpusearch(args) - + #gpusearch(args) + if args.quan_type != 'none': if args.quan_type == 'pq': train_pq(args) From 149e3e8b75ec4e3f3d0664df57ef73f89124df1d Mon Sep 17 00:00:00 2001 From: Zhou Lin Date: Tue, 3 Feb 2026 09:26:07 +0000 Subject: [PATCH 36/38] fix BKT clustering atomic_float_add issue --- AnnService/inc/Core/Common/BKTree.h | 12 ++++---- AnnService/inc/Core/Common/CommonUtils.h | 8 ++--- .../inc/Core/SPANN/ExtraDynamicSearcher.h | 1 - CMakeLists.txt | 6 ++-- Test/src/SPFreshTest.cpp | 6 ++-- Tools/OPQ/OPQ_gpu_train_infer.py | 30 +------------------ Tools/OPQ/README.md | 4 ++- 7 files changed, 20 insertions(+), 47 deletions(-) diff --git a/AnnService/inc/Core/Common/BKTree.h b/AnnService/inc/Core/Common/BKTree.h index 9addfb035..d51a75ab3 100644 --- a/AnnService/inc/Core/Common/BKTree.h +++ b/AnnService/inc/Core/Common/BKTree.h @@ -39,10 +39,10 @@ namespace SPTAG DimensionType _RD; int _TH; DistCalcMethod _M; + int8_t* reconstructVectors; T* centers; - T* newTCenters; + T* newTCenters; SizeType* counts; - char* reconstructVectors; float* newCenters; SizeType* newCounts; int* label; @@ -53,11 +53,11 @@ namespace SPTAG std::function fComputeDistance; const std::shared_ptr& m_pQuantizer; - KmeansArgs(int k, DimensionType dim, SizeType datasize, int threadnum, DistCalcMethod distMethod, const std::shared_ptr& quantizer = nullptr) : _K(k), _DK(k), _D(dim), _RD(dim), _TH(threadnum), _M(distMethod), m_pQuantizer(quantizer), reconstructVectors(nullptr) { + KmeansArgs(int k, DimensionType dim, SizeType datasize, int threadnum, DistCalcMethod distMethod, const std::shared_ptr& quantizer = nullptr) : _K(k), _DK(k), _D(dim), _RD(dim), _TH(threadnum), _M(distMethod), m_pQuantizer(quantizer), reconstructVectors(nullptr) { if (m_pQuantizer) { _RD = m_pQuantizer->ReconstructDim(); fComputeDistance = m_pQuantizer->DistanceCalcSelector(distMethod); - reconstructVectors = new char [_TH * m_pQuantizer->ReconstructSize()]; + reconstructVectors = (int8_t*)ALIGN_ALLOC(_TH * m_pQuantizer->ReconstructSize()); } else { fComputeDistance = COMMON::DistanceCalcSelector(distMethod); @@ -76,10 +76,10 @@ namespace SPTAG } ~KmeansArgs() { + if (reconstructVectors) ALIGN_FREE(reconstructVectors); ALIGN_FREE(centers); - ALIGN_FREE(newTCenters); + ALIGN_FREE(newTCenters); delete[] counts; - if (reconstructVectors) delete[] reconstructVectors; delete[] newCenters; delete[] newCounts; delete[] label; diff --git a/AnnService/inc/Core/Common/CommonUtils.h b/AnnService/inc/Core/Common/CommonUtils.h index 697eb521f..84f4f6ded 100644 --- a/AnnService/inc/Core/Common/CommonUtils.h +++ b/AnnService/inc/Core/Common/CommonUtils.h @@ -33,18 +33,18 @@ namespace SPTAG static inline float atomic_float_add(volatile float* ptr, const float operand) { union { - volatile long iOld; + volatile int iOld; float fOld; }; union { - long iNew; + int iNew; float fNew; }; while (true) { - iOld = *(volatile long *)ptr; + iOld = *(volatile int *)ptr; fNew = fOld + operand; - if (InterlockedCompareExchange((long *)ptr, iNew, iOld) == iOld) { + if (InterlockedCompareExchange((int *)ptr, iNew, iOld) == iOld) { return fNew; } } diff --git a/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h b/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h index 2167cf007..b64b5bb37 100644 --- a/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h +++ b/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h @@ -1796,7 +1796,6 @@ namespace SPTAG::SPANN { { if (!m_opt->m_checksumInRead) return true; - ErrorCode ret; for (int i = 0; i < pids.size(); i++) { if (!m_checkSum.ValidateChecksum((const char *)(postings[i].GetBuffer()), diff --git a/CMakeLists.txt b/CMakeLists.txt index 2aab627c9..d83b648b0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,9 +25,9 @@ if(${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU") if (CXX_COMPILER_VERSION VERSION_LESS 5.0) message(FATAL_ERROR "GCC version must be at least 5.0!") endif() - set (CMAKE_CXX_FLAGS "-Wall -Wunreachable-code -Wno-reorder -Wno-pessimizing-move -Wno-delete-non-virtual-dtor -Wno-sign-compare -Wno-unknown-pragmas -Wcast-align -lm -lrt -std=c++17 -fopenmp") - set (CMAKE_CXX_FLAGS_RELEASE "-DNDEBUG -O3 -mno-avx -mno-avx2") - set (CMAKE_C_FLAGS_RELEASE "-DNDEBUG -O3 -mno-avx -mno-avx2") + set (CMAKE_CXX_FLAGS "-Wall -Wunreachable-code -Wno-reorder -Wno-pessimizing-move -Wno-delete-non-virtual-dtor -Wno-sign-compare -Wno-unknown-pragmas -Wcast-align -lm -lrt -std=c++17 -fopenmp -fno-omit-frame-pointer") + set (CMAKE_CXX_FLAGS_RELEASE "-DNDEBUG -O3") + set (CMAKE_C_FLAGS_RELEASE "-DNDEBUG -O3") set (CMAKE_CXX_FLAGS_DEBUG "-g -DDEBUG") if(USE_ASAN) diff --git a/Test/src/SPFreshTest.cpp b/Test/src/SPFreshTest.cpp index a830432ff..45952f775 100644 --- a/Test/src/SPFreshTest.cpp +++ b/Test/src/SPFreshTest.cpp @@ -270,7 +270,7 @@ std::shared_ptr BuildLargeIndex(const std::string &outDirectory, st [SelectHead] isExecute=true NumberOfThreads=)" + std::to_string(maxthreads) + R"( - SelectHeadType=Random + SelectHeadType=BKT SelectThreshold=0 SplitFactor=0 SplitThreshold=0 @@ -744,9 +744,9 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c quantizedBase->Save(pquanvecset); } - index = BuildLargeIndex(indexPath, pquanvecset, pmeta, pmetaidx, dist, numThreads, numThreads, quantizer); + index = BuildLargeIndex(indexPath, pquanvecset, pmeta, pmetaidx, dist, numThreads, numThreads, quantizer); BOOST_REQUIRE(index != nullptr); - index->SetQuantizerADC(false); + index->SetQuantizerADC(true); } else { diff --git a/Tools/OPQ/OPQ_gpu_train_infer.py b/Tools/OPQ/OPQ_gpu_train_infer.py index 5a8fce7ca..3f5e45d19 100644 --- a/Tools/OPQ/OPQ_gpu_train_infer.py +++ b/Tools/OPQ/OPQ_gpu_train_infer.py @@ -331,8 +331,6 @@ def train_pq(args): if args.quan_test == 0 and len(args.output_quan_vector_file) == 0 and len(args.output_rec_vector_file) == 0: os.rename(args.output_truth, os.path.join(output_dir, 'truth.txt' + '.' + str(args.task))) - ret = subprocess.run(['ZipKDTree.exe', output_dir, args.output_truth]) - print (ret) return if len(args.output_quan_vector_file) > 0: @@ -386,8 +384,6 @@ def train_pq(args): os.rename(os.path.join(output_dir, args.output_rec_vector_file + '.' + str(args.task) + '.tmp'), os.path.join(output_dir, args.output_rec_vector_file + '.' + str(args.task))) os.rename(args.output_truth, os.path.join(output_dir, 'truth.txt' + '.' + str(args.task))) - ret = subprocess.run(['ZipKDTree.exe', output_dir, args.output_truth]) - print (ret) if args.quan_test > 0: queryreader = DataReader(args.query_file, args.dim, -1, args.query_normalize, args.data_type, args.target_type) @@ -462,7 +458,6 @@ def train_opq(args): if args.quan_test == 0 and len(args.output_quan_vector_file) == 0 and len(args.output_rec_vector_file) == 0: os.rename(args.output_truth, os.path.join(output_dir, 'truth.txt' + '.' + str(args.task))) - print (ret) return if len(args.output_quan_vector_file) > 0: @@ -519,8 +514,6 @@ def train_opq(args): os.rename(os.path.join(output_dir, args.output_rec_vector_file + '.' + str(args.task) + '.tmp'), os.path.join(output_dir, args.output_rec_vector_file + '.' + str(args.task))) os.rename(args.output_truth, os.path.join(output_dir, 'truth.txt' + '.' + str(args.task))) - ret = subprocess.run(['ZipKDTree.exe', output_dir, args.output_truth]) - print (ret) if args.quan_test > 0: queryreader = DataReader(args.query_file, args.dim, -1, args.query_normalize, args.data_type, args.target_type) @@ -681,8 +674,6 @@ def fourcc(x): if os.path.exists(os.path.join(output_dir, 'truth.txt' + '.' + str(args.task))): os.remove(os.path.join(output_dir, 'truth.txt' + '.' + str(args.task))) os.rename(args.output_truth, os.path.join(output_dir, 'truth.txt' + '.' + str(args.task))) - ret = subprocess.run(['ZipKDTree.exe', output_dir, args.output_truth]) - print (ret) if __name__ == '__main__': args = get_config() @@ -691,24 +682,6 @@ def fourcc(x): if not os.path.exists(args.output_dir): os.mkdir(args.output_dir) - if args.data_format != 'DEFAULT': - target = 'PreprocessData.exe' if args.data_format == 'BOND' else 'ProcessData.exe' - casttype = 'BYTE' - if args.data_type == 'uint8': casttype = 'UBYTE' - elif args.data_type == 'int16': casttype = 'SHORT' - elif args.data_type == 'float32': casttype = 'FLOAT' - ret = subprocess.run([target, args.data_file, os.path.join(args.output_dir, 'vectors.bin.%d' % args.task), os.path.join(args.output_dir, 'meta.bin.%d' % args.task), os.path.join(args.output_dir, 'metaindex.bin.%d' % args.task), str(args.dim), casttype, '0']) - args.data_file = os.path.join(args.output_dir, 'vectors.bin.%d' % args.task) - print(ret) - - if args.query_file[-4:] != '.bin': - casttype = 'BYTE' - if args.data_type == 'uint8': casttype = 'UBYTE' - elif args.data_type == 'int16': casttype = 'SHORT' - elif args.data_type == 'float32': casttype = 'FLOAT' - ret = subprocess.run(['SearchPreprocess.exe', '-q', args.query_file, '-o', args.query_file + '_queryVector.bin', '-v', args.query_file + '_validQuery.bin', '-d', str(args.dim), '-t', casttype, '-n', '0']) - args.query_file = args.query_file + '_queryVector.bin' - print(ret) #gpusearch(args) @@ -722,5 +695,4 @@ def fourcc(x): if args.log_dir != '': localpath = args.output_truth + '.dist\\dist.bin.' + str(args.task) - ret = subprocess.run(['CosmosFolderTransfer.exe', 'uploadStream', localpath.replace('\\', '/'), args.log_dir + '/dist/', 'ap']) - print (ret) + # upload to cloud storage diff --git a/Tools/OPQ/README.md b/Tools/OPQ/README.md index 5931a3d8d..65b6d691b 100644 --- a/Tools/OPQ/README.md +++ b/Tools/OPQ/README.md @@ -5,8 +5,10 @@ 1. Python>=3.7 2. numpy>=1.18.1 3. faiss>=1.7.0 -4. LibVQ ## Parameter Sample --data_file [input_path]\vectors.bin.0 --query_file [model_path]\query.bin --output_truth [output_path] --output_dir [output_path]\5474\cluster_unzip --task 0 --data_type float32 --k 5 --dim 1024 --B 1000000 --Q 1000 --D L2 --data_format DEFAULT --T 18 --train_samples 1000000 --quan_type opq --quan_dim 1024 --output_quantizer quantizer.bin --output_quan_vector_file dssm_vectors.bin --output_rec_vector_file vectors.bin --quan_test 1 --data_normalize 0 --query_normalize 0 + +## Example +python3 OPQ_gpu_train_infer.py --data_file perftest_vector.bin --query_file perftest_query.bin --task 0 --data_type float32 --k 5 --dim 64 --B 1000000 --Q 1000 --D L2 --data_format DEFAULT --T 20 --train_samples 1000000 --quan_type opq --quan_dim 32 --output_quantizer quantizer.bin From 3d2c8c7ecdd2dad8059491752edeb4cd3f183675 Mon Sep 17 00:00:00 2001 From: Zhou Lin Date: Wed, 4 Feb 2026 08:17:16 +0000 Subject: [PATCH 37/38] optimize the reconstruct vector allocation --- AnnService/inc/Core/Common/BKTree.h | 4 +- .../inc/Core/SPANN/ExtraDynamicSearcher.h | 62 +++++++------------ AnnService/inc/Core/SPANN/Index.h | 2 +- Test/src/SPFreshTest.cpp | 37 +---------- 4 files changed, 28 insertions(+), 77 deletions(-) diff --git a/AnnService/inc/Core/Common/BKTree.h b/AnnService/inc/Core/Common/BKTree.h index d51a75ab3..e59962bd3 100644 --- a/AnnService/inc/Core/Common/BKTree.h +++ b/AnnService/inc/Core/Common/BKTree.h @@ -39,7 +39,7 @@ namespace SPTAG DimensionType _RD; int _TH; DistCalcMethod _M; - int8_t* reconstructVectors; + uint8_t* reconstructVectors; T* centers; T* newTCenters; SizeType* counts; @@ -57,7 +57,7 @@ namespace SPTAG if (m_pQuantizer) { _RD = m_pQuantizer->ReconstructDim(); fComputeDistance = m_pQuantizer->DistanceCalcSelector(distMethod); - reconstructVectors = (int8_t*)ALIGN_ALLOC(_TH * m_pQuantizer->ReconstructSize()); + reconstructVectors = (uint8_t*)ALIGN_ALLOC(_TH * m_pQuantizer->ReconstructSize()); } else { fComputeDistance = COMMON::DistanceCalcSelector(distMethod); diff --git a/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h b/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h index b64b5bb37..f48a881ac 100644 --- a/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h +++ b/AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h @@ -265,17 +265,13 @@ namespace SPTAG::SPANN { bool IsAssumptionBroken(VectorIndex* p_index, SizeType headID, ValueType* vector, SizeType vid) { COMMON::QueryResultSet headCandidates(vector, m_opt->m_reassignK); - void* rec_query = nullptr; + std::shared_ptr rec_query; if (p_index->m_pQuantizer) { - rec_query = ALIGN_ALLOC(p_index->m_pQuantizer->ReconstructSize()); - p_index->m_pQuantizer->ReconstructVector((const uint8_t*)headCandidates.GetTarget(), rec_query); - headCandidates.SetTarget((ValueType*)rec_query, p_index->m_pQuantizer); + rec_query.reset((uint8_t*)ALIGN_ALLOC(p_index->m_pQuantizer->ReconstructSize()), [=](std::uint8_t* ptr) { ALIGN_FREE(ptr); }); + p_index->m_pQuantizer->ReconstructVector((const uint8_t*)headCandidates.GetTarget(), rec_query.get()); + headCandidates.SetTarget((ValueType*)(rec_query.get()), p_index->m_pQuantizer); } p_index->SearchIndex(headCandidates); - if (rec_query) - { - ALIGN_FREE(rec_query); - } int replicaCount = 0; BasicResult* queryResults = headCandidates.GetResults(); std::vector selections(static_cast(m_opt->m_replicaCount)); @@ -380,17 +376,13 @@ namespace SPTAG::SPANN { void QuantifySplitCaseB(ExtraWorkSpace* p_exWorkSpace, VectorIndex* p_index, SizeType headID, std::vector& newHeads, SizeType SplitHead, int split_order, int assumptionBrokenNum_top0, std::set& brokenID) { COMMON::QueryResultSet nearbyHeads((ValueType*)(p_index->GetSample(headID)), m_opt->m_reassignK); - void* rec_query = nullptr; + std::shared_ptr rec_query; if (p_index->m_pQuantizer) { - rec_query = ALIGN_ALLOC(p_index->m_pQuantizer->ReconstructSize()); - p_index->m_pQuantizer->ReconstructVector((const uint8_t*)nearbyHeads.GetTarget(), rec_query); - nearbyHeads.SetTarget((ValueType*)rec_query, p_index->m_pQuantizer); + rec_query.reset((uint8_t*)ALIGN_ALLOC(p_index->m_pQuantizer->ReconstructSize()), [=](std::uint8_t* ptr) { ALIGN_FREE(ptr); }); + p_index->m_pQuantizer->ReconstructVector((const uint8_t*)nearbyHeads.GetTarget(), rec_query.get()); + nearbyHeads.SetTarget((ValueType*)(rec_query.get()), p_index->m_pQuantizer); } p_index->SearchIndex(nearbyHeads); - if (rec_query) - { - ALIGN_FREE(rec_query); - } std::vector postingLists; std::string postingList; BasicResult* queryResults = nearbyHeads.GetResults(); @@ -1043,17 +1035,14 @@ namespace SPTAG::SPANN { } COMMON::QueryResultSet queryResults((ValueType*)(p_index->GetSample(headID)), m_opt->m_internalResultNum); - void* rec_query = nullptr; + std::shared_ptr rec_query; if (p_index->m_pQuantizer) { - rec_query = ALIGN_ALLOC(p_index->m_pQuantizer->ReconstructSize()); - p_index->m_pQuantizer->ReconstructVector((const uint8_t*)queryResults.GetTarget(), rec_query); - queryResults.SetTarget((ValueType*)rec_query, p_index->m_pQuantizer); + rec_query.reset((uint8_t*)ALIGN_ALLOC(p_index->m_pQuantizer->ReconstructSize()), [=](std::uint8_t* ptr) { ALIGN_FREE(ptr); }); + p_index->m_pQuantizer->ReconstructVector((const uint8_t*)queryResults.GetTarget(), rec_query.get()); + queryResults.SetTarget((ValueType*)(rec_query.get()), p_index->m_pQuantizer); } p_index->SearchIndex(queryResults); - if (rec_query) - { - ALIGN_FREE(rec_query); - } + std::string nextPostingList; for (int i = 1; i < queryResults.GetResultNum(); ++i) { @@ -1328,17 +1317,13 @@ namespace SPTAG::SPANN { newHeadsDist.clear(); newHeadsDist.resize(0); COMMON::QueryResultSet nearbyHeads((ValueType*)headVector, m_opt->m_reassignK); - void* rec_query = nullptr; + std::shared_ptr rec_query; if (p_index->m_pQuantizer) { - rec_query = ALIGN_ALLOC(p_index->m_pQuantizer->ReconstructSize()); - p_index->m_pQuantizer->ReconstructVector((const uint8_t*)nearbyHeads.GetTarget(), rec_query); - nearbyHeads.SetTarget((ValueType*)rec_query, p_index->m_pQuantizer); + rec_query.reset((uint8_t*)ALIGN_ALLOC(p_index->m_pQuantizer->ReconstructSize()), [=](std::uint8_t* ptr) { ALIGN_FREE(ptr); }); + p_index->m_pQuantizer->ReconstructVector((const uint8_t*)nearbyHeads.GetTarget(), rec_query.get()); + nearbyHeads.SetTarget((ValueType*)(rec_query.get()), p_index->m_pQuantizer); } p_index->SearchIndex(nearbyHeads); - if (rec_query) - { - ALIGN_FREE(rec_query); - } BasicResult* queryResults = nearbyHeads.GetResults(); for (int i = 0; i < nearbyHeads.GetResultNum(); i++) { auto vid = queryResults[i].VID; @@ -1392,17 +1377,14 @@ namespace SPTAG::SPANN { bool RNGSelection(std::vector& selections, ValueType* queryVector, VectorIndex* p_index, SizeType p_fullID, int& replicaCount, int checkHeadID = -1) { COMMON::QueryResultSet queryResults(queryVector, m_opt->m_internalResultNum); - void* rec_query = nullptr; + std::shared_ptr rec_query; if (p_index->m_pQuantizer) { - rec_query = ALIGN_ALLOC(p_index->m_pQuantizer->ReconstructSize()); - p_index->m_pQuantizer->ReconstructVector((const uint8_t*)queryResults.GetTarget(), rec_query); - queryResults.SetTarget((ValueType*)rec_query, p_index->m_pQuantizer); + rec_query.reset((uint8_t*)ALIGN_ALLOC(p_index->m_pQuantizer->ReconstructSize()), [=](std::uint8_t* ptr) { ALIGN_FREE(ptr); }); + p_index->m_pQuantizer->ReconstructVector((const uint8_t*)queryResults.GetTarget(), rec_query.get()); + queryResults.SetTarget((ValueType*)(rec_query.get()), p_index->m_pQuantizer); } p_index->SearchIndex(queryResults); - if (rec_query) - { - ALIGN_FREE(rec_query); - } + replicaCount = 0; for (int i = 0; i < queryResults.GetResultNum() && replicaCount < m_opt->m_replicaCount; ++i) { diff --git a/AnnService/inc/Core/SPANN/Index.h b/AnnService/inc/Core/SPANN/Index.h index 446cb784c..d449c7540 100644 --- a/AnnService/inc/Core/SPANN/Index.h +++ b/AnnService/inc/Core/SPANN/Index.h @@ -84,7 +84,7 @@ namespace SPTAG inline Options* GetOptions() { return &m_options; } inline SizeType GetNumSamples() const { return m_versionMap.Count(); } - inline DimensionType GetFeatureDim() const { return m_pQuantizer ? m_pQuantizer->ReconstructDim() : m_index->GetFeatureDim(); } + inline DimensionType GetFeatureDim() const { return m_index->GetFeatureDim(); } inline SizeType GetValueSize() const { return m_options.m_dim * sizeof(T); } inline int GetCurrMaxCheck() const { return m_options.m_maxCheck; } diff --git a/Test/src/SPFreshTest.cpp b/Test/src/SPFreshTest.cpp index 45952f775..f736fe69e 100644 --- a/Test/src/SPFreshTest.cpp +++ b/Test/src/SPFreshTest.cpp @@ -80,34 +80,6 @@ std::shared_ptr ConvertToFloatVectorSet(const std::shared_ptr(bytes, VectorValueType::Float, dim, count); } - void RemoveStaticSsdIndexIfDynamic(const std::shared_ptr& index, const std::string& indexPath) - { - if (!index) - return; - - std::string storage = index->GetParameter("Storage", "BuildSSDIndex"); - if (storage == "STATIC") - return; - - std::string ssdIndex = index->GetParameter("SSDIndex", "Base"); - if (ssdIndex.empty()) - return; - - std::filesystem::path ssdIndexPath = std::filesystem::path(indexPath) / ssdIndex; - if (std::filesystem::exists(ssdIndexPath)) - { - std::filesystem::remove(ssdIndexPath); - } - } - -std::shared_ptr SliceQuantizedVectorSet(const ByteArray& data, SizeType start, SizeType count, DimensionType dim) -{ - size_t offset = (size_t)start * (size_t)dim; - size_t length = (size_t)count * (size_t)dim; - ByteArray view(data.Data() + offset, length, false); - return std::make_shared(view, VectorValueType::UInt8, dim, count); -} - std::shared_ptr EnsurePQQuantizer(const std::string& quantizerFile, const std::shared_ptr& trainVectors, DimensionType quantizedDim, @@ -464,14 +436,13 @@ void InsertVectors(SPANN::Index *p_index, int insertThreads, int step std::shared_ptr meta(new MemMetadataSet( p_meta, ByteArray((std::uint8_t *)offsets, 2 * sizeof(std::uint64_t), true), 1)); // For quantized index, pass GetFeatureDim() which returns reconstruct dimension - DimensionType dim = p_index->GetFeatureDim(); - ErrorCode ret = p_index->AddIndex(addset->GetVector((SizeType)index), 1, dim, meta, true); + ErrorCode ret = p_index->AddIndex(addset->GetVector((SizeType)index), 1, addset->Dimension(), meta, true); if (ret != ErrorCode::Success) { SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "AddIndex failed. VID:%zu Dim:%d IndexDim:%d Storage:%s Error:%d\n", index, - dim, + addset->Dimension(), p_index->GetFeatureDim(), p_index->GetParameter("Storage", "BuildSSDIndex").c_str(), static_cast(ret)); @@ -809,7 +780,7 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c // Benchmark 0: Query performance before insertions BOOST_TEST_MESSAGE("\n=== Benchmark 0: Query Before Insertions ==="); - BenchmarkQueryPerformance(index, queryset, truth,truthPath, baseVectorCount, topK, SearchK, + BenchmarkQueryPerformance(index, queryset, truth, truthPath, baseVectorCount, topK, SearchK, numThreads, numQueries, 0, batches, tmpbenchmark); jsonFile << " \"benchmark0_query_before_insert\": "; BenchmarkQueryPerformance(index, queryset, truth, truthPath, baseVectorCount, topK, SearchK, @@ -818,7 +789,6 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c jsonFile.flush(); BOOST_REQUIRE(index->SaveIndex(indexPath) == ErrorCode::Success); - RemoveStaticSsdIndexIfDynamic(index, indexPath); index = nullptr; @@ -972,7 +942,6 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c start = std::chrono::high_resolution_clock::now(); BOOST_REQUIRE(cloneIndex->SaveIndex(clonePath) == ErrorCode::Success); end = std::chrono::high_resolution_clock::now(); - RemoveStaticSsdIndexIfDynamic(cloneIndex, clonePath); seconds = std::chrono::duration_cast(end - start).count() / 1000000.0f; BOOST_TEST_MESSAGE(" Save Time: " << seconds << " seconds"); From 4b2e4cff8427de8b8917a6fff39ca756e7b02625 Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Wed, 4 Feb 2026 18:49:44 +0800 Subject: [PATCH 38/38] fix Windows compiling for interlockedcompareexchange --- AnnService/inc/Core/Common/CommonUtils.h | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/AnnService/inc/Core/Common/CommonUtils.h b/AnnService/inc/Core/Common/CommonUtils.h index 84f4f6ded..23a59c30e 100644 --- a/AnnService/inc/Core/Common/CommonUtils.h +++ b/AnnService/inc/Core/Common/CommonUtils.h @@ -19,6 +19,12 @@ #define PREFETCH +#ifndef _MSC_VER +#define EXCHANGE_TYPE_32 int +#else +#define EXCHANGE_TYPE_32 long +#endif + namespace SPTAG { namespace COMMON @@ -33,18 +39,19 @@ namespace SPTAG static inline float atomic_float_add(volatile float* ptr, const float operand) { union { - volatile int iOld; + volatile EXCHANGE_TYPE_32 iOld; float fOld; }; union { - int iNew; + EXCHANGE_TYPE_32 iNew; float fNew; }; while (true) { - iOld = *(volatile int *)ptr; + iOld = *(volatile EXCHANGE_TYPE_32 *)ptr; fNew = fOld + operand; - if (InterlockedCompareExchange((int *)ptr, iNew, iOld) == iOld) { + if (InterlockedCompareExchange((EXCHANGE_TYPE_32 *)ptr, iNew, iOld) == iOld) + { return fNew; } }