furyhawk/milvus
1
0
Fork 0
mirror of https://github.com/milvus-io/milvus.git synced 2026-07-21 10:15:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: remove unnecessary batching for avoiding OOM (#47078)

Browse Source 
issue: #47001

different batch effect for same test:
| Batch Size | 峰值内存       | 构建时间       | Term Query         |
  |------------|------------|------------|--------------------|
  | 1          | 107-118 MB | 10.4-10.7s | 0.11ms (~8900 QPS) |
  | 64         | 330 MB     | 10.6s      | 0.12ms (~7900 QPS) |
  | 148        | 610 MB     | 10.7s      | 0.12ms (~8000 QPS) |
  | 304        | 1131 MB    | 11.1s      | 0.13ms (~8000 QPS) |
 | 4096       | 13500 MB   | 12.3s      | 0.14ms (~8000 QPS) |

Signed-off-by: luzhang <luzhang@zilliz.com>
Co-authored-by: luzhang <luzhang@zilliz.com>
This commit is contained in:
zhagnlu
2026-01-19 19:45:29 +08:00
committed by GitHub
co-authored by luzhang
parent bf996cb8a0
commit 925d5875fd
1 changed files with 5 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/core/thirdparty/tantivy/tantivy-binding/src/index_writer_v7/index_writer.rs
Vendored
+5 -16
View File
@@ -20,8 +20,6 @@ use crate::index_reader_c::SetBitsetFn;
use crate::index_writer::TantivyValue;
use crate::util::c_ptr_to_str;
const BATCH_SIZE: usize = 4096;
#[inline]
pub(crate) fn schema_builder_add_field(
schema_builder: &mut SchemaBuilder,
@@ -203,14 +201,16 @@ impl IndexWriterWrapperImpl {
self.add_document(document, offset)
}
/// Add json key stats - adds documents one by one
/// Tantivy's IndexWriter has internal buffering, so external batching is unnecessary
pub fn add_json_key_stats(
&mut self,
keys: &[*const c_char],
json_offsets: &[*const i64],
json_offsets_len: &[usize],
) -> Result<()> {
let mut batch = Vec::with_capacity(BATCH_SIZE);
let id_field = self.id_field.unwrap();
for i in 0..keys.len() {
let key = c_ptr_to_str(keys[i])
.map_err(|e| TantivyBindingError::InternalError(e.to_string()))?;
@@ -218,24 +218,13 @@ impl IndexWriterWrapperImpl {
let offsets = unsafe { convert_to_rust_slice!(json_offsets[i], json_offsets_len[i]) };
for offset in offsets {
batch.push(UserOperation::Add(doc!(
self.index_writer.add_document(doc!(
id_field => *offset,
self.field => key,
)));
if batch.len() >= BATCH_SIZE {
self.index_writer.run(std::mem::replace(
&mut batch,
Vec::with_capacity(BATCH_SIZE),
))?;
}
))?;
}
}
if !batch.is_empty() {
self.index_writer.run(batch)?;
}
Ok(())
}