furyhawk/milvus
1
0
Fork 0
mirror of https://github.com/milvus-io/milvus.git synced 2026-07-21 10:15:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: remove segment_loader's reserve for warmup field/index (#47481)

Browse Source 
issue: https://github.com/milvus-io/milvus/issues/41435

---------

Signed-off-by: Buqian Zheng <zhengbuqian@gmail.com>
Co-authored-by: Shawn Wang <shawn.wang@zilliz.com>
This commit is contained in:
Buqian Zheng
2026-02-03 15:41:57 +08:00
committed by GitHub
co-authored by Shawn Wang
parent 607cda567a
commit c4677defd3
1 changed files with 8 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/querynodev2/segments/segment_loader.go
+8 -17
View File
@@ -1982,11 +1982,7 @@ func estimateLoadingResourceUsageOfSegment(schema *schemapb.CollectionSchema, lo
fieldIndexInfo.GetBuildID())
}
// Use per-field index warmup policy instead of global config
warmupPolicy := getIndexWarmupPolicy(fieldSchema, fieldIndexInfo)
needWarmup := warmupPolicy == common.WarmupSync
if !multiplyFactor.TieredEvictionEnabled || needWarmup {
if !multiplyFactor.TieredEvictionEnabled {
indexMemorySize += estimateResult.MaxMemoryCost
segDiskLoadingSize += estimateResult.MaxDiskCost
}
@@ -2037,7 +2033,6 @@ func estimateLoadingResourceUsageOfSegment(schema *schemapb.CollectionSchema, lo
var legacyNilSchema bool
mmapEnabled := true
isVectorType := true
needWarmup := false
hasIndex := true
for _, fieldID := range fieldIDs {
@@ -2062,9 +2057,6 @@ func estimateLoadingResourceUsageOfSegment(schema *schemapb.CollectionSchema, lo
supportInterimIndexDataType = supportInterimIndexDataType || SupportInterimIndexDataType(fieldSchema.GetDataType())
isVectorType = isVectorType && typeutil.IsVectorType(fieldSchema.GetDataType())
// Use per-field warmup policy instead of global config
warmupPolicy := getFieldWarmupPolicy(fieldSchema)
needWarmup = needWarmup || warmupPolicy == common.WarmupSync
mmapEnabled = mmapEnabled && isDataMmapEnable(fieldSchema)
containsTimestampField = containsTimestampField || DoubleMemorySystemField(fieldSchema.GetFieldID())
doubleMomoryDataField = doubleMomoryDataField || DoubleMemoryDataType(fieldSchema.GetDataType())
@@ -2075,7 +2067,7 @@ func estimateLoadingResourceUsageOfSegment(schema *schemapb.CollectionSchema, lo
}
if !hasIndex {
if !multiplyFactor.TieredEvictionEnabled || needWarmup {
if !multiplyFactor.TieredEvictionEnabled {
interimIndexEnable := multiplyFactor.EnableInterminSegmentIndex && !isGrowingMmapEnable() && supportInterimIndexDataType
if interimIndexEnable {
segMemoryLoadingSize += uint64(float64(binlogSize) * multiplyFactor.tempSegmentIndexFactor)
@@ -2086,11 +2078,11 @@ func estimateLoadingResourceUsageOfSegment(schema *schemapb.CollectionSchema, lo
if isVectorType {
mmapVectorField := paramtable.Get().QueryNodeCfg.MmapVectorField.GetAsBool()
if mmapVectorField {
if !multiplyFactor.TieredEvictionEnabled || needWarmup {
if !multiplyFactor.TieredEvictionEnabled {
segDiskLoadingSize += binlogSize
}
} else {
if !multiplyFactor.TieredEvictionEnabled || needWarmup {
if !multiplyFactor.TieredEvictionEnabled {
segMemoryLoadingSize += binlogSize
}
}
@@ -2106,14 +2098,14 @@ func estimateLoadingResourceUsageOfSegment(schema *schemapb.CollectionSchema, lo
}
if !mmapEnabled {
if !multiplyFactor.TieredEvictionEnabled || needWarmup {
if !multiplyFactor.TieredEvictionEnabled {
segMemoryLoadingSize += binlogSize
if doubleMomoryDataField {
segMemoryLoadingSize += binlogSize
}
}
} else {
if !multiplyFactor.TieredEvictionEnabled || needWarmup {
if !multiplyFactor.TieredEvictionEnabled {
segDiskLoadingSize += uint64(getBinlogDataMemorySize(fieldBinlog))
}
}
@@ -2146,14 +2138,13 @@ func estimateLoadingResourceUsageOfSegment(schema *schemapb.CollectionSchema, lo
// PART 5: calculate size of json key stats data
jsonStatsMmapEnable := paramtable.Get().QueryNodeCfg.MmapJSONStats.GetAsBool()
needWarmup := paramtable.Get().QueryNodeCfg.TieredWarmupScalarIndex.GetValue() == "sync"
for _, jsonKeyStats := range loadInfo.GetJsonKeyStatsLogs() {
if jsonStatsMmapEnable {
if !multiplyFactor.TieredEvictionEnabled || needWarmup {
if !multiplyFactor.TieredEvictionEnabled {
segDiskLoadingSize += uint64(float64(jsonKeyStats.GetMemorySize()) * multiplyFactor.jsonKeyStatsExpansionFactor)
}
} else {
if !multiplyFactor.TieredEvictionEnabled || needWarmup {
if !multiplyFactor.TieredEvictionEnabled {
segMemoryLoadingSize += uint64(float64(jsonKeyStats.GetMemorySize()) * multiplyFactor.jsonKeyStatsExpansionFactor)
}
}