furyhawk/milvus
1
0
Fork 0
mirror of https://github.com/milvus-io/milvus.git synced 2026-07-21 10:15:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

enhance: use bump_schema_version to involve backfill and drop field compaction(#48808) (#49759)

Browse Source 
related: https://github.com/milvus-io/milvus/issues/48808

Signed-off-by: MrPresent-Han <chun.han@gmail.com>
Co-authored-by: MrPresent-Han <chun.han@gmail.com>
This commit is contained in:
Chun Han
2026-05-29 13:26:15 +08:00
committed by GitHub
co-authored by MrPresent-Han
parent dac62896c9
commit 7841f8a9b0
78 changed files with 8515 additions and 5289 deletions
Download Patch File Download Diff File Expand all files Collapse all files
configs/milvus.yaml
+4 -3
View File
@@ -710,8 +710,9 @@ dataCoord:
deltalogMaxNum: 1000 # The maxmum number of deltalog files to force trigger a LevelZero Compaction, default as 1000
expiry:
tolerance: -1 # tolerant duration in hours for expiry data, negative value means no toleration and equivalent to zero
backfill:
triggerInterval: 20 # The time interval in seconds for trigger backfill compaction
bumpSchemaVersion:
enabled: false # Enable schema version bump compaction
triggerInterval: 20 # The time interval in seconds for trigger schema bump compaction
single:
ratio:
threshold: 0.2 # The ratio threshold of a segment to trigger a single compaction, default as 0.2
@@ -774,7 +775,7 @@ dataCoord:
clusteringCompactionUsage: 65535 # slot usage of clustering compaction task, setting it to 65536 means it takes up a whole worker.
mixCompactionUsage: 4 # slot usage of mix compaction task.
l0DeleteCompactionUsage: 8 # slot usage of l0 compaction task.
backfillCompactionUsage: 1 # slot usage of backfill compaction task.
bumpSchemaVersionCompactionUsage: 1 # slot usage of schema bump compaction task.
indexTaskSlotUsage: 64 # slot usage of index task per 512mb
scalarIndexTaskSlotUsage: 16 # slot usage of scalar index task per 512mb
statsTaskSlotUsage: 8 # slot usage of stats task per 512mb
internal/datacoord/compaction_inspector.go
+10 -10
View File
@@ -41,11 +41,11 @@ import (
)
var maxCompactionTaskExecutionDuration = map[datapb.CompactionType]time.Duration{
datapb.CompactionType_MixCompaction: 30 * time.Minute,
datapb.CompactionType_Level0DeleteCompaction: 30 * time.Minute,
datapb.CompactionType_ClusteringCompaction: 60 * time.Minute,
datapb.CompactionType_SortCompaction: 20 * time.Minute,
datapb.CompactionType_BackfillCompaction: 30 * time.Minute,
datapb.CompactionType_MixCompaction: 30 * time.Minute,
datapb.CompactionType_Level0DeleteCompaction: 30 * time.Minute,
datapb.CompactionType_ClusteringCompaction: 60 * time.Minute,
datapb.CompactionType_SortCompaction: 20 * time.Minute,
datapb.CompactionType_BumpSchemaVersionCompaction: 30 * time.Minute,
}
type CompactionInspector interface {
@@ -222,7 +222,7 @@ func (c *compactionInspector) schedule() []CompactionTask {
switch t.GetTaskProto().GetType() {
case datapb.CompactionType_Level0DeleteCompaction:
l0ChannelExcludes.Insert(t.GetTaskProto().GetChannel())
case datapb.CompactionType_MixCompaction, datapb.CompactionType_SortCompaction, datapb.CompactionType_BackfillCompaction:
case datapb.CompactionType_MixCompaction, datapb.CompactionType_SortCompaction, datapb.CompactionType_BumpSchemaVersionCompaction:
mixChannelExcludes.Insert(t.GetTaskProto().GetChannel())
mixLabelExcludes.Insert(t.GetLabel())
case datapb.CompactionType_ClusteringCompaction:
@@ -263,8 +263,8 @@ func (c *compactionInspector) schedule() []CompactionTask {
}
l0ChannelExcludes.Insert(t.GetTaskProto().GetChannel())
selected = append(selected, t)
case datapb.CompactionType_MixCompaction, datapb.CompactionType_SortCompaction, datapb.CompactionType_BackfillCompaction:
// BackfillCompaction shares the same exclusion rules as Mix/Sort:
case datapb.CompactionType_MixCompaction, datapb.CompactionType_SortCompaction, datapb.CompactionType_BumpSchemaVersionCompaction:
// BumpSchemaVersionCompaction shares the same exclusion rules as Mix/Sort:
// - Channel-level mutual exclusion with L0 (L0 may write delta logs to any segment on the channel)
// - Label-level exclusion registered for Clustering awareness
if l0ChannelExcludes.Contain(t.GetTaskProto().GetChannel()) {
@@ -591,8 +591,8 @@ func (c *compactionInspector) createCompactTask(t *datapb.CompactionTask) (Compa
task = newL0CompactionTask(t, c.allocator, c.meta)
case datapb.CompactionType_ClusteringCompaction:
task = newClusteringCompactionTask(t, c.allocator, c.meta, c.handler, c.analyzeScheduler)
case datapb.CompactionType_BackfillCompaction:
task = newBackfillCompactionTask(t, c.allocator, c.meta, c.ievm, t.GetDiffFunctions())
case datapb.CompactionType_BumpSchemaVersionCompaction:
task = newBumpSchemaVersionTask(t, c.allocator, c.meta, c.ievm)
default:
return nil, merr.WrapErrIllegalCompactionPlan("illegal compaction type")
}
internal/datacoord/compaction_inspector_test.go
+89 -3
View File
@@ -368,6 +368,88 @@ func (s *CompactionPlanHandlerSuite) TestScheduleNodeWithL0Executing() {
}
}
func (s *CompactionPlanHandlerSuite) TestSchedule_BumpSchemaVersionConflictsWithExecutingL0SameChannel() {
s.SetupTest()
s.handler.executingTasks[1] = newL0CompactionTask(&datapb.CompactionTask{
PlanID: 1,
Type: datapb.CompactionType_Level0DeleteCompaction,
State: datapb.CompactionTaskState_pipelining,
Channel: "ch-1",
PartitionID: 10,
NodeID: 102,
}, nil, s.mockMeta)
s.NoError(s.handler.submitTask(newBumpSchemaVersionTask(&datapb.CompactionTask{
PlanID: 2,
Type: datapb.CompactionType_BumpSchemaVersionCompaction,
State: datapb.CompactionTaskState_pipelining,
Channel: "ch-1",
PartitionID: 10,
NodeID: 102,
}, nil, s.mockMeta, newMockVersionManager())))
gotTasks := s.handler.schedule()
s.Empty(gotTasks)
s.Equal(1, s.handler.queueTasks.Len())
}
func (s *CompactionPlanHandlerSuite) TestSchedule_BumpSchemaVersionBlocksQueuedL0SameChannel() {
s.SetupTest()
paramtable.Get().Save(paramtable.Get().DataCoordCfg.CompactionTaskPrioritizer.Key, "mix")
defer paramtable.Get().Reset(paramtable.Get().DataCoordCfg.CompactionTaskPrioritizer.Key)
s.handler.scheduler.(*task.MockGlobalScheduler).EXPECT().Enqueue(mock.Anything).Return().Once()
s.NoError(s.handler.submitTask(newBumpSchemaVersionTask(&datapb.CompactionTask{
PlanID: 2,
Type: datapb.CompactionType_BumpSchemaVersionCompaction,
State: datapb.CompactionTaskState_pipelining,
Channel: "ch-1",
PartitionID: 10,
NodeID: 102,
}, nil, s.mockMeta, newMockVersionManager())))
s.NoError(s.handler.submitTask(newL0CompactionTask(&datapb.CompactionTask{
PlanID: 1,
Type: datapb.CompactionType_Level0DeleteCompaction,
State: datapb.CompactionTaskState_pipelining,
Channel: "ch-1",
PartitionID: 10,
NodeID: 102,
}, nil, s.mockMeta)))
gotTasks := s.handler.schedule()
s.Equal([]UniqueID{2}, lo.Map(gotTasks, func(t CompactionTask, _ int) int64 {
return t.GetTaskProto().GetPlanID()
}))
s.Equal(1, s.handler.queueTasks.Len())
}
func (s *CompactionPlanHandlerSuite) TestSchedule_BumpSchemaVersionBlocksClusteringSameLabel() {
s.SetupTest()
s.handler.scheduler.(*task.MockGlobalScheduler).EXPECT().Enqueue(mock.Anything).Return().Once()
s.NoError(s.handler.submitTask(newBumpSchemaVersionTask(&datapb.CompactionTask{
PlanID: 1,
Type: datapb.CompactionType_BumpSchemaVersionCompaction,
State: datapb.CompactionTaskState_pipelining,
Channel: "ch-1",
PartitionID: 10,
NodeID: 102,
}, nil, s.mockMeta, newMockVersionManager())))
s.NoError(s.handler.submitTask(newClusteringCompactionTask(&datapb.CompactionTask{
PlanID: 2,
Type: datapb.CompactionType_ClusteringCompaction,
State: datapb.CompactionTaskState_pipelining,
Channel: "ch-1",
PartitionID: 10,
NodeID: 102,
}, nil, s.mockMeta, s.mockHandler, nil)))
gotTasks := s.handler.schedule()
s.Equal([]UniqueID{1}, lo.Map(gotTasks, func(t CompactionTask, _ int) int64 {
return t.GetTaskProto().GetPlanID()
}))
s.Equal(1, s.handler.queueTasks.Len())
}
func (s *CompactionPlanHandlerSuite) TestRemoveTasksByChannel() {
s.SetupTest()
ch := "ch1"
@@ -1022,6 +1104,10 @@ func TestCheckDelay(t *testing.T) {
StartTime: time.Now().Add(-100 * time.Minute).Unix(),
}, nil, nil, nil, nil)
handler.checkDelay(t3)
t4 := newBumpSchemaVersionTask(&datapb.CompactionTask{
StartTime: time.Now().Add(-100 * time.Minute).Unix(),
}, nil, nil, newMockVersionManager())
handler.checkDelay(t4)
}
func TestGetCompactionTasksNum(t *testing.T) {
@@ -1081,7 +1167,7 @@ func TestGetCompactionTasksNum(t *testing.T) {
})
}
func (s *CompactionPlanHandlerSuite) TestCreateCompactTask_BackfillCompaction() {
func (s *CompactionPlanHandlerSuite) TestCreateCompactTask_BumpSchemaVersionCompaction() {
s.SetupTest()
s.mockMeta.EXPECT().CheckAndSetSegmentsCompacting(mock.Anything, mock.Anything).Return(true, true).Maybe()
@@ -1093,13 +1179,13 @@ func (s *CompactionPlanHandlerSuite) TestCreateCompactTask_BackfillCompaction()
TriggerID: 1,
PlanID: 10,
Channel: "ch-1",
Type: datapb.CompactionType_BackfillCompaction,
Type: datapb.CompactionType_BumpSchemaVersionCompaction,
}
compactTask, err := handler.createCompactTask(t)
s.NoError(err)
s.NotNil(compactTask)
s.Equal(datapb.CompactionType_BackfillCompaction, compactTask.GetTaskProto().GetType())
s.Equal(datapb.CompactionType_BumpSchemaVersionCompaction, compactTask.GetTaskProto().GetType())
}
func (s *CompactionPlanHandlerSuite) TestCreateCompactTask_UnknownType() {
internal/datacoord/compaction_l0_view.go
-4
View File
@@ -45,10 +45,6 @@ type LevelZeroCompactionView struct {
var _ CompactionView = (*LevelZeroCompactionView)(nil)
// IsInlineExecutable returns false: L0 compaction is real compaction work that must
// be dispatched to the inspector.
func (v *LevelZeroCompactionView) IsInlineExecutable() bool { return false }
func (v *LevelZeroCompactionView) String() string {
l0strings := lo.Map(v.l0Segments, func(v *SegmentView, _ int) string {
return v.LevelZeroString()
internal/datacoord/compaction_policy_backfill.go
-319
View File
@@ -1,319 +0,0 @@
// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package datacoord
import (
"context"
"fmt"
"go.uber.org/zap"
"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
"github.com/milvus-io/milvus/internal/datacoord/allocator"
"github.com/milvus-io/milvus/pkg/v3/log"
"github.com/milvus-io/milvus/pkg/v3/proto/datapb"
)
// FuncDiff represents the differences in functions between two schemas.
type FuncDiff struct {
Added []*schemapb.FunctionSchema
}
type backfillCompactionPolicy struct {
meta *meta
handler Handler
allocator allocator.Allocator
}
// Ensure backfillCompactionPolicy implements CompactionPolicy interface
var _ CompactionPolicy = (*backfillCompactionPolicy)(nil)
func newBackfillCompactionPolicy(meta *meta, allocator allocator.Allocator, handler Handler) *backfillCompactionPolicy {
return &backfillCompactionPolicy{meta: meta, allocator: allocator, handler: handler}
}
func (policy *backfillCompactionPolicy) Enable() bool {
// Always enabled: both metadata-only and physical backfill are correctness requirements.
// Without backfill, segments can never reconcile missing function output fields and
// schema version consistency will never be reached, blocking GetCollectionStatistics.
return true
}
func (policy *backfillCompactionPolicy) Name() string {
return "BackfillCompaction"
}
// getMissingFunctions checks which functions in the collection schema have output fields
// missing from the segment's binlogs. Returns the list of functions that need backfill.
func (policy *backfillCompactionPolicy) getMissingFunctions(segment *SegmentInfo, schema *schemapb.CollectionSchema) []*schemapb.FunctionSchema {
existingFields := getSegmentBinlogFields(segment)
var missing []*schemapb.FunctionSchema
for _, fn := range schema.GetFunctions() {
for _, outputFieldID := range fn.GetOutputFieldIds() {
if _, ok := existingFields[outputFieldID]; !ok {
missing = append(missing, fn)
break // one missing output field is enough to mark this function
}
}
}
return missing
}
// staleFlushedSegments returns the flushed segments for collectionID whose SchemaVersion
// lags behind collectionSchemaVersion. L0 segments are excluded (deletes only, no data).
func (policy *backfillCompactionPolicy) staleFlushedSegments(collectionID int64, collectionSchemaVersion int32) []*chanPartSegments {
return GetSegmentsChanPart(policy.meta, collectionID, SegmentFilterFunc(func(segment *SegmentInfo) bool {
return isSegmentHealthy(segment) &&
isFlushed(segment) &&
!segment.isCompacting &&
!segment.GetIsImporting() &&
!segment.GetIsInvisible() &&
segment.GetLevel() != datapb.SegmentLevel_L0 &&
segment.GetSchemaVersion() < collectionSchemaVersion
}))
}
// TriggerInline returns metadata-only backfill views that can be applied without
// consuming inspector slots. Two cases qualify:
// 1. DoPhysicalBackfill=false: schema-only change, no function output to compute.
// 2. DoPhysicalBackfill=true but all function outputs already present in binlogs:
// stale SchemaVersion anomaly — self-heal by bumping the version in-place.
//
// TriggerInline is called before the inspector isFull() check so these lightweight
// updates always proceed regardless of compaction queue capacity.
func (policy *backfillCompactionPolicy) TriggerInline(ctx context.Context) (map[CompactionTriggerType][]CompactionView, error) {
collections := policy.meta.GetCollections()
events := make(map[CompactionTriggerType][]CompactionView)
for _, collection := range collections {
collectionID := collection.ID
collectionSchemaVersion := collection.Schema.GetVersion()
doPhysicalBackfill := collection.Schema.GetDoPhysicalBackfill()
partSegments := policy.staleFlushedSegments(collectionID, collectionSchemaVersion)
var views []CompactionView
for _, group := range partSegments {
for _, segment := range group.segments {
segmentID := segment.GetID()
segmentViews := GetViewsByInfo(segment)
if len(segmentViews) == 0 {
log.Ctx(ctx).Warn("GetViewsByInfo returned empty views, skip segment",
zap.Int64("segmentID", segmentID))
continue
}
if len(segmentViews) != 1 {
log.Ctx(ctx).Warn("GetViewsByInfo returned unexpected view count, using first view only",
zap.Int64("segmentID", segmentID),
zap.Int("viewCount", len(segmentViews)))
}
// Case 1: no physical data to rewrite — bump SchemaVersion only.
if !doPhysicalBackfill {
views = append(views, &BackfillSegmentsView{
label: segmentViews[0].label,
segments: segmentViews,
inlineMetaOnly: true,
targetSchemaVersion: collectionSchemaVersion,
})
continue
}
// Case 2: physical mode but all function outputs already present — self-heal.
missingFunctions := policy.getMissingFunctions(segment, collection.Schema)
if len(missingFunctions) == 0 {
log.Ctx(ctx).Error("backfill: segment has all function output fields but schema_version is stale, self-healing via inline meta update",
zap.Int64("segmentID", segmentID),
zap.Int64("collectionID", collectionID),
zap.Int32("segmentSchemaVersion", segment.GetSchemaVersion()),
zap.Int32("collectionSchemaVersion", collectionSchemaVersion))
views = append(views, &BackfillSegmentsView{
label: segmentViews[0].label,
segments: segmentViews,
inlineMetaOnly: true,
targetSchemaVersion: collectionSchemaVersion,
})
}
// Segments with missing functions are handled by Trigger() (physical backfill).
}
}
if len(views) > 0 {
events[TriggerTypeBackfill] = append(events[TriggerTypeBackfill], views...)
}
}
return events, nil
}
// Trigger returns physical backfill views for segments that are missing function output
// fields. It is only called when the inspector has available capacity (after TriggerInline
// and the isFull() gate in handleTicker).
func (policy *backfillCompactionPolicy) Trigger(ctx context.Context) (map[CompactionTriggerType][]CompactionView, error) {
collections := policy.meta.GetCollections()
events := make(map[CompactionTriggerType][]CompactionView)
for _, collection := range collections {
if !collection.Schema.GetDoPhysicalBackfill() {
// Metadata-only collections are handled entirely by TriggerInline.
continue
}
collectionID := collection.ID
collectionSchemaVersion := collection.Schema.GetVersion()
partSegments := policy.staleFlushedSegments(collectionID, collectionSchemaVersion)
var views []CompactionView
// triggerID is allocated per collection so tasks belonging to the same collection
// can be grouped and queried independently. Lazy-allocated on the first segment
// that actually needs a physical backfill task.
var collectionTriggerID int64
for _, group := range partSegments {
for _, segment := range group.segments {
segmentID := segment.GetID()
missingFunctions := policy.getMissingFunctions(segment, collection.Schema)
if len(missingFunctions) == 0 {
// Self-heal case — already handled by TriggerInline, skip here.
continue
}
// checkSchemaVersionConsistencyAtRootCoord guarantees the cluster-wide
// schema version increments by exactly 1 per DDL, and a single DDL adds
// at most one function. More than one missing function is therefore an
// invariant violation — skip the segment rather than silently backfilling
// only part of the missing state.
if len(missingFunctions) > 1 {
log.Ctx(ctx).Error("backfill: segment is missing more than one function output field — invariant violation, skipping",
zap.Int64("segmentID", segmentID),
zap.Int64("collectionID", collectionID),
zap.Int32("segmentSchemaVersion", segment.GetSchemaVersion()),
zap.Int32("collectionSchemaVersion", collectionSchemaVersion),
zap.Int("missingFunctionCount", len(missingFunctions)))
continue
}
segmentViews := GetViewsByInfo(segment)
if len(segmentViews) == 0 {
log.Ctx(ctx).Warn("GetViewsByInfo returned empty views, skip segment",
zap.Int64("segmentID", segmentID))
continue
}
if len(segmentViews) != 1 {
log.Ctx(ctx).Warn("GetViewsByInfo returned unexpected view count, using first view only",
zap.Int64("segmentID", segmentID),
zap.Int("viewCount", len(segmentViews)))
}
if collectionTriggerID == 0 {
id, err := policy.allocator.AllocID(ctx)
if err != nil {
log.Ctx(ctx).Warn("Failed to allocate triggerID for backfill, skip remaining segments in current group",
zap.Int64("collectionID", collectionID),
zap.Error(err))
break
}
collectionTriggerID = id
}
backfillFunc := missingFunctions[0]
log.Ctx(ctx).Info("Found segment missing function output fields, do physical backfill",
zap.Int64("segmentID", segmentID),
zap.Int64("collectionID", collectionID),
zap.Int32("segmentSchemaVersion", segment.GetSchemaVersion()),
zap.Int32("collectionSchemaVersion", collectionSchemaVersion),
zap.String("backfillFunction", backfillFunc.GetName()))
views = append(views, &BackfillSegmentsView{
label: segmentViews[0].label,
segments: segmentViews,
triggerID: collectionTriggerID,
funcDiff: &FuncDiff{Added: []*schemapb.FunctionSchema{backfillFunc}},
schema: collection.Schema,
})
}
}
if len(views) > 0 {
events[TriggerTypeBackfill] = append(events[TriggerTypeBackfill], views...)
}
}
return events, nil
}
type BackfillSegmentsView struct {
label *CompactionGroupLabel
segments []*SegmentView
triggerID int64
funcDiff *FuncDiff
// inlineMetaOnly marks this view as inline-executable: the trigger manager
// applies targetSchemaVersion to each segment via meta.UpdateSegment without
// dispatching any compaction task. funcDiff is unused on this path. Set when
// the segment needs no physical backfill (DoPhysicalBackfill=false on the
// collection, or all required function output fields are already present in
// the segment's binlogs).
inlineMetaOnly bool
targetSchemaVersion int32
// schema is the collection schema captured at policy-scan time. Used as
// task.Schema in SubmitBackfillViewToScheduler so that completeBackfillCompactionMutation
// advances the segment's SchemaVersion only to the version that was actually
// backfilled — not to a newer version if the live collection raced ahead between
// scan and submission (possible before the cluster-wide gate in PR #48989 lands).
schema *schemapb.CollectionSchema
}
var _ CompactionView = (*BackfillSegmentsView)(nil)
func (v *BackfillSegmentsView) GetGroupLabel() *CompactionGroupLabel {
return v.label
}
func (v *BackfillSegmentsView) GetSegmentsView() []*SegmentView {
return v.segments
}
func (v *BackfillSegmentsView) Append(segments ...*SegmentView) {
v.segments = append(v.segments, segments...)
}
func (v *BackfillSegmentsView) String() string {
if v.inlineMetaOnly {
return fmt.Sprintf("BackfillSegmentsView(meta-only): label=%s, segments=%d, targetSchemaVersion=%d",
v.label.Key(), len(v.segments), v.targetSchemaVersion)
}
return fmt.Sprintf("BackfillSegmentsView: label=%s, segments=%d, triggerID=%d",
v.label.Key(), len(v.segments), v.triggerID)
}
func (v *BackfillSegmentsView) Trigger() (CompactionView, string) {
return v, "backfill schema version mismatch"
}
func (v *BackfillSegmentsView) ForceTrigger() (CompactionView, string) {
return v.Trigger()
}
// ForceTriggerAll returns a single-element slice intentionally: the backfill policy
// creates one BackfillSegmentsView per segment, so each view already represents the
// smallest unit of work. Unlike LevelZeroCompactionView or ForceMergeSegmentView
// which aggregate many segments and need multi-round splitting, there is nothing to
// split further here.
func (v *BackfillSegmentsView) ForceTriggerAll() ([]CompactionView, string) {
view, reason := v.Trigger()
return []CompactionView{view}, reason
}
func (v *BackfillSegmentsView) GetTriggerID() int64 {
return v.triggerID
}
func (v *BackfillSegmentsView) IsInlineExecutable() bool {
return v.inlineMetaOnly
}
internal/datacoord/compaction_policy_backfill_test.go
-1004
View File
File diff suppressed because it is too large Load Diff
internal/datacoord/compaction_policy_bump_schema_version.go
+205
View File
@@ -0,0 +1,205 @@
// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package datacoord
import (
"context"
"fmt"
"go.uber.org/zap"
"google.golang.org/protobuf/proto"
"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
"github.com/milvus-io/milvus/internal/datacoord/allocator"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/pkg/v3/log"
"github.com/milvus-io/milvus/pkg/v3/proto/datapb"
"github.com/milvus-io/milvus/pkg/v3/util/paramtable"
)
type bumpSchemaVersionPolicy struct {
meta *meta
handler Handler
allocator allocator.Allocator
}
var _ CompactionPolicy = (*bumpSchemaVersionPolicy)(nil)
func newBumpSchemaVersionPolicy(meta *meta, allocator allocator.Allocator, handler Handler) *bumpSchemaVersionPolicy {
return &bumpSchemaVersionPolicy{meta: meta, allocator: allocator, handler: handler}
}
func (policy *bumpSchemaVersionPolicy) Enable() bool {
return paramtable.Get().DataCoordCfg.BumpSchemaVersionCompactionEnabled.GetAsBool()
}
func (policy *bumpSchemaVersionPolicy) Name() string {
return "BumpSchemaVersion"
}
func isSchemaBumpDataSegment(segment *SegmentInfo) bool {
return isSegmentHealthy(segment) &&
isFlushed(segment) &&
!segment.GetIsImporting() &&
!segment.GetIsInvisible() &&
segment.GetLevel() != datapb.SegmentLevel_L0
}
// staleFlushedSegments returns schema-bump-eligible flushed segments for collectionID
// whose SchemaVersion lags behind collectionSchemaVersion.
func (policy *bumpSchemaVersionPolicy) staleFlushedSegments(collectionID int64, collectionSchemaVersion int32) []*chanPartSegments {
return GetSegmentsChanPart(policy.meta, collectionID, SegmentFilterFunc(func(segment *SegmentInfo) bool {
if !isSchemaBumpDataSegment(segment) ||
segment.isCompacting ||
policy.meta.isSegmentCompactionProtected(segment.GetID()) ||
segment.GetSchemaVersion() >= collectionSchemaVersion {
return false
}
if segment.GetStorageVersion() < storage.StorageV3 || segment.GetManifestPath() == "" {
log.RatedWarn(300, "skip schema bump compaction for stale segment without V3 manifest storage",
zap.Int64("segmentID", segment.GetID()),
zap.Int64("collectionID", collectionID),
zap.Int32("segmentSchemaVersion", segment.GetSchemaVersion()),
zap.Int32("collectionSchemaVersion", collectionSchemaVersion),
zap.Int64("storageVersion", segment.GetStorageVersion()),
zap.Bool("hasManifest", segment.GetManifestPath() != ""))
return false
}
return true
}))
}
func (policy *bumpSchemaVersionPolicy) Trigger(ctx context.Context) (map[CompactionTriggerType][]CompactionView, error) {
collections := policy.meta.GetCollections()
events := make(map[CompactionTriggerType][]CompactionView)
for _, collection := range collections {
if collection.Schema == nil {
continue
}
if policy.meta.isCollectionCompactionBlocked(collection.ID) {
log.Ctx(ctx).Info("skip schema bump compaction for collection due to snapshot compaction block",
zap.Int64("collectionID", collection.ID))
continue
}
collectionID := collection.ID
capturedSchema := proto.Clone(collection.Schema).(*schemapb.CollectionSchema)
collectionSchemaVersion := capturedSchema.GetVersion()
partSegments := policy.staleFlushedSegments(collectionID, collectionSchemaVersion)
var views []CompactionView
var collectionTriggerID int64
partSegmentsLoop:
for _, group := range partSegments {
for _, segment := range group.segments {
segmentID := segment.GetID()
segmentViews := GetViewsByInfo(segment)
if len(segmentViews) == 0 {
log.Ctx(ctx).Warn("GetViewsByInfo returned empty views, skip segment",
zap.Int64("segmentID", segmentID))
continue
}
if len(segmentViews) != 1 {
log.Ctx(ctx).Warn("GetViewsByInfo returned unexpected view count, using first view only",
zap.Int64("segmentID", segmentID),
zap.Int("viewCount", len(segmentViews)))
}
if collectionTriggerID == 0 {
id, err := policy.allocator.AllocID(ctx)
if err != nil {
log.Ctx(ctx).Warn("Failed to allocate triggerID for schema version bump, skip remaining segments in current collection",
zap.Int64("collectionID", collectionID),
zap.Error(err))
break partSegmentsLoop
}
collectionTriggerID = id
}
log.Ctx(ctx).Info("Found segment needing schema version bump",
zap.Int64("segmentID", segmentID),
zap.Int64("collectionID", collectionID),
zap.Int32("segmentSchemaVersion", segment.GetSchemaVersion()),
zap.Int32("collectionSchemaVersion", collectionSchemaVersion))
views = append(views, &BumpSchemaVersionView{
label: segmentViews[0].label,
segments: segmentViews,
triggerID: collectionTriggerID,
schema: capturedSchema,
})
}
}
if len(views) > 0 {
events[TriggerTypeBumpSchemaVersion] = append(events[TriggerTypeBumpSchemaVersion], views...)
}
}
return events, nil
}
type BumpSchemaVersionView struct {
label *CompactionGroupLabel
segments []*SegmentView
triggerID int64
// schema is captured at policy-scan time so completion only advances the segment
// to the schema version that this task reconciled.
schema *schemapb.CollectionSchema
}
var _ CompactionView = (*BumpSchemaVersionView)(nil)
func (v *BumpSchemaVersionView) GetGroupLabel() *CompactionGroupLabel {
return v.label
}
func (v *BumpSchemaVersionView) GetSegmentsView() []*SegmentView {
return v.segments
}
func (v *BumpSchemaVersionView) Append(segments ...*SegmentView) {
v.segments = append(v.segments, segments...)
}
func (v *BumpSchemaVersionView) String() string {
label := "<nil>"
if v.label != nil {
label = v.label.Key()
}
schemaVersion := int32(0)
if v.schema != nil {
schemaVersion = v.schema.GetVersion()
}
return fmt.Sprintf("BumpSchemaVersionView: label=%s, segments=%d, triggerID=%d, schemaVersion=%d",
label, len(v.segments), v.triggerID, schemaVersion)
}
func (v *BumpSchemaVersionView) Trigger() (CompactionView, string) {
return v, "segment schema version behind collection schema"
}
func (v *BumpSchemaVersionView) ForceTrigger() (CompactionView, string) {
return v.Trigger()
}
func (v *BumpSchemaVersionView) ForceTriggerAll() ([]CompactionView, string) {
view, reason := v.Trigger()
return []CompactionView{view}, reason
}
func (v *BumpSchemaVersionView) GetTriggerID() int64 {
return v.triggerID
}
internal/datacoord/compaction_policy_bump_schema_version_test.go
+1143
View File
File diff suppressed because it is too large Load Diff
internal/datacoord/compaction_policy_clustering.go
-7
View File
@@ -53,10 +53,6 @@ func (policy *clusteringCompactionPolicy) Enable() bool {
Params.DataCoordCfg.ClusteringCompactionAutoEnable.GetAsBool()
}
func (policy *clusteringCompactionPolicy) TriggerInline(_ context.Context) (map[CompactionTriggerType][]CompactionView, error) {
return nil, nil
}
func (policy *clusteringCompactionPolicy) Name() string {
return "ClusteringCompactionPolicy"
}
@@ -331,9 +327,6 @@ func triggerClusteringCompactionPolicy(ctx context.Context, meta *meta, collecti
var _ CompactionView = (*ClusteringSegmentsView)(nil)
// IsInlineExecutable returns false: clustering compaction is real compaction work.
func (v *ClusteringSegmentsView) IsInlineExecutable() bool { return false }
type ClusteringSegmentsView struct {
label *CompactionGroupLabel
segments []*SegmentView
internal/datacoord/compaction_policy_l0.go
-4
View File
@@ -40,10 +40,6 @@ func (policy *l0CompactionPolicy) Enable() bool {
return Params.DataCoordCfg.EnableAutoCompaction.GetAsBool()
}
func (policy *l0CompactionPolicy) TriggerInline(_ context.Context) (map[CompactionTriggerType][]CompactionView, error) {
return nil, nil
}
func (policy *l0CompactionPolicy) Name() string {
return "L0Compaction"
}
internal/datacoord/compaction_policy_single.go
-7
View File
@@ -51,10 +51,6 @@ func (policy *singleCompactionPolicy) Enable() bool {
return Params.DataCoordCfg.EnableAutoCompaction.GetAsBool()
}
func (policy *singleCompactionPolicy) TriggerInline(_ context.Context) (map[CompactionTriggerType][]CompactionView, error) {
return nil, nil
}
func (policy *singleCompactionPolicy) Name() string {
return "SingleCompactionPolicy"
}
@@ -318,9 +314,6 @@ func (policy *singleCompactionPolicy) triggerOneCollection(ctx context.Context,
var _ CompactionView = (*MixSegmentView)(nil)
// IsInlineExecutable returns false: mix compaction is real compaction work.
func (v *MixSegmentView) IsInlineExecutable() bool { return false }
type MixSegmentView struct {
label *CompactionGroupLabel
segments []*SegmentView
internal/datacoord/compaction_policy_storage_version.go
-4
View File
@@ -60,10 +60,6 @@ func (policy *storageVersionUpgradePolicy) Enable() bool {
return paramtable.Get().DataCoordCfg.StorageVersionCompactionEnabled.GetAsBool()
}
func (policy *storageVersionUpgradePolicy) TriggerInline(_ context.Context) (map[CompactionTriggerType][]CompactionView, error) {
return nil, nil
}
func (policy *storageVersionUpgradePolicy) targetVersion() int64 {
targetVersion := storage.StorageV2
if paramtable.Get().CommonCfg.UseLoonFFI.GetAsBool() {
internal/datacoord/compaction_queue.go
+2 -2
View File
@@ -166,7 +166,7 @@ var (
return 1
case datapb.CompactionType_MixCompaction:
return 10
case datapb.CompactionType_BackfillCompaction:
case datapb.CompactionType_BumpSchemaVersionCompaction:
return 10
case datapb.CompactionType_ClusteringCompaction:
return 100
@@ -181,7 +181,7 @@ var (
return 10
case datapb.CompactionType_MixCompaction:
return 1
case datapb.CompactionType_BackfillCompaction:
case datapb.CompactionType_BumpSchemaVersionCompaction:
return 1
case datapb.CompactionType_ClusteringCompaction:
return 100
internal/datacoord/compaction_queue_test.go
+8
View File
@@ -135,6 +135,14 @@ func TestCompactionQueue(t *testing.T) {
})
}
func TestBumpSchemaVersionPrioritizer(t *testing.T) {
task := &bumpSchemaVersionTask{}
task.SetTask(&datapb.CompactionTask{Type: datapb.CompactionType_BumpSchemaVersionCompaction})
assert.Equal(t, 10, LevelPrioritizer(task))
assert.Equal(t, 1, MixFirstPrioritizer(task))
}
func TestConcurrency(t *testing.T) {
c := 10
internal/datacoord/compaction_task_backfill.go → internal/datacoord/compaction_task_bump_schema_version.go
+63 -66
View File
@@ -27,7 +27,6 @@ import (
"go.uber.org/zap"
"google.golang.org/protobuf/proto"
"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
"github.com/milvus-io/milvus/internal/compaction"
"github.com/milvus-io/milvus/internal/datacoord/allocator"
"github.com/milvus-io/milvus/internal/datacoord/session"
@@ -39,68 +38,68 @@ import (
"github.com/milvus-io/milvus/pkg/v3/util/paramtable"
)
var _ CompactionTask = (*backfillCompactionTask)(nil)
var _ CompactionTask = (*bumpSchemaVersionTask)(nil)
type backfillCompactionTask struct {
type bumpSchemaVersionTask struct {
taskProto atomic.Value // *datapb.CompactionTask
allocator allocator.Allocator
meta CompactionMeta
ievm IndexEngineVersionManager
functions []*schemapb.FunctionSchema
times *taskcommon.Times
}
func newBackfillCompactionTask(t *datapb.CompactionTask, allocator allocator.Allocator, meta CompactionMeta, ievm IndexEngineVersionManager, functions []*schemapb.FunctionSchema) *backfillCompactionTask {
task := &backfillCompactionTask{
func newBumpSchemaVersionTask(t *datapb.CompactionTask, allocator allocator.Allocator, meta CompactionMeta, ievm IndexEngineVersionManager) *bumpSchemaVersionTask {
task := &bumpSchemaVersionTask{
allocator: allocator,
meta: meta,
ievm: ievm,
functions: functions,
times: taskcommon.NewTimes(),
}
task.taskProto.Store(t)
return task
}
func (t *backfillCompactionTask) GetTaskID() int64 {
func (t *bumpSchemaVersionTask) GetTaskID() int64 {
return t.GetTaskProto().GetPlanID()
}
func (t *backfillCompactionTask) GetTaskType() taskcommon.Type {
func (t *bumpSchemaVersionTask) GetTaskType() taskcommon.Type {
return taskcommon.Compaction
}
func (t *backfillCompactionTask) GetTaskState() taskcommon.State {
func (t *bumpSchemaVersionTask) GetTaskState() taskcommon.State {
return taskcommon.FromCompactionState(t.GetTaskProto().GetState())
}
func (t *backfillCompactionTask) GetTaskProto() *datapb.CompactionTask {
func (t *bumpSchemaVersionTask) GetTaskProto() *datapb.CompactionTask {
return t.taskProto.Load().(*datapb.CompactionTask)
}
func (t *backfillCompactionTask) GetTaskSlot() int64 {
return paramtable.Get().DataCoordCfg.BackfillCompactionSlotUsage.GetAsInt64()
func (t *bumpSchemaVersionTask) GetTaskSlot() int64 {
return paramtable.Get().DataCoordCfg.BumpSchemaVersionCompactionSlotUsage.GetAsInt64()
}
func (t *backfillCompactionTask) SetTaskTime(timeType taskcommon.TimeType, time time.Time) {
func (t *bumpSchemaVersionTask) SetTaskTime(timeType taskcommon.TimeType, time time.Time) {
t.times.SetTaskTime(timeType, time)
}
func (t *backfillCompactionTask) GetTaskTime(timeType taskcommon.TimeType) time.Time {
func (t *bumpSchemaVersionTask) GetTaskTime(timeType taskcommon.TimeType) time.Time {
return timeType.GetTaskTime(t.times)
}
func (t *backfillCompactionTask) GetTaskVersion() int64 {
func (t *bumpSchemaVersionTask) GetTaskVersion() int64 {
return int64(t.GetTaskProto().GetRetryTimes())
}
func (t *backfillCompactionTask) BuildCompactionRequest() (*datapb.CompactionPlan, error) {
func (t *bumpSchemaVersionTask) BuildCompactionRequest() (*datapb.CompactionPlan, error) {
taskProto := t.GetTaskProto()
if taskProto.GetSchema() == nil {
return nil, merr.WrapErrIllegalCompactionPlan("schema bump compaction task schema is nil")
}
compactionParams, err := compaction.GenerateJSONParams(taskProto.GetSchema())
if err != nil {
return nil, err
}
log := log.With(zap.Int64("triggerID", taskProto.GetTriggerID()), zap.Int64("PlanID", taskProto.GetPlanID()), zap.Int64("collectionID", taskProto.GetCollectionID()))
plan := &datapb.CompactionPlan{
PlanID: taskProto.GetPlanID(),
StartTime: taskProto.GetStartTime(),
@@ -114,7 +113,6 @@ func (t *backfillCompactionTask) BuildCompactionRequest() (*datapb.CompactionPla
MaxSize: taskProto.GetMaxSize(),
JsonParams: compactionParams,
CurrentScalarIndexVersion: t.ievm.GetCurrentScalarIndexEngineVersion(),
Functions: t.functions,
}
segments := make([]*SegmentInfo, 0, len(taskProto.GetInputSegments()))
for _, segID := range taskProto.GetInputSegments() {
@@ -135,6 +133,7 @@ func (t *backfillCompactionTask) BuildCompactionRequest() (*datapb.CompactionPla
IsSortedByNamespace: segInfo.GetIsSortedByNamespace(),
StorageVersion: segInfo.GetStorageVersion(),
Manifest: segInfo.GetManifestPath(),
CommitTimestamp: segInfo.GetCommitTimestamp(),
})
segments = append(segments, segInfo)
}
@@ -146,24 +145,22 @@ func (t *backfillCompactionTask) BuildCompactionRequest() (*datapb.CompactionPla
plan.PreAllocatedLogIDs = logIDRange
plan.BeginLogID = logIDRange.Begin
WrapPluginContext(taskProto.GetCollectionID(), taskProto.GetSchema().GetProperties(), plan)
log.Info("Compaction handler refreshed backfill compaction plan", zap.Int64("maxSize", plan.GetMaxSize()),
zap.Any("PreAllocatedLogIDs", logIDRange), zap.Int64s("inputSegments", taskProto.GetInputSegments()))
return plan, nil
}
func (t *backfillCompactionTask) GetSlotUsage() int64 {
func (t *bumpSchemaVersionTask) GetSlotUsage() int64 {
return t.GetTaskSlot()
}
func (t *backfillCompactionTask) GetLabel() string {
func (t *bumpSchemaVersionTask) GetLabel() string {
return fmt.Sprintf("%d-%s", t.GetTaskProto().GetPartitionID(), t.GetTaskProto().GetChannel())
}
func (t *backfillCompactionTask) SetTask(task *datapb.CompactionTask) {
func (t *bumpSchemaVersionTask) SetTask(task *datapb.CompactionTask) {
t.taskProto.Store(task)
}
func (t *backfillCompactionTask) ShadowClone(opts ...compactionTaskOpt) *datapb.CompactionTask {
func (t *bumpSchemaVersionTask) ShadowClone(opts ...compactionTaskOpt) *datapb.CompactionTask {
cloned := proto.Clone(t.GetTaskProto()).(*datapb.CompactionTask)
for _, opt := range opts {
opt(cloned)
@@ -171,51 +168,51 @@ func (t *backfillCompactionTask) ShadowClone(opts ...compactionTaskOpt) *datapb.
return cloned
}
func (t *backfillCompactionTask) SetNodeID(nodeID int64) error {
func (t *bumpSchemaVersionTask) SetNodeID(nodeID int64) error {
return t.updateAndSaveTaskMeta(setNodeID(nodeID))
}
func (t *backfillCompactionTask) NeedReAssignNodeID() bool {
func (t *bumpSchemaVersionTask) NeedReAssignNodeID() bool {
return t.GetTaskProto().GetState() == datapb.CompactionTaskState_pipelining && (t.GetTaskProto().GetNodeID() == 0 || t.GetTaskProto().GetNodeID() == NullNodeID)
}
func (t *backfillCompactionTask) saveTaskMeta(task *datapb.CompactionTask) error {
func (t *bumpSchemaVersionTask) saveTaskMeta(task *datapb.CompactionTask) error {
return t.meta.SaveCompactionTask(context.TODO(), task)
}
func (t *backfillCompactionTask) SaveTaskMeta() error {
func (t *bumpSchemaVersionTask) SaveTaskMeta() error {
return t.saveTaskMeta(t.GetTaskProto())
}
func (t *backfillCompactionTask) Clean() bool {
func (t *bumpSchemaVersionTask) Clean() bool {
return t.doClean() == nil
}
func (t *backfillCompactionTask) doClean() error {
func (t *bumpSchemaVersionTask) doClean() error {
log := log.With(zap.Int64("triggerID", t.GetTaskProto().GetTriggerID()),
zap.Int64("PlanID", t.GetTaskProto().GetPlanID()),
zap.Int64("collectionID", t.GetTaskProto().GetCollectionID()))
err := t.updateAndSaveTaskMeta(setState(datapb.CompactionTaskState_cleaned))
if err != nil {
log.Warn("backfillCompactionTask fail to updateAndSaveTaskMeta", zap.Error(err))
log.Warn("bumpSchemaVersionTask fail to updateAndSaveTaskMeta", zap.Error(err))
return err
}
// resetSegmentCompacting must be the last step of Clean, to make sure resetSegmentCompacting only called once
// otherwise, it may unlock segments locked by other compaction tasks
t.resetSegmentCompacting()
log.Info("backfillCompactionTask clean done")
log.Info("bumpSchemaVersionTask clean done")
return nil
}
func (t *backfillCompactionTask) resetSegmentCompacting() {
func (t *bumpSchemaVersionTask) resetSegmentCompacting() {
t.meta.SetSegmentsCompacting(context.TODO(), t.GetTaskProto().GetInputSegments(), false)
}
func (t *backfillCompactionTask) processFailed() bool {
func (t *bumpSchemaVersionTask) processFailed() bool {
return true
}
func (t *backfillCompactionTask) CreateTaskOnWorker(nodeID int64, cluster session.Cluster) {
func (t *bumpSchemaVersionTask) CreateTaskOnWorker(nodeID int64, cluster session.Cluster) {
log := log.With(zap.Int64("triggerID", t.GetTaskProto().GetTriggerID()),
zap.Int64("PlanID", t.GetTaskProto().GetPlanID()),
zap.Int64("collectionID", t.GetTaskProto().GetCollectionID()),
@@ -223,37 +220,37 @@ func (t *backfillCompactionTask) CreateTaskOnWorker(nodeID int64, cluster sessio
plan, err := t.BuildCompactionRequest()
if err != nil {
log.Warn("backfillCompactionTask failed to build compaction request", zap.Error(err))
log.Warn("bumpSchemaVersionTask failed to build compaction request", zap.Error(err))
err = t.updateAndSaveTaskMeta(setState(datapb.CompactionTaskState_failed), setFailReason(err.Error()))
if err != nil {
log.Warn("backfillCompactionTask failed to updateAndSaveTaskMeta", zap.Error(err))
log.Warn("bumpSchemaVersionTask failed to updateAndSaveTaskMeta", zap.Error(err))
}
return
}
err = cluster.CreateCompaction(nodeID, plan, t.GetTaskProto().GetCollectionID())
if err != nil {
log.Warn("backfillCompactionTask failed to notify compaction tasks to DataNode",
log.Warn("bumpSchemaVersionTask failed to notify compaction tasks to DataNode",
zap.Int64("planID", t.GetTaskProto().GetPlanID()),
zap.Int64("nodeID", nodeID),
zap.Error(err))
err = t.updateAndSaveTaskMeta(setState(datapb.CompactionTaskState_pipelining), setNodeID(NullNodeID))
if err != nil {
log.Warn("backfillCompactionTask failed to updateAndSaveTaskMeta", zap.Error(err))
log.Warn("bumpSchemaVersionTask failed to updateAndSaveTaskMeta", zap.Error(err))
}
return
}
log.Info("backfillCompactionTask created task on worker", zap.Int64("planID", t.GetTaskProto().GetPlanID()),
log.Info("bumpSchemaVersionTask created task on worker", zap.Int64("planID", t.GetTaskProto().GetPlanID()),
zap.Int64("nodeID", nodeID))
err = t.updateAndSaveTaskMeta(setState(datapb.CompactionTaskState_executing), setNodeID(nodeID))
if err != nil {
log.Warn("backfillCompactionTask failed to updateAndSaveTaskMeta", zap.Error(err))
log.Warn("bumpSchemaVersionTask failed to updateAndSaveTaskMeta", zap.Error(err))
}
}
func (t *backfillCompactionTask) QueryTaskOnWorker(cluster session.Cluster) {
func (t *bumpSchemaVersionTask) QueryTaskOnWorker(cluster session.Cluster) {
log := log.With(zap.Int64("triggerID", t.GetTaskProto().GetTriggerID()),
zap.Int64("PlanID", t.GetTaskProto().GetPlanID()),
zap.Int64("collectionID", t.GetTaskProto().GetCollectionID()))
@@ -263,35 +260,35 @@ func (t *backfillCompactionTask) QueryTaskOnWorker(cluster session.Cluster) {
if err != nil || result == nil {
if errors.Is(err, merr.ErrNodeNotFound) {
if err := t.updateAndSaveTaskMeta(setState(datapb.CompactionTaskState_pipelining), setNodeID(NullNodeID)); err != nil {
log.Warn("backfillCompactionTask failed to updateAndSaveTaskMeta", zap.Error(err))
log.Warn("bumpSchemaVersionTask failed to updateAndSaveTaskMeta", zap.Error(err))
}
}
log.Warn("backfillCompactionTask failed to get compaction result", zap.Error(err))
log.Warn("bumpSchemaVersionTask failed to get compaction result", zap.Error(err))
return
}
switch result.GetState() {
case datapb.CompactionTaskState_completed:
if len(result.GetSegments()) == 0 {
log.Warn("backfillCompactionTask illegal compaction results: no segments returned")
log.Warn("bumpSchemaVersionTask illegal compaction results: no segments returned")
if err := t.updateAndSaveTaskMeta(setState(datapb.CompactionTaskState_failed),
setFailReason("illegal compaction results: no segments returned")); err != nil {
log.Warn("backfillCompactionTask failed to setState failed", zap.Error(err))
log.Warn("bumpSchemaVersionTask failed to setState failed", zap.Error(err))
}
return
}
err = t.meta.ValidateSegmentStateBeforeCompleteCompactionMutation(t.GetTaskProto())
if err != nil {
if saveErr := t.updateAndSaveTaskMeta(setState(datapb.CompactionTaskState_failed), setFailReason(err.Error())); saveErr != nil {
log.Warn("backfillCompactionTask failed to setState failed", zap.Error(saveErr))
log.Warn("bumpSchemaVersionTask failed to setState failed", zap.Error(saveErr))
}
return
}
if err := t.saveSegmentMeta(result); err != nil {
log.Warn("backfillCompactionTask failed to save segment meta", zap.Error(err))
log.Warn("bumpSchemaVersionTask failed to save segment meta", zap.Error(err))
if errors.Is(err, merr.ErrIllegalCompactionPlan) {
if saveErr := t.updateAndSaveTaskMeta(setState(datapb.CompactionTaskState_failed),
setFailReason(err.Error())); saveErr != nil {
log.Warn("backfillCompactionTask failed to setState failed", zap.Error(saveErr))
log.Warn("bumpSchemaVersionTask failed to setState failed", zap.Error(saveErr))
}
}
return
@@ -303,39 +300,39 @@ func (t *backfillCompactionTask) QueryTaskOnWorker(cluster session.Cluster) {
case datapb.CompactionTaskState_timeout:
err = t.updateAndSaveTaskMeta(setState(datapb.CompactionTaskState_timeout))
if err != nil {
log.Warn("backfillCompactionTask failed to updateAndSaveTaskMeta", zap.Error(err))
log.Warn("bumpSchemaVersionTask failed to updateAndSaveTaskMeta", zap.Error(err))
return
}
case datapb.CompactionTaskState_failed:
log.Warn("backfillCompactionTask fail in datanode")
log.Warn("bumpSchemaVersionTask fail in datanode")
if err := t.updateAndSaveTaskMeta(setState(datapb.CompactionTaskState_failed),
setFailReason("compaction failed in datanode")); err != nil {
log.Warn("backfillCompactionTask failed to updateAndSaveTaskMeta", zap.Error(err))
log.Warn("bumpSchemaVersionTask failed to updateAndSaveTaskMeta", zap.Error(err))
}
default:
log.Error("not support compaction task state", zap.String("state", result.GetState().String()))
reason := fmt.Sprintf("unsupported compaction state: %s", result.GetState().String())
if err = t.updateAndSaveTaskMeta(setState(datapb.CompactionTaskState_failed),
setFailReason(reason)); err != nil {
log.Warn("backfillCompactionTask failed to updateAndSaveTaskMeta", zap.Error(err))
log.Warn("bumpSchemaVersionTask failed to updateAndSaveTaskMeta", zap.Error(err))
return
}
}
}
func (t *backfillCompactionTask) DropTaskOnWorker(cluster session.Cluster) {
func (t *bumpSchemaVersionTask) DropTaskOnWorker(cluster session.Cluster) {
log := log.With(zap.Int64("triggerID", t.GetTaskProto().GetTriggerID()),
zap.Int64("PlanID", t.GetTaskProto().GetPlanID()),
zap.Int64("collectionID", t.GetTaskProto().GetCollectionID()))
if err := cluster.DropCompaction(t.GetTaskProto().GetNodeID(), t.GetTaskProto().GetPlanID()); err != nil {
log.Warn("backfillCompactionTask unable to drop compaction plan", zap.Error(err))
log.Warn("bumpSchemaVersionTask unable to drop compaction plan", zap.Error(err))
}
}
// Process performs the task's state machine
// Note: return True means exit this state machine.
// ONLY return True for Completed, Failed, Timeout
func (t *backfillCompactionTask) Process() bool {
func (t *bumpSchemaVersionTask) Process() bool {
log := log.With(zap.Int64("triggerID", t.GetTaskProto().GetTriggerID()),
zap.Int64("PlanID", t.GetTaskProto().GetPlanID()),
zap.Int64("collectionID", t.GetTaskProto().GetCollectionID()))
@@ -353,12 +350,12 @@ func (t *backfillCompactionTask) Process() bool {
}
currentState := t.GetTaskProto().GetState().String()
if currentState != lastState {
log.Info("backfill compaction task state changed", zap.String("lastState", lastState), zap.String("currentState", currentState))
log.Info("schema bump compaction task state changed", zap.String("lastState", lastState), zap.String("currentState", currentState))
}
return processResult
}
func (t *backfillCompactionTask) saveSegmentMeta(result *datapb.CompactionPlanResult) error {
func (t *bumpSchemaVersionTask) saveSegmentMeta(result *datapb.CompactionPlanResult) error {
log := log.With(zap.Int64("triggerID", t.GetTaskProto().GetTriggerID()),
zap.Int64("PlanID", t.GetTaskProto().GetPlanID()),
zap.Int64("collectionID", t.GetTaskProto().GetCollectionID()))
@@ -380,17 +377,17 @@ func (t *backfillCompactionTask) saveSegmentMeta(result *datapb.CompactionPlanRe
err = t.updateAndSaveTaskMeta(setState(datapb.CompactionTaskState_meta_saved), setResultSegments(newSegmentIDs))
if err != nil {
log.Warn("backfillCompactionTask failed to setState meta saved", zap.Error(err))
log.Warn("bumpSchemaVersionTask failed to setState meta saved", zap.Error(err))
return err
}
log.Info("backfillCompactionTask success to save segment meta")
log.Info("bumpSchemaVersionTask success to save segment meta")
return nil
}
func (t *backfillCompactionTask) processMetaSaved() bool {
func (t *bumpSchemaVersionTask) processMetaSaved() bool {
err := t.updateAndSaveTaskMeta(setState(datapb.CompactionTaskState_completed))
if err != nil {
log.Warn("backfillCompactionTask unable to processMetaSaved",
log.Warn("bumpSchemaVersionTask unable to processMetaSaved",
zap.Int64("planID", t.GetTaskProto().GetPlanID()),
zap.Error(err))
return false
@@ -398,15 +395,15 @@ func (t *backfillCompactionTask) processMetaSaved() bool {
return t.processCompleted()
}
func (t *backfillCompactionTask) processCompleted() bool {
func (t *bumpSchemaVersionTask) processCompleted() bool {
log := log.With(zap.Int64("triggerID", t.GetTaskProto().GetTriggerID()),
zap.Int64("PlanID", t.GetTaskProto().GetPlanID()),
zap.Int64("collectionID", t.GetTaskProto().GetCollectionID()))
log.Info("backfillCompactionTask processCompleted done")
log.Info("bumpSchemaVersionTask processCompleted done")
return true
}
func (t *backfillCompactionTask) updateAndSaveTaskMeta(opts ...compactionTaskOpt) error {
func (t *bumpSchemaVersionTask) updateAndSaveTaskMeta(opts ...compactionTaskOpt) error {
// if task state is completed, cleaned, failed, timeout, then do append end time and save
if t.GetTaskProto().State == datapb.CompactionTaskState_completed ||
t.GetTaskProto().State == datapb.CompactionTaskState_cleaned ||
internal/datacoord/compaction_task_backfill_test.go → internal/datacoord/compaction_task_bump_schema_version_test.go
+216 -58
View File
@@ -34,17 +34,18 @@ import (
"github.com/milvus-io/milvus/internal/metastore/kv/datacoord"
"github.com/milvus-io/milvus/internal/metastore/mocks"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/internal/storagev2/packed"
"github.com/milvus-io/milvus/pkg/v3/objectstorage"
"github.com/milvus-io/milvus/pkg/v3/proto/datapb"
taskcommon "github.com/milvus-io/milvus/pkg/v3/taskcommon"
"github.com/milvus-io/milvus/pkg/v3/util/merr"
)
func TestBackfillCompactionTaskSuite(t *testing.T) {
suite.Run(t, new(BackfillCompactionTaskSuite))
func TestBumpSchemaVersionCompactionTaskSuite(t *testing.T) {
suite.Run(t, new(BumpSchemaVersionCompactionTaskSuite))
}
type BackfillCompactionTaskSuite struct {
type BumpSchemaVersionCompactionTaskSuite struct {
suite.Suite
mockID atomic.Int64
@@ -54,7 +55,7 @@ type BackfillCompactionTaskSuite struct {
ievm IndexEngineVersionManager
}
func (s *BackfillCompactionTaskSuite) SetupTest() {
func (s *BumpSchemaVersionCompactionTaskSuite) SetupTest() {
ctx := context.Background()
cm := storage.NewLocalChunkManager(objectstorage.RootPath(""))
catalog := datacoord.NewCatalog(NewMetaMemoryKV(), "", "")
@@ -81,14 +82,14 @@ func (s *BackfillCompactionTaskSuite) SetupTest() {
s.ievm = newIndexEngineVersionManager()
}
func (s *BackfillCompactionTaskSuite) SetupSubTest() {
func (s *BumpSchemaVersionCompactionTaskSuite) SetupSubTest() {
s.SetupTest()
}
func (s *BackfillCompactionTaskSuite) generateBasicTask() *backfillCompactionTask {
func (s *BumpSchemaVersionCompactionTaskSuite) generateBasicTask() *bumpSchemaVersionTask {
schema := &schemapb.CollectionSchema{
Name: "test_backfill_collection",
Description: "test collection for backfill compaction",
Name: "test_schema_bump_collection",
Description: "test collection for schema bump compaction",
Version: 2,
Fields: []*schemapb.FieldSchema{
{
@@ -111,17 +112,12 @@ func (s *BackfillCompactionTaskSuite) generateBasicTask() *backfillCompactionTas
},
}
// Create BM25 function schema
// Note: FunctionSchema structure needs to be verified from actual proto definition
// For now, we create an empty functions slice
functions := []*schemapb.FunctionSchema{}
compactionTask := &datapb.CompactionTask{
PlanID: 1,
TriggerID: 19530,
CollectionID: 1,
PartitionID: 10,
Type: datapb.CompactionType_BackfillCompaction,
Type: datapb.CompactionType_BumpSchemaVersionCompaction,
NodeID: 1,
State: datapb.CompactionTaskState_pipelining,
Schema: schema,
@@ -134,11 +130,11 @@ func (s *BackfillCompactionTaskSuite) generateBasicTask() *backfillCompactionTas
Channel: "ch-1",
}
task := newBackfillCompactionTask(compactionTask, s.mockAlloc, s.meta, s.ievm, functions)
task := newBumpSchemaVersionTask(compactionTask, s.mockAlloc, s.meta, s.ievm)
return task
}
func (s *BackfillCompactionTaskSuite) TestBackfillCompactionTaskBasic() {
func (s *BumpSchemaVersionCompactionTaskSuite) TestBumpSchemaVersionCompactionTaskBasic() {
task := s.generateBasicTask()
// Test basic getters
@@ -155,13 +151,13 @@ func (s *BackfillCompactionTaskSuite) TestBackfillCompactionTaskBasic() {
s.Equal(int64(19530), taskProto.GetTriggerID())
s.Equal(int64(1), taskProto.GetCollectionID())
s.Equal(int64(10), taskProto.GetPartitionID())
s.Equal(datapb.CompactionType_BackfillCompaction, taskProto.GetType())
s.Equal(datapb.CompactionType_BumpSchemaVersionCompaction, taskProto.GetType())
s.Equal(datapb.CompactionTaskState_pipelining, taskProto.GetState())
s.Equal([]int64{101}, taskProto.GetInputSegments())
s.Equal([]int64{1000}, taskProto.GetResultSegments())
}
func (s *BackfillCompactionTaskSuite) TestBuildCompactionRequest() {
func (s *BumpSchemaVersionCompactionTaskSuite) TestBuildCompactionRequest() {
// Add a segment to meta
segmentID := int64(101)
err := s.meta.AddSegment(context.TODO(), &SegmentInfo{
@@ -194,16 +190,87 @@ func (s *BackfillCompactionTaskSuite) TestBuildCompactionRequest() {
// Verify plan
s.Equal(int64(1), plan.GetPlanID())
s.Equal(datapb.CompactionType_BackfillCompaction, plan.GetType())
s.Equal(datapb.CompactionType_BumpSchemaVersionCompaction, plan.GetType())
s.Equal("ch-1", plan.GetChannel())
s.Equal(1, len(plan.GetSegmentBinlogs()))
s.Equal(segmentID, plan.GetSegmentBinlogs()[0].GetSegmentID())
s.Equal(int64(1), plan.GetSegmentBinlogs()[0].GetCollectionID())
s.Equal(int64(10), plan.GetSegmentBinlogs()[0].GetPartitionID())
s.NotNil(plan.GetFunctions())
s.Require().NotNil(plan.GetSchema())
s.Equal(task.GetTaskProto().GetSchema().GetVersion(), plan.GetSchema().GetVersion())
}
func (s *BackfillCompactionTaskSuite) TestBuildCompactionRequestSegmentNotFound() {
func (s *BumpSchemaVersionCompactionTaskSuite) TestBuildCompactionRequestCarriesV3ManifestAndPreAllocatedLogs() {
segmentID := int64(101)
manifest := "manifest-v3"
commitTimestamp := uint64(5000)
err := s.meta.AddSegment(context.TODO(), &SegmentInfo{
SegmentInfo: &datapb.SegmentInfo{
ID: segmentID,
CollectionID: 1,
PartitionID: 10,
InsertChannel: "ch-1",
Level: datapb.SegmentLevel_L1,
State: commonpb.SegmentState_Flushed,
NumOfRows: 1000,
StorageVersion: storage.StorageV3,
ManifestPath: manifest,
CommitTimestamp: commitTimestamp,
Binlogs: []*datapb.FieldBinlog{
{
FieldID: 101,
Binlogs: []*datapb.Binlog{
{LogID: 1000, EntriesNum: 1000},
},
},
},
},
})
s.NoError(err)
task := s.generateBasicTask()
plan, err := task.BuildCompactionRequest()
s.NoError(err)
s.Require().Len(plan.GetSegmentBinlogs(), 1)
s.EqualValues(storage.StorageV3, plan.GetSegmentBinlogs()[0].GetStorageVersion())
s.Equal(manifest, plan.GetSegmentBinlogs()[0].GetManifest())
s.Equal(commitTimestamp, plan.GetSegmentBinlogs()[0].GetCommitTimestamp())
s.Require().NotNil(plan.GetSchema())
s.Equal(task.GetTaskProto().GetSchema().GetVersion(), plan.GetSchema().GetVersion())
s.Equal(task.GetTaskProto().GetPreAllocatedSegmentIDs(), plan.GetPreAllocatedSegmentIDs())
s.Require().NotNil(plan.GetPreAllocatedLogIDs())
s.Greater(plan.GetPreAllocatedLogIDs().GetEnd(), plan.GetPreAllocatedLogIDs().GetBegin())
s.Equal(plan.GetPreAllocatedLogIDs().GetBegin(), plan.GetBeginLogID())
}
func (s *BumpSchemaVersionCompactionTaskSuite) TestBuildCompactionRequestPreAllocateLogIDsError() {
segmentID := int64(101)
err := s.meta.AddSegment(context.TODO(), &SegmentInfo{
SegmentInfo: &datapb.SegmentInfo{
ID: segmentID,
CollectionID: 1,
PartitionID: 10,
InsertChannel: "ch-1",
Level: datapb.SegmentLevel_L1,
State: commonpb.SegmentState_Flushed,
NumOfRows: 1000,
StorageVersion: storage.StorageV3,
ManifestPath: "manifest-v3",
},
})
s.NoError(err)
allocErr := errors.New("alloc failed")
mockAlloc := allocator.NewMockAllocator(s.T())
mockAlloc.EXPECT().AllocN(mock.Anything).Return(int64(0), int64(0), allocErr).Once()
task := newBumpSchemaVersionTask(s.generateBasicTask().GetTaskProto(), mockAlloc, s.meta, s.ievm)
plan, err := task.BuildCompactionRequest()
s.ErrorIs(err, allocErr)
s.Nil(plan)
}
func (s *BumpSchemaVersionCompactionTaskSuite) TestBuildCompactionRequestSegmentNotFound() {
task := s.generateBasicTask()
// Try to build compaction request without adding segment to meta
@@ -213,7 +280,7 @@ func (s *BackfillCompactionTaskSuite) TestBuildCompactionRequestSegmentNotFound(
s.Contains(err.Error(), "segment not found")
}
func (s *BackfillCompactionTaskSuite) TestCreateTaskOnWorker() {
func (s *BumpSchemaVersionCompactionTaskSuite) TestCreateTaskOnWorker() {
s.Run("CreateTaskOnWorker fail, segment not found", func() {
task := s.generateBasicTask()
cluster := session.NewMockCluster(s.T())
@@ -288,7 +355,7 @@ func (s *BackfillCompactionTaskSuite) TestCreateTaskOnWorker() {
})
}
func (s *BackfillCompactionTaskSuite) TestQueryTaskOnWorker() {
func (s *BumpSchemaVersionCompactionTaskSuite) TestQueryTaskOnWorker() {
s.Run("QueryTaskOnWorker, node not found", func() {
task := s.generateBasicTask()
task.SetTask(task.ShadowClone(setState(datapb.CompactionTaskState_executing), setNodeID(1)))
@@ -410,15 +477,19 @@ func (s *BackfillCompactionTaskSuite) TestQueryTaskOnWorker() {
s.Run("QueryTaskOnWorker, completed success path", func() {
segmentID := int64(101)
manifest := "manifest-v3"
err := s.meta.AddSegment(context.TODO(), &SegmentInfo{
SegmentInfo: &datapb.SegmentInfo{
ID: segmentID,
CollectionID: 1,
PartitionID: 10,
InsertChannel: "ch-1",
Level: datapb.SegmentLevel_L1,
State: commonpb.SegmentState_Flushed,
NumOfRows: 1000,
ID: segmentID,
CollectionID: 1,
PartitionID: 10,
InsertChannel: "ch-1",
Level: datapb.SegmentLevel_L1,
State: commonpb.SegmentState_Flushed,
NumOfRows: 1000,
SchemaVersion: 1,
StorageVersion: storage.StorageV3,
ManifestPath: manifest,
},
isCompacting: true,
})
@@ -430,13 +501,94 @@ func (s *BackfillCompactionTaskSuite) TestQueryTaskOnWorker() {
cluster.EXPECT().QueryCompaction(mock.Anything, mock.Anything).Return(&datapb.CompactionPlanResult{
State: datapb.CompactionTaskState_completed,
Segments: []*datapb.CompactionSegment{{
SegmentID: segmentID,
InsertLogs: []*datapb.FieldBinlog{},
SegmentID: segmentID,
InsertLogs: []*datapb.FieldBinlog{},
Manifest: manifest,
StorageVersion: storage.StorageV3,
}},
}, nil).Once()
task.QueryTaskOnWorker(cluster)
// saveSegmentMeta succeeds → processMetaSaved → completed
s.Equal(datapb.CompactionTaskState_completed, task.GetTaskProto().GetState())
s.Equal([]int64{segmentID}, task.GetTaskProto().GetResultSegments())
})
s.Run("QueryTaskOnWorker, completed replacement success path", func() {
segmentID := int64(101)
newSegmentID := int64(1000)
err := s.meta.AddSegment(context.TODO(), &SegmentInfo{
SegmentInfo: &datapb.SegmentInfo{
ID: segmentID,
CollectionID: 1,
PartitionID: 10,
InsertChannel: "ch-1",
Level: datapb.SegmentLevel_L1,
State: commonpb.SegmentState_Flushed,
NumOfRows: 1000,
SchemaVersion: 1,
StorageVersion: storage.StorageV3,
},
isCompacting: true,
})
s.NoError(err)
task := s.generateBasicTask()
task.SetTask(task.ShadowClone(
setState(datapb.CompactionTaskState_executing),
setNodeID(1),
func(t *datapb.CompactionTask) {
t.PreAllocatedSegmentIDs = &datapb.IDRange{Begin: newSegmentID, End: newSegmentID + 1}
},
))
cluster := session.NewMockCluster(s.T())
cluster.EXPECT().QueryCompaction(mock.Anything, mock.Anything).Return(&datapb.CompactionPlanResult{
State: datapb.CompactionTaskState_completed,
Segments: []*datapb.CompactionSegment{{
SegmentID: newSegmentID,
NumOfRows: 5,
InsertLogs: []*datapb.FieldBinlog{{FieldID: 101, Binlogs: []*datapb.Binlog{{LogID: 1001}}}},
Manifest: "replacement-manifest-v3",
StorageVersion: storage.StorageV3,
}},
}, nil).Once()
task.QueryTaskOnWorker(cluster)
s.Equal(datapb.CompactionTaskState_completed, task.GetTaskProto().GetState())
s.Equal([]int64{newSegmentID}, task.GetTaskProto().GetResultSegments())
s.Equal(commonpb.SegmentState_Dropped, s.meta.GetSegment(context.TODO(), segmentID).GetState())
s.Equal(commonpb.SegmentState_Flushed, s.meta.GetSegment(context.TODO(), newSegmentID).GetState())
})
s.Run("QueryTaskOnWorker, completed invalid manifest marks failed", func() {
segmentID := int64(101)
err := s.meta.AddSegment(context.TODO(), &SegmentInfo{
SegmentInfo: &datapb.SegmentInfo{
ID: segmentID,
CollectionID: 1,
PartitionID: 10,
InsertChannel: "ch-1",
Level: datapb.SegmentLevel_L1,
State: commonpb.SegmentState_Flushed,
NumOfRows: 1000,
SchemaVersion: 1,
StorageVersion: storage.StorageV3,
ManifestPath: "manifest-v3",
},
isCompacting: true,
})
s.NoError(err)
task := s.generateBasicTask()
task.SetTask(task.ShadowClone(setState(datapb.CompactionTaskState_executing), setNodeID(1)))
cluster := session.NewMockCluster(s.T())
cluster.EXPECT().QueryCompaction(mock.Anything, mock.Anything).Return(&datapb.CompactionPlanResult{
State: datapb.CompactionTaskState_completed,
Segments: []*datapb.CompactionSegment{{
SegmentID: segmentID,
StorageVersion: storage.StorageV3,
}},
}, nil).Once()
task.QueryTaskOnWorker(cluster)
s.Equal(datapb.CompactionTaskState_failed, task.GetTaskProto().GetState())
s.Contains(task.GetTaskProto().GetFailReason(), "StorageV3 manifest")
})
s.Run("QueryTaskOnWorker, default unknown state", func() {
@@ -451,7 +603,7 @@ func (s *BackfillCompactionTaskSuite) TestQueryTaskOnWorker() {
})
}
func (s *BackfillCompactionTaskSuite) TestProcess() {
func (s *BumpSchemaVersionCompactionTaskSuite) TestProcess() {
s.Run("Process meta_saved state", func() {
task := s.generateBasicTask()
task.SetTask(task.ShadowClone(setState(datapb.CompactionTaskState_meta_saved)))
@@ -500,7 +652,7 @@ func (s *BackfillCompactionTaskSuite) TestProcess() {
})
}
func (s *BackfillCompactionTaskSuite) TestClean() {
func (s *BumpSchemaVersionCompactionTaskSuite) TestClean() {
task := s.generateBasicTask()
// Mark segment as compacting
s.meta.SetSegmentsCompacting(context.TODO(), []int64{101}, true)
@@ -530,7 +682,7 @@ func (s *BackfillCompactionTaskSuite) TestClean() {
s.False(seg.isCompacting)
}
func (s *BackfillCompactionTaskSuite) TestNeedReAssignNodeID() {
func (s *BumpSchemaVersionCompactionTaskSuite) TestNeedReAssignNodeID() {
s.Run("NeedReAssignNodeID, pipelining with nodeID 0", func() {
task := s.generateBasicTask()
task.SetTask(task.ShadowClone(setState(datapb.CompactionTaskState_pipelining), setNodeID(0)))
@@ -556,7 +708,7 @@ func (s *BackfillCompactionTaskSuite) TestNeedReAssignNodeID() {
})
}
func (s *BackfillCompactionTaskSuite) TestDropTaskOnWorker() {
func (s *BumpSchemaVersionCompactionTaskSuite) TestDropTaskOnWorker() {
task := s.generateBasicTask()
task.SetTask(task.ShadowClone(setState(datapb.CompactionTaskState_executing), setNodeID(1)))
cluster := session.NewMockCluster(s.T())
@@ -564,25 +716,29 @@ func (s *BackfillCompactionTaskSuite) TestDropTaskOnWorker() {
task.DropTaskOnWorker(cluster)
}
func (s *BackfillCompactionTaskSuite) TestSetNodeID() {
func (s *BumpSchemaVersionCompactionTaskSuite) TestSetNodeID() {
task := s.generateBasicTask()
err := task.SetNodeID(100)
s.NoError(err)
s.Equal(int64(100), task.GetTaskProto().GetNodeID())
}
func (s *BackfillCompactionTaskSuite) TestSaveSegmentMeta() {
func (s *BumpSchemaVersionCompactionTaskSuite) TestSaveSegmentMeta() {
s.Run("success", func() {
segmentID := int64(101)
currentManifest := packed.MarshalManifestPath("/data/segments/101", 1)
resultManifest := packed.MarshalManifestPath("/data/segments/101", 2)
err := s.meta.AddSegment(context.TODO(), &SegmentInfo{
SegmentInfo: &datapb.SegmentInfo{
ID: segmentID,
CollectionID: 1,
PartitionID: 10,
InsertChannel: "ch-1",
Level: datapb.SegmentLevel_L1,
State: commonpb.SegmentState_Flushed,
NumOfRows: 1000,
ID: segmentID,
CollectionID: 1,
PartitionID: 10,
InsertChannel: "ch-1",
Level: datapb.SegmentLevel_L1,
State: commonpb.SegmentState_Flushed,
NumOfRows: 1000,
StorageVersion: storage.StorageV3,
ManifestPath: currentManifest,
Binlogs: []*datapb.FieldBinlog{
{FieldID: 101, Binlogs: []*datapb.Binlog{{LogID: 1000, EntriesNum: 1000}}},
},
@@ -595,10 +751,12 @@ func (s *BackfillCompactionTaskSuite) TestSaveSegmentMeta() {
result := &datapb.CompactionPlanResult{
PlanID: 1,
State: datapb.CompactionTaskState_completed,
Type: datapb.CompactionType_BackfillCompaction,
Type: datapb.CompactionType_BumpSchemaVersionCompaction,
Segments: []*datapb.CompactionSegment{
{
SegmentID: segmentID,
SegmentID: segmentID,
StorageVersion: storage.StorageV3,
Manifest: resultManifest,
InsertLogs: []*datapb.FieldBinlog{
{FieldID: 101, Binlogs: []*datapb.Binlog{{LogID: 1000, EntriesNum: 1000}}},
{FieldID: 102, Binlogs: []*datapb.Binlog{{LogID: 2000, EntriesNum: 1000}}},
@@ -617,7 +775,7 @@ func (s *BackfillCompactionTaskSuite) TestSaveSegmentMeta() {
result := &datapb.CompactionPlanResult{
PlanID: 1,
State: datapb.CompactionTaskState_completed,
Type: datapb.CompactionType_BackfillCompaction,
Type: datapb.CompactionType_BumpSchemaVersionCompaction,
Segments: []*datapb.CompactionSegment{
{SegmentID: 101},
},
@@ -627,7 +785,7 @@ func (s *BackfillCompactionTaskSuite) TestSaveSegmentMeta() {
})
}
func (s *BackfillCompactionTaskSuite) TestProcessCompleted() {
func (s *BumpSchemaVersionCompactionTaskSuite) TestProcessCompleted() {
segmentID := int64(101)
err := s.meta.AddSegment(context.TODO(), &SegmentInfo{
SegmentInfo: &datapb.SegmentInfo{
@@ -650,7 +808,7 @@ func (s *BackfillCompactionTaskSuite) TestProcessCompleted() {
// processCompleted() does NOT reset the compacting flag — that is done by Clean().
}
func (s *BackfillCompactionTaskSuite) TestUpdateAndSaveTaskMeta() {
func (s *BumpSchemaVersionCompactionTaskSuite) TestUpdateAndSaveTaskMeta() {
s.Run("normal state update", func() {
task := s.generateBasicTask()
err := task.updateAndSaveTaskMeta(setState(datapb.CompactionTaskState_executing))
@@ -681,25 +839,25 @@ func (s *BackfillCompactionTaskSuite) TestUpdateAndSaveTaskMeta() {
})
}
func (s *BackfillCompactionTaskSuite) TestProcessFailed() {
func (s *BumpSchemaVersionCompactionTaskSuite) TestProcessFailed() {
task := s.generateBasicTask()
task.SetTask(task.ShadowClone(setState(datapb.CompactionTaskState_failed)))
s.True(task.processFailed())
}
func (s *BackfillCompactionTaskSuite) TestGetSlotUsage() {
func (s *BumpSchemaVersionCompactionTaskSuite) TestGetSlotUsage() {
task := s.generateBasicTask()
s.Equal(int64(1), task.GetSlotUsage())
}
func (s *BackfillCompactionTaskSuite) TestSetTaskTime() {
func (s *BumpSchemaVersionCompactionTaskSuite) TestSetTaskTime() {
task := s.generateBasicTask()
now := time.Now()
task.SetTaskTime(taskcommon.TimeQueue, now)
s.False(task.GetTaskTime(taskcommon.TimeQueue).IsZero())
}
func (s *BackfillCompactionTaskSuite) TestGetTaskState() {
func (s *BumpSchemaVersionCompactionTaskSuite) TestGetTaskState() {
s.Run("pipelining state", func() {
task := s.generateBasicTask()
task.SetTask(task.ShadowClone(setState(datapb.CompactionTaskState_pipelining)))
@@ -729,14 +887,14 @@ func (s *BackfillCompactionTaskSuite) TestGetTaskState() {
})
}
func (s *BackfillCompactionTaskSuite) TestGetTaskSlot() {
func (s *BumpSchemaVersionCompactionTaskSuite) TestGetTaskSlot() {
task := s.generateBasicTask()
slot := task.GetTaskSlot()
// GetTaskSlot reads from paramtable; default is 1
s.GreaterOrEqual(slot, int64(1))
}
func (s *BackfillCompactionTaskSuite) TestCleanError() {
func (s *BumpSchemaVersionCompactionTaskSuite) TestCleanError() {
// Make the compactionTaskMeta catalog fail on SaveCompactionTask so that doClean returns an error.
// meta.compactionTaskMeta.catalog is the catalog used by SaveCompactionTask,
// separate from meta.catalog which is used by segment operations.
@@ -749,7 +907,7 @@ func (s *BackfillCompactionTaskSuite) TestCleanError() {
s.False(result, "Clean() must return false when doClean fails")
}
func (s *BackfillCompactionTaskSuite) TestResetSegmentCompacting() {
func (s *BumpSchemaVersionCompactionTaskSuite) TestResetSegmentCompacting() {
// Add two segments and mark them as compacting.
for _, segID := range []int64{101, 102} {
err := s.meta.AddSegment(context.TODO(), &SegmentInfo{
internal/datacoord/compaction_task_meta.go
+3 -7
View File
@@ -91,18 +91,14 @@ func (csm *compactionTaskMeta) reloadFromKV() error {
// here we just mark the task as failed and wait for the compaction trigger to generate a new one.
//
// NOTE:
// - Only compaction tasks that require pre-allocated segment IDs (clustering/mix/sort, etc.)
// should be marked as failed when PreAllocatedSegmentIDs is nil.
// - Only compaction tasks that require pre-allocated segment IDs should be marked
// as failed when PreAllocatedSegmentIDs is nil.
// - Level0DeleteCompaction tasks never use PreAllocatedSegmentIDs and must be ignored here,
// otherwise unfinished L0 delete compaction tasks created before upgrade will be
// incorrectly marked as failed on reload.
// - BackfillCompaction is an in-place update of the original segment — no new segment
// ID is allocated. Without this exception, an in-progress backfill task would be killed
// on every datacoord restart, preventing backfill from ever completing under restart loops.
if !isCompactionTaskFinished(task) &&
task.PreAllocatedSegmentIDs == nil &&
task.GetType() != datapb.CompactionType_Level0DeleteCompaction &&
task.GetType() != datapb.CompactionType_BackfillCompaction {
task.GetType() != datapb.CompactionType_Level0DeleteCompaction {
log.Warn("PreAllocatedSegmentIDs is nil, mark the task as failed",
zap.Int64("taskID", task.GetPlanID()),
zap.String("type", task.GetType().String()),
internal/datacoord/compaction_task_meta_test.go
+10 -13
View File
@@ -166,21 +166,18 @@ func (suite *CompactionTaskMetaSuite) TestReloadFromKV_PreAllocatedSegmentIDsCom
suite.Equal(datapb.CompactionTaskState_failed, clusteringTasks[0].State)
}
// TestReloadFromKV_BackfillTaskSurvives verifies that an in-progress backfill compaction
// task is NOT marked failed on reload, even though it has no PreAllocatedSegmentIDs —
// backfill is in-place and never allocates new segment IDs. Without this guard, every
// datacoord restart would kill any running backfill task.
func (suite *CompactionTaskMetaSuite) TestReloadFromKV_BackfillTaskSurvives() {
backfillTask := &datapb.CompactionTask{
PlanID: 10,
TriggerID: 10,
Type: datapb.CompactionType_BackfillCompaction,
State: datapb.CompactionTaskState_executing,
// PreAllocatedSegmentIDs intentionally nil — backfill never allocates new segment IDs
// TestReloadFromKV_BumpSchemaVersionTaskSurvives verifies that an in-progress schema bump compaction
func (suite *CompactionTaskMetaSuite) TestReloadFromKV_BumpSchemaVersionTaskSurvives() {
bumpSchemaVersionTask := &datapb.CompactionTask{
PlanID: 10,
TriggerID: 10,
Type: datapb.CompactionType_BumpSchemaVersionCompaction,
State: datapb.CompactionTaskState_executing,
PreAllocatedSegmentIDs: &datapb.IDRange{Begin: 100, End: 101},
}
catalog := mocks.NewDataCoordCatalog(suite.T())
catalog.EXPECT().ListCompactionTask(mock.Anything).Return([]*datapb.CompactionTask{backfillTask}, nil).Once()
catalog.EXPECT().ListCompactionTask(mock.Anything).Return([]*datapb.CompactionTask{bumpSchemaVersionTask}, nil).Once()
meta, err := newCompactionTaskMeta(context.TODO(), catalog)
suite.NoError(err)
@@ -188,5 +185,5 @@ func (suite *CompactionTaskMetaSuite) TestReloadFromKV_BackfillTaskSurvives() {
tasks := meta.GetCompactionTasksByTriggerID(10)
suite.Equal(1, len(tasks))
suite.Equal(datapb.CompactionTaskState_executing, tasks[0].State,
"backfill task must survive reload even with nil PreAllocatedSegmentIDs")
"schema bump task must survive reload even with nil PreAllocatedSegmentIDs")
}
internal/datacoord/compaction_task_mix.go
+14 -6
View File
@@ -369,7 +369,12 @@ func (t *mixCompactionTask) SetTask(task *datapb.CompactionTask) {
func (t *mixCompactionTask) BuildCompactionRequest() (*datapb.CompactionPlan, error) {
log := log.With(zap.Int64("triggerID", t.GetTaskProto().GetTriggerID()), zap.Int64("PlanID", t.GetTaskProto().GetPlanID()), zap.Int64("collectionID", t.GetTaskProto().GetCollectionID()))
taskProto := t.taskProto.Load().(*datapb.CompactionTask)
compactionParams, err := compaction.GenerateJSONParams(taskProto.GetSchema())
taskSchema := taskProto.GetSchema()
if taskSchema == nil {
return nil, merr.WrapErrIllegalCompactionPlan("compaction task schema is nil")
}
taskSchemaVersion := taskSchema.GetVersion()
compactionParams, err := compaction.GenerateJSONParams(taskSchema)
if err != nil {
return nil, err
}
@@ -380,7 +385,7 @@ func (t *mixCompactionTask) BuildCompactionRequest() (*datapb.CompactionPlan, er
Channel: taskProto.GetChannel(),
CollectionTtl: taskProto.GetCollectionTtl(),
TotalRows: taskProto.GetTotalRows(),
Schema: taskProto.GetSchema(),
Schema: taskSchema,
PreAllocatedSegmentIDs: taskProto.GetPreAllocatedSegmentIDs(),
SlotUsage: t.GetSlotUsage(),
MaxSize: taskProto.GetMaxSize(),
@@ -389,8 +394,8 @@ func (t *mixCompactionTask) BuildCompactionRequest() (*datapb.CompactionPlan, er
}
// set analyzer resource for text match index if use ref mode
if fileresource.IsRefMode(paramtable.Get().CommonCfg.DNFileResourceMode.GetValue()) && taskProto.GetType() == datapb.CompactionType_SortCompaction && len(taskProto.GetSchema().GetFileResourceIds()) > 0 {
resources, err := t.meta.GetFileResources(context.Background(), taskProto.GetSchema().GetFileResourceIds()...)
if fileresource.IsRefMode(paramtable.Get().CommonCfg.DNFileResourceMode.GetValue()) && taskProto.GetType() == datapb.CompactionType_SortCompaction && len(taskSchema.GetFileResourceIds()) > 0 {
resources, err := t.meta.GetFileResources(context.Background(), taskSchema.GetFileResourceIds()...)
if err != nil {
log.Warn("get file resources for collection failed", zap.Int64("collectionID", taskProto.GetCollectionID()), zap.Error(err))
return nil, errors.Errorf("get file resources for sort compaction failed: %v", err)
@@ -405,6 +410,9 @@ func (t *mixCompactionTask) BuildCompactionRequest() (*datapb.CompactionPlan, er
if segInfo == nil {
return nil, merr.WrapErrSegmentNotFound(segID)
}
if taskSchemaVersion < segInfo.GetSchemaVersion() {
return nil, merr.WrapErrIllegalCompactionPlan(fmt.Sprintf("compaction task schema version %d is older than input segment %d schema version %d", taskSchemaVersion, segInfo.GetID(), segInfo.GetSchemaVersion()))
}
plan.SegmentBinlogs = append(plan.SegmentBinlogs, &datapb.CompactionSegmentBinlogs{
SegmentID: segID,
CollectionID: segInfo.GetCollectionID(),
@@ -424,7 +432,7 @@ func (t *mixCompactionTask) BuildCompactionRequest() (*datapb.CompactionPlan, er
segments = append(segments, segInfo)
}
logIDRange, err := PreAllocateBinlogIDs(t.allocator, segments, taskProto.GetSchema())
logIDRange, err := PreAllocateBinlogIDs(t.allocator, segments, taskSchema)
if err != nil {
return nil, err
}
@@ -432,7 +440,7 @@ func (t *mixCompactionTask) BuildCompactionRequest() (*datapb.CompactionPlan, er
// BeginLogID is deprecated, but still assign it for compatibility.
plan.BeginLogID = logIDRange.Begin
WrapPluginContext(taskProto.GetCollectionID(), taskProto.GetSchema().GetProperties(), plan)
WrapPluginContext(taskProto.GetCollectionID(), taskSchema.GetProperties(), plan)
log.Info("Compaction handler refreshed mix compaction plan", zap.Int64("maxSize", plan.GetMaxSize()),
zap.Any("PreAllocatedLogIDs", logIDRange), zap.Any("segID2DeltaLogs", segIDMap))
internal/datacoord/compaction_task_mix_test.go
+116
View File
@@ -10,6 +10,7 @@ import (
"github.com/stretchr/testify/suite"
"github.com/milvus-io/milvus-proto/go-api/v3/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
"github.com/milvus-io/milvus/internal/datacoord/allocator"
"github.com/milvus-io/milvus/internal/datacoord/session"
"github.com/milvus-io/milvus/pkg/v3/proto/datapb"
@@ -53,6 +54,7 @@ func (s *MixCompactionTaskSuite) TestProcessRefreshPlan_NormalMix() {
State: datapb.CompactionTaskState_executing,
InputSegments: []int64{200, 201},
ResultSegments: []int64{100, 200},
Schema: &schemapb.CollectionSchema{Version: 1},
}, nil, s.mockMeta, newMockVersionManager())
alloc := allocator.NewMockAllocator(s.T())
alloc.EXPECT().AllocN(mock.Anything).Return(100, 200, nil)
@@ -84,6 +86,7 @@ func (s *MixCompactionTaskSuite) TestProcessRefreshPlan_MixSegmentNotFound() {
NodeID: 1,
InputSegments: []int64{200, 201},
ResultSegments: []int64{100, 200},
Schema: &schemapb.CollectionSchema{Version: 1},
}, nil, s.mockMeta, newMockVersionManager())
_, err := task.BuildCompactionRequest()
s.Error(err)
@@ -91,6 +94,119 @@ func (s *MixCompactionTaskSuite) TestProcessRefreshPlan_MixSegmentNotFound() {
})
}
func (s *MixCompactionTaskSuite) TestBuildCompactionRequestSchemaVersionGuard() {
s.Run("nil_schema", func() {
task := newMixCompactionTask(&datapb.CompactionTask{
PlanID: 1,
Type: datapb.CompactionType_MixCompaction,
InputSegments: []int64{200},
}, nil, NewMockCompactionMeta(s.T()), newMockVersionManager())
_, err := task.BuildCompactionRequest()
s.Error(err)
s.ErrorIs(err, merr.ErrIllegalCompactionPlan)
})
s.Run("mix_task_schema_older_than_input", func() {
meta := NewMockCompactionMeta(s.T())
meta.EXPECT().GetHealthySegment(mock.Anything, int64(200)).Return(&SegmentInfo{SegmentInfo: &datapb.SegmentInfo{
ID: 200,
State: commonpb.SegmentState_Flushed,
SchemaVersion: 3,
}}).Once()
task := newMixCompactionTask(&datapb.CompactionTask{
PlanID: 1,
Type: datapb.CompactionType_MixCompaction,
InputSegments: []int64{200},
Schema: &schemapb.CollectionSchema{Version: 2},
}, nil, meta, newMockVersionManager())
_, err := task.BuildCompactionRequest()
s.Error(err)
s.ErrorIs(err, merr.ErrIllegalCompactionPlan)
})
s.Run("sort_task_schema_older_than_input", func() {
meta := NewMockCompactionMeta(s.T())
meta.EXPECT().GetHealthySegment(mock.Anything, int64(200)).Return(&SegmentInfo{SegmentInfo: &datapb.SegmentInfo{
ID: 200,
State: commonpb.SegmentState_Flushed,
SchemaVersion: 3,
}}).Once()
task := newMixCompactionTask(&datapb.CompactionTask{
PlanID: 1,
Type: datapb.CompactionType_SortCompaction,
InputSegments: []int64{200},
Schema: &schemapb.CollectionSchema{Version: 2},
}, nil, meta, newMockVersionManager())
task.slotUsage.Store(1)
_, err := task.BuildCompactionRequest()
s.Error(err)
s.ErrorIs(err, merr.ErrIllegalCompactionPlan)
})
for _, test := range []struct {
name string
compactionType datapb.CompactionType
taskSchema int32
inputSchema int32
storeSlotUsage bool
expectedSchema int32
}{
{
name: "mix_task_schema_newer_than_mixed_inputs_allowed",
compactionType: datapb.CompactionType_MixCompaction,
taskSchema: 4,
inputSchema: 3,
expectedSchema: 4,
},
{
name: "sort_task_schema_equal_input_allowed",
compactionType: datapb.CompactionType_SortCompaction,
taskSchema: 3,
inputSchema: 3,
storeSlotUsage: true,
expectedSchema: 3,
},
{
name: "sort_task_schema_newer_than_input_allowed",
compactionType: datapb.CompactionType_SortCompaction,
taskSchema: 4,
inputSchema: 3,
storeSlotUsage: true,
expectedSchema: 4,
},
} {
s.Run(test.name, func() {
meta := NewMockCompactionMeta(s.T())
meta.EXPECT().GetHealthySegment(mock.Anything, int64(200)).Return(&SegmentInfo{SegmentInfo: &datapb.SegmentInfo{
ID: 200,
State: commonpb.SegmentState_Flushed,
SchemaVersion: test.inputSchema,
Binlogs: []*datapb.FieldBinlog{getFieldBinlogIDs(101, 1)},
}}).Once()
task := newMixCompactionTask(&datapb.CompactionTask{
PlanID: 1,
Type: test.compactionType,
InputSegments: []int64{200},
Schema: &schemapb.CollectionSchema{Version: test.taskSchema},
}, nil, meta, newMockVersionManager())
if test.storeSlotUsage {
task.slotUsage.Store(1)
}
alloc := allocator.NewMockAllocator(s.T())
alloc.EXPECT().AllocN(mock.Anything).Return(int64(100), int64(200), nil).Once()
task.allocator = alloc
plan, err := task.BuildCompactionRequest()
s.NoError(err)
s.EqualValues(test.expectedSchema, plan.GetSchema().GetVersion())
s.Len(plan.GetSegmentBinlogs(), 1)
})
}
}
func (s *MixCompactionTaskSuite) TestProcess() {
s.Run("test process states", func() {
testCases := []struct {
internal/datacoord/compaction_trigger_v2.go
+45 -100
View File
@@ -30,6 +30,7 @@ import (
"github.com/milvus-io/milvus/internal/datacoord/session"
"github.com/milvus-io/milvus/internal/types"
"github.com/milvus-io/milvus/internal/util/sessionutil"
"github.com/milvus-io/milvus/pkg/v3/common"
"github.com/milvus-io/milvus/pkg/v3/log"
"github.com/milvus-io/milvus/pkg/v3/proto/datapb"
"github.com/milvus-io/milvus/pkg/v3/util/logutil"
@@ -49,7 +50,7 @@ const (
TriggerTypeSort
TriggerTypeForceMerge
TriggerTypeStorageVersionUpgrade
TriggerTypeBackfill
TriggerTypeBumpSchemaVersion
)
type TickerType int8
@@ -58,7 +59,7 @@ const (
L0Ticker TickerType = iota + 1
ClusteringTicker
SingleTicker
BackfillTicker
BumpSchemaVersionTicker
StorageVersionTicker
)
@@ -74,8 +75,8 @@ func (t CompactionTriggerType) GetCompactionType() datapb.CompactionType {
return datapb.CompactionType_SortCompaction
case TriggerTypeStorageVersionUpgrade:
return datapb.CompactionType_MixCompaction
case TriggerTypeBackfill:
return datapb.CompactionType_BackfillCompaction
case TriggerTypeBumpSchemaVersion:
return datapb.CompactionType_BumpSchemaVersionCompaction
default:
return datapb.CompactionType_MixCompaction
}
@@ -101,8 +102,8 @@ func (t CompactionTriggerType) String() string {
return "ForceMerge"
case TriggerTypeStorageVersionUpgrade:
return "StorageVersionUpgrade"
case TriggerTypeBackfill:
return "Backfill"
case TriggerTypeBumpSchemaVersion:
return "BumpSchemaVersion"
default:
return ""
}
@@ -112,10 +113,6 @@ func (t CompactionTriggerType) String() string {
type CompactionPolicy interface {
// Enable returns whether this compaction policy is enabled
Enable() bool
// TriggerInline returns views that can be applied inline (without inspector slots).
// Called unconditionally before the isFull() check so metadata-only updates always
// proceed regardless of inspector capacity. Non-backfill policies return an empty map.
TriggerInline(ctx context.Context) (map[CompactionTriggerType][]CompactionView, error)
// Trigger returns views that require inspector slots (actual compaction tasks).
// Only called when the inspector is not full.
Trigger(ctx context.Context) (map[CompactionTriggerType][]CompactionView, error)
@@ -149,7 +146,7 @@ type CompactionTriggerManager struct {
singlePolicy *singleCompactionPolicy
forceMergePolicy *forceMergeCompactionPolicy
upgradeStorageVersionPolicy *storageVersionUpgradePolicy
backfillPolicy *backfillCompactionPolicy
bumpSchemaVersionPolicy *bumpSchemaVersionPolicy
cancel context.CancelFunc
closeWg sync.WaitGroup
@@ -173,13 +170,13 @@ func NewCompactionTriggerManager(alloc allocator.Allocator, handler Handler, ins
m.forceMergePolicy = newForceMergeCompactionPolicy(meta, m.allocator, m.handler)
m.upgradeStorageVersionPolicy = newStorageVersionUpgradePolicy(meta, m.allocator, m.handler, versionManager)
m.backfillPolicy = newBackfillCompactionPolicy(meta, m.allocator, m.handler)
m.bumpSchemaVersionPolicy = newBumpSchemaVersionPolicy(meta, m.allocator, m.handler)
// Initialize policies map for ticker handling
m.policies[L0Ticker] = m.l0Policy
m.policies[ClusteringTicker] = m.clusteringPolicy
m.policies[SingleTicker] = m.singlePolicy
m.policies[BackfillTicker] = m.backfillPolicy
m.policies[BumpSchemaVersionTicker] = m.bumpSchemaVersionPolicy
m.policies[StorageVersionTicker] = m.upgradeStorageVersionPolicy
return m
}
@@ -227,8 +224,8 @@ func (m *CompactionTriggerManager) loop(ctx context.Context) {
defer singleTicker.Stop()
storageVersionTicker := time.NewTicker(Params.DataCoordCfg.MixCompactionTriggerInterval.GetAsDuration(time.Second))
defer storageVersionTicker.Stop()
backfillTicker := time.NewTicker(Params.DataCoordCfg.BackfillCompactionTriggerInterval.GetAsDuration(time.Second))
defer backfillTicker.Stop()
bumpSchemaVersionTicker := time.NewTicker(Params.DataCoordCfg.BumpSchemaVersionCompactionTriggerInterval.GetAsDuration(time.Second))
defer bumpSchemaVersionTicker.Stop()
log.Info("Compaction trigger manager start")
for {
select {
@@ -243,8 +240,8 @@ func (m *CompactionTriggerManager) loop(ctx context.Context) {
m.handleTicker(ctx, SingleTicker)
case <-storageVersionTicker.C:
m.handleTicker(ctx, StorageVersionTicker)
case <-backfillTicker.C:
m.handleTicker(ctx, BackfillTicker)
case <-bumpSchemaVersionTicker.C:
m.handleTicker(ctx, BumpSchemaVersionTicker)
case segID := <-getStatsTaskChSingleton():
log.Info("receive new segment to trigger sort compaction", zap.Int64("segmentID", segID))
view := m.singlePolicy.triggerSegmentSortCompaction(ctx, segID)
@@ -272,24 +269,12 @@ func (m *CompactionTriggerManager) handleTicker(ctx context.Context, tickerType
return
}
// Step 1: apply inline views unconditionally — these never need inspector slots
// (e.g. backfill metadata-only schema-version bumps) and must proceed even when
// the inspector queue is full.
inlineEvents, err := policy.TriggerInline(ctx)
if err != nil {
log.Warn("Fail to trigger inline policy", zap.String("policy", policy.Name()), zap.Error(err))
return
}
m.executeInline(ctx, inlineEvents)
// Step 2: gate normal compaction dispatch on inspector capacity.
if m.inspector.isFull() {
log.RatedInfo(10, "Skip dispatching compaction events since inspector is full",
zap.String("policy", policy.Name()))
return
}
// Step 3: trigger and dispatch views that require inspector slots.
events, err := policy.Trigger(ctx)
if err != nil {
log.Warn("Fail to trigger policy", zap.String("policy", policy.Name()), zap.Error(err))
@@ -303,41 +288,6 @@ func (m *CompactionTriggerManager) handleTicker(ctx context.Context, tickerType
}
}
// executeInline applies all views returned by TriggerInline directly inside datacoord
// without consuming inspector slots or notifying the scheduler.
func (m *CompactionTriggerManager) executeInline(ctx context.Context, events map[CompactionTriggerType][]CompactionView) {
for _, views := range events {
for _, view := range views {
m.applyInlineView(ctx, view)
}
}
}
// applyInlineView applies a CompactionView whose IsInlineExecutable() == true
// directly inside datacoord. Today this is only used by backfillCompactionPolicy
// for the metadata-only path: bump segment SchemaVersion via meta.UpdateSegment
// without producing any compaction task.
func (m *CompactionTriggerManager) applyInlineView(ctx context.Context, view CompactionView) {
bv, ok := view.(*BackfillSegmentsView)
if !ok {
log.Ctx(ctx).Warn("unexpected inline-executable view type, skip",
zap.String("actualType", fmt.Sprintf("%T", view)))
return
}
for _, sv := range bv.GetSegmentsView() {
if err := m.meta.UpdateSegment(sv.ID, SetSchemaVersion(bv.targetSchemaVersion)); err != nil {
log.Ctx(ctx).Error("failed to apply inline backfill schema version update",
zap.Int64("segmentID", sv.ID),
zap.Int32("newSchemaVersion", bv.targetSchemaVersion),
zap.Error(err))
continue
}
log.Ctx(ctx).Info("applied inline backfill schema version update",
zap.Int64("segmentID", sv.ID),
zap.Int32("newSchemaVersion", bv.targetSchemaVersion))
}
}
func (m *CompactionTriggerManager) ManualTrigger(ctx context.Context, collectionID int64, isClustering bool, isL0 bool, targetSize int64) (UniqueID, error) {
log.Ctx(ctx).Info("receive manual trigger",
zap.Int64("collectionID", collectionID),
@@ -421,8 +371,8 @@ func (m *CompactionTriggerManager) notify(ctx context.Context, eventType Compact
m.SubmitSingleViewToScheduler(ctx, outView, eventType)
case TriggerTypeForceMerge:
m.SubmitForceMergeViewToScheduler(ctx, outView)
case TriggerTypeBackfill:
m.SubmitBackfillViewToScheduler(ctx, outView)
case TriggerTypeBumpSchemaVersion:
m.SubmitBumpSchemaVersionViewToScheduler(ctx, outView)
}
}
}
@@ -696,9 +646,15 @@ func (m *CompactionTriggerManager) SubmitForceMergeViewToScheduler(ctx context.C
)
}
func (m *CompactionTriggerManager) SubmitBackfillViewToScheduler(ctx context.Context, view CompactionView) {
func (m *CompactionTriggerManager) SubmitBumpSchemaVersionViewToScheduler(ctx context.Context, view CompactionView) {
log := log.Ctx(ctx).With(zap.String("view", view.String()))
planID, _, err := m.allocator.AllocN(1)
bumpView, ok := view.(*BumpSchemaVersionView)
if !ok {
log.Warn("unexpected view type for schema bump trigger, expected *BumpSchemaVersionView",
zap.String("actualType", fmt.Sprintf("%T", view)))
return
}
planID, endID, err := m.allocator.AllocN(2)
if err != nil {
log.Warn("Failed to submit compaction view to scheduler because allocate id fail", zap.Error(err))
return
@@ -713,7 +669,12 @@ func (m *CompactionTriggerManager) SubmitBackfillViewToScheduler(ctx context.Con
return
}
if collection.IsExternal() {
log.Info("skip submitting backfill compaction for external collection", zap.Int64("collectionID", collection.ID))
log.Info("skip submitting schema bump compaction for external collection", zap.Int64("collectionID", collection.ID))
return
}
collectionTTL, err := common.GetCollectionTTLFromMap(collection.Properties)
if err != nil {
log.Warn("Failed to submit schema bump compaction because get collection ttl failed", zap.Error(err))
return
}
var totalRows int64 = 0
@@ -721,42 +682,26 @@ func (m *CompactionTriggerManager) SubmitBackfillViewToScheduler(ctx context.Con
totalRows += s.NumOfRows
}
expectedSize := getExpectedSegmentSize(m.meta, collection.ID, collection.Schema)
bfView, ok := view.(*BackfillSegmentsView)
if !ok {
log.Warn("unexpected view type for backfill trigger, expected *BackfillSegmentsView",
zap.String("actualType", fmt.Sprintf("%T", view)))
return
}
if bfView.funcDiff == nil || len(bfView.funcDiff.Added) != 1 {
funcCount := 0
if bfView.funcDiff != nil {
funcCount = len(bfView.funcDiff.Added)
}
log.Warn("backfill view must have exactly one function to backfill",
zap.Int("funcCount", funcCount))
return
}
task := &datapb.CompactionTask{
PlanID: planID,
TriggerID: bfView.triggerID,
State: datapb.CompactionTaskState_pipelining,
StartTime: time.Now().Unix(),
Type: datapb.CompactionType_BackfillCompaction,
CollectionID: view.GetGroupLabel().CollectionID,
PartitionID: view.GetGroupLabel().PartitionID,
Channel: view.GetGroupLabel().Channel,
// Use the schema frozen at scan time so completeBackfillCompactionMutation
// only advances the segment to the version that was actually backfilled.
// Re-using the live collection.Schema here risks advancing the segment's
// SchemaVersion beyond what this task backfills if the collection raced
// ahead between scan and submission (prevented in practice by PR #48989).
Schema: bfView.schema,
PlanID: planID,
TriggerID: bumpView.triggerID,
State: datapb.CompactionTaskState_pipelining,
StartTime: time.Now().Unix(),
CollectionTtl: collectionTTL.Nanoseconds(),
Type: datapb.CompactionType_BumpSchemaVersionCompaction,
CollectionID: view.GetGroupLabel().CollectionID,
PartitionID: view.GetGroupLabel().PartitionID,
Channel: view.GetGroupLabel().Channel,
Schema: bumpView.schema,
InputSegments: lo.Map(view.GetSegmentsView(), func(segmentView *SegmentView, _ int) int64 { return segmentView.ID }),
ResultSegments: []int64{},
TotalRows: totalRows,
LastStateStartTime: time.Now().Unix(),
MaxSize: expectedSize,
DiffFunctions: bfView.funcDiff.Added,
PreAllocatedSegmentIDs: &datapb.IDRange{
Begin: planID + 1,
End: endID,
},
}
err = m.inspector.enqueueCompaction(task)
if err != nil {
@@ -767,7 +712,7 @@ func (m *CompactionTriggerManager) SubmitBackfillViewToScheduler(ctx context.Con
zap.Error(err))
return
}
log.Info("Finish to submit a backfill compaction task",
log.Info("Finish to submit a schema bump compaction task",
zap.Int64("triggerID", task.GetTriggerID()),
zap.Int64("planID", task.GetPlanID()),
zap.String("type", task.GetType().String()),
internal/datacoord/compaction_trigger_v2_test.go
+95 -219
View File
@@ -4,6 +4,7 @@ import (
"context"
"strconv"
"testing"
"time"
"github.com/blang/semver/v4"
"github.com/cockroachdb/errors"
@@ -15,7 +16,6 @@ import (
"github.com/milvus-io/milvus-proto/go-api/v3/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
"github.com/milvus-io/milvus/internal/datacoord/allocator"
"github.com/milvus-io/milvus/internal/metastore/mocks"
"github.com/milvus-io/milvus/internal/metastore/model"
"github.com/milvus-io/milvus/pkg/v3/common"
"github.com/milvus-io/milvus/pkg/v3/log"
@@ -30,26 +30,20 @@ func TestCompactionTriggerManagerSuite(t *testing.T) {
// testCompactionPolicy is a minimal CompactionPolicy stub for handleTicker tests.
type testCompactionPolicy struct {
enabled bool
triggerResult map[CompactionTriggerType][]CompactionView
triggerErr error
inlineTriggerResult map[CompactionTriggerType][]CompactionView
policyName string
enabled bool
triggerResult map[CompactionTriggerType][]CompactionView
triggerErr error
policyName string
}
func (p *testCompactionPolicy) Enable() bool { return p.enabled }
func (p *testCompactionPolicy) TriggerInline(_ context.Context) (map[CompactionTriggerType][]CompactionView, error) {
return p.inlineTriggerResult, nil
}
func (p *testCompactionPolicy) Trigger(_ context.Context) (map[CompactionTriggerType][]CompactionView, error) {
return p.triggerResult, p.triggerErr
}
func (p *testCompactionPolicy) Name() string { return p.policyName }
// stubDispatchableView is a CompactionView stub for handleTicker tests that need a
// non-inline-executable view to reach the dispatch / isFull branch. All methods
// return zero values; the only method used by handleTicker is IsInlineExecutable.
// stubDispatchableView is a CompactionView stub for handleTicker tests that need
// a view to reach the dispatch / isFull branch.
type stubDispatchableView struct{}
func (stubDispatchableView) GetGroupLabel() *CompactionGroupLabel { return &CompactionGroupLabel{} }
@@ -64,7 +58,6 @@ func (stubDispatchableView) Trigger() (CompactionView, string) { retur
func (stubDispatchableView) ForceTrigger() (CompactionView, string) { return nil, "" }
func (stubDispatchableView) ForceTriggerAll() ([]CompactionView, string) { return nil, "" }
func (stubDispatchableView) GetTriggerID() int64 { return 0 }
func (stubDispatchableView) IsInlineExecutable() bool { return false }
type CompactionTriggerManagerSuite struct {
suite.Suite
@@ -432,7 +425,7 @@ func (s *CompactionTriggerManagerSuite) TestManualTriggerInvalidParams() {
s.Equal(int64(0), triggerID)
}
func (s *CompactionTriggerManagerSuite) TestSubmitBackfillViewToScheduler() {
func (s *CompactionTriggerManagerSuite) TestSubmitBumpSchemaVersionViewToScheduler() {
collectionSchema := &schemapb.CollectionSchema{
Name: "test_coll",
Fields: []*schemapb.FieldSchema{
@@ -442,61 +435,58 @@ func (s *CompactionTriggerManagerSuite) TestSubmitBackfillViewToScheduler() {
{Name: "bm25_fn", Type: schemapb.FunctionType_BM25, OutputFieldIds: []int64{100}},
},
}
backfillFunc := collectionSchema.Functions[0]
makeBackfillView := func(triggerID int64) *BackfillSegmentsView {
makeBumpSchemaVersionView := func(triggerID int64) *BumpSchemaVersionView {
segView := &SegmentView{
ID: 200,
label: s.testLabel,
NumOfRows: 1000,
}
return &BackfillSegmentsView{
return &BumpSchemaVersionView{
label: s.testLabel,
segments: []*SegmentView{segView},
triggerID: triggerID,
funcDiff: &FuncDiff{Added: []*schemapb.FunctionSchema{backfillFunc}},
schema: collectionSchema, // frozen at scan time
schema: collectionSchema,
}
}
s.Run("AllocN fails", func() {
s.SetupTest()
s.mockAlloc.EXPECT().AllocN(int64(1)).Return(int64(0), int64(0), errors.New("alloc error")).Once()
view := makeBackfillView(111)
s.mockAlloc.EXPECT().AllocN(int64(2)).Return(int64(0), int64(0), errors.New("alloc error")).Once()
view := makeBumpSchemaVersionView(111)
// Should return early with no panic — no other mock calls expected.
s.triggerManager.SubmitBackfillViewToScheduler(context.Background(), view)
s.triggerManager.SubmitBumpSchemaVersionViewToScheduler(context.Background(), view)
})
s.Run("GetCollection fails", func() {
s.SetupTest()
const planID = int64(500)
s.mockAlloc.EXPECT().AllocN(int64(1)).Return(planID, planID, nil).Once()
s.mockAlloc.EXPECT().AllocN(int64(2)).Return(planID, planID+2, nil).Once()
handler := NewNMockHandler(s.T())
handler.EXPECT().GetCollection(mock.Anything, s.testLabel.CollectionID).
Return(nil, errors.New("get collection error")).Once()
s.triggerManager.handler = handler
view := makeBackfillView(111)
s.triggerManager.SubmitBackfillViewToScheduler(context.Background(), view)
view := makeBumpSchemaVersionView(111)
s.triggerManager.SubmitBumpSchemaVersionViewToScheduler(context.Background(), view)
})
s.Run("collection is nil", func() {
s.SetupTest()
const planID = int64(501)
s.mockAlloc.EXPECT().AllocN(int64(1)).Return(planID, planID, nil).Once()
s.mockAlloc.EXPECT().AllocN(int64(2)).Return(planID, planID+2, nil).Once()
handler := NewNMockHandler(s.T())
handler.EXPECT().GetCollection(mock.Anything, s.testLabel.CollectionID).
Return(nil, nil).Once()
s.triggerManager.handler = handler
view := makeBackfillView(111)
s.triggerManager.SubmitBackfillViewToScheduler(context.Background(), view)
view := makeBumpSchemaVersionView(111)
s.triggerManager.SubmitBumpSchemaVersionViewToScheduler(context.Background(), view)
})
s.Run("collection is external", func() {
s.SetupTest()
const planID = int64(502)
s.mockAlloc.EXPECT().AllocN(int64(1)).Return(planID, planID, nil).Once()
s.mockAlloc.EXPECT().AllocN(int64(2)).Return(planID, planID+2, nil).Once()
handler := NewNMockHandler(s.T())
handler.EXPECT().GetCollection(mock.Anything, s.testLabel.CollectionID).
Return(&collectionInfo{
@@ -510,52 +500,19 @@ func (s *CompactionTriggerManagerSuite) TestSubmitBackfillViewToScheduler() {
}, nil).Once()
s.triggerManager.handler = handler
view := makeBackfillView(111)
s.triggerManager.SubmitBackfillViewToScheduler(context.Background(), view)
view := makeBumpSchemaVersionView(111)
s.triggerManager.SubmitBumpSchemaVersionViewToScheduler(context.Background(), view)
})
s.Run("funcDiff is nil", func() {
s.Run("view is not BumpSchemaVersionView", func() {
s.SetupTest()
s.meta.indexMeta = &indexMeta{indexes: make(map[UniqueID]map[UniqueID]*model.Index)}
const planID = int64(504)
s.mockAlloc.EXPECT().AllocN(int64(1)).Return(planID, planID, nil).Once()
handler := NewNMockHandler(s.T())
handler.EXPECT().GetCollection(mock.Anything, s.testLabel.CollectionID).
Return(&collectionInfo{
ID: s.testLabel.CollectionID,
Schema: collectionSchema,
}, nil).Once()
s.triggerManager.handler = handler
// Create a BackfillSegmentsView with funcDiff=nil — should log warning and return, not panic.
view := &BackfillSegmentsView{
label: s.testLabel,
segments: []*SegmentView{{ID: 200, label: s.testLabel, NumOfRows: 1000}},
triggerID: 111,
funcDiff: nil,
}
s.triggerManager.SubmitBackfillViewToScheduler(context.Background(), view)
})
s.Run("view is not BackfillSegmentsView", func() {
s.SetupTest()
s.meta.indexMeta = &indexMeta{indexes: make(map[UniqueID]map[UniqueID]*model.Index)}
const planID = int64(503)
s.mockAlloc.EXPECT().AllocN(int64(1)).Return(planID, planID, nil).Once()
handler := NewNMockHandler(s.T())
handler.EXPECT().GetCollection(mock.Anything, s.testLabel.CollectionID).
Return(&collectionInfo{
ID: s.testLabel.CollectionID,
Schema: collectionSchema,
}, nil).Once()
s.triggerManager.handler = handler
// Use LevelZeroCompactionView which is NOT a *BackfillSegmentsView.
nonBackfillView := &LevelZeroCompactionView{
// Use LevelZeroCompactionView which is NOT a *BumpSchemaVersionView.
nonBumpSchemaVersionView := &LevelZeroCompactionView{
label: s.testLabel,
l0Segments: []*SegmentView{},
}
s.triggerManager.SubmitBackfillViewToScheduler(context.Background(), nonBackfillView)
s.triggerManager.SubmitBumpSchemaVersionViewToScheduler(context.Background(), nonBumpSchemaVersionView)
})
s.Run("enqueueCompaction fails", func() {
@@ -565,7 +522,7 @@ func (s *CompactionTriggerManagerSuite) TestSubmitBackfillViewToScheduler() {
planID = int64(504)
triggerID = int64(999)
)
s.mockAlloc.EXPECT().AllocN(int64(1)).Return(planID, planID, nil).Once()
s.mockAlloc.EXPECT().AllocN(int64(2)).Return(planID, planID+2, nil).Once()
handler := NewNMockHandler(s.T())
handler.EXPECT().GetCollection(mock.Anything, s.testLabel.CollectionID).
Return(&collectionInfo{
@@ -575,8 +532,8 @@ func (s *CompactionTriggerManagerSuite) TestSubmitBackfillViewToScheduler() {
s.triggerManager.handler = handler
s.inspector.EXPECT().enqueueCompaction(mock.Anything).Return(errors.New("enqueue error")).Once()
view := makeBackfillView(triggerID)
s.triggerManager.SubmitBackfillViewToScheduler(context.Background(), view)
view := makeBumpSchemaVersionView(triggerID)
s.triggerManager.SubmitBumpSchemaVersionViewToScheduler(context.Background(), view)
})
s.Run("success", func() {
@@ -586,31 +543,74 @@ func (s *CompactionTriggerManagerSuite) TestSubmitBackfillViewToScheduler() {
planID = int64(600)
triggerID = int64(1001)
)
s.mockAlloc.EXPECT().AllocN(int64(1)).Return(planID, planID, nil).Once()
s.mockAlloc.EXPECT().AllocN(int64(2)).Return(planID, planID+2, nil).Once()
handler := NewNMockHandler(s.T())
handler.EXPECT().GetCollection(mock.Anything, s.testLabel.CollectionID).
Return(&collectionInfo{
ID: s.testLabel.CollectionID,
Schema: collectionSchema,
ID: s.testLabel.CollectionID,
Schema: collectionSchema,
Properties: map[string]string{common.CollectionTTLConfigKey: "3600"},
}, nil).Once()
s.triggerManager.handler = handler
s.inspector.EXPECT().enqueueCompaction(mock.Anything).
RunAndReturn(func(task *datapb.CompactionTask) error {
s.EqualValues(planID, task.GetPlanID())
s.EqualValues(triggerID, task.GetTriggerID())
s.Equal(datapb.CompactionType_BackfillCompaction, task.GetType())
s.Equal(datapb.CompactionType_BumpSchemaVersionCompaction, task.GetType())
s.Equal(s.testLabel.CollectionID, task.GetCollectionID())
s.Equal(s.testLabel.PartitionID, task.GetPartitionID())
s.Equal(s.testLabel.Channel, task.GetChannel())
s.Equal(collectionSchema, task.GetSchema())
s.Require().Len(task.GetDiffFunctions(), 1)
s.Equal(backfillFunc.GetName(), task.GetDiffFunctions()[0].GetName())
s.ElementsMatch([]int64{200}, task.GetInputSegments())
s.Empty(task.GetResultSegments())
s.NotZero(task.GetStartTime())
s.NotZero(task.GetLastStateStartTime())
s.NotNil(task.GetPreAllocatedSegmentIDs())
s.EqualValues(planID+1, task.GetPreAllocatedSegmentIDs().GetBegin())
s.EqualValues(planID+2, task.GetPreAllocatedSegmentIDs().GetEnd())
s.EqualValues(time.Hour.Nanoseconds(), task.GetCollectionTtl())
return nil
}).Once()
view := makeBackfillView(triggerID)
s.triggerManager.SubmitBackfillViewToScheduler(context.Background(), view)
view := makeBumpSchemaVersionView(triggerID)
s.triggerManager.SubmitBumpSchemaVersionViewToScheduler(context.Background(), view)
})
s.Run("policy to submit keeps frozen schema", func() {
s.SetupTest()
s.meta.indexMeta = &indexMeta{indexes: make(map[UniqueID]map[UniqueID]*model.Index)}
const (
planID = int64(601)
triggerID = int64(1002)
)
frozenSchema := &schemapb.CollectionSchema{
Name: "test_coll",
Version: 2,
Fields: []*schemapb.FieldSchema{{FieldID: 1, Name: "pk", DataType: schemapb.DataType_Int64, IsPrimaryKey: true}},
}
liveSchema := &schemapb.CollectionSchema{
Name: "test_coll",
Version: 3,
Fields: []*schemapb.FieldSchema{{FieldID: 1, Name: "pk", DataType: schemapb.DataType_Int64, IsPrimaryKey: true}},
}
s.mockAlloc.EXPECT().AllocN(int64(2)).Return(planID, planID+2, nil).Once()
handler := NewNMockHandler(s.T())
handler.EXPECT().GetCollection(mock.Anything, s.testLabel.CollectionID).
Return(&collectionInfo{ID: s.testLabel.CollectionID, Schema: liveSchema}, nil).Once()
s.triggerManager.handler = handler
s.inspector.EXPECT().enqueueCompaction(mock.Anything).
RunAndReturn(func(task *datapb.CompactionTask) error {
s.Equal(datapb.CompactionType_BumpSchemaVersionCompaction, task.GetType())
s.EqualValues(2, task.GetSchema().GetVersion())
s.Equal(frozenSchema, task.GetSchema())
s.NotNil(task.GetPreAllocatedSegmentIDs())
s.EqualValues(task.GetPreAllocatedSegmentIDs().GetBegin()+1, task.GetPreAllocatedSegmentIDs().GetEnd())
return nil
}).Once()
view := makeBumpSchemaVersionView(triggerID)
view.schema = frozenSchema
s.triggerManager.SubmitBumpSchemaVersionViewToScheduler(context.Background(), view)
})
}
@@ -625,27 +625,25 @@ func (s *CompactionTriggerManagerSuite) TestHandleTicker() {
s.Run("policy disabled", func() {
s.SetupTest()
mockPolicy := &testCompactionPolicy{enabled: false, policyName: "test-disabled"}
s.triggerManager.policies[BackfillTicker] = mockPolicy
s.triggerManager.handleTicker(context.Background(), BackfillTicker)
s.triggerManager.policies[BumpSchemaVersionTicker] = mockPolicy
s.triggerManager.handleTicker(context.Background(), BumpSchemaVersionTicker)
// Returns early — no inspector or trigger calls
})
s.Run("inspector full skips Trigger dispatch", func() {
// When isFull() returns true, Trigger() is not called. TriggerInline() runs
// unconditionally before the isFull() gate, so metadata-only updates always
// proceed; only physical compaction tasks are gated by inspector capacity.
// When isFull() returns true, Trigger() is not called and no schema-bump task is submitted.
s.SetupTest()
mockPolicy := &testCompactionPolicy{
enabled: true,
policyName: "test-full",
// Views in Trigger() result are NOT dispatched because isFull() returns true.
triggerResult: map[CompactionTriggerType][]CompactionView{
TriggerTypeBackfill: {stubDispatchableView{}},
TriggerTypeBumpSchemaVersion: {stubDispatchableView{}},
},
}
s.triggerManager.policies[BackfillTicker] = mockPolicy
s.triggerManager.policies[BumpSchemaVersionTicker] = mockPolicy
s.inspector.EXPECT().isFull().Return(true).Once()
s.triggerManager.handleTicker(context.Background(), BackfillTicker)
s.triggerManager.handleTicker(context.Background(), BumpSchemaVersionTicker)
})
s.Run("policy trigger error returns before isFull check", func() {
@@ -655,10 +653,10 @@ func (s *CompactionTriggerManagerSuite) TestHandleTicker() {
policyName: "test-err",
triggerErr: errors.New("trigger error"),
}
s.triggerManager.policies[BackfillTicker] = mockPolicy
s.triggerManager.policies[BumpSchemaVersionTicker] = mockPolicy
// isFull() IS called now (step 2, before Trigger). Trigger() errors → no notify().
s.inspector.EXPECT().isFull().Return(false).Once()
s.triggerManager.handleTicker(context.Background(), BackfillTicker)
s.triggerManager.handleTicker(context.Background(), BumpSchemaVersionTicker)
})
s.Run("policy trigger returns no events skips isFull check", func() {
@@ -669,10 +667,10 @@ func (s *CompactionTriggerManagerSuite) TestHandleTicker() {
// Nil result — Trigger() returns nothing, notify() is never called.
triggerResult: nil,
}
s.triggerManager.policies[BackfillTicker] = mockPolicy
s.triggerManager.policies[BumpSchemaVersionTicker] = mockPolicy
// isFull() IS called now (step 2 gate before Trigger). Trigger returns nil → no notify.
s.inspector.EXPECT().isFull().Return(false).Once()
s.triggerManager.handleTicker(context.Background(), BackfillTicker)
s.triggerManager.handleTicker(context.Background(), BumpSchemaVersionTicker)
})
s.Run("policy trigger returns events dispatched when inspector not full", func() {
@@ -681,136 +679,14 @@ func (s *CompactionTriggerManagerSuite) TestHandleTicker() {
enabled: true,
policyName: "test-events",
triggerResult: map[CompactionTriggerType][]CompactionView{
TriggerTypeBackfill: {stubDispatchableView{}},
TriggerTypeBumpSchemaVersion: {stubDispatchableView{}},
},
}
s.triggerManager.policies[BackfillTicker] = mockPolicy
s.triggerManager.policies[BumpSchemaVersionTicker] = mockPolicy
s.inspector.EXPECT().isFull().Return(false).Once()
// stubDispatchableView.Trigger() returns nil, so notify() short-circuits
// before reaching SubmitBackfillViewToScheduler. We only care here that
// before reaching SubmitBumpSchemaVersionViewToScheduler. We only care here that
// isFull was consulted and the dispatch path was entered.
s.triggerManager.handleTicker(context.Background(), BackfillTicker)
})
s.Run("inline-executable backfill view applied regardless of inspector full", func() {
// Regression for backfillCompactionPolicy: pure column additions emit an
// inline-executable BackfillSegmentsView (inlineMetaOnly=true). The trigger
// manager must apply it via meta.UpdateSegment without consulting isFull —
// otherwise schema versions never converge under inspector pressure.
s.SetupTest()
// Wire a mock catalog so meta.UpdateSegment can call catalog.AlterSegments.
mockCatalog := mocks.NewDataCoordCatalog(s.T())
mockCatalog.EXPECT().AlterSegments(mock.Anything, mock.Anything, mock.Anything).
Return(nil).Once()
s.triggerManager.meta.catalog = mockCatalog
segmentID := int64(424242)
s.triggerManager.meta.segments.SetSegment(segmentID, &SegmentInfo{
SegmentInfo: &datapb.SegmentInfo{
ID: segmentID,
CollectionID: s.testLabel.CollectionID,
State: commonpb.SegmentState_Flushed,
SchemaVersion: 1,
},
})
// Inline views come from TriggerInline(), not Trigger(). Trigger() returns nothing.
// The inline view is applied before isFull() is even consulted, so schema version
// convergence is guaranteed regardless of inspector pressure.
mockPolicy := &testCompactionPolicy{
enabled: true,
policyName: "test-meta-update",
inlineTriggerResult: map[CompactionTriggerType][]CompactionView{
TriggerTypeBackfill: {
&BackfillSegmentsView{
label: s.testLabel,
segments: []*SegmentView{{ID: segmentID, label: s.testLabel}},
inlineMetaOnly: true,
targetSchemaVersion: 5,
},
},
},
// triggerResult is nil: no compaction tasks to dispatch
}
s.triggerManager.policies[BackfillTicker] = mockPolicy
// isFull() is consulted for the Trigger() gate — simulate a full inspector to
// prove inline views are unaffected by inspector pressure.
s.inspector.EXPECT().isFull().Return(true).Once()
s.triggerManager.handleTicker(context.Background(), BackfillTicker)
updated := s.triggerManager.meta.segments.GetSegment(segmentID)
s.Require().NotNil(updated)
s.Equal(int32(5), updated.GetSchemaVersion(),
"inline view must be applied to bump segment schema version")
})
s.Run("applyInlineView with wrong view type logs warning and returns", func() {
// When TriggerInline() returns a non-BackfillSegmentsView in the inline events,
// applyInlineView must log a warning and return without panic.
s.SetupTest()
mockPolicy := &testCompactionPolicy{
enabled: true,
policyName: "test-wrong-inline-type",
inlineTriggerResult: map[CompactionTriggerType][]CompactionView{
TriggerTypeBackfill: {
// stubDispatchableView is NOT *BackfillSegmentsView — triggers the wrong-type branch.
stubDispatchableView{},
},
},
// No compaction tasks to dispatch.
}
s.triggerManager.policies[BackfillTicker] = mockPolicy
// isFull() is consulted for the Trigger() gate.
s.inspector.EXPECT().isFull().Return(true).Once()
// Should complete without panic.
s.triggerManager.handleTicker(context.Background(), BackfillTicker)
})
s.Run("applyInlineView UpdateSegment fails logs error and continues", func() {
// When meta.UpdateSegment fails (catalog error), applyInlineView logs the error
// and continues rather than aborting the whole inline pass.
s.SetupTest()
// Wire a catalog that rejects AlterSegments.
mockCatalog := mocks.NewDataCoordCatalog(s.T())
mockCatalog.EXPECT().AlterSegments(mock.Anything, mock.Anything, mock.Anything).
Return(errors.New("catalog error")).Once()
s.triggerManager.meta.catalog = mockCatalog
segmentID := int64(424243)
s.triggerManager.meta.segments.SetSegment(segmentID, &SegmentInfo{
SegmentInfo: &datapb.SegmentInfo{
ID: segmentID,
CollectionID: s.testLabel.CollectionID,
State: commonpb.SegmentState_Flushed,
SchemaVersion: 1,
},
})
mockPolicy := &testCompactionPolicy{
enabled: true,
policyName: "test-update-segment-fail",
inlineTriggerResult: map[CompactionTriggerType][]CompactionView{
TriggerTypeBackfill: {
&BackfillSegmentsView{
label: s.testLabel,
segments: []*SegmentView{{ID: segmentID, label: s.testLabel}},
inlineMetaOnly: true,
targetSchemaVersion: 9,
},
},
},
}
s.triggerManager.policies[BackfillTicker] = mockPolicy
s.inspector.EXPECT().isFull().Return(true).Once()
// Should not panic despite the catalog error.
s.triggerManager.handleTicker(context.Background(), BackfillTicker)
// Schema version must remain unchanged because UpdateSegment failed.
updated := s.triggerManager.meta.segments.GetSegment(segmentID)
s.Require().NotNil(updated)
s.Equal(int32(1), updated.GetSchemaVersion(),
"schema version must stay 1 when UpdateSegment fails")
s.triggerManager.handleTicker(context.Background(), BumpSchemaVersionTicker)
})
}
internal/datacoord/compaction_view.go
-6
View File
@@ -36,12 +36,6 @@ type CompactionView interface {
ForceTrigger() (CompactionView, string)
ForceTriggerAll() ([]CompactionView, string)
GetTriggerID() int64
// IsInlineExecutable reports whether this view should be applied directly inside
// datacoord by CompactionTriggerManager rather than dispatched as a compaction
// task to the inspector. Inline views consume no inspector slot and do not depend
// on a free queue, so they must not be gated by inspector pressure.
// Default: false (most views are real compaction work).
IsInlineExecutable() bool
}
type FullViews struct {
internal/datacoord/compaction_view_forcemerge.go
-3
View File
@@ -32,9 +32,6 @@ const (
defaultToleranceMB = 0.05
)
// IsInlineExecutable returns false: force merge is real compaction work.
func (v *ForceMergeSegmentView) IsInlineExecutable() bool { return false }
// static segment view, only algothrims here, no IO
type ForceMergeSegmentView struct {
label *CompactionGroupLabel
internal/datacoord/index_inspector.go
+37 -43
View File
@@ -18,7 +18,6 @@ package datacoord
import (
"context"
"fmt"
"sync"
"time"
@@ -32,7 +31,6 @@ import (
"github.com/milvus-io/milvus/internal/util/vecindexmgr"
"github.com/milvus-io/milvus/pkg/v3/log"
"github.com/milvus-io/milvus/pkg/v3/proto/datapb"
"github.com/milvus-io/milvus/pkg/v3/util/merr"
"github.com/milvus-io/milvus/pkg/v3/util/paramtable"
)
@@ -158,50 +156,17 @@ func (i *indexInspector) createIndexesForSegment(ctx context.Context, segment *S
indexes := i.meta.indexMeta.GetIndexesForCollection(segment.CollectionID, "")
indexIDToSegIndexes := i.meta.indexMeta.GetSegmentIndexes(segment.CollectionID, segment.ID)
// MinSchemaVersion enforcement is only applied while the collection has an in-flight
// physical backfill (e.g. AlterCollectionSchema adding a BM25 function): until backfill
// writes the new field's data, an index built on the empty binlog would silently return
// wrong results. For metadata-only schema changes (e.g. nullable AddCollectionField),
// the index builder handles null/default values, so no delay is needed.
// Within the physical-backfill branch we still double-check segment binlogs because a
// function output field that was backfilled in a previous schema version is already
// present in this segment.
coll := i.meta.GetCollection(segment.CollectionID)
collectionInPhysicalBackfill := coll != nil && coll.Schema != nil && coll.Schema.GetDoPhysicalBackfill()
var segmentBinlogFields map[int64]struct{}
for _, index := range indexes {
if _, ok := indexIDToSegIndexes[index.IndexID]; ok {
continue
}
if collectionInPhysicalBackfill && index.MinSchemaVersion > segment.GetSchemaVersion() {
if segmentBinlogFields == nil {
segmentBinlogFields = getSegmentBinlogFields(segment)
}
if _, hasField := segmentBinlogFields[index.FieldID]; !hasField {
log.Ctx(ctx).Info("skip index creation: field data not yet present in segment, waiting for physical backfill",
zap.Int64("segmentID", segment.ID),
zap.Int64("indexID", index.IndexID),
zap.Int64("fieldID", index.FieldID),
zap.Int32("segSchemaVersion", segment.GetSchemaVersion()),
zap.Int32("indexMinSchemaVersion", index.MinSchemaVersion))
continue
}
// Transient inconsistency window: backfill has already written the field's
// binlog data to this segment, but the metadata tick has not yet bumped
// segment.SchemaVersion to match. The window should close within one backfill
// tick. Bail out with an error so the inspector retries on the next tick by
// which time the metadata is expected to have caught up. Sustained occurrences
// indicate the metadata-update path is stuck and warrant investigation.
log.Ctx(ctx).Warn("inconsistent state: field binlogs present but segment schema version still behind, will retry",
zap.Int64("segmentID", segment.ID),
zap.Int64("indexID", index.IndexID),
zap.Int64("fieldID", index.FieldID),
zap.Int32("segSchemaVersion", segment.GetSchemaVersion()),
zap.Int32("indexMinSchemaVersion", index.MinSchemaVersion))
return merr.WrapErrServiceInternal(
fmt.Sprintf("segment schema version %d behind index min schema version %d while field %d binlogs already present, retry next tick",
segment.GetSchemaVersion(), index.MinSchemaVersion, index.FieldID))
if segmentBinlogFields == nil {
segmentBinlogFields = getSegmentBinlogFields(segment)
}
if !i.canCreateIndexForSegment(ctx, segment, index, segmentBinlogFields) {
continue
}
if err := i.createIndexForSegment(ctx, segment, index.IndexID); err != nil {
log.Ctx(ctx).Warn("create index for segment fail", zap.Int64("segmentID", segment.ID),
@@ -213,11 +178,16 @@ func (i *indexInspector) createIndexesForSegment(ctx context.Context, segment *S
}
// getSegmentBinlogFields returns the set of field IDs that have binlog data in the segment.
// binlog.GetFieldID() is a columnGroupID, not a real field ID; the actual field IDs
// are always in binlog.GetChildFields().
// StorageV2/V3 column groups report real field IDs through ChildFields; legacy entries may use FieldID directly.
func getSegmentBinlogFields(segment *SegmentInfo) map[int64]struct{} {
result := make(map[int64]struct{})
for _, binlog := range segment.GetBinlogs() {
if len(binlog.GetChildFields()) == 0 {
if segment.GetStorageVersion() == storage.StorageV1 {
result[binlog.GetFieldID()] = struct{}{}
}
continue
}
for _, childFieldID := range binlog.GetChildFields() {
result[childFieldID] = struct{}{}
}
@@ -225,6 +195,30 @@ func getSegmentBinlogFields(segment *SegmentInfo) map[int64]struct{} {
return result
}
func (i *indexInspector) canCreateIndexForSegment(ctx context.Context, segment *SegmentInfo, index *model.Index, segmentBinlogFields map[int64]struct{}) bool {
_, hasField := segmentBinlogFields[index.FieldID]
if !i.isFunctionOutputField(segment.CollectionID, index.FieldID) || hasField {
return true
}
log.Ctx(ctx).Debug("function output field has no binlog, skip create index", zap.Int64("segmentID", segment.ID), zap.Int64("fieldID", index.FieldID), zap.Int64("indexID", index.IndexID))
return false
}
func (i *indexInspector) isFunctionOutputField(collectionID, fieldID int64) bool {
collection := i.meta.GetCollection(collectionID)
if collection == nil || collection.Schema == nil {
return false
}
for _, functionSchema := range collection.Schema.GetFunctions() {
for _, outputFieldID := range functionSchema.GetOutputFieldIds() {
if outputFieldID == fieldID {
return true
}
}
}
return false
}
func (i *indexInspector) createIndexForSegment(ctx context.Context, segment *SegmentInfo, indexID UniqueID) error {
log.Info("create index for segment", zap.Int64("segmentID", segment.ID), zap.Int64("indexID", indexID))
buildID, err := i.allocator.AllocID(context.Background())
internal/datacoord/index_inspector_test.go
+128 -76
View File
@@ -32,6 +32,7 @@ import (
mocks2 "github.com/milvus-io/milvus/internal/metastore/mocks"
"github.com/milvus-io/milvus/internal/metastore/model"
"github.com/milvus-io/milvus/internal/mocks"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/pkg/v3/common"
"github.com/milvus-io/milvus/pkg/v3/proto/datapb"
"github.com/milvus-io/milvus/pkg/v3/proto/workerpb"
@@ -71,6 +72,9 @@ func TestIndexInspector_inspect(t *testing.T) {
NumOfRows: 3000,
State: commonpb.SegmentState_Flushed,
IsSorted: true,
Binlogs: []*datapb.FieldBinlog{
{FieldID: 101},
},
},
}
meta.segments.SetSegment(segment.GetID(), segment)
@@ -303,6 +307,9 @@ func TestIndexInspector_CreateIndexesForSegment_ExternalUnsorted(t *testing.T) {
CollectionID: 2,
State: commonpb.SegmentState_Flushed,
IsSorted: false,
Binlogs: []*datapb.FieldBinlog{
{FieldID: 101},
},
},
}
m.segments.SetSegment(segment.GetID(), segment)
@@ -405,6 +412,40 @@ func TestGetSegmentBinlogFields(t *testing.T) {
assert.NotContains(t, fields, int64(1))
})
t.Run("legacy FieldID fallback when ChildFields empty", func(t *testing.T) {
segment := &SegmentInfo{
SegmentInfo: &datapb.SegmentInfo{
StorageVersion: storage.StorageV1,
Binlogs: []*datapb.FieldBinlog{
{FieldID: 101},
{FieldID: 0, ChildFields: []int64{100, 102}},
},
},
}
fields := getSegmentBinlogFields(segment)
assert.Contains(t, fields, int64(101))
assert.Contains(t, fields, int64(100))
assert.Contains(t, fields, int64(102))
assert.NotContains(t, fields, int64(0))
})
t.Run("packed storage ignores FieldID fallback when ChildFields empty", func(t *testing.T) {
segment := &SegmentInfo{
SegmentInfo: &datapb.SegmentInfo{
StorageVersion: storage.StorageV3,
Binlogs: []*datapb.FieldBinlog{
{FieldID: 101},
{FieldID: 0, ChildFields: []int64{100, 102}},
},
},
}
fields := getSegmentBinlogFields(segment)
assert.NotContains(t, fields, int64(101))
assert.Contains(t, fields, int64(100))
assert.Contains(t, fields, int64(102))
assert.NotContains(t, fields, int64(0))
})
t.Run("empty binlogs", func(t *testing.T) {
segment := &SegmentInfo{
SegmentInfo: &datapb.SegmentInfo{Binlogs: nil},
@@ -414,7 +455,7 @@ func TestGetSegmentBinlogFields(t *testing.T) {
})
}
func TestIndexInspector_MinSchemaVersionEnforcement(t *testing.T) {
func TestIndexInspector_FunctionOutputBinlogGate(t *testing.T) {
paramtable.Init()
paramtable.Get().Save(paramtable.Get().DataCoordCfg.EnableSortCompaction.Key, "false")
defer paramtable.Get().Reset(paramtable.Get().DataCoordCfg.EnableSortCompaction.Key)
@@ -441,31 +482,15 @@ func TestIndexInspector_MinSchemaVersionEnforcement(t *testing.T) {
}
inspector := newIndexInspector(ctx, notifyChan, m, scheduler, alloc, handler, storageCli, versionManager)
collID := int64(10)
// Index on field 102 (function output) with MinSchemaVersion=2
m.indexMeta.indexes[collID] = map[UniqueID]*model.Index{
5: {
CollectionID: collID,
FieldID: 102,
IndexID: 5,
IndexName: "bm25_idx",
MinSchemaVersion: 2,
},
}
m.collections.Insert(collID, &collectionInfo{
ID: collID,
Schema: &schemapb.CollectionSchema{
// DoPhysicalBackfill=true is required for the MinSchemaVersion gate to engage.
// Without it the gate is bypassed and indexes are built immediately (the
// metadata-only schema-change path).
DoPhysicalBackfill: true,
Fields: []*schemapb.FieldSchema{
{FieldID: 100, Name: "pk", DataType: schemapb.DataType_Int64},
{FieldID: 101, Name: "text", DataType: schemapb.DataType_VarChar},
{FieldID: 102, Name: "sparse", DataType: schemapb.DataType_SparseFloatVector},
{FieldID: 103, Name: "new_vector", DataType: schemapb.DataType_FloatVector},
},
Functions: []*schemapb.FunctionSchema{
{Name: "bm25_fn", OutputFieldIds: []int64{102}},
@@ -473,67 +498,16 @@ func TestIndexInspector_MinSchemaVersionEnforcement(t *testing.T) {
},
})
t.Run("skip index when field data missing from segment", func(t *testing.T) {
// Segment with SchemaVersion=1, binlogs have fields 100+101 but NOT 102.
// In physical backfill mode, the gate skips index creation until the field's
// binlog data is written by backfill.
segment := &SegmentInfo{
SegmentInfo: &datapb.SegmentInfo{
ID: 1,
CollectionID: collID,
State: commonpb.SegmentState_Flushed,
IsSorted: true,
SchemaVersion: 1,
Binlogs: []*datapb.FieldBinlog{
{FieldID: 0, ChildFields: []int64{100, 101}},
},
},
t.Run("create function output index when binlog exists", func(t *testing.T) {
m.indexMeta.indexes[collID] = map[UniqueID]*model.Index{
5: {CollectionID: collID, FieldID: 102, IndexID: 5, IndexName: "bm25_idx"},
}
m.segments.SetSegment(segment.GetID(), segment)
err := inspector.createIndexesForSegment(ctx, segment)
assert.NoError(t, err)
// Index should NOT have been created — field 102 data is missing
assert.True(t, m.indexMeta.IsUnIndexedSegment(collID, segment.GetID()))
})
t.Run("return error when field binlogs present but segment schema version still behind", func(t *testing.T) {
// Segment with SchemaVersion=1 (behind index.MinSchemaVersion=2) BUT field 102
// already exists in binlogs. This is a transient inconsistency window: backfill
// has written the field data but the metadata-update tick has not yet bumped
// segment.SchemaVersion. The inspector must NOT proceed to build the index in
// this window — it returns an error and retries on the next tick by which time
// the metadata is expected to have caught up.
segment := &SegmentInfo{
SegmentInfo: &datapb.SegmentInfo{
ID: 2,
CollectionID: collID,
State: commonpb.SegmentState_Flushed,
IsSorted: true,
SchemaVersion: 1,
Binlogs: []*datapb.FieldBinlog{
{FieldID: 0, ChildFields: []int64{100, 101, 102}},
},
},
}
m.segments.SetSegment(segment.GetID(), segment)
err := inspector.createIndexesForSegment(ctx, segment)
assert.Error(t, err)
assert.Contains(t, err.Error(), "segment schema version")
// Index should NOT have been created
assert.True(t, m.indexMeta.IsUnIndexedSegment(collID, segment.GetID()))
})
t.Run("create index normally when MinSchemaVersion matches", func(t *testing.T) {
// Segment with SchemaVersion=2, matches MinSchemaVersion — should proceed normally
segment := &SegmentInfo{
SegmentInfo: &datapb.SegmentInfo{
ID: 3,
CollectionID: collID,
State: commonpb.SegmentState_Flushed,
IsSorted: true,
SchemaVersion: 2,
ID: 1,
CollectionID: collID,
State: commonpb.SegmentState_Flushed,
IsSorted: true,
Binlogs: []*datapb.FieldBinlog{
{FieldID: 0, ChildFields: []int64{100, 101, 102}},
},
@@ -547,5 +521,83 @@ func TestIndexInspector_MinSchemaVersionEnforcement(t *testing.T) {
err := inspector.createIndexesForSegment(ctx, segment)
assert.NoError(t, err)
assert.Contains(t, m.indexMeta.GetSegmentIndexes(collID, segment.GetID()), UniqueID(5))
})
t.Run("skip function output index when binlog is missing", func(t *testing.T) {
m.indexMeta.indexes[collID] = map[UniqueID]*model.Index{
6: {CollectionID: collID, FieldID: 102, IndexID: 6, IndexName: "bm25_idx_missing_binlog"},
}
segment := &SegmentInfo{
SegmentInfo: &datapb.SegmentInfo{
ID: 2,
CollectionID: collID,
State: commonpb.SegmentState_Flushed,
IsSorted: true,
Binlogs: []*datapb.FieldBinlog{
{FieldID: 0, ChildFields: []int64{100, 101}},
},
},
}
m.segments.SetSegment(segment.GetID(), segment)
err := inspector.createIndexesForSegment(ctx, segment)
assert.NoError(t, err)
assert.NotContains(t, m.indexMeta.GetSegmentIndexes(collID, segment.GetID()), UniqueID(6))
})
t.Run("create non function output index when binlog is missing", func(t *testing.T) {
m.indexMeta.indexes[collID] = map[UniqueID]*model.Index{
7: {CollectionID: collID, FieldID: 103, IndexID: 7, IndexName: "new_vector_idx"},
}
segment := &SegmentInfo{
SegmentInfo: &datapb.SegmentInfo{
ID: 3,
CollectionID: collID,
State: commonpb.SegmentState_Flushed,
IsSorted: true,
Binlogs: []*datapb.FieldBinlog{
{FieldID: 0, ChildFields: []int64{100, 101}},
},
},
}
m.segments.SetSegment(segment.GetID(), segment)
alloc.EXPECT().AllocID(mock.Anything).Return(int64(12347), nil).Once()
catalog.EXPECT().CreateSegmentIndex(mock.Anything, mock.Anything).Return(nil).Once()
scheduler.EXPECT().Enqueue(mock.Anything).Return().Once()
err := inspector.createIndexesForSegment(ctx, segment)
assert.NoError(t, err)
assert.Contains(t, m.indexMeta.GetSegmentIndexes(collID, segment.GetID()), UniqueID(7))
})
t.Run("create ready normal index while skipping missing function output", func(t *testing.T) {
m.indexMeta.indexes[collID] = map[UniqueID]*model.Index{
8: {CollectionID: collID, FieldID: 101, IndexID: 8, IndexName: "text_idx"},
9: {CollectionID: collID, FieldID: 102, IndexID: 9, IndexName: "bm25_idx_missing_binlog"},
}
segment := &SegmentInfo{
SegmentInfo: &datapb.SegmentInfo{
ID: 4,
CollectionID: collID,
State: commonpb.SegmentState_Flushed,
IsSorted: true,
Binlogs: []*datapb.FieldBinlog{
{FieldID: 0, ChildFields: []int64{100, 101}},
},
},
}
m.segments.SetSegment(segment.GetID(), segment)
alloc.EXPECT().AllocID(mock.Anything).Return(int64(12348), nil).Once()
catalog.EXPECT().CreateSegmentIndex(mock.Anything, mock.Anything).Return(nil).Once()
scheduler.EXPECT().Enqueue(mock.Anything).Return().Once()
err := inspector.createIndexesForSegment(ctx, segment)
assert.NoError(t, err)
segIndexes := m.indexMeta.GetSegmentIndexes(collID, segment.GetID())
assert.Contains(t, segIndexes, UniqueID(8))
assert.NotContains(t, segIndexes, UniqueID(9))
})
}
internal/datacoord/index_service.go
+1 -5
View File
@@ -299,10 +299,6 @@ func (s *Server) CreateIndex(ctx context.Context, req *indexpb.CreateIndexReques
CreateTime: req.GetTimestamp(),
IsAutoIndex: req.GetIsAutoIndex(),
UserIndexParams: req.GetUserIndexParams(),
// MinSchemaVersion prevents building an index on a function-output column (e.g. BM25 sparse
// vector) before backfill has written that column into old segments. The inspector skips any
// segment whose SchemaVersion is still below this value.
MinSchemaVersion: schema.GetVersion(),
}
// Validate the index params.
if err := ValidateIndexParams(index); err != nil {
@@ -332,7 +328,7 @@ func (s *Server) CreateIndex(ctx context.Context, req *indexpb.CreateIndexReques
}
log.Info("CreateIndex successfully",
zap.String("IndexName", index.IndexName), zap.Int64("fieldID", index.FieldID),
zap.Int64("IndexID", index.IndexID), zap.Int32("MinSchemaVersion", index.MinSchemaVersion),
zap.Int64("IndexID", index.IndexID),
zap.Strings("channels", channels))
metrics.IndexRequestCounter.WithLabelValues(metrics.SuccessLabel).Inc()
return merr.Success(), nil
internal/datacoord/meta.go
+144 -56
View File
@@ -2163,6 +2163,11 @@ func (m *meta) completeMixCompactionMutation(
log = log.With(zap.Int64s("compactFrom", compactFromSegIDs))
if t.GetSchema() == nil {
return nil, nil, merr.WrapErrIllegalCompactionPlan("mix compaction task schema is nil")
}
outputSchemaVersion := t.GetSchema().GetVersion()
// Compaction normalizes import segments: row timestamps in the output
// binlogs are already rewritten to commit_ts by the compactor, so
// recalculateSegmentPosition will pick up commit_ts from output binlogs.
@@ -2210,7 +2215,7 @@ func (m *meta) completeMixCompactionMutation(
ManifestPath: compactToSegment.GetManifest(),
IsSortedByNamespace: compactToSegment.GetIsSortedByNamespace(),
ExpirQuantiles: compactToSegment.GetExpirQuantiles(),
SchemaVersion: t.GetSchema().GetVersion(),
SchemaVersion: outputSchemaVersion,
CommitTimestamp: 0, // Normalized: row timestamps already rewritten
})
@@ -2337,8 +2342,8 @@ func (m *meta) CompleteCompactionMutation(ctx context.Context, t *datapb.Compact
return m.completeClusterCompactionMutation(t, result)
case datapb.CompactionType_SortCompaction:
return m.completeSortCompactionMutation(t, result)
case datapb.CompactionType_BackfillCompaction:
return m.completeBackfillCompactionMutation(t, result)
case datapb.CompactionType_BumpSchemaVersionCompaction:
return m.completeBumpSchemaVersionCompactionMutation(t, result)
}
return nil, nil, merr.WrapErrIllegalCompactionPlan("illegal compaction type")
}
@@ -2856,6 +2861,11 @@ func (m *meta) completeSortCompactionMutation(
normalizePositionTimestamp(oldSegment.GetStartPosition(), commitTs),
normalizePositionTimestamp(oldSegment.GetDmlPosition(), commitTs))
if t.GetSchema() == nil {
return nil, nil, merr.WrapErrIllegalCompactionPlan("sort compaction task schema is nil")
}
outputSchemaVersion := t.GetSchema().GetVersion()
segmentInfo := &datapb.SegmentInfo{
CollectionID: oldSegment.GetCollectionID(),
PartitionID: oldSegment.GetPartitionID(),
@@ -2885,7 +2895,7 @@ func (m *meta) completeSortCompactionMutation(
ManifestPath: resultSegment.GetManifest(),
ExpirQuantiles: resultSegment.GetExpirQuantiles(),
IsSortedByNamespace: resultSegment.GetIsSortedByNamespace(),
SchemaVersion: oldSegment.GetSchemaVersion(),
SchemaVersion: outputSchemaVersion,
CommitTimestamp: 0, // Normalized: row timestamps already rewritten
}
@@ -2921,7 +2931,7 @@ func (m *meta) completeSortCompactionMutation(
return []*SegmentInfo{segment}, metricMutation, nil
}
func (m *meta) completeBackfillCompactionMutation(
func (m *meta) completeBumpSchemaVersionCompactionMutation(
t *datapb.CompactionTask,
result *datapb.CompactionPlanResult,
) ([]*SegmentInfo, *segMetricMutation, error) {
@@ -2933,12 +2943,12 @@ func (m *meta) completeBackfillCompactionMutation(
metricMutation := &segMetricMutation{stateChange: make(map[string]map[string]map[string]map[string]int)}
// Backfill compaction updates the existing segment, not creating a new one
// Schema bump compaction has one input and one result.
if len(t.GetInputSegments()) != 1 {
return nil, nil, merr.WrapErrIllegalCompactionPlan("backfill compaction should have exactly one input segment")
return nil, nil, merr.WrapErrIllegalCompactionPlan("schema bump compaction should have exactly one input segment")
}
if len(result.GetSegments()) != 1 {
return nil, nil, merr.WrapErrIllegalCompactionPlan("backfill compaction result should have exactly one segment")
return nil, nil, merr.WrapErrIllegalCompactionPlan("schema bump compaction result should have exactly one segment")
}
segmentID := t.GetInputSegments()[0]
@@ -2958,59 +2968,61 @@ func (m *meta) completeBackfillCompactionMutation(
}
resultSegment := result.GetSegments()[0]
if t.GetSchema() == nil {
return nil, nil, merr.WrapErrIllegalCompactionPlan("schema bump compaction requires task schema")
}
newSchemaVersion := t.GetSchema().GetVersion()
if newSchemaVersion < oldSegment.GetSchemaVersion() {
return nil, nil, merr.WrapErrIllegalCompactionPlan("schema bump compaction schema version is older than input segment")
}
if oldSegment.GetIsInvisible() {
return nil, nil, merr.WrapErrIllegalCompactionPlan("schema bump compaction input segment should not be invisible")
}
if resultSegment.GetNumOfRows() == 0 && resultSegment.GetSegmentID() != segmentID {
if resultSegment.GetStorageVersion() < storage.StorageV3 {
return nil, nil, merr.WrapErrIllegalCompactionPlan("schema bump compaction result should contain a StorageV3 segment")
}
return m.completeBumpSchemaVersionReplacementMutation(log, metricMutation, t, oldSegment, resultSegment, newSchemaVersion)
}
resultManifest := resultSegment.GetManifest()
if resultSegment.GetStorageVersion() < storage.StorageV3 || resultManifest == "" {
return nil, nil, merr.WrapErrIllegalCompactionPlan("schema bump compaction result should contain a StorageV3 manifest")
}
if resultSegment.GetSegmentID() != segmentID {
return nil, nil, merr.WrapErrIllegalCompactionPlan(fmt.Sprintf("backfill compaction result segment ID %d does not match input segment ID %d", resultSegment.GetSegmentID(), segmentID))
return m.completeBumpSchemaVersionReplacementMutation(log, metricMutation, t, oldSegment, resultSegment, newSchemaVersion)
}
currentManifest := oldSegment.GetManifestPath()
if currentManifest == "" {
return nil, nil, merr.WrapErrIllegalCompactionPlan("schema bump compaction input segment should contain a StorageV3 manifest")
}
if currentManifest != resultManifest {
manifestCompare, err := packed.CompareManifestPath(resultManifest, currentManifest)
if err != nil {
return nil, nil, merr.WrapErrIllegalCompactionPlan(fmt.Sprintf("schema bump compaction result manifest is not comparable with current manifest: %v", err))
}
if manifestCompare <= 0 {
return nil, nil, merr.WrapErrIllegalCompactionPlan("schema bump compaction result manifest is not newer than current manifest")
}
}
// Clone the segment for update
cloned := oldSegment.Clone()
// Replace binlogs with the merged result (original fields + new function output field).
// For V3 segments, buildMergedLogsV3 assigns LogID!=0 so buildBinlogKvs validation passes
// and getSegmentBinlogFields can detect the new field via ChildFields.
cloned.Binlogs = resultSegment.GetInsertLogs()
// Update BM25 stats logs: for V2 segments, stats are separate files tracked in Bm25Statslogs.
// Merge so that stats for previously backfilled BM25 fields are preserved when a second
// AlterCollectionSchema adds another BM25 function. Before merging, filter out entries
// whose (fieldID, logID) already exist so that crash-replay — where the same result is
// applied twice because datacoord crashed between the etcd write and the task state
// transition — does not produce duplicate stats entries.
// For V3 segments (manifest-based), BM25 stats are embedded in the manifest and
// Bm25Logs in the result is nil — skip updating Bm25Statslogs to avoid clearing existing stats.
if resultSegment.GetManifest() == "" {
dedupedBm25Logs := filterDuplicateFieldBinlogs(cloned.GetBm25Statslogs(), resultSegment.GetBm25Logs())
cloned.Bm25Statslogs = mergeFieldBinlogs(cloned.GetBm25Statslogs(), dedupedBm25Logs)
}
// Update SchemaVersion from task schema.
// t.Schema is set from collection.Schema at task creation time and should never be nil.
// A nil schema here indicates data corruption (e.g. etcd serialization loss); we warn
// and skip the update so the segment's schemaVersion remains stale, causing the next
// backfill trigger to re-evaluate and re-submit the task.
if t.GetSchema() == nil {
log.Warn("backfill compaction task has nil schema, skipping schemaVersion update — segment will be re-evaluated on next trigger",
if newSchemaVersion > cloned.GetSchemaVersion() {
cloned.SchemaVersion = newSchemaVersion
log.Info("meta update: update schema version for schema bump compaction",
zap.Int64("segmentID", segmentID),
zap.Int64("planID", t.GetPlanID()))
} else {
newSchemaVersion := t.GetSchema().GetVersion()
if newSchemaVersion > cloned.GetSchemaVersion() {
cloned.SchemaVersion = newSchemaVersion
log.Info("meta update: update schema version for backfill compaction",
zap.Int64("segmentID", segmentID),
zap.Int32("oldSchemaVersion", oldSegment.GetSchemaVersion()),
zap.Int32("newSchemaVersion", newSchemaVersion))
}
zap.Int32("oldSchemaVersion", oldSegment.GetSchemaVersion()),
zap.Int32("newSchemaVersion", newSchemaVersion))
}
// Update StorageVersion only when result has manifest (true V3 segment).
// V2 segments on V3 clusters stay V2 — backfill forces V2 path to avoid
// creating partial manifests that would corrupt segment loading.
// Update ManifestPath for V3 segments — the merged manifest contains both
// original columns and new function output columns + BM25 stats.
if resultSegment.GetManifest() != "" {
cloned.StorageVersion = resultSegment.GetStorageVersion()
cloned.ManifestPath = resultSegment.GetManifest()
cloned.StorageVersion = resultSegment.GetStorageVersion()
cloned.ManifestPath = resultManifest
if !proto.Equal(oldSegment.SegmentInfo, cloned.SegmentInfo) {
cloned.DataVersion = oldSegment.GetDataVersion() + 1
}
// Prepare binlogs increment for catalog update
@@ -3021,25 +3033,101 @@ func (m *meta) completeBackfillCompactionMutation(
log = log.With(zap.Int64("segmentID", segmentID),
zap.Int32("oldSchemaVersion", oldSegment.GetSchemaVersion()),
zap.Int32("newSchemaVersion", cloned.GetSchemaVersion()),
zap.Int("newInsertLogsCount", len(resultSegment.GetInsertLogs())),
zap.Int("newBm25LogsCount", len(resultSegment.GetBm25Logs())))
zap.Int("newInsertLogsCount", len(resultSegment.GetInsertLogs())))
log.Info("meta update: prepare for complete backfill compaction mutation - complete",
log.Info("meta update: prepare for complete schema bump compaction mutation - complete",
zap.Int64("num rows", cloned.GetNumOfRows()))
// Save to catalog
if err := m.catalog.AlterSegments(m.ctx, []*datapb.SegmentInfo{cloned.SegmentInfo}, binlogsIncrement); err != nil {
log.Warn("fail to alter segment for backfill compaction", zap.Error(err))
log.Warn("fail to alter segment for schema bump compaction", zap.Error(err))
return nil, nil, err
}
// Update in-memory meta
m.segments.SetSegment(segmentID, cloned)
log.Info("meta update: alter in memory meta after backfill compaction - complete")
log.Info("meta update: alter in memory meta after schema bump compaction - complete")
return []*SegmentInfo{cloned}, metricMutation, nil
}
func (m *meta) completeBumpSchemaVersionReplacementMutation(
log *log.MLogger,
metricMutation *segMetricMutation,
t *datapb.CompactionTask,
oldSegment *SegmentInfo,
resultSegment *datapb.CompactionSegment,
schemaVersion int32,
) ([]*SegmentInfo, *segMetricMutation, error) {
idRange := t.GetPreAllocatedSegmentIDs()
if idRange == nil || idRange.GetBegin()+1 != idRange.GetEnd() || resultSegment.GetSegmentID() != idRange.GetBegin() {
return nil, nil, merr.WrapErrIllegalCompactionPlan(fmt.Sprintf("schema bump replacement result segment ID %d does not match the pre-allocated segment ID", resultSegment.GetSegmentID()))
}
dropped := oldSegment.Clone()
dropped.Compacted = true
updateSegStateAndPrepareMetrics(dropped, commonpb.SegmentState_Dropped, metricMutation)
startPos, dmlPos := recalculateSegmentPosition(resultSegment.GetInsertLogs(), oldSegment.GetInsertChannel(), oldSegment.GetStartPosition(), oldSegment.GetDmlPosition())
newSegment := NewSegmentInfo(&datapb.SegmentInfo{
ID: resultSegment.GetSegmentID(),
CollectionID: oldSegment.GetCollectionID(),
PartitionID: oldSegment.GetPartitionID(),
InsertChannel: oldSegment.GetInsertChannel(),
MaxRowNum: oldSegment.GetMaxRowNum(),
LastExpireTime: oldSegment.GetLastExpireTime(),
StartPosition: startPos,
DmlPosition: dmlPos,
IsImporting: oldSegment.GetIsImporting(),
State: commonpb.SegmentState_Flushed,
Level: oldSegment.GetLevel(),
LastLevel: oldSegment.GetLastLevel(),
PartitionStatsVersion: oldSegment.GetPartitionStatsVersion(),
LastPartitionStatsVersion: oldSegment.GetLastPartitionStatsVersion(),
CreatedByCompaction: true,
IsInvisible: false,
StorageVersion: resultSegment.GetStorageVersion(),
NumOfRows: resultSegment.GetNumOfRows(),
Binlogs: resultSegment.GetInsertLogs(),
Statslogs: resultSegment.GetField2StatslogPaths(),
TextStatsLogs: resultSegment.GetTextStatsLogs(),
Bm25Statslogs: resultSegment.GetBm25Logs(),
Deltalogs: resultSegment.GetDeltalogs(),
CompactionFrom: []int64{oldSegment.GetID()},
IsSorted: oldSegment.GetIsSorted(),
ManifestPath: resultSegment.GetManifest(),
ExpirQuantiles: resultSegment.GetExpirQuantiles(),
IsSortedByNamespace: oldSegment.GetIsSortedByNamespace(),
SchemaVersion: schemaVersion,
})
if newSegment.GetNumOfRows() > 0 {
metricMutation.addNewSeg(newSegment.GetState(), newSegment.GetLevel(), newSegment.GetIsSorted(), newSegment.GetStorageVersion(), newSegment.GetNumOfRows())
} else {
newSegment.State = commonpb.SegmentState_Dropped
newSegment.DroppedAt = uint64(time.Now().UnixNano())
}
binlogsIncrement := metastore.BinlogsIncrement{Segment: newSegment.SegmentInfo}
log = log.With(
zap.Int64("oldSegmentID", oldSegment.GetID()),
zap.Int64("newSegmentID", newSegment.GetID()),
zap.Int32("oldSchemaVersion", oldSegment.GetSchemaVersion()),
zap.Int32("newSchemaVersion", newSegment.GetSchemaVersion()),
zap.Int("newInsertLogsCount", len(resultSegment.GetInsertLogs())),
)
log.Info("meta update: prepare replacement for schema bump full rewrite", zap.Int64("num rows", newSegment.GetNumOfRows()))
if err := m.catalog.AlterSegments(m.ctx, []*datapb.SegmentInfo{dropped.SegmentInfo, newSegment.SegmentInfo}, binlogsIncrement); err != nil {
log.Warn("fail to alter replacement segments for schema bump compaction", zap.Error(err))
return nil, nil, err
}
m.segments.SetSegment(dropped.GetID(), dropped)
m.segments.SetSegment(newSegment.GetID(), newSegment)
log.Info("meta update: alter in memory meta after schema bump full rewrite replacement - complete")
return []*SegmentInfo{newSegment}, metricMutation, nil
}
func (m *meta) getSegmentsMetrics(collectionID int64) []*metricsinfo.Segment {
m.segMu.RLock()
defer m.segMu.RUnlock()
internal/datacoord/meta_test.go
+895 -200
View File
File diff suppressed because it is too large Load Diff
internal/datanode/compactor/backfill_compactor.go
-930
View File
@@ -1,930 +0,0 @@
// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package compactor
import (
"context"
"fmt"
sio "io"
"path"
"strconv"
"time"
"github.com/apache/arrow/go/v17/arrow"
"github.com/apache/arrow/go/v17/arrow/array"
"github.com/apache/arrow/go/v17/arrow/memory"
"github.com/cockroachdb/errors"
"github.com/samber/lo"
"go.opentelemetry.io/otel"
"go.uber.org/zap"
"google.golang.org/protobuf/proto"
"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
"github.com/milvus-io/milvus/internal/allocator"
"github.com/milvus-io/milvus/internal/compaction"
"github.com/milvus-io/milvus/internal/metastore/kv/binlog"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/internal/storagecommon"
"github.com/milvus-io/milvus/internal/storagev2/packed"
"github.com/milvus-io/milvus/internal/util/function"
"github.com/milvus-io/milvus/internal/util/hookutil"
"github.com/milvus-io/milvus/pkg/v3/common"
"github.com/milvus-io/milvus/pkg/v3/log"
"github.com/milvus-io/milvus/pkg/v3/proto/datapb"
"github.com/milvus-io/milvus/pkg/v3/proto/indexcgopb"
"github.com/milvus-io/milvus/pkg/v3/proto/indexpb"
"github.com/milvus-io/milvus/pkg/v3/util/funcutil"
"github.com/milvus-io/milvus/pkg/v3/util/merr"
"github.com/milvus-io/milvus/pkg/v3/util/metautil"
"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
)
// backfillWriter abstracts the packed writer interface for V2/V3 compatibility.
// Both packed.PackedWriter (V2) and packed.FFIPackedWriter (V3) implement WriteRecordBatch,
// but their Close signatures differ, so we use wrappers for both.
// Manifest() returns the V3 manifest path after Close(); V2 always returns "".
type backfillWriter interface {
WriteRecordBatch(arrow.Record) error
Close() error
Manifest() string
}
// ffiWriterWrapper wraps packed.FFIPackedWriter to implement backfillWriter.
// FFIPackedWriter itself knows nothing about manifests; this wrapper closes
// the writer to collect column groups, then commits them to the existing
// manifest via packed.CommitManifestUpdates.
type ffiWriterWrapper struct {
writer *packed.FFIPackedWriter
basePath string
baseVersion int64
storageConfig *indexpb.StorageConfig
manifest string
}
func (w *ffiWriterWrapper) WriteRecordBatch(r arrow.Record) error {
return w.writer.WriteRecordBatch(r)
}
func (w *ffiWriterWrapper) Close() error {
out, err := w.writer.Close()
if err != nil {
return err
}
if out == nil {
return nil
}
defer out.Destroy()
newPath, err := packed.CommitManifestUpdates(w.basePath, w.baseVersion, w.storageConfig,
&packed.ManifestUpdates{NewFiles: out})
if err != nil {
return err
}
w.manifest = newPath
return nil
}
func (w *ffiWriterWrapper) Manifest() string {
return w.manifest
}
// v2WriterWrapper wraps packed.PackedWriter to implement backfillWriter.
// Manifest() always returns "" for V2 segments (no manifest concept).
// fileSizes is populated by Close() via CloseAndTell, indexed by column group order.
type v2WriterWrapper struct {
writer *packed.PackedWriter
numGroups int
fileSizes []int64
}
func (w *v2WriterWrapper) WriteRecordBatch(r arrow.Record) error {
return w.writer.WriteRecordBatch(r)
}
func (w *v2WriterWrapper) Close() error {
sizes, err := w.writer.CloseAndTell(w.numGroups)
if err != nil {
return err
}
w.fileSizes = sizes
return nil
}
func (w *v2WriterWrapper) Manifest() string {
return ""
}
type backfillCompactionTask struct {
ctx context.Context
cancel context.CancelFunc
plan *datapb.CompactionPlan
compactionParams compaction.Params
done chan struct{}
logIDAlloc allocator.Interface
functionRunner function.FunctionRunner
chunkManager storage.ChunkManager
}
func (t *backfillCompactionTask) Compact() (*datapb.CompactionPlanResult, error) {
if !funcutil.CheckCtxValid(t.ctx) {
return nil, t.ctx.Err()
}
ctx, span := otel.Tracer(typeutil.DataNodeRole).Start(t.ctx, fmt.Sprintf("BackfillCompact-%d", t.GetPlanID()))
defer span.End()
if err := t.preCompact(); err != nil {
log.Ctx(ctx).Warn("failed to preCompact", zap.Error(err))
return nil, err
}
defer t.functionRunner.Close()
compactStart := time.Now()
log := log.Ctx(ctx).With(
zap.Int64("planID", t.GetPlanID()),
zap.Int64("collectionID", t.GetCollection()),
)
log.Info("backfill compact start",
zap.Int64("segmentID", t.plan.GetSegmentBinlogs()[0].GetSegmentID()),
zap.Int32("collectionSchemaVersion", t.plan.GetSchema().GetVersion()),
zap.Int("numFunctions", len(t.plan.GetFunctions())),
)
result, err := t.runBackfillFunction(ctx, t.functionRunner)
if err != nil {
log.Warn("backfill compact failed", zap.Error(err), zap.Duration("compact cost", time.Since(compactStart)))
return nil, err
}
log.Info("backfill compact done", zap.Duration("compact cost", time.Since(compactStart)))
return result, nil
}
// preCompact validates the compaction plan, checks context, and sets up the function runner.
func (t *backfillCompactionTask) preCompact() error {
if ok := funcutil.CheckCtxValid(t.ctx); !ok {
return t.ctx.Err()
}
// Check segment binlogs: must have exactly one segment
if len(t.plan.GetSegmentBinlogs()) != 1 {
return errors.Newf("backfill compaction plan is illegal, must have exactly one segment, but got %d segments, planID = %d", len(t.plan.GetSegmentBinlogs()), t.GetPlanID())
}
segment := t.plan.GetSegmentBinlogs()[0]
if segment.GetManifest() == "" && len(segment.GetFieldBinlogs()) == 0 {
return errors.Newf("compaction plan is illegal, segment's field binlogs are empty, planID = %d, segmentID = %d", t.GetPlanID(), segment.GetSegmentID())
}
backfillFunctions := t.plan.GetFunctions()
if len(backfillFunctions) != 1 {
return errors.New("backfill functions should be exactly one")
}
backfillFunction := backfillFunctions[0]
functionRunner, err := function.NewFunctionRunner(t.plan.GetSchema(), backfillFunction)
if err != nil {
return err
}
if functionRunner == nil {
return errors.New("failed to set up backfill function runner")
}
// Validate function runner
if err := t.checkFunctionRunner(functionRunner); err != nil {
functionRunner.Close()
return err
}
t.functionRunner = functionRunner
return nil
}
func (t *backfillCompactionTask) checkFunctionRunner(functionRunner function.FunctionRunner) error {
switch functionRunner.GetSchema().GetType() {
case schemapb.FunctionType_BM25:
functionSchema := functionRunner.GetSchema()
// 1. Check inputFieldIDs: must have exactly one, and type must be varchar
inputFieldIDs := functionSchema.GetInputFieldIds()
if len(inputFieldIDs) != 1 {
return errors.New("bm25 function should have exactly one input field")
}
inputFieldID := inputFieldIDs[0]
inputField := typeutil.GetField(t.plan.GetSchema(), inputFieldID)
if inputField == nil {
return errors.New("input field not found in schema")
}
if inputField.GetDataType() != schemapb.DataType_VarChar && inputField.GetDataType() != schemapb.DataType_Text {
return errors.New("input field data type must be varchar or text for bm25 function backfill")
}
// 2. Check outputFieldIDs: must have exactly one, and type must be SparseFloatVector
outputFieldIDs := functionSchema.GetOutputFieldIds()
if len(outputFieldIDs) != 1 {
return errors.New("bm25 function should have exactly one output field")
}
outputFieldID := outputFieldIDs[0]
outputField := typeutil.GetField(t.plan.GetSchema(), outputFieldID)
if outputField == nil {
return errors.New("output field not found in schema")
}
if outputField.GetDataType() != schemapb.DataType_SparseFloatVector {
return errors.New("output field data type must be sparse float vector for bm25 function backfill")
}
return nil
default:
return errors.New("unsupported function type")
}
}
func (t *backfillCompactionTask) runBackfillFunction(ctx context.Context, functionRunner function.FunctionRunner) (*datapb.CompactionPlanResult, error) {
switch functionRunner.GetSchema().GetType() {
case schemapb.FunctionType_BM25:
return t.runBm25Function(ctx, functionRunner)
default:
return nil, errors.New("unsupported function type")
}
}
func (t *backfillCompactionTask) openBinlogReader(inputFieldID int64) (storage.RecordReader, error) {
inputField := typeutil.GetField(t.plan.GetSchema(), inputFieldID)
segment := t.plan.GetSegmentBinlogs()[0]
collectionID := segment.GetCollectionID()
partitionID := segment.GetPartitionID()
segmentID := segment.GetSegmentID()
inputSchema := &schemapb.CollectionSchema{Fields: []*schemapb.FieldSchema{
inputField,
}}
if segment.GetManifest() != "" {
return storage.NewManifestRecordReader(t.ctx,
segment.GetManifest(),
inputSchema,
storage.WithCollectionID(collectionID),
storage.WithVersion(segment.GetStorageVersion()),
storage.WithDownloader(t.chunkManager.MultiRead),
storage.WithStorageConfig(t.compactionParams.StorageConfig),
)
}
if err := binlog.DecompressBinLogWithRootPath(t.compactionParams.StorageConfig.GetRootPath(),
storage.InsertBinlog, collectionID, partitionID,
segmentID, segment.GetFieldBinlogs()); err != nil {
log.Ctx(t.ctx).Warn("Decompress insert binlog error", zap.Error(err))
return nil, err
}
return storage.NewBinlogRecordReader(t.ctx, segment.GetFieldBinlogs(), inputSchema,
storage.WithCollectionID(collectionID),
storage.WithVersion(segment.GetStorageVersion()),
storage.WithDownloader(t.chunkManager.MultiRead),
storage.WithStorageConfig(t.compactionParams.StorageConfig),
)
}
func (t *backfillCompactionTask) processBatch(functionRunner function.FunctionRunner, inputStrs []string, outputFieldID int64) (*storage.InsertData, int, error) {
// run function on this batch
output, err := functionRunner.BatchRun(inputStrs)
if err != nil {
return nil, 0, err
}
if len(output) != 1 {
return nil, 0, errors.New("bm25 function backfill should return exactly one output")
}
outputSparseArray, ok := output[0].(*schemapb.SparseFloatArray)
if !ok {
return nil, 0, errors.New("unexpected output type from BM25 function runner, expected SparseFloatArray")
}
// build output field data
outputFieldData := &storage.SparseFloatVectorFieldData{
SparseFloatArray: schemapb.SparseFloatArray{
Contents: outputSparseArray.GetContents(),
Dim: outputSparseArray.GetDim(),
},
}
// build insert data
insertData := &storage.InsertData{
Data: make(map[int64]storage.FieldData),
}
insertData.Data[outputFieldID] = outputFieldData
sparseFieldMemorySize := proto.Size(&outputFieldData.SparseFloatArray)
return insertData, sparseFieldMemorySize, nil
}
// backfillWriterResult holds the writer and associated metadata needed after writing.
type backfillWriterResult struct {
writer backfillWriter
arrowSchema *arrow.Schema
outputSchema *schemapb.CollectionSchema
columnGroups []storagecommon.ColumnGroup
paths []string // V2 only: log paths for buildMergedLogs
truePaths []string // V2 only: true paths for log path resolution
logIDs []int64 // V2 only: log IDs corresponding to each column group path, for LogID in FieldBinlog
fileSizes []int64 // V2 only: on-disk compressed sizes from CloseAndTell, indexed by column group order
storageVersion int64
basePath string // V3 only: parsed from existing manifest in setupWriter
v3Stats []packed.StatEntry // V3 only: stats to commit atomically with column groups
}
func (t *backfillCompactionTask) setupWriter(outputField *schemapb.FieldSchema, outputFieldID int64, segment *datapb.CompactionSegmentBinlogs, collectionID, partitionID, segmentID int64) (*backfillWriterResult, error) {
outputSchema := &schemapb.CollectionSchema{Fields: []*schemapb.FieldSchema{
outputField,
}}
arrowSchema, err := storage.ConvertToArrowSchema(outputSchema, true)
if err != nil {
return nil, err
}
if segment.GetManifest() == "" && len(segment.GetFieldBinlogs()) == 0 {
return nil, errors.New("segment field binlogs is empty, wrong state for compaction segments")
}
newColumnGroups := []storagecommon.ColumnGroup{
{
GroupID: outputFieldID,
Columns: []int{0},
Fields: []int64{outputFieldID},
},
}
var pluginContext *indexcgopb.StoragePluginContext
if hookutil.IsClusterEncryptionEnabled() {
ez := hookutil.GetEzByCollProperties(t.plan.GetSchema().GetProperties(), collectionID)
if ez != nil {
unsafe := hookutil.GetCipher().GetUnsafeKey(ez.EzID, ez.CollectionID)
if len(unsafe) > 0 {
pluginContext = &indexcgopb.StoragePluginContext{
EncryptionZoneId: ez.EzID,
CollectionId: ez.CollectionID,
EncryptionKey: string(unsafe),
}
}
}
}
// Determine effective storage version: V3 only if segment already has a manifest.
// V2 segments on V3 clusters must stay V2 — a partial manifest (only new columns)
// would cause segcore to ignore original binlog data, corrupting the segment.
storageVersion := t.compactionParams.StorageVersion
existingManifest := segment.GetManifest()
if storageVersion == storage.StorageV3 && existingManifest == "" {
storageVersion = storage.StorageV2 // force V2 path for V2 segment on V3 cluster
}
result := &backfillWriterResult{
arrowSchema: arrowSchema,
outputSchema: outputSchema,
columnGroups: newColumnGroups,
storageVersion: storageVersion,
}
if storageVersion == storage.StorageV3 {
// V3: extend existing manifest by appending new column groups.
// Parse basePath and version from the segment's current manifest,
// so the transaction reads the existing manifest and merges new columns in.
basePath, existingVersion, err := packed.UnmarshalManifestPath(existingManifest)
if err != nil {
return nil, merr.WrapErrServiceInternal("failed to parse existing manifest for V3 backfill", err.Error())
}
ffiWriter, err := packed.NewFFIPackedWriter(basePath, arrowSchema, newColumnGroups, t.compactionParams.StorageConfig, pluginContext)
if err != nil {
return nil, err
}
// The output field is a new addition to the existing manifest (backfill).
// Use AddColumnGroup semantics so Loon does not require the count to match
// the existing groups in the manifest.
ffiWriter.AsNewColumnGroups()
result.writer = &ffiWriterWrapper{
writer: ffiWriter,
basePath: basePath,
baseVersion: existingVersion,
storageConfig: t.compactionParams.StorageConfig,
}
result.basePath = basePath
} else {
// V2: use PackedWriter with explicit file paths.
// TODO(backfill): no file-size rotation — all rows land in one Parquet file per
// column group regardless of segment size. Fix by splitting at BinLogMaxSize (64MB)
// and allocating a new LogID+path per chunk, like MultiSegmentWriter.rotateWriter.
logIdStart, _, err := t.logIDAlloc.Alloc(uint32(len(newColumnGroups)))
if err != nil {
return nil, err
}
paths := []string{}
logIDs := []int64{}
for _, columnGroup := range newColumnGroups {
p := metautil.BuildInsertLogPath(t.compactionParams.StorageConfig.GetRootPath(), collectionID, partitionID, segmentID, columnGroup.GroupID, logIdStart)
paths = append(paths, p)
logIDs = append(logIDs, logIdStart)
logIdStart++
}
truePaths := lo.Map(paths, func(p string, _ int) string {
if t.compactionParams.StorageConfig.GetStorageType() == "local" {
return p
}
return path.Join(t.compactionParams.StorageConfig.GetBucketName(), p)
})
writer, err := packed.NewPackedWriter(truePaths, arrowSchema, packed.DefaultWriteBufferSize, packed.DefaultMultiPartUploadSize, newColumnGroups, t.compactionParams.StorageConfig, pluginContext)
if err != nil {
return nil, err
}
result.writer = &v2WriterWrapper{writer: writer, numGroups: len(newColumnGroups)}
result.paths = paths
result.truePaths = truePaths
result.logIDs = logIDs
}
return result, nil
}
func (t *backfillCompactionTask) writeBatch(writer backfillWriter, arrowSchema *arrow.Schema, insertData *storage.InsertData, outputSchema *schemapb.CollectionSchema) error {
builder := array.NewRecordBuilder(memory.DefaultAllocator, arrowSchema)
defer builder.Release()
err := storage.BuildRecord(builder, insertData, outputSchema)
if err != nil {
return err
}
arrowRecord := builder.NewRecord()
defer arrowRecord.Release()
return writer.WriteRecordBatch(arrowRecord)
}
func (t *backfillCompactionTask) updateStats(stats *storage.BM25Stats, collectionID, partitionID, segmentID, outputFieldID int64, writerResult *backfillWriterResult) ([]byte, string, int64, error) {
cli := t.chunkManager
statsID, _, err := t.logIDAlloc.Alloc(uint32(1))
if err != nil {
return nil, "", 0, err
}
bytes, err := stats.Serialize()
if err != nil {
return nil, "", 0, err
}
if writerResult.storageVersion == storage.StorageV3 {
// V3: write stats file under manifest basePath/_stats/ directory.
// Stats will be registered in manifest via AddStatsToManifest after Close().
// basePath was already parsed from the manifest in setupWriter — reuse it here.
basePath := writerResult.basePath
statsRelPath := fmt.Sprintf("_stats/bm25.%d/%d", outputFieldID, statsID)
absStatsPath := path.Join(basePath, statsRelPath)
if err := cli.Write(t.ctx, absStatsPath, bytes); err != nil {
return nil, "", 0, merr.WrapErrServiceInternal("failed to write V3 BM25 stats", err.Error())
}
writerResult.v3Stats = append(writerResult.v3Stats, packed.StatEntry{
Key: fmt.Sprintf("bm25.%d", outputFieldID),
Files: []string{absStatsPath}, // C++ converts absolute to relative at commit
Metadata: map[string]string{
"memory_size": strconv.FormatInt(int64(len(bytes)), 10),
},
})
return bytes, "", 0, nil // no separate bm25LogPathForResult for V3; statsID unused
}
// V2: write stats file to bm25_stats path, return path and ID for result metadata
statsPath := metautil.JoinIDPath(collectionID, partitionID, segmentID, outputFieldID, statsID)
bm25LogPath := path.Join(cli.RootPath(), common.SegmentBm25LogPath, statsPath)
if err := cli.Write(t.ctx, bm25LogPath, bytes); err != nil {
return nil, "", 0, err
}
bm25LogPathForResult := metautil.BuildBm25LogPath(
t.compactionParams.StorageConfig.GetRootPath(),
collectionID, partitionID, segmentID, outputFieldID, statsID)
if t.compactionParams.StorageConfig.GetStorageType() != "local" {
bm25LogPathForResult = path.Join(t.compactionParams.StorageConfig.GetBucketName(), bm25LogPathForResult)
}
return bytes, bm25LogPathForResult, statsID, nil
}
func (t *backfillCompactionTask) buildMergedLogsV2(segment *datapb.CompactionSegmentBinlogs, writerResult *backfillWriterResult, sparseFieldMemorySize int, totalRows int64, bytes []byte, bm25LogPathForResult string, bm25LogID int64, outputFieldID int64) ([]*datapb.FieldBinlog, []*datapb.FieldBinlog, error) {
newInsertLogs := make(map[int64]*datapb.FieldBinlog)
for i, columnGroup := range writerResult.columnGroups {
fileSize := writerResult.fileSizes[i]
logPath := writerResult.paths[i]
truePath := writerResult.truePaths[i]
if t.compactionParams.StorageConfig.GetStorageType() != "local" {
logPath = truePath
}
fieldBinlog := &datapb.FieldBinlog{
FieldID: columnGroup.GroupID,
ChildFields: columnGroup.Fields,
Binlogs: []*datapb.Binlog{
{
LogSize: fileSize,
MemorySize: int64(sparseFieldMemorySize),
LogPath: logPath,
LogID: writerResult.logIDs[i],
EntriesNum: totalRows,
},
},
}
newInsertLogs[columnGroup.GroupID] = fieldBinlog
}
return t.finalizeMergedLogs(segment, newInsertLogs, totalRows, bytes, bm25LogPathForResult, bm25LogID, outputFieldID)
}
func (t *backfillCompactionTask) buildMergedLogsV3(segment *datapb.CompactionSegmentBinlogs, writerResult *backfillWriterResult, sparseFieldMemorySize int, totalRows int64, bytes []byte, bm25LogPathForResult string, bm25LogID int64, outputFieldID int64) ([]*datapb.FieldBinlog, []*datapb.FieldBinlog, error) {
// V3: data is tracked by the manifest. Binlog entries in etcd serve as field-presence
// markers used by getSegmentBinlogFields (backfill detection). Each entry must have
// LogID!=0 and LogPath=="" to pass buildBinlogKvs validation — allocate a real ID from
// logIDAlloc (same allocator used for V2 LogIDs) so the entry is properly persisted.
logIDStart, _, err := t.logIDAlloc.Alloc(uint32(len(writerResult.columnGroups)))
if err != nil {
return nil, nil, err
}
newInsertLogs := make(map[int64]*datapb.FieldBinlog)
for i, columnGroup := range writerResult.columnGroups {
fieldBinlog := &datapb.FieldBinlog{
FieldID: columnGroup.GroupID,
ChildFields: columnGroup.Fields,
Binlogs: []*datapb.Binlog{
{
MemorySize: int64(sparseFieldMemorySize),
EntriesNum: totalRows,
LogID: logIDStart + int64(i),
},
},
}
newInsertLogs[columnGroup.GroupID] = fieldBinlog
}
return t.finalizeMergedLogs(segment, newInsertLogs, totalRows, bytes, bm25LogPathForResult, bm25LogID, outputFieldID)
}
func (t *backfillCompactionTask) finalizeMergedLogs(segment *datapb.CompactionSegmentBinlogs, newInsertLogs map[int64]*datapb.FieldBinlog, totalRows int64, bytes []byte, bm25LogPathForResult string, bm25LogID int64, outputFieldID int64) ([]*datapb.FieldBinlog, []*datapb.FieldBinlog, error) {
// build new Bm25Logs
bm25FileSize := int64(len(bytes))
newBm25Logs := []*datapb.FieldBinlog{
{
FieldID: outputFieldID,
Binlogs: []*datapb.Binlog{
{
LogSize: bm25FileSize,
MemorySize: bm25FileSize,
LogPath: bm25LogPathForResult,
LogID: bm25LogID,
EntriesNum: totalRows,
},
},
},
}
// Crash-replay guard: if DC crashed after AlterSegments but before the task-state
// transition, the output field is already in etcd — skip it to stay idempotent.
originalInsertLogs := segment.GetFieldBinlogs()
existingFields := make(map[int64]struct{}, len(originalInsertLogs)*2)
for _, fb := range originalInsertLogs {
existingFields[fb.GetFieldID()] = struct{}{}
for _, childID := range fb.GetChildFields() {
existingFields[childID] = struct{}{}
}
}
newInsertLogsList := storage.SortFieldBinlogs(newInsertLogs)
dedupedNew := newInsertLogsList[:0]
for _, fb := range newInsertLogsList {
if _, ok := existingFields[fb.GetFieldID()]; ok {
log.Warn("backfill crash-replay: output field already in segment binlogs, skipping duplicate",
zap.Int64("fieldID", fb.GetFieldID()),
zap.Int64("segmentID", segment.GetSegmentID()))
continue
}
childDup := false
for _, childID := range fb.GetChildFields() {
if _, ok := existingFields[childID]; ok {
childDup = true
break
}
}
if childDup {
log.Warn("backfill crash-replay: output field already in segment binlogs, skipping duplicate",
zap.Int64("fieldID", fb.GetFieldID()),
zap.Int64("segmentID", segment.GetSegmentID()))
continue
}
dedupedNew = append(dedupedNew, fb)
}
mergedInsertLogs := make([]*datapb.FieldBinlog, 0, len(originalInsertLogs)+len(dedupedNew))
mergedInsertLogs = append(mergedInsertLogs, originalInsertLogs...)
mergedInsertLogs = append(mergedInsertLogs, dedupedNew...)
return mergedInsertLogs, newBm25Logs, nil
}
func (t *backfillCompactionTask) runBm25Function(ctx context.Context, functionRunner function.FunctionRunner) (*datapb.CompactionPlanResult, error) {
// 1. set up function schema — duplicate checks from checkFunctionRunner so this path cannot panic
// if schema/runner state diverges or call sites change.
functionSchema := functionRunner.GetSchema()
inputFieldIDs := functionSchema.GetInputFieldIds()
if len(inputFieldIDs) != 1 {
return nil, errors.Newf("bm25 backfill: expected exactly one input field, got %d (planID=%d)", len(inputFieldIDs), t.plan.GetPlanID())
}
inputFieldID := inputFieldIDs[0]
outputFieldIDs := functionSchema.GetOutputFieldIds()
if len(outputFieldIDs) != 1 {
return nil, errors.Newf("bm25 backfill: expected exactly one output field, got %d (planID=%d)", len(outputFieldIDs), t.plan.GetPlanID())
}
outputFieldID := outputFieldIDs[0]
outputField := typeutil.GetField(t.plan.GetSchema(), outputFieldID)
// 2. get segment info
segment := t.plan.GetSegmentBinlogs()[0]
collectionID := segment.GetCollectionID()
partitionID := segment.GetPartitionID()
segmentID := segment.GetSegmentID()
log := log.Ctx(ctx).With(
zap.Int64("planID", t.plan.GetPlanID()),
zap.Int64("collectionID", collectionID),
zap.Int64("partitionID", partitionID),
zap.Int64("segmentID", segmentID),
zap.Int64("inputFieldID", inputFieldID),
zap.Int64("outputFieldID", outputFieldID),
)
// Track durations for each heavy operation
var readDuration, computeDuration, writeDuration, updateStatsDuration time.Duration
// 3. open binlog reader
reader, err := t.openBinlogReader(inputFieldID)
if err != nil {
return nil, err
}
defer reader.Close()
// 4. set up writer
writerResult, err := t.setupWriter(outputField, outputFieldID, segment, collectionID, partitionID, segmentID)
if err != nil {
return nil, err
}
// Log effective storage version (may differ from cluster version for V2-origin segments).
log.Info("backfill writer setup",
zap.Int64("effectiveStorageVersion", writerResult.storageVersion),
zap.Int64("clusterStorageVersion", t.compactionParams.StorageVersion),
zap.Bool("segmentHasManifest", segment.GetManifest() != ""),
zap.Int("inputFieldBinlogCount", len(segment.GetFieldBinlogs())),
zap.Strings("v2WritePaths", writerResult.paths),
)
writerClosed := false
defer func() {
if !writerClosed {
writerResult.writer.Close()
}
}()
// 5. batch loop: read → compute → write → accumulate stats
_, span := otel.Tracer(typeutil.DataNodeRole).Start(ctx, "BackfillCompact.batchProcess")
stats := storage.NewBM25Stats()
var totalRows int64
var totalSparseMemorySize int
for {
// read one batch
readStart := time.Now()
record, err := reader.Next()
if err != nil {
if err == sio.EOF {
readDuration += time.Since(readStart)
break
}
span.End()
return nil, err
}
readDuration += time.Since(readStart)
// extract input strings from this batch
col := record.Column(inputFieldID)
if col == nil {
record.Release()
span.End()
return nil, merr.WrapErrServiceInternal(fmt.Sprintf("input field %d not found in record", inputFieldID))
}
recordStr, ok := col.(*array.String)
if !ok {
record.Release()
span.End()
return nil, merr.WrapErrServiceInternal(fmt.Sprintf("input field %d data type must be varchar or text for bm25 function backfill, got %T", inputFieldID, col))
}
batchStrs := make([]string, record.Len())
for i := 0; i < record.Len(); i++ {
batchStrs[i] = recordStr.Value(i)
}
record.Release()
// compute BM25 for this batch
computeStart := time.Now()
insertData, batchMemSize, err := t.processBatch(functionRunner, batchStrs, outputFieldID)
computeDuration += time.Since(computeStart)
if err != nil {
span.End()
return nil, err
}
// accumulate stats from this batch
outputFieldData := insertData.Data[outputFieldID].(*storage.SparseFloatVectorFieldData)
stats.AppendBytes(outputFieldData.GetContents()...)
// write this batch
writeStart := time.Now()
if err := t.writeBatch(writerResult.writer, writerResult.arrowSchema, insertData, writerResult.outputSchema); err != nil {
span.End()
return nil, err
}
writeDuration += time.Since(writeStart)
totalRows += int64(len(batchStrs))
totalSparseMemorySize += batchMemSize
}
span.End()
// 6. write stats file (must happen before Close for V3 — stats are committed atomically)
_, span2 := otel.Tracer(typeutil.DataNodeRole).Start(ctx, "BackfillCompact.updateStats")
startTime := time.Now()
bytes, bm25LogPathForResult, bm25LogID, err := t.updateStats(stats, collectionID, partitionID, segmentID, outputFieldID, writerResult)
updateStatsDuration = time.Since(startTime)
span2.End()
if err != nil {
return nil, err
}
log.Info("backfill bm25 stats written",
zap.String("bm25LogPath", bm25LogPathForResult),
zap.Int64("bm25LogID", bm25LogID),
zap.Int("bm25StatsBytes", len(bytes)),
zap.Int64("storageVersion", writerResult.storageVersion),
zap.Int("v3StatsCount", len(writerResult.v3Stats)),
)
// Close writer — for V3, this commits new column groups to the existing manifest
// (version N → N+1). BM25 stats are added separately via AddStatsToManifest.
if err := writerResult.writer.Close(); err != nil {
return nil, err
}
writerClosed = true
// For V2: capture on-disk compressed file sizes written by CloseAndTell.
if v2w, ok := writerResult.writer.(*v2WriterWrapper); ok {
writerResult.fileSizes = v2w.fileSizes
log.Info("backfill V2 insert binlogs written",
zap.Strings("logPaths", writerResult.paths),
zap.Int64s("logIDs", writerResult.logIDs),
zap.Int64s("fileSizeBytes", v2w.fileSizes),
zap.Int64("totalRows", totalRows),
)
}
// Read the manifest produced by Close(); "" for V2, real path for V3.
manifestPath := writerResult.writer.Manifest()
if manifestPath != "" {
log.Info("backfill V3 writer closed, manifest produced", zap.String("manifestPath", manifestPath))
}
// For V3: register BM25 stats in manifest (version N+1 → N+2).
// Channel-level scheduler exclusion guarantees no concurrent manifest modification.
if writerResult.storageVersion == storage.StorageV3 && len(writerResult.v3Stats) > 0 {
newManifest, err := packed.AddStatsToManifest(
manifestPath, t.compactionParams.StorageConfig, writerResult.v3Stats)
if err != nil {
return nil, merr.WrapErrServiceInternal("failed to add BM25 stats to V3 manifest", err.Error())
}
log.Info("backfill V3 bm25 stats added to manifest",
zap.String("oldManifest", manifestPath),
zap.String("newManifest", newManifest),
)
manifestPath = newManifest
}
// 7. build merged logs
var mergedInsertLogs, mergedBm25Logs []*datapb.FieldBinlog
if writerResult.storageVersion == storage.StorageV3 {
mergedInsertLogs, mergedBm25Logs, err = t.buildMergedLogsV3(segment, writerResult, totalSparseMemorySize, totalRows, bytes, bm25LogPathForResult, bm25LogID, outputFieldID)
} else {
mergedInsertLogs, mergedBm25Logs, err = t.buildMergedLogsV2(segment, writerResult, totalSparseMemorySize, totalRows, bytes, bm25LogPathForResult, bm25LogID, outputFieldID)
}
if err != nil {
return nil, err
}
// 8. manifest path is already set from Close() and optionally updated by AddStatsToManifest above.
// 9. return compaction result
// For V3: BM25 stats are embedded in the manifest (committed atomically with
// column groups), so Bm25Logs should be nil — PackSegmentLoadInfo skips
// Bm25Logs when ManifestPath is set, relying on StatsResolver to read from manifest.
resultBm25Logs := mergedBm25Logs
if writerResult.storageVersion == storage.StorageV3 {
resultBm25Logs = nil
}
ret := &datapb.CompactionPlanResult{
PlanID: t.plan.GetPlanID(),
State: datapb.CompactionTaskState_completed,
Segments: []*datapb.CompactionSegment{
{
SegmentID: segmentID,
NumOfRows: totalRows,
InsertLogs: mergedInsertLogs,
Bm25Logs: resultBm25Logs,
Field2StatslogPaths: segment.GetField2StatslogPaths(),
Deltalogs: segment.GetDeltalogs(),
Channel: segment.GetInsertChannel(),
StorageVersion: writerResult.storageVersion,
Manifest: manifestPath,
},
},
Type: t.plan.GetType(),
}
log.Info("backfill compaction completed",
zap.Int64("numOfRows", totalRows),
zap.Int("mergedInsertLogsCount", len(mergedInsertLogs)),
zap.Int("mergedBm25LogsCount", len(mergedBm25Logs)),
zap.Int("resultBm25LogsCount", len(resultBm25Logs)),
zap.String("manifestPath", manifestPath),
zap.Int64("effectiveStorageVersion", writerResult.storageVersion),
zap.Int32("collectionSchemaVersion", t.plan.GetSchema().GetVersion()),
zap.Duration("readDuration", readDuration),
zap.Duration("computeDuration", computeDuration),
zap.Duration("writeDuration", writeDuration),
zap.Duration("updateStatsDuration", updateStatsDuration),
)
return ret, nil
}
func (t *backfillCompactionTask) Complete() {
if t.done != nil {
select {
case t.done <- struct{}{}:
default:
}
}
}
func (t *backfillCompactionTask) Stop() {
if t.cancel != nil {
t.cancel()
}
if t.done != nil {
<-t.done
}
}
func (t *backfillCompactionTask) GetPlanID() typeutil.UniqueID {
return t.plan.GetPlanID()
}
func (t *backfillCompactionTask) GetCollection() typeutil.UniqueID {
// Get collection ID from the first segment binlog
if len(t.plan.GetSegmentBinlogs()) > 0 {
return t.plan.GetSegmentBinlogs()[0].GetCollectionID()
}
return 0
}
func (t *backfillCompactionTask) GetChannelName() string {
return t.plan.GetChannel()
}
func (t *backfillCompactionTask) GetCompactionType() datapb.CompactionType {
return t.plan.GetType()
}
func (t *backfillCompactionTask) GetSlotUsage() int64 {
return t.plan.GetSlotUsage()
}
func (t *backfillCompactionTask) GetStorageConfig() *indexpb.StorageConfig {
return t.compactionParams.StorageConfig
}
var _ Compactor = (*backfillCompactionTask)(nil)
func NewBackfillCompactionTask(ctx context.Context, cm storage.ChunkManager, plan *datapb.CompactionPlan, compactionParams compaction.Params) *backfillCompactionTask {
ctx, cancel := context.WithCancel(ctx)
return &backfillCompactionTask{
ctx: ctx,
cancel: cancel,
plan: plan,
compactionParams: compactionParams,
done: make(chan struct{}, 1),
logIDAlloc: allocator.NewLocalAllocator(plan.GetPreAllocatedLogIDs().GetBegin(), plan.GetPreAllocatedLogIDs().GetEnd()),
chunkManager: cm,
}
}
internal/datanode/compactor/backfill_compactor_test.go
-783
View File
@@ -1,783 +0,0 @@
// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package compactor
import (
"context"
"math"
"testing"
"github.com/cockroachdb/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
"go.uber.org/zap"
"github.com/milvus-io/milvus-proto/go-api/v3/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
"github.com/milvus-io/milvus/internal/allocator"
"github.com/milvus-io/milvus/internal/compaction"
"github.com/milvus-io/milvus/internal/mocks/flushcommon/mock_util"
mock_storage "github.com/milvus-io/milvus/internal/mocks/mock_storage"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/internal/storagecommon"
"github.com/milvus-io/milvus/internal/util/function"
"github.com/milvus-io/milvus/internal/util/initcore"
"github.com/milvus-io/milvus/pkg/v3/common"
"github.com/milvus-io/milvus/pkg/v3/log"
"github.com/milvus-io/milvus/pkg/v3/proto/datapb"
"github.com/milvus-io/milvus/pkg/v3/proto/etcdpb"
"github.com/milvus-io/milvus/pkg/v3/util/paramtable"
"github.com/milvus-io/milvus/pkg/v3/util/tsoutil"
)
func TestBackfillCompactionTaskSuite(t *testing.T) {
suite.Run(t, new(BackfillCompactionTaskSuite))
}
type BackfillCompactionTaskSuite struct {
suite.Suite
mockBinlogIO *mock_util.MockBinlogIO
meta *etcdpb.CollectionMeta
multiSegWriter *MultiSegmentWriter
task *backfillCompactionTask
}
func (s *BackfillCompactionTaskSuite) SetupSuite() {
paramtable.Get().Init(paramtable.NewBaseTable())
}
func (s *BackfillCompactionTaskSuite) setupTest() {
s.mockBinlogIO = mock_util.NewMockBinlogIO(s.T())
s.meta = genTestCollectionMetaWithBM25()
params, err := compaction.GenerateJSONParams(s.meta.GetSchema())
if err != nil {
panic(err)
}
plan := &datapb.CompactionPlan{
PlanID: 999,
SegmentBinlogs: []*datapb.CompactionSegmentBinlogs{{
CollectionID: 1,
PartitionID: 1,
SegmentID: 100,
FieldBinlogs: nil,
Field2StatslogPaths: nil,
Deltalogs: nil,
InsertChannel: "test_channel",
StorageVersion: storage.StorageV2,
}},
Type: datapb.CompactionType_BackfillCompaction,
Schema: s.meta.GetSchema(),
PreAllocatedSegmentIDs: &datapb.IDRange{Begin: 19531, End: math.MaxInt64},
PreAllocatedLogIDs: &datapb.IDRange{Begin: 9530, End: 19530},
MaxSize: 64 * 1024 * 1024,
JsonParams: params,
Functions: []*schemapb.FunctionSchema{{
Name: "BM25",
Id: 100,
Type: schemapb.FunctionType_BM25,
InputFieldNames: []string{"text"},
InputFieldIds: []int64{101},
OutputFieldNames: []string{"sparse"},
OutputFieldIds: []int64{102},
}},
TotalRows: 3,
}
cm, err := storage.NewChunkManagerFactoryWithParam(paramtable.Get()).NewPersistentStorageChunkManager(context.Background())
if err != nil {
panic(err)
}
s.task = NewBackfillCompactionTask(context.Background(), cm, plan, compaction.GenParams())
}
func (s *BackfillCompactionTaskSuite) SetupTest() {
// Align with sort_compaction_test fixture: use a per-test temp dir as the local
// storage root and disable Loon FFI. Without UseLoonFFI=false, the loon local
// filesystem double-prepends the configured root_path onto already-absolute log
// paths produced by MultiSegmentWriter, causing the reader to look for files at
// /rootPath/rootPath/insert_log/... and fail to open them.
paramtable.Get().Save("common.storage.enablev2", "true")
paramtable.Get().Save(paramtable.Get().CommonCfg.StorageType.Key, "local")
paramtable.Get().Save(paramtable.Get().CommonCfg.UseLoonFFI.Key, "false")
paramtable.Get().Save(paramtable.Get().LocalStorageCfg.Path.Key, s.T().TempDir())
initcore.InitStorageV2FileSystem(paramtable.Get())
s.setupTest()
}
func (s *BackfillCompactionTaskSuite) TearDownTest() {
paramtable.Get().Reset(paramtable.Get().CommonCfg.StorageType.Key)
paramtable.Get().Reset(paramtable.Get().CommonCfg.UseLoonFFI.Key)
paramtable.Get().Reset(paramtable.Get().LocalStorageCfg.Path.Key)
paramtable.Get().Reset("common.storage.enablev2")
initcore.CleanArrowFileSystem()
}
func (s *BackfillCompactionTaskSuite) prepareBackfillCompaction() {
segID := int64(100)
s.mockBinlogIO.EXPECT().Upload(mock.Anything, mock.Anything).Return(nil).Maybe()
// Create multi segment writer with varchar field (input) but without sparse vector field (output)
// This simulates a segment that needs backfill
s.initSegBufferForBackfill(segID)
// Close the writer to finalize segments
err := s.multiSegWriter.Close()
s.Require().NoError(err)
// Get compaction segments
segments := s.multiSegWriter.GetCompactionSegments()
s.Require().Equal(1, len(segments), "should create exactly one segment")
segment := segments[0]
// Use the actual segmentID allocated by MultiSegmentWriter
actualSegID := segment.GetSegmentID()
s.task.plan.SegmentBinlogs = []*datapb.CompactionSegmentBinlogs{
{
CollectionID: 1,
PartitionID: 1,
SegmentID: actualSegID,
FieldBinlogs: segment.GetInsertLogs(),
InsertChannel: "test_channel",
StorageVersion: segment.GetStorageVersion(),
Manifest: segment.GetManifest(),
},
}
// Write blobs to disk from the uploaded blobs
// Note: MultiSegmentWriter handles upload internally, but we can still write to disk for testing
log.Info("created segment with MultiSegmentWriter",
zap.Int64("segmentID", segID),
zap.Int("numRows", int(segment.GetNumOfRows())),
zap.Int("insertLogsCount", len(segment.GetInsertLogs())))
}
func (s *BackfillCompactionTaskSuite) initSegBufferForBackfill(segID int64) {
// Create schema without sparse vector field (output field)
// Only include input field (varchar)
schema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{
FieldID: common.RowIDField,
Name: "row_id",
DataType: schemapb.DataType_Int64,
},
{
FieldID: common.TimeStampField,
Name: "Timestamp",
DataType: schemapb.DataType_Int64,
},
{
FieldID: 100,
Name: "pk",
DataType: schemapb.DataType_Int64,
IsPrimaryKey: true,
},
{
FieldID: 101,
Name: "text",
DataType: schemapb.DataType_VarChar,
TypeParams: []*commonpb.KeyValuePair{
{
Key: common.MaxLengthKey,
Value: "128",
},
},
},
},
}
// Create allocator for MultiSegmentWriter
// Use segID as the starting point for segmentID allocation to ensure consistent paths
segIDAlloc := allocator.NewLocalAllocator(segID, math.MaxInt64)
logIDAlloc := allocator.NewLocalAllocator(9530, 19530)
compAlloc := NewCompactionAllocator(segIDAlloc, logIDAlloc)
// Create MultiSegmentWriter with StorageV2
multiSegWriter, err := NewMultiSegmentWriter(
context.Background(),
s.mockBinlogIO,
compAlloc,
64*1024*1024, // 64MB segment size
schema,
s.task.compactionParams,
1000, // maxRows
PartitionID,
CollectionID,
"test_channel",
compactionBatchSize,
storage.WithStorageConfig(s.task.compactionParams.StorageConfig),
storage.WithVersion(storage.StorageV2),
)
s.Require().NoError(err)
// Write test data with varchar field
for i := 0; i < 3; i++ {
v := storage.Value{
PK: storage.NewInt64PrimaryKey(segID + int64(i)),
Timestamp: int64(tsoutil.ComposeTSByTime(getMilvusBirthday(), 0)),
Value: map[int64]interface{}{
common.RowIDField: segID + int64(i),
common.TimeStampField: int64(tsoutil.ComposeTSByTime(getMilvusBirthday(), 0)),
100: segID + int64(i),
101: "test string " + string(rune('0'+i)),
},
}
err = multiSegWriter.WriteValue(&v)
s.Require().NoError(err)
}
s.multiSegWriter = multiSegWriter
}
func (s *BackfillCompactionTaskSuite) TestBackfillCompactionSuccess() {
s.prepareBackfillCompaction()
result, err := s.task.Compact()
s.NoError(err)
s.NotNil(result)
s.Equal(s.task.plan.GetPlanID(), result.GetPlanID())
s.Equal(datapb.CompactionTaskState_completed, result.GetState())
s.Equal(1, len(result.GetSegments()))
segment := result.GetSegments()[0]
// SegmentID should match the one allocated by MultiSegmentWriter
s.NotZero(segment.GetSegmentID())
s.EqualValues(3, segment.GetNumOfRows())
s.NotEmpty(segment.GetInsertLogs())
// For V3 segments (with manifest), BM25 stats are embedded in the manifest
// atomically with the column groups. PackSegmentLoadInfo relies on StatsResolver
// to read them from the manifest instead of Bm25Logs.
// For V2 segments (no manifest), BM25 stats are returned in Bm25Logs as usual.
if segment.GetManifest() != "" {
s.Empty(segment.GetBm25Logs())
s.NotEmpty(segment.GetManifest())
} else {
s.NotEmpty(segment.GetBm25Logs())
s.Equal(1, len(segment.GetBm25Logs()))
// V2 BM25 stats binlog must have non-zero LogID so buildBinlogKvs does not reject it.
s.NotZero(segment.GetBm25Logs()[0].GetBinlogs()[0].GetLogID(),
"V2 BM25 stats binlog must have non-zero LogID")
}
// The backfilled output field (ID=102) must carry a non-zero LogID in its insert binlog.
// buildBinlogKvs requires LogID!=0 or LogPath!="" for V2; this guards against regression
// where LogID is accidentally left at zero (pre-existing fields are exempt — they were
// written by the test setup without LogID allocation).
const backfillOutputFieldID = int64(102)
for _, fl := range segment.GetInsertLogs() {
if fl.GetFieldID() == backfillOutputFieldID {
s.Require().NotEmpty(fl.GetBinlogs(), "backfilled output field must have at least one binlog entry")
s.NotZero(fl.GetBinlogs()[0].GetLogID(),
"V2 backfill insert binlog (fieldID=%d) must have non-zero LogID", backfillOutputFieldID)
}
}
}
func (s *BackfillCompactionTaskSuite) TestBackfillCompactionInvalidFunctionCount() {
s.prepareBackfillCompaction()
// Test with zero functions
s.task.plan.Functions = []*schemapb.FunctionSchema{}
_, err := s.task.Compact()
s.Error(err)
s.Contains(err.Error(), "backfill functions should be exactly one")
// Test with multiple functions
s.task.plan.Functions = []*schemapb.FunctionSchema{
{Type: schemapb.FunctionType_BM25},
{Type: schemapb.FunctionType_BM25},
}
_, err = s.task.Compact()
s.Error(err)
s.Contains(err.Error(), "backfill functions should be exactly one")
}
func (s *BackfillCompactionTaskSuite) TestBackfillCompactionInvalidInputField() {
s.prepareBackfillCompaction()
// Test with wrong input field type (Int64 instead of VarChar)
s.task.plan.Functions = []*schemapb.FunctionSchema{{
Name: "BM25",
Type: schemapb.FunctionType_BM25,
InputFieldIds: []int64{100}, // Int64 field instead of VarChar
OutputFieldIds: []int64{102},
}}
_, err := s.task.Compact()
s.Error(err)
s.Contains(err.Error(), "input field data type must be varchar")
}
func (s *BackfillCompactionTaskSuite) TestBackfillCompactionInvalidOutputField() {
s.prepareBackfillCompaction()
// Test with wrong output field type (VarChar instead of SparseFloatVector)
s.task.plan.Functions = []*schemapb.FunctionSchema{{
Name: "BM25",
Type: schemapb.FunctionType_BM25,
InputFieldIds: []int64{101},
OutputFieldIds: []int64{101}, // VarChar field instead of SparseFloatVector
}}
_, err := s.task.Compact()
s.Error(err)
s.Contains(err.Error(), "output field data type must be sparse float vector")
}
func (s *BackfillCompactionTaskSuite) TestBackfillCompactionMultipleInputFields() {
s.prepareBackfillCompaction()
// Test with multiple input fields
s.task.plan.Functions = []*schemapb.FunctionSchema{{
Name: "BM25",
Type: schemapb.FunctionType_BM25,
InputFieldIds: []int64{101, 115}, // Multiple input fields
OutputFieldIds: []int64{102},
}}
_, err := s.task.Compact()
s.Error(err)
s.Contains(err.Error(), "bm25 function should have exactly one input field")
}
func (s *BackfillCompactionTaskSuite) TestBackfillCompactionMultipleOutputFields() {
s.prepareBackfillCompaction()
// Test with multiple output fields
s.task.plan.Functions = []*schemapb.FunctionSchema{{
Name: "BM25",
Type: schemapb.FunctionType_BM25,
InputFieldIds: []int64{101},
OutputFieldIds: []int64{102, 114}, // Multiple output fields
}}
_, err := s.task.Compact()
s.Error(err)
s.Contains(err.Error(), "bm25 function should only have one output field")
}
func (s *BackfillCompactionTaskSuite) TestBackfillCompactionInputFieldNotFound() {
s.prepareBackfillCompaction()
// Test with non-existent input field ID
s.task.plan.Functions = []*schemapb.FunctionSchema{{
Name: "BM25",
Type: schemapb.FunctionType_BM25,
InputFieldIds: []int64{999}, // Non-existent field ID
OutputFieldIds: []int64{102},
}}
_, err := s.task.Compact()
s.Error(err)
s.Contains(err.Error(), "input field not found in schema")
}
func (s *BackfillCompactionTaskSuite) TestBackfillCompactionOutputFieldNotFound() {
s.prepareBackfillCompaction()
// Test with non-existent output field ID
s.task.plan.Functions = []*schemapb.FunctionSchema{{
Name: "BM25",
Type: schemapb.FunctionType_BM25,
InputFieldIds: []int64{101},
OutputFieldIds: []int64{999}, // Non-existent field ID
}}
_, err := s.task.Compact()
s.Error(err)
s.Contains(err.Error(), "no output field")
}
func (s *BackfillCompactionTaskSuite) TestBackfillCompactionUnsupportedFunctionType() {
s.prepareBackfillCompaction()
// Test with unsupported function type
s.task.plan.Functions = []*schemapb.FunctionSchema{{
Name: "Unknown",
Type: schemapb.FunctionType_Unknown,
InputFieldIds: []int64{101},
OutputFieldIds: []int64{102},
}}
_, err := s.task.Compact()
s.Error(err)
s.Contains(err.Error(), "unknown functionRunner type")
}
func (s *BackfillCompactionTaskSuite) TestBackfillCompactionEmptySegmentBinlogs() {
// Test with empty segment binlogs
s.task.plan.SegmentBinlogs = nil
_, err := s.task.Compact()
s.Error(err)
}
func (s *BackfillCompactionTaskSuite) TestBackfillCompactionEmptyColumnGroups() {
s.prepareBackfillCompaction()
// Create segment binlogs with empty field binlogs to trigger empty field binlogs error
s.task.plan.SegmentBinlogs = []*datapb.CompactionSegmentBinlogs{
{
CollectionID: 1,
PartitionID: 1,
SegmentID: 100,
FieldBinlogs: []*datapb.FieldBinlog{}, // Empty field binlogs
InsertChannel: "test_channel",
StorageVersion: storage.StorageV2,
},
}
_, err := s.task.Compact()
s.Error(err)
s.Contains(err.Error(), "segment's field binlogs are empty")
}
func (s *BackfillCompactionTaskSuite) TestBackfillCompactionMultipleSegments() {
s.prepareBackfillCompaction()
// Test with multiple segments (should fail, must have exactly one)
s.task.plan.SegmentBinlogs = []*datapb.CompactionSegmentBinlogs{
{
CollectionID: 1,
PartitionID: 1,
SegmentID: 100,
FieldBinlogs: []*datapb.FieldBinlog{{FieldID: 101}},
InsertChannel: "test_channel",
StorageVersion: storage.StorageV2,
},
{
CollectionID: 1,
PartitionID: 1,
SegmentID: 200,
FieldBinlogs: []*datapb.FieldBinlog{{FieldID: 101}},
InsertChannel: "test_channel",
StorageVersion: storage.StorageV2,
},
}
_, err := s.task.Compact()
s.Error(err)
s.Contains(err.Error(), "must have exactly one segment")
s.Contains(err.Error(), "but got 2 segments")
}
func (s *BackfillCompactionTaskSuite) TestBackfillCompactionContextCanceled() {
// Cancel context before compact (before prepareBackfillCompaction)
ctx, cancel := context.WithCancel(context.Background())
cancel()
s.task.ctx = ctx
s.task.cancel = cancel
// Compact should detect context cancel at the beginning
_, err := s.task.Compact()
s.Error(err)
s.ErrorIs(err, context.Canceled)
}
func TestBackfillCompactionTaskBasic(t *testing.T) {
ctx := context.Background()
meta := genTestCollectionMetaWithBM25()
params, err := compaction.GenerateJSONParams(meta.GetSchema())
assert.NoError(t, err)
plan := &datapb.CompactionPlan{
PlanID: 1000,
SegmentBinlogs: []*datapb.CompactionSegmentBinlogs{{
CollectionID: 1,
PartitionID: 1,
SegmentID: 200,
InsertChannel: "test_channel",
StorageVersion: storage.StorageV2,
}},
Type: datapb.CompactionType_BackfillCompaction,
Schema: meta.GetSchema(),
PreAllocatedSegmentIDs: &datapb.IDRange{Begin: 19531, End: math.MaxInt64},
PreAllocatedLogIDs: &datapb.IDRange{Begin: 9530, End: 19530},
MaxSize: 64 * 1024 * 1024,
JsonParams: params,
Functions: []*schemapb.FunctionSchema{{
Name: "BM25",
Type: schemapb.FunctionType_BM25,
InputFieldIds: []int64{101},
OutputFieldIds: []int64{102},
}},
TotalRows: 0,
Channel: "test_channel",
SlotUsage: 1,
}
mockCM := mock_storage.NewMockChunkManager(t)
task := NewBackfillCompactionTask(ctx, mockCM, plan, compaction.GenParams())
assert.NotNil(t, task)
assert.Equal(t, int64(1000), task.GetPlanID())
assert.Equal(t, int64(1), task.GetCollection())
assert.Equal(t, "test_channel", task.GetChannelName())
assert.Equal(t, datapb.CompactionType_BackfillCompaction, task.GetCompactionType())
assert.Equal(t, int64(1), task.GetSlotUsage())
}
func (s *BackfillCompactionTaskSuite) TestProcessBatch() {
s.Run("success", func() {
s.setupTest()
mockRunner := function.NewMockFunctionRunner(s.T())
sparseArray := &schemapb.SparseFloatArray{
Contents: [][]byte{{1, 2, 3, 4}},
Dim: 100,
}
mockRunner.EXPECT().BatchRun([]string{"hello world"}).Return([]interface{}{sparseArray}, nil)
insertData, memSize, err := s.task.processBatch(mockRunner, []string{"hello world"}, 102)
s.NoError(err)
s.Greater(memSize, 0)
s.NotNil(insertData)
s.NotNil(insertData.Data[102])
})
s.Run("BatchRun error", func() {
s.setupTest()
mockRunner := function.NewMockFunctionRunner(s.T())
mockRunner.EXPECT().BatchRun([]string{"test"}).Return(nil, errors.New("batch run failed"))
_, _, err := s.task.processBatch(mockRunner, []string{"test"}, 102)
s.Error(err)
s.Contains(err.Error(), "batch run failed")
})
s.Run("wrong output count", func() {
s.setupTest()
mockRunner := function.NewMockFunctionRunner(s.T())
mockRunner.EXPECT().BatchRun([]string{"test"}).Return([]interface{}{nil, nil}, nil)
_, _, err := s.task.processBatch(mockRunner, []string{"test"}, 102)
s.Error(err)
s.Contains(err.Error(), "exactly one output")
})
s.Run("wrong output type", func() {
s.setupTest()
mockRunner := function.NewMockFunctionRunner(s.T())
mockRunner.EXPECT().BatchRun([]string{"test"}).Return([]interface{}{"not_sparse"}, nil)
_, _, err := s.task.processBatch(mockRunner, []string{"test"}, 102)
s.Error(err)
s.Contains(err.Error(), "unexpected output type")
})
}
func (s *BackfillCompactionTaskSuite) TestFinalizeMergedLogs() {
s.setupTest()
segment := &datapb.CompactionSegmentBinlogs{
SegmentID: 1,
FieldBinlogs: []*datapb.FieldBinlog{
{
FieldID: 100,
Binlogs: []*datapb.Binlog{
{LogPath: "original/100", EntriesNum: 1000},
},
},
},
}
newInsertLogs := map[int64]*datapb.FieldBinlog{
102: {
FieldID: 102,
Binlogs: []*datapb.Binlog{
{LogPath: "new/102", EntriesNum: 1000, MemorySize: 500},
},
},
}
bm25Bytes := []byte{1, 2, 3, 4, 5}
mergedInsert, mergedBm25, err := s.task.finalizeMergedLogs(segment, newInsertLogs, 1000, bm25Bytes, "bm25/stats/path", 77, 102)
s.NoError(err)
// Check merged insert logs contain both original and new
s.Equal(2, len(mergedInsert))
s.Equal(int64(100), mergedInsert[0].GetFieldID())
s.Equal(int64(102), mergedInsert[1].GetFieldID())
// Check BM25 logs
s.Equal(1, len(mergedBm25))
s.Equal(int64(102), mergedBm25[0].GetFieldID())
s.Equal(int64(5), mergedBm25[0].GetBinlogs()[0].GetLogSize())
s.Equal("bm25/stats/path", mergedBm25[0].GetBinlogs()[0].GetLogPath())
s.Equal(int64(1000), mergedBm25[0].GetBinlogs()[0].GetEntriesNum())
// LogID must be populated so buildBinlogKvs does not reject it with "invalid log id"
s.Equal(int64(77), mergedBm25[0].GetBinlogs()[0].GetLogID())
}
// TestFinalizeMergedLogsCrashReplay verifies that re-running finalizeMergedLogs
// when the segment already contains the output field (DC crashed after AlterSegments
// but before task-state transition) does not produce duplicate InsertLog entries.
func (s *BackfillCompactionTaskSuite) TestFinalizeMergedLogsCrashReplay() {
s.setupTest()
// Segment state after first successful run: field 102 already present in etcd.
segment := &datapb.CompactionSegmentBinlogs{
SegmentID: 1,
FieldBinlogs: []*datapb.FieldBinlog{
{FieldID: 100, Binlogs: []*datapb.Binlog{{LogPath: "original/100", EntriesNum: 1000}}},
{FieldID: 102, Binlogs: []*datapb.Binlog{{LogPath: "first-run/102", EntriesNum: 1000, LogID: 99}}},
},
}
// Second run allocates a new logID for the same output field.
newInsertLogs := map[int64]*datapb.FieldBinlog{
102: {FieldID: 102, Binlogs: []*datapb.Binlog{{LogPath: "second-run/102", EntriesNum: 1000, LogID: 100}}},
}
mergedInsert, _, err := s.task.finalizeMergedLogs(segment, newInsertLogs, 1000, []byte{1}, "bm25/path", 77, 102)
s.NoError(err)
// Must have exactly 2 entries (field 100 + field 102), not 3.
s.Require().Equal(2, len(mergedInsert))
var fieldIDs []int64
for _, fb := range mergedInsert {
fieldIDs = append(fieldIDs, fb.GetFieldID())
}
s.ElementsMatch([]int64{100, 102}, fieldIDs)
// Retained entry must be the first-run binlog, not the replay copy.
for _, fb := range mergedInsert {
if fb.GetFieldID() == 102 {
s.Equal("first-run/102", fb.GetBinlogs()[0].GetLogPath())
}
}
}
// TestFinalizeMergedLogsV3CrashReplay verifies crash-replay dedup for V3 column groups
// where the output field is identified via ChildFields rather than FieldID directly.
func (s *BackfillCompactionTaskSuite) TestFinalizeMergedLogsV3CrashReplay() {
s.setupTest()
// V3: column group 102 already present in segment after first run.
segment := &datapb.CompactionSegmentBinlogs{
SegmentID: 1,
FieldBinlogs: []*datapb.FieldBinlog{
{FieldID: 100, Binlogs: []*datapb.Binlog{{LogPath: "original/100"}}},
{FieldID: 102, ChildFields: []int64{102}, Binlogs: []*datapb.Binlog{{LogPath: "first-run/cg102"}}},
},
}
newInsertLogs := map[int64]*datapb.FieldBinlog{
102: {FieldID: 102, ChildFields: []int64{102}, Binlogs: []*datapb.Binlog{{LogPath: "second-run/cg102"}}},
}
mergedInsert, _, err := s.task.finalizeMergedLogs(segment, newInsertLogs, 0, []byte{1}, "bm25/path", 0, 102)
s.NoError(err)
s.Require().Equal(2, len(mergedInsert))
for _, fb := range mergedInsert {
if fb.GetFieldID() == 102 {
s.Equal("first-run/cg102", fb.GetBinlogs()[0].GetLogPath())
}
}
}
func (s *BackfillCompactionTaskSuite) TestBuildMergedLogsV2() {
s.setupTest()
segment := &datapb.CompactionSegmentBinlogs{
SegmentID: 1,
FieldBinlogs: []*datapb.FieldBinlog{
{FieldID: 100, Binlogs: []*datapb.Binlog{{LogPath: "original/100", EntriesNum: 1000}}},
},
}
// Simulate a V2 writer result: one column group with explicit paths, logIDs, and file sizes.
writerResult := &backfillWriterResult{
columnGroups: []storagecommon.ColumnGroup{
{GroupID: 102, Fields: []int64{102}},
},
paths: []string{"/local/insert_log/1/1/1/102/77"},
truePaths: []string{"/local/insert_log/1/1/1/102/77"},
logIDs: []int64{77},
fileSizes: []int64{2048},
storageVersion: storage.StorageV2,
}
mergedInsert, mergedBm25, err := s.task.buildMergedLogsV2(segment, writerResult, 512, 1000, []byte{1, 2, 3}, "bm25/path/55", 55, 102)
s.NoError(err)
// Expect original field 100 + new field 102.
s.Require().Equal(2, len(mergedInsert))
s.Equal(int64(100), mergedInsert[0].GetFieldID())
s.Equal(int64(102), mergedInsert[1].GetFieldID())
newBinlog := mergedInsert[1].GetBinlogs()[0]
s.Equal(int64(2048), newBinlog.GetLogSize(), "LogSize should match file size from CloseAndTell")
s.Equal(int64(512), newBinlog.GetMemorySize(), "MemorySize should match sparse field memory size")
s.Equal(int64(1000), newBinlog.GetEntriesNum())
s.Equal(int64(77), newBinlog.GetLogID(), "LogID must match allocated log ID")
s.Equal("/local/insert_log/1/1/1/102/77", newBinlog.GetLogPath())
// BM25 stats log.
s.Require().Equal(1, len(mergedBm25))
s.Equal(int64(102), mergedBm25[0].GetFieldID())
s.Equal("bm25/path/55", mergedBm25[0].GetBinlogs()[0].GetLogPath())
s.Equal(int64(55), mergedBm25[0].GetBinlogs()[0].GetLogID())
s.Equal(int64(3), mergedBm25[0].GetBinlogs()[0].GetLogSize(), "BM25 log size should equal len(bytes)")
}
func (s *BackfillCompactionTaskSuite) TestBuildMergedLogsV3() {
s.setupTest()
segment := &datapb.CompactionSegmentBinlogs{
SegmentID: 1,
FieldBinlogs: []*datapb.FieldBinlog{
{FieldID: 100, Binlogs: []*datapb.Binlog{{LogPath: "original/100", EntriesNum: 1000}}},
},
}
writerResult := &backfillWriterResult{
columnGroups: []storagecommon.ColumnGroup{
{GroupID: 102, Fields: []int64{102}},
},
}
mergedInsert, mergedBm25, err := s.task.buildMergedLogsV3(segment, writerResult, 512, 1000, []byte{1, 2, 3}, "bm25/path", 0, 102)
s.NoError(err)
// Original + new
s.Equal(2, len(mergedInsert))
s.Equal(int64(100), mergedInsert[0].GetFieldID())
s.Equal(int64(102), mergedInsert[1].GetFieldID())
s.Equal(int64(512), mergedInsert[1].GetBinlogs()[0].GetMemorySize())
s.Equal(int64(1000), mergedInsert[1].GetBinlogs()[0].GetEntriesNum())
// V3 binlog presence marker must carry a non-zero LogID so buildBinlogKvs validation
// passes (requires LogID!=0 AND LogPath=="" for V3 segments).
s.NotZero(mergedInsert[1].GetBinlogs()[0].GetLogID(),
"V3 backfill binlog presence marker must have non-zero LogID")
s.Equal(1, len(mergedBm25))
s.Equal(int64(102), mergedBm25[0].GetFieldID())
}
func (s *BackfillCompactionTaskSuite) TestCompleteAndStop() {
s.setupTest()
s.task.Complete()
s.task.Stop()
}
func (s *BackfillCompactionTaskSuite) TestGetStorageConfig() {
s.setupTest()
s.NotNil(s.task.GetStorageConfig())
}
internal/datanode/compactor/bump_schema_version_compactor.go
+1052
View File
File diff suppressed because it is too large Load Diff
internal/datanode/compactor/bump_schema_version_compactor_test.go
+1103
View File
File diff suppressed because it is too large Load Diff
internal/datanode/compactor/clustering_compactor.go
+29 -71
View File
@@ -616,37 +616,26 @@ func (t *clusteringCompactionTask) mappingSegment(
return merr.WrapErrIllegalCompactionPlan()
}
var rr storage.RecordReader
if segment.GetManifest() != "" {
rr, err = storage.NewManifestRecordReader(ctx,
segment.GetManifest(),
t.plan.Schema,
storage.WithDownloader(func(ctx context.Context, paths []string) ([][]byte, error) {
return t.binlogIO.Download(ctx, paths)
}),
storage.WithCollectionID(t.GetCollection()),
storage.WithVersion(segment.StorageVersion),
storage.WithBufferSize(t.bufferSize),
storage.WithStorageConfig(t.compactionParams.StorageConfig),
)
} else {
rr, err = storage.NewBinlogRecordReader(ctx,
segment.GetFieldBinlogs(),
t.plan.Schema,
storage.WithDownloader(func(ctx context.Context, paths []string) ([][]byte, error) {
return t.binlogIO.Download(ctx, paths)
}),
storage.WithCollectionID(t.GetCollection()),
storage.WithVersion(segment.StorageVersion),
storage.WithBufferSize(t.bufferSize),
storage.WithStorageConfig(t.compactionParams.StorageConfig),
)
}
rr, existingFields, err := newCompactionSegmentRecordReader(ctx, segment, t.plan.Schema, t.compactionParams.StorageConfig,
storage.WithDownloader(func(ctx context.Context, paths []string) ([][]byte, error) {
return t.binlogIO.Download(ctx, paths)
}),
storage.WithCollectionID(t.GetCollection()),
storage.WithVersion(segment.StorageVersion),
storage.WithBufferSize(t.bufferSize),
storage.WithStorageConfig(t.compactionParams.StorageConfig),
)
if err != nil {
log.Warn("new binlog record reader wrong", zap.Error(err))
return err
}
materializer, err := NewRecordMaterializer(t.plan.Schema, t.plan.Schema.GetFunctions(), existingFields)
if err != nil {
rr.Close()
log.Warn("new record materializer wrong", zap.Error(err))
return err
}
rr = newMaterializedRecordReader(rr, materializer)
defer rr.Close()
hasTTLField := t.ttlFieldID >= common.StartOfUserFieldID
@@ -923,8 +912,6 @@ func (t *clusteringCompactionTask) scalarAnalyzeSegment(
log.Debug("binlogNum", zap.Int("binlogNum", binlogNum))
expiredFilter := compaction.NewEntityFilter(nil, t.plan.GetCollectionTtl(), t.currentTime, segment.GetCommitTimestamp())
binlogs := make([]*datapb.FieldBinlog, 0)
requiredFields := typeutil.NewSet[int64]()
requiredFields.Insert(0, 1, t.primaryKeyField.GetFieldID(), t.clusteringKeyField.GetFieldID())
if t.ttlFieldID >= common.StartOfUserFieldID {
@@ -933,49 +920,20 @@ func (t *clusteringCompactionTask) scalarAnalyzeSegment(
selectedFields := lo.Filter(t.plan.GetSchema().GetFields(), func(field *schemapb.FieldSchema, _ int) bool {
return requiredFields.Contain(field.GetFieldID())
})
readSchema := proto.Clone(t.plan.GetSchema()).(*schemapb.CollectionSchema)
readSchema.Fields = selectedFields
readSchema.StructArrayFields = nil
switch segment.GetStorageVersion() {
case storage.StorageV1:
for _, fieldBinlog := range segment.GetFieldBinlogs() {
if requiredFields.Contain(fieldBinlog.GetFieldID()) {
binlogs = append(binlogs, fieldBinlog)
}
}
case storage.StorageV2, storage.StorageV3:
binlogs = segment.GetFieldBinlogs()
default:
log.Warn("unsupported storage version", zap.Int64("storage version", segment.GetStorageVersion()))
return nil, fmt.Errorf("unsupported storage version %d", segment.GetStorageVersion())
}
var rr storage.RecordReader
var err error
if segment.GetManifest() != "" {
rr, err = storage.NewManifestRecordReader(ctx,
segment.GetManifest(),
t.plan.GetSchema(),
storage.WithDownloader(func(ctx context.Context, paths []string) ([][]byte, error) {
return t.binlogIO.Download(ctx, paths)
}),
storage.WithVersion(segment.StorageVersion),
storage.WithBufferSize(t.bufferSize),
storage.WithStorageConfig(t.compactionParams.StorageConfig),
storage.WithNeededFields(requiredFields),
storage.WithCollectionID(t.GetCollection()),
)
} else {
rr, err = storage.NewBinlogRecordReader(ctx,
binlogs,
t.plan.GetSchema(),
storage.WithDownloader(func(ctx context.Context, paths []string) ([][]byte, error) {
return t.binlogIO.Download(ctx, paths)
}),
storage.WithVersion(segment.StorageVersion),
storage.WithBufferSize(t.bufferSize),
storage.WithStorageConfig(t.compactionParams.StorageConfig),
storage.WithNeededFields(requiredFields),
storage.WithCollectionID(t.GetCollection()),
)
}
rr, _, err := newCompactionSegmentRecordReader(ctx, segment, readSchema, t.compactionParams.StorageConfig,
storage.WithDownloader(func(ctx context.Context, paths []string) ([][]byte, error) {
return t.binlogIO.Download(ctx, paths)
}),
storage.WithVersion(segment.StorageVersion),
storage.WithBufferSize(t.bufferSize),
storage.WithStorageConfig(t.compactionParams.StorageConfig),
storage.WithNeededFields(requiredFields),
storage.WithCollectionID(t.GetCollection()),
)
if err != nil {
log.Warn("new binlog record reader wrong", zap.Error(err))
return make(map[interface{}]int64), err
internal/datanode/compactor/clustering_compactor_test.go
+53 -11
View File
@@ -387,12 +387,20 @@ func (s *ClusteringCompactionTaskSuite) TestScalarCompactionNormalByMemoryLimit(
func (s *ClusteringCompactionTaskSuite) prepareCompactionWithBM25FunctionTask() {
s.SetupTest()
s.prepareCompactionWithBM25OutputTask(10240, false)
}
func (s *ClusteringCompactionTaskSuite) prepareCompactionWithMissingBM25OutputTask(rowNum int) {
s.prepareCompactionWithBM25OutputTask(rowNum, true)
}
func (s *ClusteringCompactionTaskSuite) prepareCompactionWithBM25OutputTask(rowNum int, removeBM25Output bool) {
schema := genCollectionSchemaWithBM25()
var segmentID int64 = 1001
segWriter, err := NewSegmentWriter(schema, 1000, compactionBatchSize, segmentID, PartitionID, CollectionID, []int64{102})
segmentID := int64(1001)
segWriter, err := NewSegmentWriter(schema, int64(rowNum), compactionBatchSize, segmentID, PartitionID, CollectionID, []int64{102})
s.Require().NoError(err)
for i := 0; i < 10240; i++ {
for i := 0; i < rowNum; i++ {
v := storage.Value{
PK: storage.NewInt64PrimaryKey(int64(i)),
Timestamp: int64(tsoutil.ComposeTSByTime(getMilvusBirthday(), 0)),
@@ -404,13 +412,12 @@ func (s *ClusteringCompactionTaskSuite) prepareCompactionWithBM25FunctionTask()
segWriter.FlushAndIsFull()
kvs, fBinlogs, err := serializeWrite(context.TODO(), s.mockAlloc, segWriter)
s.NoError(err)
s.mockBinlogIO.EXPECT().Download(mock.Anything, mock.Anything).RunAndReturn(func(ctx context.Context, strings []string) ([][]byte, error) {
result := make([][]byte, 0, len(strings))
for _, path := range strings {
result = append(result, kvs[path])
}
return result, nil
s.Require().NoError(err)
if removeBM25Output {
removeFieldBinlogForTest(kvs, fBinlogs, 102)
}
s.mockBinlogIO.EXPECT().Download(mock.Anything, mock.Anything).RunAndReturn(func(ctx context.Context, paths []string) ([][]byte, error) {
return downloadValuesForPathsForTest(kvs, paths)
})
s.plan.SegmentBinlogs = []*datapb.CompactionSegmentBinlogs{
@@ -421,7 +428,6 @@ func (s *ClusteringCompactionTaskSuite) prepareCompactionWithBM25FunctionTask()
},
}
s.task.bm25FieldIds = []int64{102}
s.task.plan.Schema = schema
s.task.plan.ClusteringKeyField = 100
s.task.plan.PreferSegmentRows = 2048
@@ -490,6 +496,42 @@ func (s *ClusteringCompactionTaskSuite) TestCompactionWithBM25Function() {
s.Equal(totalRowNum, bm25RowNum)
}
func (s *ClusteringCompactionTaskSuite) TestScalarClusteringMaterializesMissingBM25OutputFromOldSegment() {
s.prepareCompactionWithMissingBM25OutputTask(3)
result, err := s.task.Compact()
s.Require().NoError(err)
s.Require().NotNil(result)
s.EqualValues(3, lo.SumBy(result.GetSegments(), func(segment *datapb.CompactionSegment) int64 {
return segment.GetNumOfRows()
}))
bm25Rows := int64(0)
for _, segment := range result.GetSegments() {
bm25Rows += fieldBinlogEntriesForTest(segment.GetBm25Logs(), 102)
}
s.EqualValues(3, bm25Rows)
}
func (s *ClusteringCompactionTaskSuite) TestScalarAnalyzeSegmentFiltersDroppedOrMissingFields() {
s.prepareCompactionWithMissingBM25OutputTask(2)
s.task.plan.ClusteringKeyField = 101
s.task.plan.SegmentBinlogs[0].FieldBinlogs = append(s.task.plan.SegmentBinlogs[0].FieldBinlogs, &datapb.FieldBinlog{
FieldID: common.StartOfUserFieldID + 1000,
Binlogs: []*datapb.Binlog{{
LogPath: "dropped-field-should-not-be-read",
}},
})
err := s.task.init()
s.Require().NoError(err)
defer s.task.cleanUp(context.Background())
analyzeResult, err := s.task.scalarAnalyzeSegment(context.Background(), s.task.plan.SegmentBinlogs[0])
s.Require().NoError(err)
s.Equal(map[interface{}]int64{"varchar": 2}, analyzeResult)
}
func genRow(magic int64) map[int64]interface{} {
ts := tsoutil.ComposeTSByTime(getMilvusBirthday(), 0)
return map[int64]interface{}{
internal/datanode/compactor/compactor_common.go
+343 -9
View File
@@ -18,23 +18,366 @@ package compactor
import (
"context"
"fmt"
"sort"
"strconv"
"sync"
"time"
"github.com/samber/lo"
"go.opentelemetry.io/otel"
"go.uber.org/zap"
"golang.org/x/sync/errgroup"
"google.golang.org/protobuf/proto"
"github.com/milvus-io/milvus-proto/go-api/v3/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
"github.com/milvus-io/milvus/internal/allocator"
"github.com/milvus-io/milvus/internal/compaction"
"github.com/milvus-io/milvus/internal/datanode/util"
"github.com/milvus-io/milvus/internal/metastore/kv/binlog"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/internal/storagev2/packed"
"github.com/milvus-io/milvus/internal/util/analyzer"
"github.com/milvus-io/milvus/internal/util/fileresource"
"github.com/milvus-io/milvus/internal/util/indexcgowrapper"
"github.com/milvus-io/milvus/pkg/v3/common"
"github.com/milvus-io/milvus/pkg/v3/log"
"github.com/milvus-io/milvus/pkg/v3/proto/datapb"
"github.com/milvus-io/milvus/pkg/v3/proto/indexcgopb"
"github.com/milvus-io/milvus/pkg/v3/proto/indexpb"
"github.com/milvus-io/milvus/pkg/v3/util/metautil"
"github.com/milvus-io/milvus/pkg/v3/util/tsoutil"
"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
)
const compactionBatchSize = 100
func createTextIndex(ctx context.Context,
cm storage.ChunkManager,
plan *datapb.CompactionPlan,
compactionParams compaction.Params,
storageVersion int64,
collectionID int64,
partitionID int64,
segmentID int64,
taskID int64,
segment *datapb.CompactionSegment,
) (map[int64]*datapb.TextIndexStats, error) {
log := log.Ctx(ctx).With(
zap.Int64("collectionID", collectionID),
zap.Int64("partitionID", partitionID),
zap.Int64("segmentID", segmentID),
)
fieldBinlogs := lo.GroupBy(segment.GetInsertLogs(), func(binlog *datapb.FieldBinlog) int64 {
return binlog.GetFieldID()
})
getInsertFiles := func(fieldID int64) ([]string, error) {
if storageVersion == storage.StorageV2 || storageVersion == storage.StorageV3 {
return []string{}, nil
}
binlogs, ok := fieldBinlogs[fieldID]
if !ok {
return nil, fmt.Errorf("field binlog not found for field %d", fieldID)
}
result := make([]string, 0, len(binlogs))
for _, binlog := range binlogs {
for _, file := range binlog.GetBinlogs() {
result = append(result, metautil.BuildInsertLogPath(compactionParams.StorageConfig.GetRootPath(),
collectionID, partitionID, segmentID, fieldID, file.GetLogID()))
}
}
return result, nil
}
newStorageConfig, err := util.ParseStorageConfig(compactionParams.StorageConfig)
if err != nil {
return nil, err
}
var (
mu sync.Mutex
textIndexLogs = make(map[int64]*datapb.TextIndexStats)
)
eg, egCtx := errgroup.WithContext(ctx)
var analyzerExtraInfo string
if len(plan.GetFileResources()) > 0 {
err := fileresource.GlobalFileManager.Download(ctx, cm, plan.GetFileResources()...)
if err != nil {
return nil, err
}
defer fileresource.GlobalFileManager.Release(plan.GetFileResources()...)
analyzerExtraInfo, err = analyzer.BuildExtraResourceInfo(compactionParams.StorageConfig.GetRootPath(), plan.GetFileResources())
if err != nil {
return nil, err
}
}
for _, field := range plan.GetSchema().GetFields() {
field := field
h := typeutil.CreateFieldSchemaHelper(field)
if !h.EnableMatch() {
continue
}
log.Info("field enable match, ready to create text index", zap.Int64("field id", field.GetFieldID()))
eg.Go(func() error {
files, err := getInsertFiles(field.GetFieldID())
if err != nil {
return err
}
statsBasePath := metautil.BuildTextIndexPrefix(compactionParams.StorageConfig.GetRootPath(),
plan.GetPlanID(), 0, collectionID, partitionID, segmentID, field.GetFieldID())
if segment.GetManifest() != "" {
basePath, _, err := packed.UnmarshalManifestPath(segment.GetManifest())
if err != nil {
return fmt.Errorf("failed to unmarshal manifest path for text_index basePath: %w", err)
}
statsBasePath = fmt.Sprintf("%s/_stats/text_index.%d", basePath, field.GetFieldID())
}
buildIndexParams := &indexcgopb.BuildIndexInfo{
BuildID: taskID,
CollectionID: collectionID,
PartitionID: partitionID,
SegmentID: segmentID,
IndexVersion: 0,
InsertFiles: files,
FieldSchema: field,
StorageConfig: newStorageConfig,
CurrentScalarIndexVersion: common.ClampScalarIndexVersion(plan.GetCurrentScalarIndexVersion()),
StorageVersion: storageVersion,
Manifest: segment.GetManifest(),
StatsBasePath: statsBasePath,
IndexParams: []*commonpb.KeyValuePair{
{Key: "index_type", Value: "INVERTED"},
{Key: "is_text_match", Value: "true"},
},
}
if len(analyzerExtraInfo) > 0 {
buildIndexParams.AnalyzerExtraInfo = analyzerExtraInfo
}
if storageVersion == storage.StorageV2 || storageVersion == storage.StorageV3 {
buildIndexParams.SegmentInsertFiles = util.GetSegmentInsertFiles(
segment.GetInsertLogs(),
compactionParams.StorageConfig,
collectionID,
partitionID,
segmentID)
}
index, err := indexcgowrapper.CreateIndex(egCtx, buildIndexParams)
if err != nil {
return err
}
defer index.Delete()
indexStats, err := index.UpLoad()
if err != nil {
return err
}
uploaded := make(map[string]int64)
for _, info := range indexStats.GetSerializedIndexInfos() {
uploaded[info.FileName] = info.FileSize
}
statsFiles := metautil.BuildStatsFilePaths(statsBasePath, lo.Keys(uploaded))
mu.Lock()
totalSize := lo.SumBy(lo.Values(uploaded), func(fileSize int64) int64 { return fileSize })
textIndexLogs[field.GetFieldID()] = &datapb.TextIndexStats{
FieldID: field.GetFieldID(),
Version: 0,
BuildID: taskID,
Files: statsFiles,
LogSize: totalSize,
MemorySize: totalSize,
CurrentScalarIndexVersion: common.ClampScalarIndexVersion(plan.GetCurrentScalarIndexVersion()),
}
mu.Unlock()
log.Info("field enable match, create text index done",
zap.Int64("segmentID", segmentID),
zap.Int64("field id", field.GetFieldID()),
zap.Strings("files", statsFiles),
)
return nil
})
}
if err := eg.Wait(); err != nil {
return nil, err
}
return textIndexLogs, nil
}
// Storage readers do not share compaction's schema-reconciliation contract, so filter physical fields before opening them.
func newCompactionSegmentRecordReader(ctx context.Context, segment *datapb.CompactionSegmentBinlogs, schema *schemapb.CollectionSchema, storageConfig *indexpb.StorageConfig, opts ...storage.RwOption) (storage.RecordReader, map[int64]struct{}, error) {
existingFields, err := compactionSegmentStorageFields(segment, storageConfig)
if err != nil {
return nil, nil, err
}
return newCompactionSegmentRecordReaderWithFields(ctx, segment, schema, storageConfig, existingFields, opts...)
}
func compactionSegmentStorageFields(segment *datapb.CompactionSegmentBinlogs, storageConfig *indexpb.StorageConfig) (map[int64]struct{}, error) {
if segment.GetManifest() != "" {
return packed.GetManifestFieldIDs(segment.GetManifest(), storageConfig)
}
return compactionSegmentBinlogFields(segment), nil
}
func compactionSegmentBinlogFields(segment *datapb.CompactionSegmentBinlogs) map[int64]struct{} {
fields := make(map[int64]struct{})
for _, fieldBinlog := range segment.GetFieldBinlogs() {
if len(fieldBinlog.GetChildFields()) == 0 {
fields[fieldBinlog.GetFieldID()] = struct{}{}
continue
}
for _, childFieldID := range fieldBinlog.GetChildFields() {
fields[childFieldID] = struct{}{}
}
}
return fields
}
func collectionSchemaFields(schema *schemapb.CollectionSchema) map[int64]struct{} {
fields := make(map[int64]struct{}, typeutil.GetTotalFieldsNum(schema))
for _, field := range typeutil.GetAllFieldSchemas(schema) {
fields[field.GetFieldID()] = struct{}{}
}
return fields
}
func missingSchemaFunctions(schema *schemapb.CollectionSchema, existingFields map[int64]struct{}) []*schemapb.FunctionSchema {
var missing []*schemapb.FunctionSchema
for _, functionSchema := range schema.GetFunctions() {
for _, outputFieldID := range functionSchema.GetOutputFieldIds() {
if _, ok := existingFields[outputFieldID]; !ok {
missing = append(missing, functionSchema)
break
}
}
}
return missing
}
func droppedSchemaFieldIDs(schema *schemapb.CollectionSchema, existingFields map[int64]struct{}) []int64 {
targetFields := collectionSchemaFields(schema)
dropped := make([]int64, 0)
for fieldID := range existingFields {
if fieldID < common.StartOfUserFieldID {
continue
}
if _, ok := targetFields[fieldID]; !ok {
dropped = append(dropped, fieldID)
}
}
sort.Slice(dropped, func(i, j int) bool { return dropped[i] < dropped[j] })
return dropped
}
func segmentDroppedFieldIDs(schema *schemapb.CollectionSchema, segment *datapb.CompactionSegmentBinlogs, storageConfig *indexpb.StorageConfig) ([]int64, error) {
existingFields, err := compactionSegmentStorageFields(segment, storageConfig)
if err != nil {
return nil, err
}
return droppedSchemaFieldIDs(schema, existingFields), nil
}
func newCompactionSegmentRecordReaderWithFields(ctx context.Context, segment *datapb.CompactionSegmentBinlogs, schema *schemapb.CollectionSchema, storageConfig *indexpb.StorageConfig, existingFields map[int64]struct{}, opts ...storage.RwOption) (storage.RecordReader, map[int64]struct{}, error) {
readSchema := compactionReadSchema(schema, existingFields)
if segment.GetManifest() != "" {
reader, err := storage.NewManifestRecordReader(ctx, segment.GetManifest(), readSchema, opts...)
return reader, existingFields, err
}
readFields := collectionSchemaFields(readSchema)
fieldBinlogs := filterCompactionFieldBinlogs(segment.GetFieldBinlogs(), readFields)
rootPath := ""
if storageConfig != nil {
rootPath = storageConfig.GetRootPath()
}
if err := binlog.DecompressBinLogWithRootPath(rootPath, storage.InsertBinlog,
segment.GetCollectionID(), segment.GetPartitionID(), segment.GetSegmentID(), fieldBinlogs); err != nil {
return nil, nil, err
}
reader, err := storage.NewBinlogRecordReader(ctx, fieldBinlogs, readSchema, opts...)
return reader, existingFields, err
}
func compactionReadSchema(schema *schemapb.CollectionSchema, existingFields map[int64]struct{}) *schemapb.CollectionSchema {
if schema == nil {
return nil
}
readSchema := proto.Clone(schema).(*schemapb.CollectionSchema)
fields := make([]*schemapb.FieldSchema, 0, len(readSchema.GetFields()))
for _, field := range readSchema.GetFields() {
if compactionFieldReadable(field, existingFields) {
fields = append(fields, field)
}
}
readSchema.Fields = fields
structFields := make([]*schemapb.StructArrayFieldSchema, 0, len(readSchema.GetStructArrayFields()))
for _, structField := range readSchema.GetStructArrayFields() {
childFields := make([]*schemapb.FieldSchema, 0, len(structField.GetFields()))
for _, field := range structField.GetFields() {
if compactionFieldReadable(field, existingFields) {
childFields = append(childFields, field)
}
}
if len(childFields) > 0 {
structField.Fields = childFields
structFields = append(structFields, structField)
}
}
readSchema.StructArrayFields = structFields
return readSchema
}
func compactionFieldReadable(field *schemapb.FieldSchema, existingFields map[int64]struct{}) bool {
_, ok := existingFields[field.GetFieldID()]
return ok
}
func filterCompactionFieldBinlogs(fieldBinlogs []*datapb.FieldBinlog, readFields map[int64]struct{}) []*datapb.FieldBinlog {
filtered := make([]*datapb.FieldBinlog, 0, len(fieldBinlogs))
for _, fieldBinlog := range fieldBinlogs {
if compactionFieldBinlogReadable(fieldBinlog, readFields) {
filtered = append(filtered, fieldBinlog)
}
}
return filtered
}
func compactionFieldBinlogReadable(fieldBinlog *datapb.FieldBinlog, readFields map[int64]struct{}) bool {
if fieldBinlog == nil {
return false
}
if _, ok := readFields[fieldBinlog.GetFieldID()]; ok {
return true
}
for _, childFieldID := range fieldBinlog.GetChildFields() {
if _, ok := readFields[childFieldID]; ok {
return true
}
}
return false
}
type EntityFilter struct {
deletedPkTs map[interface{}]typeutil.Timestamp // pk2ts
ttl int64 // nanoseconds
@@ -153,15 +496,6 @@ func serializeWrite(ctx context.Context, allocator allocator.Interface, writer *
return
}
func mergeFieldBinlogs(base, paths map[typeutil.UniqueID]*datapb.FieldBinlog) {
for fID, fpath := range paths {
if _, ok := base[fID]; !ok {
base[fID] = &datapb.FieldBinlog{FieldID: fID, Binlogs: make([]*datapb.Binlog, 0)}
}
base[fID].Binlogs = append(base[fID].Binlogs, fpath.GetBinlogs()...)
}
}
func getTTLFieldID(schema *schemapb.CollectionSchema) int64 {
ttlFieldName := ""
for _, pair := range schema.GetProperties() {
internal/datanode/compactor/compactor_common_test.go
+158
View File
@@ -0,0 +1,158 @@
package compactor
import (
"testing"
"github.com/cockroachdb/errors"
"github.com/stretchr/testify/require"
"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
"github.com/milvus-io/milvus/pkg/v3/common"
"github.com/milvus-io/milvus/pkg/v3/proto/datapb"
)
func TestCompactionSegmentBinlogFieldsUsesChildFields(t *testing.T) {
fields := compactionSegmentBinlogFields(&datapb.CompactionSegmentBinlogs{
FieldBinlogs: []*datapb.FieldBinlog{
{FieldID: 900, ChildFields: []int64{102, 103}},
{FieldID: 104},
},
})
require.Contains(t, fields, int64(102))
require.Contains(t, fields, int64(103))
require.Contains(t, fields, int64(104))
require.NotContains(t, fields, int64(900))
}
func TestFilterCompactionFieldBinlogsKeepsChildFieldMatch(t *testing.T) {
fieldBinlogs := []*datapb.FieldBinlog{
nil,
{FieldID: 900, ChildFields: []int64{102, 103}},
{FieldID: 200},
}
filtered := filterCompactionFieldBinlogs(fieldBinlogs, map[int64]struct{}{102: {}})
require.Len(t, filtered, 1)
require.EqualValues(t, 900, filtered[0].GetFieldID())
require.Equal(t, []int64{102, 103}, filtered[0].GetChildFields())
}
func TestCompactionReadSchemaFiltersMissingScalarAndStructChildren(t *testing.T) {
schema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{FieldID: 100, Name: "pk", DataType: schemapb.DataType_Int64},
{FieldID: 101, Name: "missing", DataType: schemapb.DataType_Int64},
},
StructArrayFields: []*schemapb.StructArrayFieldSchema{
{
FieldID: 200,
Name: "struct_with_child",
Fields: []*schemapb.FieldSchema{
{FieldID: 201, Name: "child_present", DataType: schemapb.DataType_Int64},
{FieldID: 202, Name: "child_missing", DataType: schemapb.DataType_Int64},
},
},
{
FieldID: 300,
Name: "struct_without_child",
Fields: []*schemapb.FieldSchema{
{FieldID: 301, Name: "child_missing", DataType: schemapb.DataType_Int64},
},
},
},
}
readSchema := compactionReadSchema(schema, map[int64]struct{}{100: {}, 201: {}})
require.NotNil(t, readSchema)
require.Len(t, readSchema.GetFields(), 1)
require.EqualValues(t, 100, readSchema.GetFields()[0].GetFieldID())
require.Len(t, readSchema.GetStructArrayFields(), 1)
require.EqualValues(t, 200, readSchema.GetStructArrayFields()[0].GetFieldID())
require.Len(t, readSchema.GetStructArrayFields()[0].GetFields(), 1)
require.EqualValues(t, 201, readSchema.GetStructArrayFields()[0].GetFields()[0].GetFieldID())
}
func TestCompactionReadSchemaNilSchema(t *testing.T) {
require.Nil(t, compactionReadSchema(nil, map[int64]struct{}{}))
}
func TestMissingSchemaFunctionsAndDroppedFields(t *testing.T) {
functionSchema := &schemapb.FunctionSchema{
Name: "bm25",
Type: schemapb.FunctionType_BM25,
InputFieldIds: []int64{100},
OutputFieldIds: []int64{101},
}
schema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{FieldID: 100, Name: "text", DataType: schemapb.DataType_VarChar},
{FieldID: 101, Name: "sparse", DataType: schemapb.DataType_SparseFloatVector},
},
Functions: []*schemapb.FunctionSchema{functionSchema},
}
droppedUserField := int64(common.StartOfUserFieldID + 1000)
systemField := int64(common.StartOfUserFieldID - 1)
existingFields := map[int64]struct{}{
100: {},
droppedUserField: {},
systemField: {},
}
missingFunctions := missingSchemaFunctions(schema, existingFields)
require.Len(t, missingFunctions, 1)
require.Equal(t, functionSchema.GetName(), missingFunctions[0].GetName())
dropped := droppedSchemaFieldIDs(schema, existingFields)
require.Equal(t, []int64{droppedUserField}, dropped)
}
func removeFieldBinlogForTest(kvs map[string][]byte, fieldBinlogs map[int64]*datapb.FieldBinlog, fieldID int64) {
for _, binlog := range fieldBinlogs[fieldID].GetBinlogs() {
delete(kvs, binlog.GetLogPath())
}
delete(fieldBinlogs, fieldID)
}
func downloadValuesForPathsForTest(kvs map[string][]byte, paths []string) ([][]byte, error) {
values := make([][]byte, 0, len(paths))
for _, path := range paths {
value, ok := kvs[path]
if !ok {
return nil, errors.Newf("unexpected download path %s", path)
}
values = append(values, value)
}
return values, nil
}
func TestFieldBinlogEntriesForTestUsesChildFields(t *testing.T) {
fieldBinlogs := []*datapb.FieldBinlog{
{FieldID: 0, ChildFields: []int64{101, 107}, Binlogs: []*datapb.Binlog{{EntriesNum: 3}}},
{FieldID: 108, Binlogs: []*datapb.Binlog{{EntriesNum: 5}}},
}
require.EqualValues(t, 3, fieldBinlogEntriesForTest(fieldBinlogs, 107))
require.EqualValues(t, 5, fieldBinlogEntriesForTest(fieldBinlogs, 108))
require.EqualValues(t, 0, fieldBinlogEntriesForTest(fieldBinlogs, 109))
}
func fieldBinlogEntriesForTest(fieldBinlogs []*datapb.FieldBinlog, fieldID int64) int64 {
var entries int64
for _, fieldBinlog := range fieldBinlogs {
matchesField := fieldBinlog.GetFieldID() == fieldID
for _, childFieldID := range fieldBinlog.GetChildFields() {
if childFieldID == fieldID {
matchesField = true
break
}
}
if !matchesField {
continue
}
for _, binlog := range fieldBinlog.GetBinlogs() {
entries += binlog.GetEntriesNum()
}
}
return entries
}
internal/datanode/compactor/executor.go
+2 -2
View File
@@ -91,8 +91,8 @@ func getTaskSlotUsage(task Compactor) int64 {
taskSlotUsage = paramtable.Get().DataCoordCfg.MixCompactionSlotUsage.GetAsInt64()
case datapb.CompactionType_Level0DeleteCompaction:
taskSlotUsage = paramtable.Get().DataCoordCfg.L0DeleteCompactionSlotUsage.GetAsInt64()
case datapb.CompactionType_BackfillCompaction:
taskSlotUsage = paramtable.Get().DataCoordCfg.BackfillCompactionSlotUsage.GetAsInt64()
case datapb.CompactionType_BumpSchemaVersionCompaction:
taskSlotUsage = paramtable.Get().DataCoordCfg.BumpSchemaVersionCompactionSlotUsage.GetAsInt64()
}
log.Warn("illegal task slot usage, change it to a default value",
zap.Int64("illegalSlotUsage", task.GetSlotUsage()),
internal/datanode/compactor/executor_test.go
+7 -2
View File
@@ -73,8 +73,6 @@ func TestCompactionExecutor(t *testing.T) {
})
t.Run("Test_Enqueue_DefaultSlotUsage", func(t *testing.T) {
ex := NewExecutor()
testCases := []struct {
name string
compactionType datapb.CompactionType
@@ -95,10 +93,16 @@ func TestCompactionExecutor(t *testing.T) {
compactionType: datapb.CompactionType_ClusteringCompaction,
expectedSlotUsage: paramtable.Get().DataCoordCfg.ClusteringCompactionSlotUsage.GetAsInt64(),
},
{
name: "BumpSchemaVersionCompaction",
compactionType: datapb.CompactionType_BumpSchemaVersionCompaction,
expectedSlotUsage: paramtable.Get().DataCoordCfg.BumpSchemaVersionCompactionSlotUsage.GetAsInt64(),
},
}
for i, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
ex := NewExecutor()
mockC := NewMockCompactor(t)
mockC.EXPECT().GetPlanID().Return(int64(i + 10))
mockC.EXPECT().GetSlotUsage().Return(int64(0)).Times(2)
@@ -107,6 +111,7 @@ func TestCompactionExecutor(t *testing.T) {
succeed, err := ex.Enqueue(mockC)
assert.True(t, succeed)
assert.NoError(t, err)
assert.Equal(t, tc.expectedSlotUsage, ex.Slots())
})
}
})
internal/datanode/compactor/merge_sort.go
+20 -26
View File
@@ -63,31 +63,31 @@ func mergeSortMultipleSegments(ctx context.Context,
hasTTLField := ttlFieldID >= common.StartOfUserFieldID
segmentReaders := make([]storage.RecordReader, len(binlogs))
defer func() {
for _, r := range segmentReaders {
if r != nil {
r.Close()
}
}
}()
segmentFilters := make([]compaction.EntityFilter, len(binlogs))
for i, s := range binlogs {
var reader storage.RecordReader
if s.GetManifest() != "" {
reader, err = storage.NewManifestRecordReader(ctx,
s.GetManifest(),
plan.GetSchema(),
storage.WithCollectionID(collectionID),
storage.WithDownloader(binlogIO.Download),
storage.WithVersion(s.StorageVersion),
storage.WithStorageConfig(compactionParams.StorageConfig),
)
} else {
reader, err = storage.NewBinlogRecordReader(ctx,
s.GetFieldBinlogs(),
plan.GetSchema(),
storage.WithCollectionID(collectionID),
storage.WithDownloader(binlogIO.Download),
storage.WithVersion(s.StorageVersion),
storage.WithStorageConfig(compactionParams.StorageConfig),
)
}
reader, existingFields, err := newCompactionSegmentRecordReader(ctx, s, plan.GetSchema(), compactionParams.StorageConfig,
storage.WithCollectionID(collectionID),
storage.WithDownloader(binlogIO.Download),
storage.WithVersion(s.StorageVersion),
storage.WithStorageConfig(compactionParams.StorageConfig),
)
if err != nil {
return nil, err
}
materializer, err := NewRecordMaterializer(writerSchema, writerSchema.GetFunctions(), existingFields)
if err != nil {
reader.Close()
return nil, err
}
reader = newMaterializedRecordReader(reader, materializer)
segmentReaders[i] = wrapReaderWithTimestampOverwrite(reader, s.GetCommitTimestamp())
delta, err := compaction.ComposeDeleteFromDeltalogs(ctx, pkField.DataType, s,
storage.WithDownloader(binlogIO.Download),
@@ -98,12 +98,6 @@ func mergeSortMultipleSegments(ctx context.Context,
segmentFilters[i] = compaction.NewEntityFilter(delta, collectionTTL, currentTime, s.GetCommitTimestamp())
}
defer func() {
for _, r := range segmentReaders {
r.Close()
}
}()
var predicate func(r storage.Record, ri, i int) bool
switch pkField.DataType {
case schemapb.DataType_Int64:
internal/datanode/compactor/mix_compactor.go
+24 -20
View File
@@ -262,32 +262,25 @@ func (t *mixCompactionTask) writeSegment(ctx context.Context,
}
entityFilter := compaction.NewEntityFilter(delta, t.plan.GetCollectionTtl(), t.currentTime, seg.GetCommitTimestamp())
var reader storage.RecordReader
if seg.GetManifest() != "" {
reader, err = storage.NewManifestRecordReader(ctx,
seg.GetManifest(),
t.plan.GetSchema(),
storage.WithCollectionID(t.collectionID),
storage.WithDownloader(t.binlogIO.Download),
storage.WithVersion(seg.GetStorageVersion()),
storage.WithStorageConfig(t.compactionParams.StorageConfig),
)
} else {
reader, err = storage.NewBinlogRecordReader(ctx,
seg.GetFieldBinlogs(),
t.plan.GetSchema(),
storage.WithCollectionID(t.collectionID),
storage.WithDownloader(t.binlogIO.Download),
storage.WithVersion(seg.GetStorageVersion()),
storage.WithStorageConfig(t.compactionParams.StorageConfig),
)
}
reader, existingFields, err := newCompactionSegmentRecordReader(ctx, seg, t.plan.GetSchema(), t.compactionParams.StorageConfig,
storage.WithCollectionID(t.collectionID),
storage.WithDownloader(t.binlogIO.Download),
storage.WithVersion(seg.GetStorageVersion()),
storage.WithStorageConfig(t.compactionParams.StorageConfig),
)
if err != nil {
log.Warn("compact wrong, failed to new insert binlogs reader", zap.Error(err))
return
}
defer reader.Close()
materializer, err := NewRecordMaterializer(writerSchema, writerSchema.GetFunctions(), existingFields)
if err != nil {
log.Warn("compact wrong, failed to init record materializer", zap.Error(err))
return
}
defer materializer.Close()
hasTTLField := t.ttlFieldID >= common.StartOfUserFieldID
var totalRowsRead int64
@@ -304,6 +297,14 @@ func (t *mixCompactionTask) writeSegment(ctx context.Context,
}
}
baseRecord := r
r, err = materializer.Wrap(baseRecord)
if err != nil {
baseRecord.Release()
log.Warn("compact wrong, failed to materialize record", zap.Error(err))
return
}
totalRowsRead += int64(r.Len())
var (
@@ -368,6 +369,7 @@ func (t *mixCompactionTask) writeSegment(ctx context.Context,
return mWriter.Write(out)
}()
if err != nil {
releaseWrappedRecord(r, baseRecord)
return 0, 0, err
}
}
@@ -378,9 +380,11 @@ func (t *mixCompactionTask) writeSegment(ctx context.Context,
out.Release()
}
if err != nil {
releaseWrappedRecord(r, baseRecord)
return 0, 0, err
}
}
releaseWrappedRecord(r, baseRecord)
}
deltalogDeleteEntriesCount := len(delta)
internal/datanode/compactor/mix_compactor_test.go
+148 -60
View File
@@ -51,6 +51,122 @@ func TestMixCompactionTaskSuite(t *testing.T) {
suite.Run(t, new(MixCompactionTaskStorageV1Suite))
}
func newMixCompactionStorageV1SuiteForDirectTest(t *testing.T) *MixCompactionTaskStorageV1Suite {
s := &MixCompactionTaskStorageV1Suite{}
s.SetT(t)
s.SetupSuite()
paramtable.Get().Save(paramtable.Get().CommonCfg.StorageType.Key, "local")
paramtable.Get().Save(paramtable.Get().CommonCfg.UseLoonFFI.Key, "false")
t.Cleanup(func() {
paramtable.Get().Reset(paramtable.Get().CommonCfg.StorageType.Key)
paramtable.Get().Reset(paramtable.Get().CommonCfg.UseLoonFFI.Key)
s.TearDownTest()
})
s.SetupTest()
return s
}
func (s *MixCompactionTaskStorageV1Suite) prepareMissingBM25OutputSegments(isSorted bool) {
s.SetupBM25()
segments := []int64{5, 6, 7}
alloc := allocator.NewLocalAllocator(7777777, math.MaxInt64)
s.mockBinlogIO.EXPECT().Upload(mock.Anything, mock.Anything).Return(nil)
s.task.plan.SegmentBinlogs = make([]*datapb.CompactionSegmentBinlogs, 0, len(segments))
for _, segID := range segments {
s.initSegBufferWithBM25(segID)
kvs, fBinlogs, err := serializeWrite(context.TODO(), alloc, s.segWriter)
s.Require().NoError(err)
removeFieldBinlogForTest(kvs, fBinlogs, 102)
s.mockBinlogIO.EXPECT().Download(mock.Anything, mock.MatchedBy(func(keys []string) bool {
left, right := lo.Difference(keys, lo.Keys(kvs))
return len(left) == 0 && len(right) == 0
})).RunAndReturn(func(ctx context.Context, keys []string) ([][]byte, error) {
return downloadValuesForPathsForTest(kvs, keys)
}).Once()
s.task.plan.SegmentBinlogs = append(s.task.plan.SegmentBinlogs, &datapb.CompactionSegmentBinlogs{
CollectionID: 1,
SegmentID: segID,
FieldBinlogs: lo.Values(fBinlogs),
IsSorted: isSorted,
})
}
}
func TestMixCompactionMaterializesMissingBM25OutputNoDelete(t *testing.T) {
s := newMixCompactionStorageV1SuiteForDirectTest(t)
s.prepareMissingBM25OutputSegments(false)
result, err := s.task.Compact()
s.NoError(err)
s.NotNil(result)
segment := result.GetSegments()[0]
s.EqualValues(3, segment.GetNumOfRows())
s.EqualValues(3, fieldBinlogEntriesForTest(segment.GetBm25Logs(), 102))
}
func TestMixCompactionMaterializesMissingFieldWithDeleteFilter(t *testing.T) {
s := newMixCompactionStorageV1SuiteForDirectTest(t)
segmentID := int64(4)
alloc := allocator.NewLocalAllocator(888888, math.MaxInt64)
s.initMultiRowsSegBuffer(segmentID, 2, 1)
kvs, fBinlogs, err := serializeWrite(context.TODO(), alloc, s.segWriter)
s.Require().NoError(err)
removeFieldBinlogForTest(kvs, fBinlogs, StringField)
deleteTs := tsoutil.ComposeTSByTime(getMilvusBirthday().Add(10*time.Second), 0)
blob, err := getInt64DeltaBlobs(segmentID, []int64{segmentID}, []uint64{deleteTs})
s.Require().NoError(err)
deltaPath := "deltalog/missing-field-delete"
s.mockBinlogIO.EXPECT().Download(mock.Anything, []string{deltaPath}).
Return([][]byte{blob.GetValue()}, nil).Once()
s.mockBinlogIO.EXPECT().Download(mock.Anything, mock.MatchedBy(func(keys []string) bool {
left, right := lo.Difference(keys, lo.Keys(kvs))
return len(left) == 0 && len(right) == 0
})).RunAndReturn(func(ctx context.Context, keys []string) ([][]byte, error) {
return downloadValuesForPathsForTest(kvs, keys)
}).Once()
s.mockBinlogIO.EXPECT().Upload(mock.Anything, mock.Anything).Return(nil).Maybe()
s.task.plan.SegmentBinlogs = []*datapb.CompactionSegmentBinlogs{{
CollectionID: 1,
SegmentID: segmentID,
FieldBinlogs: lo.Values(fBinlogs),
Deltalogs: []*datapb.FieldBinlog{{
Binlogs: []*datapb.Binlog{{LogPath: deltaPath}},
}},
}}
result, err := s.task.Compact()
s.NoError(err)
s.NotNil(result)
s.Require().Len(result.GetSegments(), 1)
segment := result.GetSegments()[0]
s.EqualValues(1, segment.GetNumOfRows())
s.EqualValues(1, fieldBinlogEntriesForTest(segment.GetInsertLogs(), StringField))
}
func TestMixCompactionMergeSortMaterializesMissingBM25Output(t *testing.T) {
s := newMixCompactionStorageV1SuiteForDirectTest(t)
mergeSortKey := paramtable.Get().DataNodeCfg.UseMergeSort.Key
paramtable.Get().Save(mergeSortKey, "true")
t.Cleanup(func() { paramtable.Get().Reset(mergeSortKey) })
s.prepareMissingBM25OutputSegments(true)
s.task.compactionParams = compaction.GenParams()
result, err := s.task.Compact()
s.NoError(err)
s.NotNil(result)
segment := result.GetSegments()[0]
s.EqualValues(3, segment.GetNumOfRows())
s.True(segment.GetIsSorted())
s.EqualValues(3, fieldBinlogEntriesForTest(segment.GetBm25Logs(), 102))
}
type MixCompactionTaskStorageV1Suite struct {
suite.Suite
@@ -450,12 +566,18 @@ func (s *MixCompactionTaskStorageV1Suite) prepareSplitMergeEntityExpired() {
s.task.plan.CollectionTtl = int64(collTTL)
alloc := allocator.NewLocalAllocator(888888, math.MaxInt64)
kvs, _, err := serializeWrite(context.TODO(), alloc, s.segWriter)
kvs, fBinlogs, err := serializeWrite(context.TODO(), alloc, s.segWriter)
s.Require().NoError(err)
s.mockBinlogIO.EXPECT().Download(mock.Anything, mock.Anything).RunAndReturn(
s.mockBinlogIO.EXPECT().Download(mock.Anything, mock.MatchedBy(func(paths []string) bool {
left, right := lo.Difference(paths, lo.Keys(kvs))
return len(left) == 0 && len(right) == 0
})).RunAndReturn(
func(ctx context.Context, paths []string) ([][]byte, error) {
s.Require().Equal(len(paths), len(kvs))
return lo.Values(kvs), nil
res := make([][]byte, 0, len(paths))
for _, path := range paths {
res = append(res, kvs[path])
}
return res, nil
})
s.mockBinlogIO.EXPECT().Upload(mock.Anything, mock.Anything).Return(nil).Maybe()
@@ -463,17 +585,7 @@ func (s *MixCompactionTaskStorageV1Suite) prepareSplitMergeEntityExpired() {
s.task.partitionID = PartitionID
s.task.maxRows = 1000
fieldBinlogs := make([]*datapb.FieldBinlog, 0, len(kvs))
for k := range kvs {
fieldBinlogs = append(fieldBinlogs, &datapb.FieldBinlog{
Binlogs: []*datapb.Binlog{
{
LogPath: k,
},
},
})
}
s.task.plan.SegmentBinlogs[0].FieldBinlogs = fieldBinlogs
s.task.plan.SegmentBinlogs[0].FieldBinlogs = lo.Values(fBinlogs)
}
func (s *MixCompactionTaskStorageV1Suite) TestSplitMergeEntityExpired() {
@@ -549,30 +661,18 @@ func (s *MixCompactionTaskStorageV1Suite) TestMergeNoExpirationLackBinlog() {
}
}
insertPaths := lo.Keys(kvs)
insertBytes := func() [][]byte {
res := make([][]byte, 0, len(insertPaths))
for _, path := range insertPaths {
res = append(res, kvs[path])
}
return res
}()
s.mockBinlogIO.EXPECT().Download(mock.Anything, insertPaths).RunAndReturn(
s.mockBinlogIO.EXPECT().Download(mock.Anything, mock.MatchedBy(func(paths []string) bool {
left, right := lo.Difference(paths, lo.Keys(kvs))
return len(left) == 0 && len(right) == 0
})).RunAndReturn(
func(ctx context.Context, paths []string) ([][]byte, error) {
s.Require().Equal(len(paths), len(kvs))
return insertBytes, nil
res := make([][]byte, 0, len(paths))
for _, path := range paths {
res = append(res, kvs[path])
}
return res, nil
})
fieldBinlogs := make([]*datapb.FieldBinlog, 0, len(insertPaths))
for _, k := range insertPaths {
fieldBinlogs = append(fieldBinlogs, &datapb.FieldBinlog{
Binlogs: []*datapb.Binlog{
{
LogPath: k,
},
},
})
}
s.task.plan.SegmentBinlogs[0].FieldBinlogs = fieldBinlogs
s.task.plan.SegmentBinlogs[0].FieldBinlogs = lo.Values(fBinlogs)
s.mockBinlogIO.EXPECT().Upload(mock.Anything, mock.Anything).Return(nil).Maybe()
@@ -606,7 +706,7 @@ func (s *MixCompactionTaskStorageV1Suite) TestMergeNoExpiration() {
}
alloc := allocator.NewLocalAllocator(888888, math.MaxInt64)
kvs, _, err := serializeWrite(context.TODO(), alloc, s.segWriter)
kvs, fBinlogs, err := serializeWrite(context.TODO(), alloc, s.segWriter)
s.Require().NoError(err)
for _, test := range tests {
@@ -631,30 +731,18 @@ func (s *MixCompactionTaskStorageV1Suite) TestMergeNoExpiration() {
}
}
insertPaths := lo.Keys(kvs)
insertBytes := func() [][]byte {
res := make([][]byte, 0, len(insertPaths))
for _, path := range insertPaths {
res = append(res, kvs[path])
}
return res
}()
s.mockBinlogIO.EXPECT().Download(mock.Anything, insertPaths).RunAndReturn(
s.mockBinlogIO.EXPECT().Download(mock.Anything, mock.MatchedBy(func(paths []string) bool {
left, right := lo.Difference(paths, lo.Keys(kvs))
return len(left) == 0 && len(right) == 0
})).RunAndReturn(
func(ctx context.Context, paths []string) ([][]byte, error) {
s.Require().Equal(len(paths), len(kvs))
return insertBytes, nil
res := make([][]byte, 0, len(paths))
for _, path := range paths {
res = append(res, kvs[path])
}
return res, nil
})
fieldBinlogs := make([]*datapb.FieldBinlog, 0, len(insertPaths))
for _, k := range insertPaths {
fieldBinlogs = append(fieldBinlogs, &datapb.FieldBinlog{
Binlogs: []*datapb.Binlog{
{
LogPath: k,
},
},
})
}
s.task.plan.SegmentBinlogs[0].FieldBinlogs = fieldBinlogs
s.task.plan.SegmentBinlogs[0].FieldBinlogs = lo.Values(fBinlogs)
s.mockBinlogIO.EXPECT().Upload(mock.Anything, mock.Anything).Return(nil).Maybe()
internal/datanode/compactor/record_materializer.go
+528
View File
@@ -0,0 +1,528 @@
// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package compactor
import (
"fmt"
"github.com/apache/arrow/go/v17/arrow"
"github.com/apache/arrow/go/v17/arrow/array"
"github.com/apache/arrow/go/v17/arrow/memory"
"github.com/cockroachdb/errors"
"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/internal/util/function"
"github.com/milvus-io/milvus/pkg/v3/common"
"github.com/milvus-io/milvus/pkg/v3/util/merr"
"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
)
type FunctionMaterializer interface {
Materialize(rec storage.Record) (map[int64]arrow.Array, error)
Close()
}
type rowRange struct {
start int
end int
}
type recordSelection struct {
ranges []rowRange
length int
}
func (s *recordSelection) Len() int {
if s == nil {
return 0
}
return s.length
}
type RecordMaterializer struct {
materializers []FunctionMaterializer
missingFields []*schemapb.FieldSchema
schema *schemapb.CollectionSchema
}
func NewRecordMaterializer(schema *schemapb.CollectionSchema, functions []*schemapb.FunctionSchema, existingFields map[int64]struct{}) (*RecordMaterializer, error) {
materializer := &RecordMaterializer{schema: schema}
materializedFields := make(map[int64]struct{})
for _, functionSchema := range functions {
outputIndexes := functionOutputIndexesToMaterialize(functionSchema, existingFields)
if len(outputIndexes) == 0 {
continue
}
for _, outputIndex := range outputIndexes {
materializedFields[functionSchema.GetOutputFieldIds()[outputIndex]] = struct{}{}
}
runner, err := function.NewFunctionRunner(schema, functionSchema)
if err != nil {
materializer.Close()
return nil, err
}
if runner == nil {
materializer.Close()
return nil, errors.Newf("failed to set up function runner for %s", functionSchema.GetName())
}
functionMaterializer, err := newFunctionMaterializer(schema, runner, outputIndexes, true)
if err != nil {
runner.Close()
materializer.Close()
return nil, err
}
materializer.materializers = append(materializer.materializers, functionMaterializer)
}
materializer.missingFields = missingNonMaterializedSchemaFields(schema, existingFields, materializedFields)
return materializer, nil
}
func (m *RecordMaterializer) Wrap(rec storage.Record) (storage.Record, error) {
return m.WrapWithSelection(rec, nil)
}
func (m *RecordMaterializer) WrapWithSelection(rec storage.Record, selection *recordSelection) (storage.Record, error) {
base := rec
var selected *selectedRecord
if selection != nil {
selected = newSelectedRecord(rec, m.schema, selection)
base = selected
}
if !m.hasMaterialization() {
return base, nil
}
computed := make(map[int64]arrow.Array)
for _, materializer := range m.materializers {
arrays, err := materializer.Materialize(base)
if err != nil {
releaseArrowArrays(computed)
if base != rec {
base.Release()
}
if selected != nil && selected.err != nil {
return nil, selected.err
}
return nil, err
}
if selected != nil && selected.err != nil {
releaseArrowArrays(computed)
base.Release()
return nil, selected.err
}
for fieldID, arr := range arrays {
computed[fieldID] = arr
}
}
for _, field := range m.missingFields {
fieldID := field.GetFieldID()
if _, ok := computed[fieldID]; ok {
continue
}
arr, err := storage.GenerateEmptyArrayFromSchema(field, base.Len())
if err != nil {
releaseArrowArrays(computed)
if base != rec {
base.Release()
}
return nil, err
}
computed[fieldID] = arr
}
if len(computed) == 0 {
return base, nil
}
return &materializedRecord{base: base, computed: computed}, nil
}
func (m *RecordMaterializer) Close() {
if m == nil {
return
}
for _, materializer := range m.materializers {
materializer.Close()
}
}
func (m *RecordMaterializer) hasMaterialization() bool {
return m != nil && (len(m.materializers) > 0 || len(m.missingFields) > 0)
}
type materializedRecord struct {
base storage.Record
computed map[int64]arrow.Array
}
var _ storage.Record = (*materializedRecord)(nil)
func (r *materializedRecord) Column(fieldID storage.FieldID) arrow.Array {
if col, ok := r.computed[fieldID]; ok {
return col
}
return r.base.Column(fieldID)
}
func (r *materializedRecord) Len() int {
return r.base.Len()
}
func (r *materializedRecord) Retain() {
r.base.Retain()
for _, col := range r.computed {
col.Retain()
}
}
func (r *materializedRecord) retainBase() {
r.base.Retain()
}
func (r *materializedRecord) Release() {
r.base.Release()
for _, col := range r.computed {
col.Release()
}
}
type selectedRecord struct {
base storage.Record
fields map[int64]*schemapb.FieldSchema
selection *recordSelection
columns map[int64]arrow.Array
err error
}
var _ storage.Record = (*selectedRecord)(nil)
func newSelectedRecord(base storage.Record, schema *schemapb.CollectionSchema, selection *recordSelection) *selectedRecord {
fields := make(map[int64]*schemapb.FieldSchema)
for _, field := range typeutil.GetAllFieldSchemas(schema) {
fields[field.GetFieldID()] = field
}
return &selectedRecord{
base: base,
fields: fields,
selection: selection,
columns: make(map[int64]arrow.Array),
}
}
func (r *selectedRecord) Column(fieldID storage.FieldID) arrow.Array {
if col, ok := r.columns[fieldID]; ok {
return col
}
field := r.fields[fieldID]
if field == nil {
return nil
}
builder := storage.NewRecordBuilder(&schemapb.CollectionSchema{Fields: []*schemapb.FieldSchema{field}})
defer builder.Release()
for _, rowRange := range r.selection.ranges {
if err := builder.Append(r.base, rowRange.start, rowRange.end); err != nil {
r.err = err
return nil
}
}
selected := builder.Build()
defer selected.Release()
col := selected.Column(fieldID)
if col == nil {
r.err = errors.Newf("selected record field %d not found", fieldID)
return nil
}
col.Retain()
r.columns[fieldID] = col
return col
}
func (r *selectedRecord) Len() int {
return r.selection.Len()
}
func (r *selectedRecord) Retain() {
r.base.Retain()
for _, col := range r.columns {
col.Retain()
}
}
func (r *selectedRecord) Release() {
r.base.Release()
for _, col := range r.columns {
col.Release()
}
}
type materializedRecordReader struct {
base storage.RecordReader
materializer *RecordMaterializer
current storage.Record
}
var _ storage.RecordReader = (*materializedRecordReader)(nil)
func newMaterializedRecordReader(base storage.RecordReader, materializer *RecordMaterializer) storage.RecordReader {
if !materializer.hasMaterialization() {
return base
}
return &materializedRecordReader{base: base, materializer: materializer}
}
func (r *materializedRecordReader) Next() (storage.Record, error) {
if r.current != nil {
r.current.Release()
r.current = nil
}
rec, err := r.base.Next()
if err != nil {
return nil, err
}
wrapped, err := r.materializer.Wrap(rec)
if err != nil {
rec.Release()
return nil, err
}
if materialized, ok := wrapped.(*materializedRecord); ok {
materialized.retainBase()
} else {
wrapped.Retain()
}
r.current = wrapped
return wrapped, nil
}
func (r *materializedRecordReader) Close() error {
if r.current != nil {
r.current.Release()
r.current = nil
}
r.materializer.Close()
return r.base.Close()
}
type bm25FunctionMaterializer struct {
runner function.FunctionRunner
inputFieldIDs []int64
outputFieldIDs []int64
missingOutputIndexes []int
outputFields map[int64]*schemapb.FieldSchema
ownRunner bool
}
var _ FunctionMaterializer = (*bm25FunctionMaterializer)(nil)
func newFunctionMaterializer(schema *schemapb.CollectionSchema, runner function.FunctionRunner, missingOutputIndexes []int, ownRunner bool) (FunctionMaterializer, error) {
functionSchema := runner.GetSchema()
switch functionSchema.GetType() {
case schemapb.FunctionType_BM25:
return newBM25FunctionMaterializer(schema, runner, missingOutputIndexes, ownRunner)
default:
return nil, errors.Newf("unsupported function type %s", functionSchema.GetType().String())
}
}
func newBM25FunctionMaterializer(schema *schemapb.CollectionSchema, runner function.FunctionRunner, missingOutputIndexes []int, ownRunner bool) (*bm25FunctionMaterializer, error) {
functionSchema := runner.GetSchema()
inputFields := runner.GetInputFields()
if len(inputFields) == 0 {
return nil, errors.New("bm25 function should have input fields")
}
inputFieldIDs := make([]int64, 0, len(inputFields))
for _, inputField := range inputFields {
if inputField == nil || typeutil.GetField(schema, inputField.GetFieldID()) == nil {
return nil, errors.New("input field not found in schema")
}
if inputField.GetDataType() != schemapb.DataType_VarChar && inputField.GetDataType() != schemapb.DataType_Text {
return nil, errors.New("input field data type must be varchar or text for bm25 function materialization")
}
inputFieldIDs = append(inputFieldIDs, inputField.GetFieldID())
}
outputFieldIDs := functionSchema.GetOutputFieldIds()
if len(outputFieldIDs) == 0 {
return nil, errors.New("bm25 function should have output fields")
}
outputFields := make(map[int64]*schemapb.FieldSchema, len(outputFieldIDs))
for _, outputFieldID := range outputFieldIDs {
outputField := typeutil.GetField(schema, outputFieldID)
if outputField == nil {
return nil, errors.New("output field not found in schema")
}
if outputField.GetDataType() != schemapb.DataType_SparseFloatVector {
return nil, errors.New("output field data type must be sparse float vector for bm25 function materialization")
}
if outputField.GetNullable() {
return nil, errors.Newf("function output field cannot be nullable: function %s, field %s", functionSchema.GetName(), outputField.GetName())
}
outputFields[outputFieldID] = outputField
}
return &bm25FunctionMaterializer{
runner: runner,
inputFieldIDs: inputFieldIDs,
outputFieldIDs: outputFieldIDs,
missingOutputIndexes: missingOutputIndexes,
outputFields: outputFields,
ownRunner: ownRunner,
}, nil
}
func (m *bm25FunctionMaterializer) Materialize(rec storage.Record) (map[int64]arrow.Array, error) {
inputs := make([]any, 0, len(m.inputFieldIDs))
for _, inputFieldID := range m.inputFieldIDs {
input, err := stringInputsFromRecord(rec, inputFieldID)
if err != nil {
return nil, err
}
inputs = append(inputs, input)
}
outputs, err := m.runner.BatchRun(inputs...)
if err != nil {
return nil, err
}
if len(outputs) != len(m.outputFieldIDs) {
return nil, errors.Newf("bm25 function materialization expects %d outputs, got %d", len(m.outputFieldIDs), len(outputs))
}
result := make(map[int64]arrow.Array, len(m.missingOutputIndexes))
for _, outputIndex := range m.missingOutputIndexes {
outputFieldID := m.outputFieldIDs[outputIndex]
outputSparseArray, ok := outputs[outputIndex].(*schemapb.SparseFloatArray)
if !ok {
releaseArrowArrays(result)
return nil, errors.Newf("unexpected output type from BM25 function runner, expected SparseFloatArray, got %T", outputs[outputIndex])
}
arr, err := buildSparseFloatVectorArrowArray(m.outputFields[outputFieldID], outputSparseArray, rec.Len())
if err != nil {
releaseArrowArrays(result)
return nil, err
}
result[outputFieldID] = arr
}
return result, nil
}
func (m *bm25FunctionMaterializer) Close() {
if m.ownRunner && m.runner != nil {
m.runner.Close()
}
}
func functionOutputIndexesToMaterialize(functionSchema *schemapb.FunctionSchema, existingFields map[int64]struct{}) []int {
outputFieldIDs := functionSchema.GetOutputFieldIds()
indexes := make([]int, 0, len(outputFieldIDs))
hasMissingOutput := false
for idx, outputFieldID := range outputFieldIDs {
indexes = append(indexes, idx)
if _, ok := existingFields[outputFieldID]; !ok {
hasMissingOutput = true
}
}
if !hasMissingOutput {
return nil
}
return indexes
}
func missingNonMaterializedSchemaFields(schema *schemapb.CollectionSchema, existingFields map[int64]struct{}, materializedFields map[int64]struct{}) []*schemapb.FieldSchema {
missing := make([]*schemapb.FieldSchema, 0)
for _, field := range typeutil.GetAllFieldSchemas(schema) {
fieldID := field.GetFieldID()
if common.IsSystemField(fieldID) {
continue
}
if _, ok := existingFields[fieldID]; ok {
continue
}
if _, ok := materializedFields[fieldID]; ok {
continue
}
missing = append(missing, field)
}
return missing
}
func stringInputsFromRecord(rec storage.Record, fieldID int64) ([]string, error) {
col := rec.Column(fieldID)
if col == nil {
return nil, merr.WrapErrServiceInternal(fmt.Sprintf("input field %d not found in record", fieldID))
}
inputs := make([]string, rec.Len())
switch values := col.(type) {
case *array.String:
for i := 0; i < rec.Len(); i++ {
if values.IsValid(i) {
inputs[i] = values.Value(i)
}
}
case *array.Binary:
return nil, merr.WrapErrServiceInternal("cannot materialize bm25 from text binary values without lob decoding")
default:
return nil, merr.WrapErrServiceInternal(fmt.Sprintf("input field %d data type must be varchar or text for bm25 function materialization, got %T", fieldID, col))
}
return inputs, nil
}
func buildSparseFloatVectorArrowArray(field *schemapb.FieldSchema, outputSparseArray *schemapb.SparseFloatArray, rowCount int) (arrow.Array, error) {
if len(outputSparseArray.GetContents()) != rowCount {
return nil, errors.Newf("bm25 function output row count mismatch, expected %d, got %d", rowCount, len(outputSparseArray.GetContents()))
}
outputSchema := &schemapb.CollectionSchema{Fields: []*schemapb.FieldSchema{field}}
arrowSchema, err := storage.ConvertToArrowSchema(outputSchema, true)
if err != nil {
return nil, err
}
builder := array.NewRecordBuilder(memory.DefaultAllocator, arrowSchema)
defer builder.Release()
fieldData := &storage.SparseFloatVectorFieldData{
SparseFloatArray: schemapb.SparseFloatArray{
Contents: outputSparseArray.GetContents(),
Dim: outputSparseArray.GetDim(),
},
}
insertData := &storage.InsertData{Data: map[int64]storage.FieldData{
field.GetFieldID(): fieldData,
}}
if err := storage.BuildRecord(builder, insertData, outputSchema); err != nil {
return nil, err
}
record := builder.NewRecord()
defer record.Release()
col := record.Column(0)
col.Retain()
return col, nil
}
func releaseArrowArrays(arrays map[int64]arrow.Array) {
for _, arr := range arrays {
arr.Release()
}
}
func releaseWrappedRecord(wrapped storage.Record, base storage.Record) {
if wrapped != base {
wrapped.Release()
return
}
base.Release()
}
internal/datanode/compactor/record_materializer_test.go
+448
View File
@@ -0,0 +1,448 @@
package compactor
import (
"testing"
"github.com/apache/arrow/go/v17/arrow"
"github.com/apache/arrow/go/v17/arrow/array"
"github.com/apache/arrow/go/v17/arrow/memory"
"github.com/cockroachdb/errors"
"github.com/stretchr/testify/require"
"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/internal/util/function"
"github.com/milvus-io/milvus/pkg/v3/common"
"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
)
type materializerTestRecord struct {
columns map[storage.FieldID]arrow.Array
len int
releaseCount int
retainCount int
}
func (r *materializerTestRecord) Column(fieldID storage.FieldID) arrow.Array {
return r.columns[fieldID]
}
func (r *materializerTestRecord) Len() int {
return r.len
}
func (r *materializerTestRecord) Release() {
r.releaseCount++
}
func (r *materializerTestRecord) Retain() {
r.retainCount++
}
type materializerTestReader struct {
records []*materializerTestRecord
idx int
closed bool
}
func (r *materializerTestReader) Next() (storage.Record, error) {
if r.idx >= len(r.records) {
return nil, errors.New("no more records")
}
record := r.records[r.idx]
r.idx++
return record, nil
}
func (r *materializerTestReader) Close() error {
r.closed = true
return nil
}
type selectedColumnMaterializer struct {
fieldID int64
}
func (m selectedColumnMaterializer) Materialize(rec storage.Record) (map[int64]arrow.Array, error) {
rec.Column(m.fieldID)
return nil, nil
}
func (m selectedColumnMaterializer) Close() {}
type materializerTestFunctionRunner struct {
schema *schemapb.FunctionSchema
inputFields []*schemapb.FieldSchema
outputFields []*schemapb.FieldSchema
outputs []any
err error
closed bool
inputs []any
}
func (r *materializerTestFunctionRunner) BatchRun(inputs ...any) ([]any, error) {
r.inputs = inputs
return r.outputs, r.err
}
func (r *materializerTestFunctionRunner) GetSchema() *schemapb.FunctionSchema {
return r.schema
}
func (r *materializerTestFunctionRunner) GetOutputFields() []*schemapb.FieldSchema {
return r.outputFields
}
func (r *materializerTestFunctionRunner) GetInputFields() []*schemapb.FieldSchema {
return r.inputFields
}
func (r *materializerTestFunctionRunner) Close() {
r.closed = true
}
var _ function.FunctionRunner = (*materializerTestFunctionRunner)(nil)
func TestRecordMaterializerWrapNoOpWhenAllFieldsExist(t *testing.T) {
schema := &schemapb.CollectionSchema{Fields: []*schemapb.FieldSchema{
{FieldID: 100, Name: "text", DataType: schemapb.DataType_VarChar},
{FieldID: 101, Name: "score", DataType: schemapb.DataType_Int64},
}}
materializer, err := NewRecordMaterializer(schema, nil, map[int64]struct{}{100: {}, 101: {}})
require.NoError(t, err)
defer materializer.Close()
record := &materializerTestRecord{len: 2}
wrapped, err := materializer.Wrap(record)
require.NoError(t, err)
require.Same(t, record, wrapped)
}
func TestRecordMaterializerWrapFillsNullableMissingFields(t *testing.T) {
schema := &schemapb.CollectionSchema{Fields: []*schemapb.FieldSchema{
{FieldID: 100, Name: "text", DataType: schemapb.DataType_VarChar},
{FieldID: 101, Name: "added", DataType: schemapb.DataType_Int64, Nullable: true},
}}
materializer, err := NewRecordMaterializer(schema, nil, map[int64]struct{}{100: {}})
require.NoError(t, err)
defer materializer.Close()
record := &materializerTestRecord{len: 3}
wrapped, err := materializer.Wrap(record)
require.NoError(t, err)
require.NotSame(t, record, wrapped)
column := wrapped.Column(101)
require.NotNil(t, column)
require.Equal(t, 3, column.Len())
require.Equal(t, 3, column.NullN())
wrapped.Release()
}
func TestRecordMaterializerWrapWithSelectionMaterializesKeptRowsOnly(t *testing.T) {
schema := &schemapb.CollectionSchema{Fields: []*schemapb.FieldSchema{
{FieldID: 100, Name: "text", DataType: schemapb.DataType_VarChar},
{FieldID: 101, Name: "added", DataType: schemapb.DataType_Int64, Nullable: true},
}}
materializer, err := NewRecordMaterializer(schema, nil, map[int64]struct{}{100: {}})
require.NoError(t, err)
defer materializer.Close()
input := newStringArray(t, []string{"drop-0", "keep-1", "drop-2", "keep-3"})
defer input.Release()
record := &materializerTestRecord{len: 4, columns: map[storage.FieldID]arrow.Array{100: input}}
selection := &recordSelection{ranges: []rowRange{{start: 1, end: 2}, {start: 3, end: 4}}, length: 2}
wrapped, err := materializer.WrapWithSelection(record, selection)
require.NoError(t, err)
require.Equal(t, 2, wrapped.Len())
textColumn := wrapped.Column(100).(*array.String)
require.Equal(t, []string{"keep-1", "keep-3"}, []string{textColumn.Value(0), textColumn.Value(1)})
addedColumn := wrapped.Column(101)
require.NotNil(t, addedColumn)
require.Equal(t, 2, addedColumn.Len())
require.Equal(t, 2, addedColumn.NullN())
wrapped.Release()
require.Equal(t, 1, record.releaseCount)
}
func TestRecordMaterializerWrapWithSelectionReturnsLazyColumnError(t *testing.T) {
schema := &schemapb.CollectionSchema{Fields: []*schemapb.FieldSchema{
{FieldID: 100, Name: "text", DataType: schemapb.DataType_VarChar},
}}
materializer := &RecordMaterializer{
schema: schema,
materializers: []FunctionMaterializer{selectedColumnMaterializer{fieldID: 100}},
}
input := newInt64Array(t, []int64{1, 2})
defer input.Release()
record := &materializerTestRecord{len: 2, columns: map[storage.FieldID]arrow.Array{100: input}}
selection := &recordSelection{ranges: []rowRange{{start: 0, end: 1}}, length: 1}
wrapped, err := materializer.WrapWithSelection(record, selection)
require.Nil(t, wrapped)
require.ErrorContains(t, err, "failed to append value")
}
func TestRecordMaterializerWrapSkipsMissingSystemFields(t *testing.T) {
schema := &schemapb.CollectionSchema{Fields: []*schemapb.FieldSchema{
{FieldID: common.RowIDField, Name: common.RowIDFieldName, DataType: schemapb.DataType_Int64},
{FieldID: common.TimeStampField, Name: common.TimeStampFieldName, DataType: schemapb.DataType_Int64},
{FieldID: 100, Name: "pk", DataType: schemapb.DataType_Int64},
}}
materializer, err := NewRecordMaterializer(schema, nil, map[int64]struct{}{100: {}})
require.NoError(t, err)
defer materializer.Close()
record := &materializerTestRecord{len: 3}
wrapped, err := materializer.Wrap(record)
require.NoError(t, err)
require.Same(t, record, wrapped)
}
func TestRecordMaterializerWrapFailsForMissingNonNullableField(t *testing.T) {
schema := &schemapb.CollectionSchema{Fields: []*schemapb.FieldSchema{
{FieldID: 100, Name: "text", DataType: schemapb.DataType_VarChar},
{FieldID: 101, Name: "required", DataType: schemapb.DataType_Int64},
}}
materializer, err := NewRecordMaterializer(schema, nil, map[int64]struct{}{100: {}})
require.NoError(t, err)
defer materializer.Close()
_, err = materializer.Wrap(&materializerTestRecord{len: 3})
require.Error(t, err)
require.ErrorContains(t, err, "missing field data required")
}
func TestBM25FunctionMaterializerMaterializesSparseOutput(t *testing.T) {
schema, functionSchema, inputField, outputField := materializerBM25Schema()
runner := &materializerTestFunctionRunner{
schema: functionSchema,
inputFields: []*schemapb.FieldSchema{inputField},
outputFields: []*schemapb.FieldSchema{outputField},
outputs: []any{&schemapb.SparseFloatArray{
Dim: 16,
Contents: [][]byte{
typeutil.CreateSparseFloatRow([]uint32{1}, []float32{0.5}),
typeutil.CreateSparseFloatRow([]uint32{2}, []float32{0.8}),
},
}},
}
materializer, err := newBM25FunctionMaterializer(schema, runner, []int{0}, false)
require.NoError(t, err)
defer materializer.Close()
input := newStringArray(t, []string{"hello", "world"})
defer input.Release()
record := &materializerTestRecord{len: 2, columns: map[storage.FieldID]arrow.Array{100: input}}
arrays, err := materializer.Materialize(record)
require.NoError(t, err)
defer releaseArrowArrays(arrays)
require.Contains(t, arrays, int64(101))
require.Equal(t, 2, arrays[101].Len())
require.Equal(t, []any{[]string{"hello", "world"}}, runner.inputs)
}
func TestBM25FunctionMaterializerRejectsBinaryTextInputWithoutLOBDecoding(t *testing.T) {
schema, functionSchema, inputField, outputField := materializerBM25Schema()
inputField.DataType = schemapb.DataType_Text
runner := &materializerTestFunctionRunner{
schema: functionSchema,
inputFields: []*schemapb.FieldSchema{inputField},
outputFields: []*schemapb.FieldSchema{outputField},
outputs: []any{&schemapb.SparseFloatArray{
Dim: 16,
Contents: [][]byte{typeutil.CreateSparseFloatRow([]uint32{1}, []float32{0.5})},
}},
}
materializer, err := newBM25FunctionMaterializer(schema, runner, []int{0}, false)
require.NoError(t, err)
defer materializer.Close()
input := newBinaryArray(t, [][]byte{[]byte("encoded-lob-ref")})
defer input.Release()
_, err = materializer.Materialize(&materializerTestRecord{len: 1, columns: map[storage.FieldID]arrow.Array{100: input}})
require.ErrorContains(t, err, "cannot materialize bm25 from text binary values without lob decoding")
require.Nil(t, runner.inputs)
}
func TestBM25FunctionMaterializerMaterializesNullableInputAsNonNullableOutput(t *testing.T) {
schema, functionSchema, inputField, outputField := materializerBM25Schema()
inputField.Nullable = true
firstRow := typeutil.CreateSparseFloatRow([]uint32{1}, []float32{0.5})
emptyRow := typeutil.CreateSparseFloatRow(nil, nil)
thirdRow := typeutil.CreateSparseFloatRow([]uint32{2}, []float32{0.8})
runner := &materializerTestFunctionRunner{
schema: functionSchema,
inputFields: []*schemapb.FieldSchema{inputField},
outputFields: []*schemapb.FieldSchema{outputField},
outputs: []any{&schemapb.SparseFloatArray{
Dim: 16,
Contents: [][]byte{firstRow, emptyRow, thirdRow},
}},
}
materializer, err := newBM25FunctionMaterializer(schema, runner, []int{0}, false)
require.NoError(t, err)
defer materializer.Close()
input := newNullableStringArray(t, []string{"hello", "", "world"}, []bool{true, false, true})
defer input.Release()
record := &materializerTestRecord{len: 3, columns: map[storage.FieldID]arrow.Array{100: input}}
arrays, err := materializer.Materialize(record)
require.NoError(t, err)
defer releaseArrowArrays(arrays)
output := arrays[101]
require.Equal(t, 3, output.Len())
require.Zero(t, output.NullN())
require.True(t, output.IsValid(0))
require.True(t, output.IsValid(1))
require.True(t, output.IsValid(2))
binaryOutput := output.(*array.Binary)
require.Equal(t, firstRow, binaryOutput.Value(0))
require.Equal(t, emptyRow, binaryOutput.Value(1))
require.Equal(t, thirdRow, binaryOutput.Value(2))
require.Equal(t, []any{[]string{"hello", "", "world"}}, runner.inputs)
}
func TestBM25FunctionMaterializerRejectsNullableOutput(t *testing.T) {
schema, functionSchema, inputField, outputField := materializerBM25Schema()
outputField.Nullable = true
runner := &materializerTestFunctionRunner{
schema: functionSchema,
inputFields: []*schemapb.FieldSchema{inputField},
outputFields: []*schemapb.FieldSchema{outputField},
}
_, err := newBM25FunctionMaterializer(schema, runner, []int{0}, false)
require.ErrorContains(t, err, "function output field cannot be nullable")
}
func TestBM25FunctionMaterializerRejectsBadRunnerOutput(t *testing.T) {
schema, functionSchema, inputField, outputField := materializerBM25Schema()
newMaterializer := func(outputs []any) *bm25FunctionMaterializer {
runner := &materializerTestFunctionRunner{
schema: functionSchema,
inputFields: []*schemapb.FieldSchema{inputField},
outputFields: []*schemapb.FieldSchema{outputField},
outputs: outputs,
}
materializer, err := newBM25FunctionMaterializer(schema, runner, []int{0}, false)
require.NoError(t, err)
return materializer
}
t.Run("output count mismatch", func(t *testing.T) {
input := newStringArray(t, []string{"hello"})
defer input.Release()
_, err := newMaterializer(nil).Materialize(&materializerTestRecord{len: 1, columns: map[storage.FieldID]arrow.Array{100: input}})
require.ErrorContains(t, err, "expects 1 outputs")
})
t.Run("output type not sparse float array", func(t *testing.T) {
input := newStringArray(t, []string{"hello"})
defer input.Release()
_, err := newMaterializer([]any{&schemapb.FloatArray{Data: []float32{1}}}).Materialize(&materializerTestRecord{len: 1, columns: map[storage.FieldID]arrow.Array{100: input}})
require.ErrorContains(t, err, "unexpected output type")
})
t.Run("output row count mismatch", func(t *testing.T) {
input := newStringArray(t, []string{"hello", "world"})
defer input.Release()
_, err := newMaterializer([]any{&schemapb.SparseFloatArray{
Dim: 16,
Contents: [][]byte{typeutil.CreateSparseFloatRow([]uint32{1}, []float32{0.5})},
}}).Materialize(&materializerTestRecord{len: 2, columns: map[storage.FieldID]arrow.Array{100: input}})
require.ErrorContains(t, err, "bm25 function output row count mismatch")
})
t.Run("input field column missing", func(t *testing.T) {
_, err := newMaterializer([]any{}).Materialize(&materializerTestRecord{len: 1, columns: map[storage.FieldID]arrow.Array{}})
require.ErrorContains(t, err, "input field 100 not found")
})
t.Run("input column not string or binary", func(t *testing.T) {
input := newInt64Array(t, []int64{1})
defer input.Release()
_, err := newMaterializer([]any{}).Materialize(&materializerTestRecord{len: 1, columns: map[storage.FieldID]arrow.Array{100: input}})
require.ErrorContains(t, err, "data type must be varchar or text")
})
}
func TestFunctionOutputIndexesToMaterializePartialStateReturnsAllOutputs(t *testing.T) {
functionSchema := &schemapb.FunctionSchema{OutputFieldIds: []int64{101, 102}}
require.Nil(t, functionOutputIndexesToMaterialize(functionSchema, map[int64]struct{}{101: {}, 102: {}}))
require.Equal(t, []int{0, 1}, functionOutputIndexesToMaterialize(functionSchema, map[int64]struct{}{101: {}}))
}
func TestMaterializedRecordReaderReleasesPreviousRecordOnNextAndClose(t *testing.T) {
schema := &schemapb.CollectionSchema{Fields: []*schemapb.FieldSchema{
{FieldID: 100, Name: "text", DataType: schemapb.DataType_VarChar},
{FieldID: 101, Name: "added", DataType: schemapb.DataType_Int64, Nullable: true},
}}
materializer, err := NewRecordMaterializer(schema, nil, map[int64]struct{}{100: {}})
require.NoError(t, err)
first := &materializerTestRecord{len: 1}
second := &materializerTestRecord{len: 1}
base := &materializerTestReader{records: []*materializerTestRecord{first, second}}
reader := newMaterializedRecordReader(base, materializer)
_, err = reader.Next()
require.NoError(t, err)
require.Equal(t, 0, first.releaseCount)
_, err = reader.Next()
require.NoError(t, err)
require.Equal(t, 1, first.releaseCount)
require.Equal(t, 0, second.releaseCount)
require.NoError(t, reader.Close())
require.Equal(t, 1, second.releaseCount)
require.True(t, base.closed)
}
func materializerBM25Schema() (*schemapb.CollectionSchema, *schemapb.FunctionSchema, *schemapb.FieldSchema, *schemapb.FieldSchema) {
inputField := &schemapb.FieldSchema{FieldID: 100, Name: "text", DataType: schemapb.DataType_VarChar}
outputField := &schemapb.FieldSchema{FieldID: 101, Name: "sparse", DataType: schemapb.DataType_SparseFloatVector}
functionSchema := &schemapb.FunctionSchema{
Name: "bm25",
Type: schemapb.FunctionType_BM25,
InputFieldIds: []int64{100},
OutputFieldIds: []int64{101},
}
return &schemapb.CollectionSchema{Fields: []*schemapb.FieldSchema{inputField, outputField}, Functions: []*schemapb.FunctionSchema{functionSchema}}, functionSchema, inputField, outputField
}
func newStringArray(t *testing.T, values []string) *array.String {
builder := array.NewStringBuilder(memory.DefaultAllocator)
t.Cleanup(builder.Release)
builder.AppendValues(values, nil)
return builder.NewStringArray()
}
func newNullableStringArray(t *testing.T, values []string, valid []bool) *array.String {
builder := array.NewStringBuilder(memory.DefaultAllocator)
t.Cleanup(builder.Release)
builder.AppendValues(values, valid)
return builder.NewStringArray()
}
func newBinaryArray(t *testing.T, values [][]byte) *array.Binary {
builder := array.NewBinaryBuilder(memory.DefaultAllocator, arrow.BinaryTypes.Binary)
t.Cleanup(builder.Release)
builder.AppendValues(values, nil)
return builder.NewBinaryArray()
}
func newInt64Array(t *testing.T, values []int64) *array.Int64 {
builder := array.NewInt64Builder(memory.DefaultAllocator)
t.Cleanup(builder.Release)
builder.AppendValues(values, nil)
return builder.NewInt64Array()
}
internal/datanode/compactor/sort_compaction.go
+15 -184
View File
@@ -20,7 +20,6 @@ import (
"context"
"fmt"
"runtime/debug"
"sync"
"time"
"github.com/apache/arrow/go/v17/arrow/array"
@@ -28,29 +27,21 @@ import (
"github.com/samber/lo"
"go.opentelemetry.io/otel"
"go.uber.org/zap"
"golang.org/x/sync/errgroup"
"github.com/milvus-io/milvus-proto/go-api/v3/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
"github.com/milvus-io/milvus/internal/allocator"
"github.com/milvus-io/milvus/internal/compaction"
"github.com/milvus-io/milvus/internal/datanode/util"
"github.com/milvus-io/milvus/internal/flushcommon/io"
"github.com/milvus-io/milvus/internal/metastore/kv/binlog"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/internal/storagev2/packed"
"github.com/milvus-io/milvus/internal/util/analyzer"
"github.com/milvus-io/milvus/internal/util/fileresource"
"github.com/milvus-io/milvus/internal/util/indexcgowrapper"
"github.com/milvus-io/milvus/pkg/v3/common"
"github.com/milvus-io/milvus/pkg/v3/log"
"github.com/milvus-io/milvus/pkg/v3/metrics"
"github.com/milvus-io/milvus/pkg/v3/proto/datapb"
"github.com/milvus-io/milvus/pkg/v3/proto/indexcgopb"
"github.com/milvus-io/milvus/pkg/v3/proto/indexpb"
"github.com/milvus-io/milvus/pkg/v3/util/funcutil"
"github.com/milvus-io/milvus/pkg/v3/util/merr"
"github.com/milvus-io/milvus/pkg/v3/util/metautil"
"github.com/milvus-io/milvus/pkg/v3/util/paramtable"
"github.com/milvus-io/milvus/pkg/v3/util/timerecord"
"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
@@ -250,28 +241,25 @@ func (t *sortCompactionTask) sortSegment(ctx context.Context) (*datapb.Compactio
}
phaseStart = time.Now()
var rr storage.RecordReader
// use manifest reader if manifest presents
if t.manifest != "" {
rr, err = storage.NewManifestRecordReader(ctx, t.manifest, t.plan.Schema,
storage.WithVersion(t.segmentStorageVersion),
storage.WithDownloader(t.binlogIO.Download),
storage.WithStorageConfig(t.compactionParams.StorageConfig),
storage.WithCollectionID(t.collectionID),
)
} else {
rr, err = storage.NewBinlogRecordReader(ctx, t.insertLogs, t.plan.Schema,
storage.WithVersion(t.segmentStorageVersion),
storage.WithDownloader(t.binlogIO.Download),
storage.WithStorageConfig(t.compactionParams.StorageConfig),
storage.WithCollectionID(t.collectionID),
)
}
rr, existingFields, err := newCompactionSegmentRecordReader(ctx, t.plan.GetSegmentBinlogs()[0], t.plan.Schema, t.compactionParams.StorageConfig,
storage.WithVersion(t.segmentStorageVersion),
storage.WithDownloader(t.binlogIO.Download),
storage.WithStorageConfig(t.compactionParams.StorageConfig),
storage.WithCollectionID(t.collectionID),
)
if err != nil {
log.Warn("error creating insert binlog reader", zap.Error(err))
srw.Close()
return nil, err
}
materializer, err := NewRecordMaterializer(writerSchema, writerSchema.GetFunctions(), existingFields)
if err != nil {
log.Warn("error creating record materializer", zap.Error(err))
rr.Close()
srw.Close()
return nil, err
}
rr = newMaterializedRecordReader(rr, materializer)
defer rr.Close()
initReaderCost := time.Since(phaseStart)
@@ -556,164 +544,7 @@ func (t *sortCompactionTask) createTextIndex(ctx context.Context,
taskID int64,
segment *datapb.CompactionSegment,
) (map[int64]*datapb.TextIndexStats, error) {
log := log.Ctx(ctx).With(
zap.Int64("collectionID", collectionID),
zap.Int64("partitionID", partitionID),
zap.Int64("segmentID", segmentID),
)
fieldBinlogs := lo.GroupBy(segment.GetInsertLogs(), func(binlog *datapb.FieldBinlog) int64 {
return binlog.GetFieldID()
})
getInsertFiles := func(fieldID int64) ([]string, error) {
if t.storageVersion == storage.StorageV2 || t.storageVersion == storage.StorageV3 {
return []string{}, nil
}
binlogs, ok := fieldBinlogs[fieldID]
if !ok {
return nil, fmt.Errorf("field binlog not found for field %d", fieldID)
}
result := make([]string, 0, len(binlogs))
for _, binlog := range binlogs {
for _, file := range binlog.GetBinlogs() {
result = append(result, metautil.BuildInsertLogPath(t.compactionParams.StorageConfig.GetRootPath(),
collectionID, partitionID, segmentID, fieldID, file.GetLogID()))
}
}
return result, nil
}
newStorageConfig, err := util.ParseStorageConfig(t.compactionParams.StorageConfig)
if err != nil {
return nil, err
}
// Concurrent create text index for all match-enabled fields
var (
mu sync.Mutex
textIndexLogs = make(map[int64]*datapb.TextIndexStats)
)
eg, egCtx := errgroup.WithContext(ctx)
var analyzerExtraInfo string
if len(t.plan.GetFileResources()) > 0 {
err := fileresource.GlobalFileManager.Download(ctx, t.cm, t.plan.GetFileResources()...)
if err != nil {
return nil, err
}
defer fileresource.GlobalFileManager.Release(t.plan.GetFileResources()...)
analyzerExtraInfo, err = analyzer.BuildExtraResourceInfo(t.compactionParams.StorageConfig.GetRootPath(), t.plan.GetFileResources())
if err != nil {
return nil, err
}
}
for _, field := range t.plan.GetSchema().GetFields() {
field := field
h := typeutil.CreateFieldSchemaHelper(field)
if !h.EnableMatch() {
continue
}
log.Info("field enable match, ready to create text index", zap.Int64("field id", field.GetFieldID()))
eg.Go(func() error {
files, err := getInsertFiles(field.GetFieldID())
if err != nil {
return err
}
// Compute statsBasePath so C++ uploads text index to the same location
// that is returned in TextStatsLogs.Files.
statsBasePath := metautil.BuildTextIndexPrefix(t.compactionParams.StorageConfig.GetRootPath(),
t.GetPlanID(), 0, collectionID, partitionID, segmentID, field.GetFieldID())
if segment.GetManifest() != "" {
basePath, _, err := packed.UnmarshalManifestPath(segment.GetManifest())
if err != nil {
return fmt.Errorf("failed to unmarshal manifest path for text_index basePath: %w", err)
}
statsBasePath = fmt.Sprintf("%s/_stats/text_index.%d", basePath, field.GetFieldID())
}
buildIndexParams := &indexcgopb.BuildIndexInfo{
BuildID: t.GetPlanID(),
CollectionID: collectionID,
PartitionID: partitionID,
SegmentID: segmentID,
IndexVersion: 0, // always zero
InsertFiles: files,
FieldSchema: field,
StorageConfig: newStorageConfig,
CurrentScalarIndexVersion: common.ClampScalarIndexVersion(t.plan.GetCurrentScalarIndexVersion()),
StorageVersion: t.storageVersion,
Manifest: segment.GetManifest(),
StatsBasePath: statsBasePath,
IndexParams: []*commonpb.KeyValuePair{
{Key: "index_type", Value: "INVERTED"},
{Key: "is_text_match", Value: "true"},
},
}
if len(analyzerExtraInfo) > 0 {
buildIndexParams.AnalyzerExtraInfo = analyzerExtraInfo
}
if t.storageVersion == storage.StorageV2 || t.storageVersion == storage.StorageV3 {
buildIndexParams.SegmentInsertFiles = util.GetSegmentInsertFiles(
segment.GetInsertLogs(),
t.compactionParams.StorageConfig,
collectionID,
partitionID,
segmentID)
}
index, err := indexcgowrapper.CreateIndex(egCtx, buildIndexParams)
if err != nil {
return err
}
defer index.Delete()
indexStats, err := index.UpLoad()
if err != nil {
return err
}
uploaded := make(map[string]int64)
for _, info := range indexStats.GetSerializedIndexInfos() {
uploaded[info.FileName] = info.FileSize
}
// TextMatch upload returns relative filenames. Store full paths in
// metadata/task results for mixed-version compatibility.
statsFiles := metautil.BuildStatsFilePaths(statsBasePath, lo.Keys(uploaded))
mu.Lock()
totalSize := lo.SumBy(lo.Values(uploaded), func(fileSize int64) int64 { return fileSize })
textIndexLogs[field.GetFieldID()] = &datapb.TextIndexStats{
FieldID: field.GetFieldID(),
Version: 0,
BuildID: taskID,
Files: statsFiles,
LogSize: totalSize,
MemorySize: totalSize,
CurrentScalarIndexVersion: common.ClampScalarIndexVersion(t.plan.GetCurrentScalarIndexVersion()),
}
mu.Unlock()
log.Info("field enable match, create text index done",
zap.Int64("segmentID", segmentID),
zap.Int64("field id", field.GetFieldID()),
zap.Strings("files", statsFiles),
)
return nil
})
}
if err := eg.Wait(); err != nil {
return nil, err
}
return textIndexLogs, nil
return createTextIndex(ctx, t.cm, t.plan, t.compactionParams, t.storageVersion, collectionID, partitionID, segmentID, taskID, segment)
}
// initLOBCompactionContext initializes the LOB compaction context for TEXT columns.
internal/datanode/compactor/sort_compaction_test.go
+59 -3
View File
@@ -216,7 +216,7 @@ func (s *SortCompactionTaskSuite) TestSortCompactionBasic() {
func (s *SortCompactionTaskSuite) TestSortCompactionWithBM25() {
s.setupBM25Test()
s.prepareSortCompactionWithBM25Task()
s.prepareSortCompactionWithBM25Task(false)
result, err := s.task.Compact()
s.NoError(err)
@@ -234,6 +234,57 @@ func (s *SortCompactionTaskSuite) TestSortCompactionWithBM25() {
s.Empty(segment.Deltalogs)
}
func (s *SortCompactionTaskSuite) TestSortCompactionMaterializesMissingBM25OutputFromOldSegment() {
s.setupBM25Test()
s.prepareSortCompactionWithBM25Task(true)
result, err := s.task.Compact()
s.NoError(err)
s.NotNil(result)
segment := result.GetSegments()[0]
s.EqualValues(1, segment.GetNumOfRows())
s.True(segment.GetIsSorted())
s.EqualValues(1, fieldBinlogEntriesForTest(segment.GetBm25Logs(), 102))
}
func (s *SortCompactionTaskSuite) TestSortCompactionMaterializesNullableAddedFieldFromOldSegment() {
segmentID := int64(1001)
alloc := allocator.NewLocalAllocator(100, math.MaxInt64)
s.mockBinlogIO.EXPECT().Upload(mock.Anything, mock.Anything).Return(nil)
s.initSegBuffer(1, segmentID)
kvs, fBinlogs, err := serializeWrite(context.TODO(), alloc, s.segWriter)
s.Require().NoError(err)
removeFieldBinlogForTest(kvs, fBinlogs, StringField)
s.mockBinlogIO.EXPECT().Download(mock.Anything, mock.MatchedBy(func(keys []string) bool {
left, right := lo.Difference(keys, lo.Keys(kvs))
return len(left) == 0 && len(right) == 0
})).RunAndReturn(func(ctx context.Context, keys []string) ([][]byte, error) {
return downloadValuesForPathsForTest(kvs, keys)
}).Once()
s.task.plan.TotalRows = 1
s.task.plan.SegmentBinlogs = []*datapb.CompactionSegmentBinlogs{
{
SegmentID: segmentID,
CollectionID: CollectionID,
PartitionID: PartitionID,
FieldBinlogs: lo.Values(fBinlogs),
},
}
result, err := s.task.Compact()
s.NoError(err)
s.NotNil(result)
segment := result.GetSegments()[0]
s.EqualValues(1, segment.GetNumOfRows())
s.True(segment.GetIsSorted())
s.EqualValues(1, fieldBinlogEntriesForTest(segment.GetInsertLogs(), StringField))
}
func (s *SortCompactionTaskSuite) setupBM25Test() {
s.mockBinlogIO = mock_util.NewMockBinlogIO(s.T())
s.mockChunkManager = mocks.NewChunkManager(s.T())
@@ -267,7 +318,7 @@ func (s *SortCompactionTaskSuite) setupBM25Test() {
s.task.binlogIO = s.mockBinlogIO
}
func (s *SortCompactionTaskSuite) prepareSortCompactionWithBM25Task() {
func (s *SortCompactionTaskSuite) prepareSortCompactionWithBM25Task(removeBM25Output bool) {
segmentID := int64(1001)
alloc := allocator.NewLocalAllocator(100, math.MaxInt64)
s.mockBinlogIO.EXPECT().Upload(mock.Anything, mock.Anything).Return(nil)
@@ -275,11 +326,16 @@ func (s *SortCompactionTaskSuite) prepareSortCompactionWithBM25Task() {
s.initSegBufferWithBM25(segmentID)
kvs, fBinlogs, err := serializeWrite(context.TODO(), alloc, s.segWriter)
s.Require().NoError(err)
if removeBM25Output {
removeFieldBinlogForTest(kvs, fBinlogs, 102)
}
s.mockBinlogIO.EXPECT().Download(mock.Anything, mock.MatchedBy(func(keys []string) bool {
left, right := lo.Difference(keys, lo.Keys(kvs))
return len(left) == 0 && len(right) == 0
})).Return(lo.Values(kvs), nil).Once()
})).RunAndReturn(func(ctx context.Context, keys []string) ([][]byte, error) {
return downloadValuesForPathsForTest(kvs, keys)
}).Once()
s.task.plan.SegmentBinlogs = []*datapb.CompactionSegmentBinlogs{
{
internal/datanode/services.go
+2 -2
View File
@@ -302,8 +302,8 @@ func (node *DataNode) CompactionV2(ctx context.Context, req *datapb.CompactionPl
compactionParams,
sortFields,
)
case datapb.CompactionType_BackfillCompaction:
task = compactor.NewBackfillCompactionTask(taskCtx, cm, req, compactionParams)
case datapb.CompactionType_BumpSchemaVersionCompaction:
task = compactor.NewBumpSchemaVersionCompactionTask(taskCtx, cm, req, compactionParams)
default:
log.Warn("Unknown compaction type", zap.String("type", req.GetType().String()))
return merr.Status(merr.WrapErrParameterInvalidMsg("Unknown compaction type: %v", req.GetType().String())), nil
internal/datanode/services_test.go
+29
View File
@@ -33,6 +33,7 @@ import (
"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
"github.com/milvus-io/milvus/internal/compaction"
"github.com/milvus-io/milvus/internal/datanode/compactor"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/internal/util/sessionutil"
"github.com/milvus-io/milvus/pkg/v3/common"
"github.com/milvus-io/milvus/pkg/v3/log"
@@ -284,6 +285,34 @@ func (s *DataNodeServicesSuite) TestCompaction() {
s.T().Logf("status=%v", resp)
})
s.Run("bump schema version compaction", func() {
node := s.node
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
jsonParams, err := compaction.GenerateJSONParams(&schemapb.CollectionSchema{})
s.Require().NoError(err)
req := &datapb.CompactionPlan{
PlanID: 1001,
Channel: dmChannelName,
SegmentBinlogs: []*datapb.CompactionSegmentBinlogs{{
SegmentID: 102,
Level: datapb.SegmentLevel_L1,
StorageVersion: storage.StorageV3,
Manifest: "manifest",
}},
Type: datapb.CompactionType_BumpSchemaVersionCompaction,
BeginLogID: 100,
PreAllocatedLogIDs: &datapb.IDRange{Begin: 200, End: 2000},
JsonParams: jsonParams,
}
resp, err := node.CompactionV2(ctx, req)
s.NoError(err)
s.True(merr.Ok(resp))
})
s.Run("beginLogID is invalid", func() {
node := s.node
ctx, cancel := context.WithCancel(context.Background())
internal/metastore/model/collection.go
+2 -9
View File
@@ -58,7 +58,6 @@ type Collection struct {
FileResourceIds []int64
ExternalSource string
ExternalSpec string
DoPhysicalBackfill bool
}
type ShardInfo struct {
@@ -101,7 +100,6 @@ func (c *Collection) ShallowClone() *Collection {
FileResourceIds: c.FileResourceIds,
ExternalSource: c.ExternalSource,
ExternalSpec: c.ExternalSpec,
DoPhysicalBackfill: c.DoPhysicalBackfill,
}
}
@@ -143,14 +141,13 @@ func (c *Collection) Clone() *Collection {
FileResourceIds: slices.Clone(c.FileResourceIds),
ExternalSource: c.ExternalSource,
ExternalSpec: c.ExternalSpec,
DoPhysicalBackfill: c.DoPhysicalBackfill,
}
}
// ToCollectionSchemaPB returns a schemapb.CollectionSchema populated from the
// current Collection. All schema-level fields are copied verbatim — callers
// override Version, Properties, EnableDynamicField, DoPhysicalBackfill, etc.
// after the call when the operation requires a different value.
// override Version, Properties, EnableDynamicField, etc. after the call when
// the operation requires a different value.
//
// Centralizing the conversion here ensures that newly added schema fields are
// propagated consistently across every rootcoord broadcast/response path.
@@ -170,7 +167,6 @@ func (c *Collection) ToCollectionSchemaPB() *schemapb.CollectionSchema {
FileResourceIds: c.FileResourceIds,
ExternalSource: c.ExternalSource,
ExternalSpec: c.ExternalSpec,
DoPhysicalBackfill: c.DoPhysicalBackfill,
}
}
@@ -223,7 +219,6 @@ func (c *Collection) ApplyUpdates(header *message.AlterCollectionMessageHeader,
c.SchemaVersion = updates.Schema.Version
c.ExternalSource = updates.Schema.ExternalSource
c.ExternalSpec = updates.Schema.ExternalSpec
c.DoPhysicalBackfill = updates.Schema.DoPhysicalBackfill
case message.FieldMaskCollectionExternalSpec:
// Defensive: only overwrite when the update carries a value.
// Legacy WAL messages from before the atomic-tuple invariant may
@@ -297,7 +292,6 @@ func UnmarshalCollectionModel(coll *pb.CollectionInfo) *Collection {
FileResourceIds: coll.Schema.GetFileResourceIds(),
ExternalSource: coll.Schema.ExternalSource,
ExternalSpec: coll.Schema.ExternalSpec,
DoPhysicalBackfill: coll.Schema.DoPhysicalBackfill,
}
}
@@ -353,7 +347,6 @@ func marshalCollectionModelWithConfig(coll *Collection, c *config) *pb.Collectio
FileResourceIds: coll.FileResourceIds,
ExternalSource: coll.ExternalSource,
ExternalSpec: coll.ExternalSpec,
DoPhysicalBackfill: coll.DoPhysicalBackfill,
}
if c.withFields {
internal/metastore/model/collection_test.go
+36 -2
View File
@@ -669,6 +669,42 @@ func TestApplyUpdates_ExternalSpecMaskOnlyOverwriteNonEmpty(t *testing.T) {
})
}
func TestCollection_IgnoresDoPhysicalBackfill(t *testing.T) {
coll := UnmarshalCollectionModel(&pb.CollectionInfo{
ID: colID,
Schema: &schemapb.CollectionSchema{
Name: colName,
DoPhysicalBackfill: true,
},
})
assert.False(t, coll.ToCollectionSchemaPB().GetDoPhysicalBackfill())
assert.False(t, MarshalCollectionModel(coll).GetSchema().GetDoPhysicalBackfill())
coll.ApplyUpdates(
&message.AlterCollectionMessageHeader{UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{message.FieldMaskCollectionSchema}}},
&message.AlterCollectionMessageBody{Updates: &messagespb.AlterCollectionMessageUpdates{Schema: &schemapb.CollectionSchema{
Version: 3,
Fields: []*schemapb.FieldSchema{filedSchemaPb},
Functions: []*schemapb.FunctionSchema{functionSchemaPb},
StructArrayFields: []*schemapb.StructArrayFieldSchema{structFieldPb},
DoPhysicalBackfill: true,
}}},
)
assert.False(t, coll.ToCollectionSchemaPB().GetDoPhysicalBackfill())
marshaled := MarshalCollectionModelWithOption(coll, WithFields(), WithPartitions(), WithStructArrayFields())
assert.False(t, marshaled.GetSchema().GetDoPhysicalBackfill())
assert.Len(t, marshaled.GetSchema().GetFields(), 1)
assert.Len(t, marshaled.GetSchema().GetStructArrayFields(), 1)
schema := coll.ToCollectionSchemaPB()
assert.False(t, schema.GetDoPhysicalBackfill())
assert.EqualValues(t, 3, schema.GetVersion())
assert.Len(t, schema.GetFields(), 1)
assert.Len(t, schema.GetFunctions(), 1)
}
func TestCollection_ToCollectionSchemaPB(t *testing.T) {
// All schema-level fields populated with non-zero values so a future
// addition that forgets to wire ToCollectionSchemaPB will trip an
@@ -690,7 +726,6 @@ func TestCollection_ToCollectionSchemaPB(t *testing.T) {
FileResourceIds: []int64{11, 22},
ExternalSource: "s3://bucket/dataset",
ExternalSpec: `{"format":"parquet"}`,
DoPhysicalBackfill: true,
}
schema := coll.ToCollectionSchemaPB()
@@ -709,5 +744,4 @@ func TestCollection_ToCollectionSchemaPB(t *testing.T) {
assert.Equal(t, []int64{11, 22}, schema.GetFileResourceIds())
assert.Equal(t, "s3://bucket/dataset", schema.GetExternalSource())
assert.Equal(t, `{"format":"parquet"}`, schema.GetExternalSpec())
assert.True(t, schema.GetDoPhysicalBackfill())
}
internal/metastore/model/index.go
+34 -38
View File
@@ -8,18 +8,17 @@ import (
)
type Index struct {
TenantID string
CollectionID int64
FieldID int64
IndexID int64
IndexName string
IsDeleted bool
CreateTime uint64
TypeParams []*commonpb.KeyValuePair
IndexParams []*commonpb.KeyValuePair
IsAutoIndex bool
UserIndexParams []*commonpb.KeyValuePair
MinSchemaVersion int32
TenantID string
CollectionID int64
FieldID int64
IndexID int64
IndexName string
IsDeleted bool
CreateTime uint64
TypeParams []*commonpb.KeyValuePair
IndexParams []*commonpb.KeyValuePair
IsAutoIndex bool
UserIndexParams []*commonpb.KeyValuePair
}
func UnmarshalIndexModel(indexInfo *indexpb.FieldIndex) *Index {
@@ -28,17 +27,16 @@ func UnmarshalIndexModel(indexInfo *indexpb.FieldIndex) *Index {
}
return &Index{
CollectionID: indexInfo.IndexInfo.GetCollectionID(),
FieldID: indexInfo.IndexInfo.GetFieldID(),
IndexID: indexInfo.IndexInfo.GetIndexID(),
IndexName: indexInfo.IndexInfo.GetIndexName(),
IsDeleted: indexInfo.GetDeleted(),
CreateTime: indexInfo.CreateTime,
TypeParams: indexInfo.IndexInfo.GetTypeParams(),
IndexParams: indexInfo.IndexInfo.GetIndexParams(),
IsAutoIndex: indexInfo.IndexInfo.GetIsAutoIndex(),
UserIndexParams: indexInfo.IndexInfo.GetUserIndexParams(),
MinSchemaVersion: indexInfo.GetMinSchemaVersion(),
CollectionID: indexInfo.IndexInfo.GetCollectionID(),
FieldID: indexInfo.IndexInfo.GetFieldID(),
IndexID: indexInfo.IndexInfo.GetIndexID(),
IndexName: indexInfo.IndexInfo.GetIndexName(),
IsDeleted: indexInfo.GetDeleted(),
CreateTime: indexInfo.CreateTime,
TypeParams: indexInfo.IndexInfo.GetTypeParams(),
IndexParams: indexInfo.IndexInfo.GetIndexParams(),
IsAutoIndex: indexInfo.IndexInfo.GetIsAutoIndex(),
UserIndexParams: indexInfo.IndexInfo.GetUserIndexParams(),
}
}
@@ -58,9 +56,8 @@ func MarshalIndexModel(index *Index) *indexpb.FieldIndex {
IsAutoIndex: index.IsAutoIndex,
UserIndexParams: index.UserIndexParams,
},
Deleted: index.IsDeleted,
CreateTime: index.CreateTime,
MinSchemaVersion: index.MinSchemaVersion,
Deleted: index.IsDeleted,
CreateTime: index.CreateTime,
}
}
@@ -116,18 +113,17 @@ func MarshalIndexModel(index *Index) *indexpb.FieldIndex {
func CloneIndex(index *Index) *Index {
clonedIndex := &Index{
TenantID: index.TenantID,
CollectionID: index.CollectionID,
FieldID: index.FieldID,
IndexID: index.IndexID,
IndexName: index.IndexName,
IsDeleted: index.IsDeleted,
CreateTime: index.CreateTime,
TypeParams: make([]*commonpb.KeyValuePair, len(index.TypeParams)),
IndexParams: make([]*commonpb.KeyValuePair, len(index.IndexParams)),
IsAutoIndex: index.IsAutoIndex,
UserIndexParams: make([]*commonpb.KeyValuePair, len(index.UserIndexParams)),
MinSchemaVersion: index.MinSchemaVersion,
TenantID: index.TenantID,
CollectionID: index.CollectionID,
FieldID: index.FieldID,
IndexID: index.IndexID,
IndexName: index.IndexName,
IsDeleted: index.IsDeleted,
CreateTime: index.CreateTime,
TypeParams: make([]*commonpb.KeyValuePair, len(index.TypeParams)),
IndexParams: make([]*commonpb.KeyValuePair, len(index.IndexParams)),
IsAutoIndex: index.IsAutoIndex,
UserIndexParams: make([]*commonpb.KeyValuePair, len(index.UserIndexParams)),
}
for i, param := range index.TypeParams {
clonedIndex.TypeParams[i] = proto.Clone(param).(*commonpb.KeyValuePair)
internal/proxy/impl.go
-4
View File
@@ -1601,10 +1601,6 @@ func (node *Proxy) AlterCollectionField(ctx context.Context, request *milvuspb.A
return merr.Status(err), nil
}
// TODO(#48808): gate AlterCollectionField against in-progress backfill once segment schema-version
// propagation for DoPhysicalBackfill=false DDLs is synchronous. Same timing issue as
// AddCollectionField — backfill tick may not have fired before the next DDL arrives.
act := &alterCollectionFieldTask{
ctx: ctx,
Condition: NewTaskCondition(ctx),
internal/proxy/service_provider.go
-1
View File
@@ -215,7 +215,6 @@ func (node *CachedProxyServiceProvider) DescribeCollection(ctx context.Context,
ExternalSource: c.schema.ExternalSource,
ExternalSpec: c.schema.ExternalSpec,
Version: c.schema.Version,
DoPhysicalBackfill: c.schema.DoPhysicalBackfill,
}
// Restore struct field names from internal format (structName[fieldName]) to original format
internal/proxy/service_provider_test.go
+39
View File
@@ -39,6 +39,45 @@ func TestNewInterceptor(t *testing.T) {
assert.Equal(t, "can't find collection[database=test][collection=test]", resp.Status.Reason)
}
func TestCachedProxyServiceProvider_DescribeCollection_IgnoresLegacyDoPhysicalBackfill(t *testing.T) {
ctx := context.Background()
origCache := globalMetaCache
defer func() { globalMetaCache = origCache }()
dbName := "test_db"
collectionName := "test_collection"
collectionID := int64(1000)
schema := &schemapb.CollectionSchema{
Name: collectionName,
DoPhysicalBackfill: true,
Fields: []*schemapb.FieldSchema{{
FieldID: common.StartOfUserFieldID,
Name: "id",
IsPrimaryKey: true,
DataType: schemapb.DataType_Int64,
}},
}
mockCache := &MockCache{}
mockCache.EXPECT().GetCollectionID(mock.Anything, dbName, collectionName).Return(collectionID, nil)
mockCache.EXPECT().GetCollectionInfo(mock.Anything, dbName, collectionName, collectionID).Return(&collectionInfo{
collID: collectionID,
schema: newSchemaInfo(schema),
shardsNum: common.DefaultShardsNum,
}, nil)
globalMetaCache = mockCache
provider := &CachedProxyServiceProvider{Proxy: &Proxy{}}
resp, err := provider.DescribeCollection(ctx, &milvuspb.DescribeCollectionRequest{
DbName: dbName,
CollectionName: collectionName,
})
assert.NoError(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, resp.GetStatus().GetErrorCode())
assert.False(t, resp.GetSchema().GetDoPhysicalBackfill())
}
func TestCachedProxyServiceProvider_DescribeCollection_FilterNamespaceField(t *testing.T) {
ctx := context.Background()
internal/proxy/task.go
+3 -9
View File
@@ -1060,6 +1060,9 @@ func (t *alterCollectionSchemaTask) PreExecute(ctx context.Context) error {
if len(funcSchemas) != 1 || funcSchemas[0] == nil {
return merr.WrapErrParameterInvalidMsg("For now, exactly one function schema is required in alter schema task")
}
if funcSchemas[0].GetType() != schemapb.FunctionType_BM25 {
return merr.WrapErrParameterInvalidMsg("For now, only BM25 function is supported in alter schema task")
}
if len(fieldInfos) == 0 {
return merr.WrapErrParameterInvalidMsg("fieldInfos is empty, function output fields are required")
}
@@ -1097,15 +1100,6 @@ func (t *alterCollectionSchemaTask) PreExecute(ctx context.Context) error {
}
}
// Physical backfill is currently only implemented for BM25 in the datanode backfill
// compactor. Reject unsupported types early so the request never reaches RootCoord
// and no segment is left in an unrecoverable stale-schema state.
if addRequest.GetDoPhysicalBackfill() && funcSchemas[0].GetType() != schemapb.FunctionType_BM25 {
return merr.WrapErrParameterInvalidMsg(
"physical backfill is currently only supported for BM25 functions, got %s",
funcSchemas[0].GetType().String())
}
// Validate function-field type compatibility (e.g., BM25 requires varchar input,
// SparseFloatVector output). Construct a merged schema with old fields + new fields
// + new function, then validate only the new function to avoid re-checking existing
internal/proxy/task_test.go
+16 -23
View File
@@ -7418,33 +7418,26 @@ func TestAlterCollectionSchemaTask(t *testing.T) {
assert.ErrorIs(t, err, merr.ErrParameterInvalid)
})
t.Run("PreExecute rejects non-BM25 function with DoPhysicalBackfill", func(t *testing.T) {
req := &milvuspb.AlterCollectionSchemaRequest{
DbName: dbName,
CollectionName: collectionName,
Action: &milvuspb.AlterCollectionSchemaRequest_Action{
Op: &milvuspb.AlterCollectionSchemaRequest_Action_AddRequest{
AddRequest: &milvuspb.AlterCollectionSchemaRequest_AddRequest{
DoPhysicalBackfill: true,
FieldInfos: []*milvuspb.AlterCollectionSchemaRequest_FieldInfo{
{FieldSchema: sparseOutputField},
},
FuncSchema: []*schemapb.FunctionSchema{
{
Name: "text_embedding_func",
Type: schemapb.FunctionType_TextEmbedding,
InputFieldNames: []string{"text"},
OutputFieldNames: []string{"sparse_bm25"},
},
},
},
},
},
}
t.Run("PreExecute rejects non-BM25 function", func(t *testing.T) {
req := buildValidRequest()
addRequest := req.GetAction().GetAddRequest()
functionSchema := proto.Clone(addRequest.GetFuncSchema()[0]).(*schemapb.FunctionSchema)
functionSchema.Type = schemapb.FunctionType_MinHash
addRequest.FuncSchema = []*schemapb.FunctionSchema{functionSchema}
task := buildTask(req, oldSchema)
err := task.PreExecute(ctx)
assert.Error(t, err)
assert.ErrorIs(t, err, merr.ErrParameterInvalid)
assert.ErrorContains(t, err, "only BM25 function is supported")
})
t.Run("PreExecute ignores legacy DoPhysicalBackfill", func(t *testing.T) {
schema := proto.Clone(oldSchema).(*schemapb.CollectionSchema)
schema.DoPhysicalBackfill = true
task := buildTask(buildValidRequest(), schema)
err := task.PreExecute(ctx)
assert.NoError(t, err)
})
t.Run("PreExecute happy path", func(t *testing.T) {
internal/proxy/util_test.go
+53 -3
View File
@@ -2863,7 +2863,27 @@ func TestValidateFunction(t *testing.T) {
assert.Contains(t, err.Error(), "output field not found")
})
t.Run("Valid function schema - nullable input field", func(t *testing.T) {
t.Run("Invalid function schema - nullable BM25 output field", func(t *testing.T) {
schema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{Name: "input_field", DataType: schemapb.DataType_VarChar, TypeParams: []*commonpb.KeyValuePair{{Key: "enable_analyzer", Value: "true"}}, Nullable: true},
{Name: "output_field", DataType: schemapb.DataType_SparseFloatVector, Nullable: true},
},
Functions: []*schemapb.FunctionSchema{
{
Name: "bm25_func",
Type: schemapb.FunctionType_BM25,
InputFieldNames: []string{"input_field"},
OutputFieldNames: []string{"output_field"},
},
},
}
err := validateFunction(schema, "", false)
assert.Error(t, err)
assert.Contains(t, err.Error(), "function output field cannot be nullable")
})
t.Run("Valid create schema function - nullable BM25 input with non-nullable output field", func(t *testing.T) {
schema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{Name: "input_field", DataType: schemapb.DataType_VarChar, TypeParams: []*commonpb.KeyValuePair{{Key: "enable_analyzer", Value: "true"}}, Nullable: true},
@@ -2942,7 +2962,7 @@ func TestValidateFunction(t *testing.T) {
assert.Contains(t, err.Error(), "function output field cannot be partition key or clustering key")
})
t.Run("Invalid function schema - nullable output field", func(t *testing.T) {
t.Run("Invalid create schema function - non-nullable BM25 input with nullable output field", func(t *testing.T) {
schema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{Name: "input_field", DataType: schemapb.DataType_VarChar, TypeParams: []*commonpb.KeyValuePair{{Key: "enable_analyzer", Value: "true"}}},
@@ -5240,7 +5260,7 @@ func TestMinHashFunction(t *testing.T) {
t.Run("MinHash function without permutations ", func(t *testing.T) {
schema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{Name: "text_field", DataType: schemapb.DataType_VarChar},
{Name: "text_field", DataType: schemapb.DataType_VarChar, Nullable: true},
{
Name: "minhash_output",
DataType: schemapb.DataType_BinaryVector,
@@ -5272,6 +5292,36 @@ func TestMinHashFunction(t *testing.T) {
assert.NoError(t, err)
})
t.Run("create schema allows nullable mismatch", func(t *testing.T) {
schema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{Name: "text_field", DataType: schemapb.DataType_VarChar, Nullable: true},
{
Name: "minhash_output",
DataType: schemapb.DataType_BinaryVector,
TypeParams: []*commonpb.KeyValuePair{
{Key: common.DimKey, Value: "4096"},
},
},
},
Functions: []*schemapb.FunctionSchema{
{
Name: "text_to_minhash",
Type: schemapb.FunctionType_MinHash,
InputFieldNames: []string{"text_field"},
OutputFieldNames: []string{"minhash_output"},
Params: []*commonpb.KeyValuePair{
{Key: "num_hashes", Value: "128"},
{Key: "shingle_size", Value: "3"},
},
},
},
}
err := validateFunction(schema, "", false)
assert.NoError(t, err)
})
t.Run("miss num_hashes", func(t *testing.T) {
createSchema := func() *schemapb.CollectionSchema {
return &schemapb.CollectionSchema{
internal/rootcoord/broker.go
+1 -1
View File
@@ -273,7 +273,7 @@ func (b *ServerBroker) BroadcastAlteredCollection(ctx context.Context, collectio
log.Ctx(ctx).Info("done to broadcast request to alter collection",
zap.String("collectionName", colMeta.Name), zap.Int64("collectionID", dcReq.GetCollectionID()),
zap.Any("props", colMeta.Properties), zap.Any("fields", colMeta.Fields),
zap.Int32("schemaVersion", colMeta.SchemaVersion), zap.Bool("doPhysicalBackfill", colMeta.DoPhysicalBackfill))
zap.Int32("schemaVersion", colMeta.SchemaVersion))
return nil
}
internal/rootcoord/ddl_callbacks_alter_collection_schema.go
+42 -14
View File
@@ -59,14 +59,11 @@ func (c *Core) broadcastAlterCollectionSchema(ctx context.Context, req *milvuspb
return merr.WrapErrParameterInvalidMsg("For now, exactly one function schema is supported in alter schema task")
}
functionSchema := funcSchemas[0]
// Physical backfill is currently only implemented for BM25 functions in the datanode
// backfill_compactor. Proxy performs the same check; this is a defense-in-depth guard
// for requests that bypass the Proxy (e.g. direct gRPC to RootCoord).
if addRequest.GetDoPhysicalBackfill() && functionSchema.GetType() != schemapb.FunctionType_BM25 {
return merr.WrapErrParameterInvalidMsg(
"physical backfill is currently only supported for BM25 functions, got %s",
functionSchema.GetType().String())
if err := checkAlterSchemaFunctionAllowed(functionSchema); err != nil {
return err
}
if err := validateAlterSchemaFunctionInputOutput(functionSchema); err != nil {
return err
}
if len(fieldInfos) == 0 {
@@ -85,6 +82,15 @@ func (c *Core) broadcastAlterCollectionSchema(ctx context.Context, req *milvuspb
if err := checkFieldSchema(fieldSchemas); err != nil {
return errors.Wrap(err, "failed to check field schema")
}
newFieldNames := make(map[string]struct{}, len(fieldSchemas))
for _, fieldSchema := range fieldSchemas {
newFieldNames[fieldSchema.GetName()] = struct{}{}
}
for _, outputFieldName := range functionSchema.GetOutputFieldNames() {
if _, ok := newFieldNames[outputFieldName]; !ok {
return merr.WrapErrParameterInvalidMsg("function output field %q must be one of the newly-added fields", outputFieldName)
}
}
// 3. check if the fields already exist
fieldNameSet := make(map[string]struct{})
@@ -97,6 +103,12 @@ func (c *Core) broadcastAlterCollectionSchema(ctx context.Context, req *milvuspb
}
fieldNameSet[fieldSchema.Name] = struct{}{}
}
outputFieldNames := typeutil.NewSet[string](functionSchema.GetOutputFieldNames()...)
for _, fieldSchema := range fieldSchemas {
if outputFieldNames.Contain(fieldSchema.GetName()) && fieldSchema.GetNullable() {
return merr.WrapErrParameterInvalidMsg("function output field cannot be nullable: function %s, field %s", functionSchema.GetName(), fieldSchema.GetName())
}
}
// 4. check if the function already exists
for _, function := range coll.Functions {
@@ -139,13 +151,7 @@ func (c *Core) broadcastAlterCollectionSchema(ctx context.Context, req *milvuspb
// 6. build new collection schema.
schema := coll.ToCollectionSchemaPB()
// Version is incremented by 1. No CAS is needed here because Proxy's
// checkSchemaVersionConsistency gate blocks new AlterCollectionSchema calls
// until the previous backfill reaches 100% consistency, and DDL requests
// are serialized through a single DDL queue — so concurrent schema changes
// that could produce duplicate version numbers are impossible.
schema.Version = coll.SchemaVersion + 1
schema.DoPhysicalBackfill = addRequest.GetDoPhysicalBackfill()
schema.Fields = append(schema.Fields, fieldSchemas...)
schema.Functions = append(schema.Functions, functionSchema)
if err := typeutil.ValidateExternalCollectionGeneratedColumns(schema); err != nil {
@@ -182,3 +188,25 @@ func (c *Core) broadcastAlterCollectionSchema(ctx context.Context, req *milvuspb
}
return nil
}
func checkAlterSchemaFunctionAllowed(functionSchema *schemapb.FunctionSchema) error {
switch functionSchema.GetType() {
case schemapb.FunctionType_BM25:
return nil
default:
return merr.WrapErrParameterInvalidMsg("For now, only BM25 function is supported in alter schema task")
}
}
func validateAlterSchemaFunctionInputOutput(functionSchema *schemapb.FunctionSchema) error {
switch functionSchema.GetType() {
case schemapb.FunctionType_BM25:
inputCount := len(functionSchema.GetInputFieldNames())
if (inputCount != 1 && inputCount != 2) || len(functionSchema.GetOutputFieldNames()) != 1 {
return merr.WrapErrParameterInvalidMsg("BM25 function should have one or two input fields and exactly one output field")
}
return nil
default:
return merr.WrapErrParameterInvalidMsg("unsupported function type in alter schema task: %s", functionSchema.GetType().String())
}
}
internal/rootcoord/ddl_callbacks_alter_collection_schema_test.go
+55 -65
View File
@@ -29,10 +29,11 @@ import (
"github.com/milvus-io/milvus/pkg/v3/common"
"github.com/milvus-io/milvus/pkg/v3/util/funcutil"
"github.com/milvus-io/milvus/pkg/v3/util/merr"
"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
)
// buildAlterSchemaReq constructs a valid AlterCollectionSchemaRequest with an add operation.
func buildAlterSchemaReq(dbName, collName, inputField, outputField, funcName string, doBackfill bool) *milvuspb.AlterCollectionSchemaRequest {
func buildAlterSchemaReq(dbName, collName, inputField, outputField, funcName string) *milvuspb.AlterCollectionSchemaRequest {
outputFieldSchema := &schemapb.FieldSchema{
Name: outputField,
DataType: schemapb.DataType_SparseFloatVector,
@@ -53,8 +54,7 @@ func buildAlterSchemaReq(dbName, collName, inputField, outputField, funcName str
FieldInfos: []*milvuspb.AlterCollectionSchemaRequest_FieldInfo{
{FieldSchema: outputFieldSchema},
},
FuncSchema: []*schemapb.FunctionSchema{functionSchema},
DoPhysicalBackfill: doBackfill,
FuncSchema: []*schemapb.FunctionSchema{functionSchema},
},
},
},
@@ -112,7 +112,7 @@ func TestDDLCallbacksBroadcastAlterCollectionSchema(t *testing.T) {
Op: &milvuspb.AlterCollectionSchemaRequest_Action_AddRequest{
AddRequest: &milvuspb.AlterCollectionSchemaRequest_AddRequest{
FuncSchema: []*schemapb.FunctionSchema{
{Name: "fn1", Type: schemapb.FunctionType_BM25},
{Name: "fn1", Type: schemapb.FunctionType_BM25, InputFieldNames: []string{"field1"}, OutputFieldNames: []string{"sparse1"}},
},
FieldInfos: []*milvuspb.AlterCollectionSchemaRequest_FieldInfo{},
},
@@ -121,9 +121,7 @@ func TestDDLCallbacksBroadcastAlterCollectionSchema(t *testing.T) {
})
require.ErrorIs(t, merr.CheckRPCCall(resp.GetAlterStatus(), err), merr.ErrParameterInvalid)
// case 4.1: physical backfill with non-BM25 function must be rejected.
// Otherwise the backfill task would fail at datanode with "unsupported function type"
// and permanently block subsequent schema-change DDLs via the consistency gate.
// case 5: BM25 arity invalid
resp, err = core.AlterCollectionSchema(ctx, &milvuspb.AlterCollectionSchemaRequest{
DbName: dbName,
CollectionName: collectionName,
@@ -131,34 +129,20 @@ func TestDDLCallbacksBroadcastAlterCollectionSchema(t *testing.T) {
Op: &milvuspb.AlterCollectionSchemaRequest_Action_AddRequest{
AddRequest: &milvuspb.AlterCollectionSchemaRequest_AddRequest{
FuncSchema: []*schemapb.FunctionSchema{
{
Name: "minhash_fn",
Type: schemapb.FunctionType_MinHash,
InputFieldNames: []string{"text_input"},
OutputFieldNames: []string{"sparse_minhash"},
},
{Name: "fn_bad_arity", Type: schemapb.FunctionType_BM25, InputFieldNames: []string{"field1"}},
},
FieldInfos: []*milvuspb.AlterCollectionSchemaRequest_FieldInfo{
{FieldSchema: &schemapb.FieldSchema{
Name: "sparse_minhash",
DataType: schemapb.DataType_SparseFloatVector,
IsFunctionOutput: true,
}},
{FieldSchema: &schemapb.FieldSchema{Name: "sparse_bad_arity", DataType: schemapb.DataType_SparseFloatVector}},
},
DoPhysicalBackfill: true,
},
},
},
})
require.ErrorIs(t, merr.CheckRPCCall(resp.GetAlterStatus(), err), merr.ErrParameterInvalid)
require.Contains(t, resp.GetAlterStatus().GetReason(), "physical backfill is currently only supported for BM25 functions")
arityErr := merr.CheckRPCCall(resp.GetAlterStatus(), err)
require.ErrorIs(t, arityErr, merr.ErrParameterInvalid)
require.ErrorContains(t, arityErr, "one or two input fields and exactly one output field")
// case 4.2: non-BM25 function with DoPhysicalBackfill=false should NOT be rejected by
// the type check (it may still fail later for other reasons, but the type check must pass).
// This path never invokes the backfill_compactor, so "unsupported type" cannot be triggered.
// We don't assert success here because downstream validation may reject for unrelated
// reasons in this minimal test setup — we only assert that the error, if any, is not the
// physical-backfill type rejection.
// case 6: fieldSchema nil in fieldInfos
resp, err = core.AlterCollectionSchema(ctx, &milvuspb.AlterCollectionSchemaRequest{
DbName: dbName,
CollectionName: collectionName,
@@ -166,41 +150,7 @@ func TestDDLCallbacksBroadcastAlterCollectionSchema(t *testing.T) {
Op: &milvuspb.AlterCollectionSchemaRequest_Action_AddRequest{
AddRequest: &milvuspb.AlterCollectionSchemaRequest_AddRequest{
FuncSchema: []*schemapb.FunctionSchema{
{
Name: "minhash_fn_nophys",
Type: schemapb.FunctionType_MinHash,
InputFieldNames: []string{"text_input"},
OutputFieldNames: []string{"sparse_minhash2"},
},
},
FieldInfos: []*milvuspb.AlterCollectionSchemaRequest_FieldInfo{
{FieldSchema: &schemapb.FieldSchema{
Name: "sparse_minhash2",
DataType: schemapb.DataType_SparseFloatVector,
IsFunctionOutput: true,
}},
},
DoPhysicalBackfill: false,
},
},
},
})
// Whatever happens, it must not be the BM25-only rejection.
if err := merr.CheckRPCCall(resp.GetAlterStatus(), err); err != nil {
require.NotContains(t, resp.GetAlterStatus().GetReason(),
"physical backfill is currently only supported for BM25 functions",
"non-BM25 with DoPhysicalBackfill=false must not be rejected by the BM25-only type check")
}
// case 5: fieldSchema nil in fieldInfos
resp, err = core.AlterCollectionSchema(ctx, &milvuspb.AlterCollectionSchemaRequest{
DbName: dbName,
CollectionName: collectionName,
Action: &milvuspb.AlterCollectionSchemaRequest_Action{
Op: &milvuspb.AlterCollectionSchemaRequest_Action_AddRequest{
AddRequest: &milvuspb.AlterCollectionSchemaRequest_AddRequest{
FuncSchema: []*schemapb.FunctionSchema{
{Name: "fn1", Type: schemapb.FunctionType_BM25},
{Name: "fn1", Type: schemapb.FunctionType_BM25, InputFieldNames: []string{"field1"}, OutputFieldNames: []string{"sparse_nil_field"}},
},
FieldInfos: []*milvuspb.AlterCollectionSchemaRequest_FieldInfo{
{FieldSchema: nil},
@@ -230,10 +180,30 @@ func TestDDLCallbacksBroadcastAlterCollectionSchema(t *testing.T) {
})
require.ErrorIs(t, merr.CheckRPCCall(resp.GetAlterStatus(), err), merr.ErrParameterInvalid)
// case 7: output field points to an existing field while FieldInfos adds a different field
resp, err = core.AlterCollectionSchema(ctx, &milvuspb.AlterCollectionSchemaRequest{
DbName: dbName,
CollectionName: collectionName,
Action: &milvuspb.AlterCollectionSchemaRequest_Action{
Op: &milvuspb.AlterCollectionSchemaRequest_Action_AddRequest{
AddRequest: &milvuspb.AlterCollectionSchemaRequest_AddRequest{
FuncSchema: []*schemapb.FunctionSchema{
{Name: "fn_output_existing_field", Type: schemapb.FunctionType_BM25, InputFieldNames: []string{"field1"}, OutputFieldNames: []string{"field1"}},
},
FieldInfos: []*milvuspb.AlterCollectionSchemaRequest_FieldInfo{
{FieldSchema: &schemapb.FieldSchema{Name: "sparse_output_existing_bypass", DataType: schemapb.DataType_SparseFloatVector, IsFunctionOutput: true}},
},
},
},
},
})
require.ErrorIs(t, merr.CheckRPCCall(resp.GetAlterStatus(), err), merr.ErrParameterInvalid)
// Add a VARCHAR input field so that the happy-path call below can succeed.
varcharFieldSchema := &schemapb.FieldSchema{
Name: "text_input",
DataType: schemapb.DataType_VarChar,
Nullable: true,
TypeParams: []*commonpb.KeyValuePair{
{Key: common.MaxLengthKey, Value: "256"},
},
@@ -247,17 +217,37 @@ func TestDDLCallbacksBroadcastAlterCollectionSchema(t *testing.T) {
})
require.NoError(t, merr.CheckRPCCall(addFieldResp, err))
// Non-BM25 function is not supported by schema evolution backfill.
nonBM25Req := buildAlterSchemaReq(dbName, collectionName, "text_input", "sparse_output_non_bm25", "minhash_fn")
nonBM25Req.GetAction().GetAddRequest().GetFuncSchema()[0].Type = schemapb.FunctionType_MinHash
resp, err = core.AlterCollectionSchema(ctx, nonBM25Req)
alterErr := merr.CheckRPCCall(resp.GetAlterStatus(), err)
require.ErrorIs(t, alterErr, merr.ErrParameterInvalid)
require.ErrorContains(t, alterErr, "only BM25 function is supported")
nullableOutputReq := buildAlterSchemaReq(dbName, collectionName, "text_input", "sparse_output_nullable", "bm25_nullable_output")
nullableOutputReq.GetAction().GetAddRequest().GetFieldInfos()[0].GetFieldSchema().Nullable = true
resp, err = core.AlterCollectionSchema(ctx, nullableOutputReq)
alterErr = merr.CheckRPCCall(resp.GetAlterStatus(), err)
require.ErrorIs(t, alterErr, merr.ErrParameterInvalid)
require.ErrorContains(t, alterErr, "function output field cannot be nullable")
// happy path: add sparse vector output field + BM25 function → schema version bumps (AddCollectionField already bumped to 1, so now 2)
firstAlterReq := buildAlterSchemaReq(dbName, collectionName, "text_input", "sparse_output", "bm25_fn", false)
firstAlterReq := buildAlterSchemaReq(dbName, collectionName, "text_input", "sparse_output", "bm25_fn")
resp, err = core.AlterCollectionSchema(ctx, firstAlterReq)
require.NoError(t, merr.CheckRPCCall(resp.GetAlterStatus(), err))
assertSchemaVersion(t, ctx, core, dbName, collectionName, 2)
// happy path with DoPhysicalBackfill=true: flag propagates through broadcast, schema version bumps to 3
secondAlterReq := buildAlterSchemaReq(dbName, collectionName, "text_input", "sparse_output2", "bm25_fn2", true)
// second happy path: schema version bumps to 3
secondAlterReq := buildAlterSchemaReq(dbName, collectionName, "text_input", "sparse_output2", "bm25_fn2")
resp, err = core.AlterCollectionSchema(ctx, secondAlterReq)
require.NoError(t, merr.CheckRPCCall(resp.GetAlterStatus(), err))
assertSchemaVersion(t, ctx, core, dbName, collectionName, 3)
updated, err := core.meta.GetCollectionByName(ctx, dbName, collectionName, typeutil.MaxTimestamp, false)
require.NoError(t, err)
schema := updated.ToCollectionSchemaPB()
require.False(t, schema.GetDoPhysicalBackfill())
require.EqualValues(t, 3, schema.GetVersion())
// case 7: function already exists (same name "bm25_fn")
resp, err = core.AlterCollectionSchema(ctx, &milvuspb.AlterCollectionSchemaRequest{
internal/rootcoord/meta_table.go
-1
View File
@@ -1101,7 +1101,6 @@ func (mt *MetaTable) AlterCollection(ctx context.Context, result message.Broadca
zap.Int64("oldCollectionID", oldColl.CollectionID),
zap.Bool("dbChanged", dbChanged),
zap.Uint64("ts", newColl.UpdateTimestamp),
zap.Bool("doPhysicalBackfill", newColl.DoPhysicalBackfill),
zap.Int32("schemaVersion", newColl.SchemaVersion),
)
return nil
internal/storage/arrow_util.go
+7 -1
View File
@@ -271,7 +271,7 @@ func GenerateEmptyArrayFromSchema(schema *schemapb.FieldSchema, numRows int) (ar
if schema.GetNullable() && isNullableDenseVectorArrowType(schema.GetDataType()) {
arrowType = arrow.BinaryTypes.Binary
}
builder := array.NewBuilder(memory.DefaultAllocator, arrowType) // serdeEntry[schema.GetDataType()].newBuilder()
builder := array.NewBuilder(memory.DefaultAllocator, arrowType)
if schema.GetDefaultValue() != nil {
switch schema.GetDataType() {
case schemapb.DataType_Bool:
@@ -373,6 +373,12 @@ func (b *RecordBuilder) GetSize() uint64 {
return b.size
}
func (b *RecordBuilder) Release() {
for _, builder := range b.builders {
builder.Release()
}
}
func (b *RecordBuilder) Build() Record {
arrays := make([]arrow.Array, len(b.builders))
fields := make([]arrow.Field, len(b.builders))
internal/storage/arrow_util_test.go
+35
View File
@@ -21,6 +21,7 @@ import (
"testing"
"github.com/apache/arrow/go/v17/arrow"
"github.com/apache/arrow/go/v17/arrow/array"
"github.com/stretchr/testify/assert"
"github.com/milvus-io/milvus-proto/go-api/v3/commonpb"
@@ -246,6 +247,7 @@ func TestGenerateEmptyArrayFromSchemaNullableDenseVectorUsesBinaryArrow(t *testi
}
arr, err := GenerateEmptyArrayFromSchema(field, 3)
defer arr.Release()
assert.NoError(t, err)
assert.Equal(t, arrow.BinaryTypes.Binary, arr.DataType())
@@ -255,6 +257,39 @@ func TestGenerateEmptyArrayFromSchemaNullableDenseVectorUsesBinaryArrow(t *testi
}
}
func TestGenerateEmptyArrayFromSchemaNullableVectorAppendsToRecordBuilder(t *testing.T) {
field := &schemapb.FieldSchema{
FieldID: 100,
Name: "embeddings_new",
DataType: schemapb.DataType_FloatVector,
Nullable: true,
TypeParams: []*commonpb.KeyValuePair{
{Key: "dim", Value: "4"},
},
}
arr, err := GenerateEmptyArrayFromSchema(field, 3)
assert.NoError(t, err)
defer arr.Release()
assert.IsType(t, &array.Binary{}, arr)
rec := NewSimpleArrowRecord(array.NewRecord(
arrow.NewSchema([]arrow.Field{{Name: field.Name, Type: arr.DataType(), Nullable: true}}, nil),
[]arrow.Array{arr},
int64(arr.Len()),
), map[FieldID]int{field.FieldID: 0})
defer rec.Release()
rb := NewRecordBuilder(&schemapb.CollectionSchema{Fields: []*schemapb.FieldSchema{field}})
defer rb.Release()
assert.NoError(t, rb.Append(rec, 0, arr.Len()))
rebuilt := rb.Build()
defer rebuilt.Release()
assert.Equal(t, arr.Len(), rebuilt.Column(field.FieldID).Len())
for i := 0; i < arr.Len(); i++ {
assert.True(t, rebuilt.Column(field.FieldID).IsNull(i))
}
}
func TestRecordBuilderNullableDenseVectorPreservesDimMetadata(t *testing.T) {
schema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
internal/storage/record_reader.go
+12
View File
@@ -309,6 +309,16 @@ var _ RecordReader = (*CompositeBinlogRecordReader)(nil)
func (crr *CompositeBinlogRecordReader) Next() (Record, error) {
recs := make([]arrow.Array, len(crr.fields))
releaseRecsOnError := true
defer func() {
if releaseRecsOnError {
for _, rec := range recs {
if rec != nil {
rec.Release()
}
}
}
}()
nonExistingFields := make([]*schemapb.FieldSchema, 0)
nRows := 0
for _, f := range crr.fields {
@@ -319,6 +329,7 @@ func (crr *CompositeBinlogRecordReader) Next() (Record, error) {
}
r := crr.rrs[idx].Record()
recs[idx] = r.Column(0)
recs[idx].Retain()
if nRows == 0 {
nRows = int(r.NumRows())
}
@@ -337,6 +348,7 @@ func (crr *CompositeBinlogRecordReader) Next() (Record, error) {
}
recs[crr.index[f.FieldID]] = arr
}
releaseRecsOnError = false
return &compositeRecord{
index: crr.index,
recs: recs,
internal/storage/record_writer.go
+36 -4
View File
@@ -272,6 +272,12 @@ func (pw *packedRecordBatchWriter) Close() (packed.WriterOutput, error) {
return out, nil
}
func (pw *packedRecordBatchWriter) AsNewColumnGroups() {
if pw.writer != nil {
pw.writer.AsNewColumnGroups()
}
}
func NewPackedRecordBatchWriter(
basePath string,
schema *schemapb.CollectionSchema,
@@ -281,10 +287,36 @@ func NewPackedRecordBatchWriter(
storageConfig *indexpb.StorageConfig,
storagePluginContext *indexcgopb.StoragePluginContext,
) (*packedRecordBatchWriter, error) {
// Validate PK field exists before proceeding
_, err := typeutil.GetPrimaryFieldSchema(schema)
if err != nil {
return nil, err
return newPackedRecordBatchWriter(basePath, schema, bufferSize, multiPartUploadSize, columnGroups, storageConfig, storagePluginContext, true)
}
func NewPartialPackedRecordBatchWriter(
basePath string,
schema *schemapb.CollectionSchema,
bufferSize int64,
multiPartUploadSize int64,
columnGroups []storagecommon.ColumnGroup,
storageConfig *indexpb.StorageConfig,
storagePluginContext *indexcgopb.StoragePluginContext,
) (*packedRecordBatchWriter, error) {
return newPackedRecordBatchWriter(basePath, schema, bufferSize, multiPartUploadSize, columnGroups, storageConfig, storagePluginContext, false)
}
func newPackedRecordBatchWriter(
basePath string,
schema *schemapb.CollectionSchema,
bufferSize int64,
multiPartUploadSize int64,
columnGroups []storagecommon.ColumnGroup,
storageConfig *indexpb.StorageConfig,
storagePluginContext *indexcgopb.StoragePluginContext,
validatePK bool,
) (*packedRecordBatchWriter, error) {
if validatePK {
_, err := typeutil.GetPrimaryFieldSchema(schema)
if err != nil {
return nil, err
}
}
arrowSchema, err := ConvertToArrowSchema(schema, true)
internal/storagev2/packed/manifest_ffi.go
+36
View File
@@ -26,6 +26,7 @@ import "C"
import (
"fmt"
"strconv"
"unsafe"
"go.uber.org/zap"
@@ -174,6 +175,41 @@ func createColumnGroups(
return outColumnGroups, nil
}
func GetManifestFieldIDs(manifestPath string, storageConfig *indexpb.StorageConfig) (map[int64]struct{}, error) {
manifest, err := GetManifestHandle(manifestPath, storageConfig)
if err != nil {
return nil, err
}
defer C.loon_manifest_destroy(manifest)
fields := make(map[int64]struct{})
cgroups := &manifest.column_groups
if cgroups.column_group_array == nil && cgroups.num_of_column_groups > 0 {
return nil, fmt.Errorf("column_group_array is nil but num_of_column_groups is %d", cgroups.num_of_column_groups)
}
cgArray := unsafe.Slice(cgroups.column_group_array, int(cgroups.num_of_column_groups))
for i := range cgArray {
cg := &cgArray[i]
if cg.columns == nil {
continue
}
columns := unsafe.Slice(cg.columns, int(cg.num_of_columns))
for _, column := range columns {
if column == nil {
continue
}
columnName := C.GoString(column)
fieldID, err := strconv.ParseInt(columnName, 10, 64)
if err != nil {
return nil, fmt.Errorf("invalid manifest column name %q: %w", columnName, err)
}
fields[fieldID] = struct{}{}
}
}
return fields, nil
}
// ReadFragmentsFromManifest reads fragment info from a manifest path.
// This function wraps loon_exttable_read_manifest and walks its column groups.
//
internal/storagev2/packed/packed_writer_ffi.go
+23
View File
@@ -144,6 +144,10 @@ func NewFFIPackedWriter(basePath string, schema *arrow.Schema, columnGroups []st
err = HandleLoonFFIResult(result)
if err != nil {
if writerHandle != 0 {
C.loon_writer_destroy(writerHandle)
}
FreeProperties(cProperties)
return nil, err
}
@@ -164,6 +168,21 @@ func (pw *FFIPackedWriter) AsNewColumnGroups() *FFIPackedWriter {
return pw
}
// Destroy releases writer resources without committing pending output.
func (pw *FFIPackedWriter) Destroy() {
if pw == nil {
return
}
if pw.cWriterHandle != 0 {
C.loon_writer_destroy(pw.cWriterHandle)
pw.cWriterHandle = 0
}
if pw.cProperties != nil {
FreeProperties(pw.cProperties)
pw.cProperties = nil
}
}
func (pw *FFIPackedWriter) WriteRecordBatch(recordBatch arrow.Record) error {
var caa cdata.CArrowArray
var cas cdata.CArrowSchema
@@ -240,6 +259,10 @@ func (pw *FFIPackedWriter) Close() (WriterOutput, error) {
}
pw.closed = true
defer func() {
if pw.cWriterHandle != 0 {
C.loon_writer_destroy(pw.cWriterHandle)
pw.cWriterHandle = 0
}
if pw.cProperties != nil {
C.loon_properties_free(pw.cProperties)
pw.cProperties = nil
internal/storagev2/packed/packed_writer_ffi_test.go
+123
View File
@@ -16,6 +16,129 @@ import (
"github.com/milvus-io/milvus/pkg/v3/util/paramtable"
)
func TestFFIPackedWriterDestroyIsIdempotent(t *testing.T) {
var nilWriter *FFIPackedWriter
require.NotPanics(t, func() {
nilWriter.Destroy()
})
writer := &FFIPackedWriter{}
require.NotPanics(t, func() {
writer.Destroy()
writer.Destroy()
})
_, err := writer.Close()
require.Error(t, err)
}
func TestFFIPackedWriter_AsNewColumnGroupsAddsFields(t *testing.T) {
writer := &FFIPackedWriter{}
returned := writer.AsNewColumnGroups()
require.Same(t, writer, returned)
assert.True(t, writer.addNewColumnGroups)
}
func TestGetManifestFieldIDs_InvalidManifestPath(t *testing.T) {
fields, err := GetManifestFieldIDs("not-a-manifest-path", nil)
require.Error(t, err)
assert.Nil(t, fields)
}
func TestGetManifestFieldIDs_InvalidColumnName(t *testing.T) {
paramtable.Init()
pt := paramtable.Get()
pt.Save(pt.CommonCfg.StorageType.Key, "local")
pt.Save(pt.LocalStorageCfg.Path.Key, t.TempDir())
t.Cleanup(func() {
pt.Reset(pt.CommonCfg.StorageType.Key)
pt.Reset(pt.LocalStorageCfg.Path.Key)
})
schema := arrow.NewSchema([]arrow.Field{
{
Name: "bad_column",
Type: arrow.PrimitiveTypes.Int64,
Nullable: false,
Metadata: arrow.NewMetadata([]string{ArrowFieldIdMetadataKey}, []string{"100"}),
},
}, nil)
columnGroups := []storagecommon.ColumnGroup{{Columns: []int{0}, GroupID: storagecommon.DefaultShortColumnGroupID}}
basePath := "files/packed_writer_invalid_column/1"
cfg := CreateStorageConfig()
writer, err := NewFFIPackedWriter(basePath, schema, columnGroups, cfg, nil)
require.NoError(t, err)
builder := array.NewRecordBuilder(memory.DefaultAllocator, schema)
defer builder.Release()
builder.Field(0).(*array.Int64Builder).Append(1)
record := builder.NewRecord()
defer record.Release()
require.NoError(t, writer.WriteRecordBatch(record))
out, err := writer.Close()
require.NoError(t, err)
defer out.Destroy()
manifest, err := CommitManifestUpdates(basePath, ManifestEarliest, cfg, &ManifestUpdates{NewFiles: out})
require.NoError(t, err)
fields, err := GetManifestFieldIDs(manifest, cfg)
require.ErrorContains(t, err, "invalid manifest column name")
assert.Nil(t, fields)
}
func TestGetManifestFieldIDs_FromPackedWriterManifest(t *testing.T) {
paramtable.Init()
pt := paramtable.Get()
pt.Save(pt.CommonCfg.StorageType.Key, "local")
pt.Save(pt.LocalStorageCfg.Path.Key, t.TempDir())
t.Cleanup(func() {
pt.Reset(pt.CommonCfg.StorageType.Key)
pt.Reset(pt.LocalStorageCfg.Path.Key)
})
schema := arrow.NewSchema([]arrow.Field{
{
Name: "100",
Type: arrow.PrimitiveTypes.Int64,
Nullable: false,
Metadata: arrow.NewMetadata([]string{ArrowFieldIdMetadataKey}, []string{"100"}),
},
{
Name: "101",
Type: arrow.PrimitiveTypes.Int64,
Nullable: false,
Metadata: arrow.NewMetadata([]string{ArrowFieldIdMetadataKey}, []string{"101"}),
},
}, nil)
columnGroups := []storagecommon.ColumnGroup{{Columns: []int{0, 1}, GroupID: storagecommon.DefaultShortColumnGroupID}}
basePath := "files/packed_writer_field_ids/1"
cfg := CreateStorageConfig()
writer, err := NewFFIPackedWriter(basePath, schema, columnGroups, cfg, nil)
require.NoError(t, err)
builder := array.NewRecordBuilder(memory.DefaultAllocator, schema)
defer builder.Release()
builder.Field(0).(*array.Int64Builder).Append(1)
builder.Field(1).(*array.Int64Builder).Append(1000)
record := builder.NewRecord()
defer record.Release()
require.NoError(t, writer.WriteRecordBatch(record))
out, err := writer.Close()
require.NoError(t, err)
defer out.Destroy()
manifest, err := CommitManifestUpdates(basePath, ManifestEarliest, cfg, &ManifestUpdates{NewFiles: out})
require.NoError(t, err)
fields, err := GetManifestFieldIDs(manifest, cfg)
require.NoError(t, err)
assert.Contains(t, fields, int64(100))
assert.Contains(t, fields, int64(101))
_, err = writer.Close()
require.ErrorContains(t, err, "FFIPackedWriter already closed")
}
func TestPackedFFIWriter(t *testing.T) {
paramtable.Init()
pt := paramtable.Get()
pkg/proto/data_coord.proto
+1 -1
View File
@@ -680,7 +680,7 @@ enum CompactionType {
SortCompaction = 9;
PartitionKeySortCompaction = 10;
ClusteringPartitionKeySortCompaction = 11;
BackfillCompaction = 12;
BumpSchemaVersionCompaction = 12;
}
message CompactionStateRequest {
pkg/proto/datapb/data_coord.pb.go
+630 -629
View File
File diff suppressed because it is too large Load Diff
pkg/util/paramtable/component_param.go
+48 -38
View File
@@ -4973,26 +4973,27 @@ type dataCoordConfig struct {
CompactionTaskQueueCapacity ParamItem `refreshable:"false"`
CompactionPreAllocateIDExpansionFactor ParamItem `refreshable:"false"`
CompactionRPCTimeout ParamItem `refreshable:"true"`
CompactionMaxParallelTasks ParamItem `refreshable:"true"`
CompactionWorkerParallelTasks ParamItem `refreshable:"true"`
CompactionMaxFullSegmentThreshold ParamItem `refreshable:"true"`
CompactionForceMergeDataNodeMemoryFactor ParamItem `refreshable:"true"`
CompactionForceMergeQueryNodeMemoryFactor ParamItem `refreshable:"true"`
MinSegmentToMerge ParamItem `refreshable:"true"`
SegmentSmallProportion ParamItem `refreshable:"true"`
SegmentCompactableProportion ParamItem `refreshable:"true"`
SegmentExpansionRate ParamItem `refreshable:"true"`
CompactionTimeoutInSeconds ParamItem `refreshable:"true"` // deprecated
CompactionDropToleranceInSeconds ParamItem `refreshable:"true"`
CompactionGCIntervalInSeconds ParamItem `refreshable:"true"`
CompactionCheckIntervalInSeconds ParamItem `refreshable:"false"` // deprecated
CompactionScheduleInterval ParamItem `refreshable:"false"`
MixCompactionTriggerInterval ParamItem `refreshable:"false"`
L0CompactionTriggerInterval ParamItem `refreshable:"false"`
GlobalCompactionInterval ParamItem `refreshable:"false"`
CompactionExpiryTolerance ParamItem `refreshable:"true"`
BackfillCompactionTriggerInterval ParamItem `refreshable:"true"`
CompactionRPCTimeout ParamItem `refreshable:"true"`
CompactionMaxParallelTasks ParamItem `refreshable:"true"`
CompactionWorkerParallelTasks ParamItem `refreshable:"true"`
CompactionMaxFullSegmentThreshold ParamItem `refreshable:"true"`
CompactionForceMergeDataNodeMemoryFactor ParamItem `refreshable:"true"`
CompactionForceMergeQueryNodeMemoryFactor ParamItem `refreshable:"true"`
MinSegmentToMerge ParamItem `refreshable:"true"`
SegmentSmallProportion ParamItem `refreshable:"true"`
SegmentCompactableProportion ParamItem `refreshable:"true"`
SegmentExpansionRate ParamItem `refreshable:"true"`
CompactionTimeoutInSeconds ParamItem `refreshable:"true"` // deprecated
CompactionDropToleranceInSeconds ParamItem `refreshable:"true"`
CompactionGCIntervalInSeconds ParamItem `refreshable:"true"`
CompactionCheckIntervalInSeconds ParamItem `refreshable:"false"` // deprecated
CompactionScheduleInterval ParamItem `refreshable:"false"`
MixCompactionTriggerInterval ParamItem `refreshable:"false"`
L0CompactionTriggerInterval ParamItem `refreshable:"false"`
GlobalCompactionInterval ParamItem `refreshable:"false"`
CompactionExpiryTolerance ParamItem `refreshable:"true"`
BumpSchemaVersionCompactionEnabled ParamItem `refreshable:"true"`
BumpSchemaVersionCompactionTriggerInterval ParamItem `refreshable:"true"`
SingleCompactionRatioThreshold ParamItem `refreshable:"true"`
SingleCompactionDeltaLogMaxSize ParamItem `refreshable:"true"`
@@ -5097,14 +5098,14 @@ type dataCoordConfig struct {
GracefulStopTimeout ParamItem `refreshable:"true"`
ClusteringCompactionSlotUsage ParamItem `refreshable:"true"`
MixCompactionSlotUsage ParamItem `refreshable:"true"`
L0DeleteCompactionSlotUsage ParamItem `refreshable:"true"`
BackfillCompactionSlotUsage ParamItem `refreshable:"true"`
IndexTaskSlotUsage ParamItem `refreshable:"true"`
ScalarIndexTaskSlotUsage ParamItem `refreshable:"true"`
StatsTaskSlotUsage ParamItem `refreshable:"true"`
AnalyzeTaskSlotUsage ParamItem `refreshable:"true"`
ClusteringCompactionSlotUsage ParamItem `refreshable:"true"`
MixCompactionSlotUsage ParamItem `refreshable:"true"`
L0DeleteCompactionSlotUsage ParamItem `refreshable:"true"`
BumpSchemaVersionCompactionSlotUsage ParamItem `refreshable:"true"`
IndexTaskSlotUsage ParamItem `refreshable:"true"`
ScalarIndexTaskSlotUsage ParamItem `refreshable:"true"`
StatsTaskSlotUsage ParamItem `refreshable:"true"`
AnalyzeTaskSlotUsage ParamItem `refreshable:"true"`
EnableSortCompaction ParamItem `refreshable:"true"`
TaskCheckInterval ParamItem `refreshable:"true"`
@@ -5594,14 +5595,23 @@ During compaction, the size of segment # of rows is able to exceed segment max #
}
p.CompactionExpiryTolerance.Init(base.mgr)
p.BackfillCompactionTriggerInterval = ParamItem{
Key: "dataCoord.compaction.backfill.triggerInterval",
Version: "2.6.2",
Doc: "The time interval in seconds for trigger backfill compaction",
p.BumpSchemaVersionCompactionEnabled = ParamItem{
Key: "dataCoord.compaction.bumpSchemaVersion.enabled",
Version: "3.0.0",
DefaultValue: "false",
Doc: "Enable schema version bump compaction",
Export: true,
}
p.BumpSchemaVersionCompactionEnabled.Init(base.mgr)
p.BumpSchemaVersionCompactionTriggerInterval = ParamItem{
Key: "dataCoord.compaction.bumpSchemaVersion.triggerInterval",
Version: "3.0.0",
Doc: "The time interval in seconds for trigger schema bump compaction",
DefaultValue: "20",
Export: true,
}
p.BackfillCompactionTriggerInterval.Init(base.mgr)
p.BumpSchemaVersionCompactionTriggerInterval.Init(base.mgr)
p.MixCompactionTriggerInterval = ParamItem{
Key: "dataCoord.compaction.mix.triggerInterval",
@@ -6424,10 +6434,10 @@ if param targetScalarIndexVersion is not set, the default value is -1, which mea
}
p.L0DeleteCompactionSlotUsage.Init(base.mgr)
p.BackfillCompactionSlotUsage = ParamItem{
Key: "dataCoord.slot.backfillCompactionUsage",
Version: "2.6.9",
Doc: "slot usage of backfill compaction task.",
p.BumpSchemaVersionCompactionSlotUsage = ParamItem{
Key: "dataCoord.slot.bumpSchemaVersionCompactionUsage",
Version: "3.0.0",
Doc: "slot usage of schema bump compaction task.",
DefaultValue: "1",
PanicIfEmpty: false,
Export: true,
@@ -6439,7 +6449,7 @@ if param targetScalarIndexVersion is not set, the default value is -1, which mea
return strconv.Itoa(slot)
},
}
p.BackfillCompactionSlotUsage.Init(base.mgr)
p.BumpSchemaVersionCompactionSlotUsage.Init(base.mgr)
p.IndexTaskSlotUsage = ParamItem{
Key: "dataCoord.slot.indexTaskSlotUsage",
pkg/util/paramtable/component_param_test.go
+24
View File
@@ -32,6 +32,30 @@ func shouldPanic(t *testing.T, name string, f func()) {
t.Errorf("%s should have panicked", name)
}
func TestComponentParam_DataCoordBumpSchemaVersionCompactionParams(t *testing.T) {
Init()
params := Get()
params.Reset(params.DataCoordCfg.BumpSchemaVersionCompactionEnabled.Key)
params.Reset(params.DataCoordCfg.BumpSchemaVersionCompactionTriggerInterval.Key)
params.Reset(params.DataCoordCfg.BumpSchemaVersionCompactionSlotUsage.Key)
t.Cleanup(func() {
params.Reset(params.DataCoordCfg.BumpSchemaVersionCompactionEnabled.Key)
params.Reset(params.DataCoordCfg.BumpSchemaVersionCompactionTriggerInterval.Key)
params.Reset(params.DataCoordCfg.BumpSchemaVersionCompactionSlotUsage.Key)
})
assert.False(t, params.DataCoordCfg.BumpSchemaVersionCompactionEnabled.GetAsBool())
assert.Equal(t, time.Second*20, params.DataCoordCfg.BumpSchemaVersionCompactionTriggerInterval.GetAsDuration(time.Second))
assert.EqualValues(t, 1, params.DataCoordCfg.BumpSchemaVersionCompactionSlotUsage.GetAsInt64())
params.Save(params.DataCoordCfg.BumpSchemaVersionCompactionSlotUsage.Key, "5")
assert.EqualValues(t, 5, params.DataCoordCfg.BumpSchemaVersionCompactionSlotUsage.GetAsInt64())
params.Save(params.DataCoordCfg.BumpSchemaVersionCompactionSlotUsage.Key, "0")
assert.EqualValues(t, 1, params.DataCoordCfg.BumpSchemaVersionCompactionSlotUsage.GetAsInt64())
params.Save(params.DataCoordCfg.BumpSchemaVersionCompactionSlotUsage.Key, "-1")
assert.EqualValues(t, 1, params.DataCoordCfg.BumpSchemaVersionCompactionSlotUsage.GetAsInt64())
}
func TestComponentParam(t *testing.T) {
Init()
params := Get()
tests/python_client/base/client_v2_base.py
-88
View File
@@ -1590,94 +1590,6 @@ class TestMilvusClientV2Base(Base):
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
return res, check_result
def wait_schema_version_consistent(self, client, collection_name, timeout=30, poll_interval=0.01):
"""
Poll get_collection_stats until the schema version consistency gate would pass.
Background: After a previous schema-change DDL (AlterCollectionSchema or
AddCollectionField), DataCoord's backfill segment-version propagation runs on
a periodic tick. Until the tick fires, a back-to-back schema-change call is
rejected by the consistency gate at the Proxy / RootCoord with an error like
"schema version consistency check failed: N/M segments have caught up".
E2E tests that issue successive schema changes need to wait until the gate
would pass; this helper polls the same stats keys the gate reads.
Pass condition (mirrors checkSchemaVersionConsistency):
- both keys absent → schema version is 0, no backfill in progress → pass
- schema_version_consistent_segments == schema_version_total_segments → pass
Note: pymilvus MilvusClient.get_collection_stats returns a dict[str, str]
(with row_count converted to int), so we read the keys directly from the dict.
Args:
client: pymilvus MilvusClient
collection_name: target collection
timeout: max seconds to wait before giving up (default 30s)
poll_interval: sleep between polls in seconds (default 10ms)
Raises:
AssertionError on timeout, with the last observed counts for debugging.
"""
consistent_key = "schema_version_consistent_segments"
total_key = "schema_version_total_segments"
deadline = time.time() + timeout
last_consistent, last_total = None, None
while time.time() < deadline:
stats, _ = self.get_collection_stats(client, collection_name)
if not stats:
time.sleep(poll_interval)
continue
consistent = stats.get(consistent_key)
total = stats.get(total_key)
# Both absent → schema v0 path, gate passes trivially.
if consistent is None and total is None:
return
# Both present and equal → backfill caught up.
if consistent is not None and total is not None and int(consistent) == int(total):
return
last_consistent, last_total = consistent, total
time.sleep(poll_interval)
raise AssertionError(
f"wait_schema_version_consistent timed out after {timeout}s for collection "
f"{collection_name}: consistent={last_consistent}, total={last_total}"
)
def add_collection_field_wait_schema_version_consistency(
self,
client,
collection_name,
field_name,
data_type,
desc="",
timeout=None,
check_task=None,
check_items=None,
wait_timeout=30,
poll_interval=0.01,
**kwargs,
):
"""
Wrapper around add_collection_field that first waits for the schema-version
consistency gate to pass. Use this in E2E tests that issue successive
schema-change DDLs back-to-back, where the previous call's backfill
segment-version propagation tick may not have fired yet.
See wait_schema_version_consistent for the polling logic and pass conditions.
All add_collection_field arguments are forwarded unchanged.
"""
self.wait_schema_version_consistent(client, collection_name, timeout=wait_timeout, poll_interval=poll_interval)
return self.add_collection_field(
client,
collection_name,
field_name,
data_type,
desc=desc,
timeout=timeout,
check_task=check_task,
check_items=check_items,
**kwargs,
)
# ====================== Snapshot ======================
@trace()
def create_snapshot(
tests/python_client/milvus_client/test_add_field_feature.py
+6 -18
View File
@@ -71,8 +71,7 @@ class TestMilvusClientAddFieldFeature(TestMilvusClientV2Base):
is_clustering_key=True,
mmap_enabled=True,
)
# Wait for previous schema bump's backfill segment-version propagation tick to fire.
self.add_collection_field_wait_schema_version_consistency(
self.add_collection_field(
client,
collection_name,
field_name="field_new_var",
@@ -351,11 +350,8 @@ class TestMilvusClientAddFieldFeature(TestMilvusClientV2Base):
rows = cf.gen_row_data_by_schema(nb=ct.default_nb, schema=new_collection_info, start=ct.default_nb)
self.insert(client, collection_name, rows)
# 4. add one more vector field and index data
# Wait for the previous add_collection_field's backfill segment-version
# propagation tick to fire before the second schema change — otherwise
# the consistency gate at Proxy / RootCoord rejects this call.
new_vec_field_name_2 = "embeddings_2"
self.add_collection_field_wait_schema_version_consistency(
self.add_collection_field(
client,
collection_name,
field_name=new_vec_field_name_2,
@@ -1701,10 +1697,8 @@ class TestMilvusClientAddFieldFeatureInvalid(TestMilvusClientV2Base):
self.create_collection(client, collection_name, dim)
collections = self.list_collections(client)[0]
assert collection_name in collections
# Each loop iteration bumps the schema version; wait for the previous bump's
# backfill segment-version propagation tick to fire before the next call.
for i in range(62):
self.add_collection_field_wait_schema_version_consistency(
self.add_collection_field(
client,
collection_name,
field_name=f"{field_name}_{i}",
@@ -1712,9 +1706,7 @@ class TestMilvusClientAddFieldFeatureInvalid(TestMilvusClientV2Base):
nullable=True,
max_length=64,
)
# Final err_res call: the gate must be passed BEFORE this call so the error
# returned is the intended "max fields exceeded" error, not a consistency error.
self.add_collection_field_wait_schema_version_consistency(
self.add_collection_field(
client,
collection_name,
field_name=field_name,
@@ -1743,10 +1735,8 @@ class TestMilvusClientAddFieldFeatureInvalid(TestMilvusClientV2Base):
self.create_collection(client, collection_name, dim)
collections = self.list_collections(client)[0]
assert collection_name in collections
# Each loop iteration bumps the schema version; wait for the previous bump's
# backfill segment-version propagation tick to fire before the next call.
for i in range(ct.max_vector_field_num - 1):
self.add_collection_field_wait_schema_version_consistency(
self.add_collection_field(
client,
collection_name,
field_name=f"{field_name}_{i}",
@@ -1754,9 +1744,7 @@ class TestMilvusClientAddFieldFeatureInvalid(TestMilvusClientV2Base):
dim=dim,
nullable=True,
)
# Final err_res call: wait for consistency so the error we assert is the
# real "max vector fields exceeded" error, not a spurious consistency error.
self.add_collection_field_wait_schema_version_consistency(
self.add_collection_field(
client,
collection_name,
field_name=field_name,
tests/python_client/milvus_client/test_milvus_client_alter.py
+2 -4
View File
@@ -355,8 +355,7 @@ class TestMilvusClientAlterCollection(TestMilvusClientV2Base):
"pk_name": default_primary_key_field_name,
"limit": default_limit})
# 9. add new field same as dynamic field name
# Wait for step-3 add_collection_field's backfill segment-version propagation tick.
self.add_collection_field_wait_schema_version_consistency(
self.add_collection_field(
client, collection_name, field_name=default_dynamic_field_name,
data_type=DataType.INT64, nullable=True, default_value=default_value)
# 10. query using filter with dynamic field and new field
@@ -703,8 +702,7 @@ class TestMilvusClientAlterCollectionField(TestMilvusClientV2Base):
field_params={"nullable": False},
check_task=CheckTasks.err_res, check_items=error)
# add a nullable varchar field to the collection
# Wait for previous add_collection_field's backfill segment-version propagation tick.
self.add_collection_field_wait_schema_version_consistency(
self.add_collection_field(
client, collection_name, field_name="varchar_3",
data_type=DataType.VARCHAR, max_length=64, nullable=True)
# try to alert the new added nullable varchar field to non-nullable varchar field
tests/python_client/milvus_client/test_milvus_client_alter_warmup.py
+1 -4
View File
@@ -3012,10 +3012,7 @@ class TestMilvusClientWarmupAsync(TestMilvusClientV2Base):
DataType.VARCHAR, max_length=1024,
nullable=True, warmup="async",
enable_analyzer=True, enable_match=True)
# Wait for the previous AddCollectionField's backfill segment-version propagation
# tick to fire before issuing the next one — otherwise the consistency gate at
# the Proxy / RootCoord rejects this call with "schema version consistency check failed".
self.add_collection_field_wait_schema_version_consistency(
self.add_collection_field(
client, collection_name, "json_1",
DataType.JSON, nullable=True, warmup="async")
tests/python_client/milvus_client/test_milvus_client_collection.py
+1 -2
View File
@@ -937,8 +937,7 @@ class TestMilvusClientCollectionValid(TestMilvusClientV2Base):
nullable=True,
is_cluster_key=True,
)
# Wait for previous schema bump's backfill segment-version propagation tick.
self.add_collection_field_wait_schema_version_consistency(
self.add_collection_field(
client,
collection_name,
field_name="field_new_var",
tests/python_client/milvus_client/test_milvus_client_index.py
+1 -2
View File
@@ -411,8 +411,7 @@ class TestMilvusClientIndexValid(TestMilvusClientV2Base):
if add_field:
self.add_collection_field(client, collection_name, field_name="field_int", data_type=DataType.INT32,
nullable=True)
# Wait for previous schema bump's backfill segment-version propagation tick.
self.add_collection_field_wait_schema_version_consistency(
self.add_collection_field(
client, collection_name, field_name="field_varchar", data_type=DataType.VARCHAR,
nullable=True, max_length=64)
self.release_collection(client, collection_name)