fix: Use actual data timestamps for imported segment positions (#47276)

This PR:
1. Modifies the import mechanism to use actual timestamps from the
imported data for segment positions instead of using the channel
checkpoint.
2. Fix writeDelta() to extract actual min/max timestamps from
deltaData.Tss instead of using pack.tsFrom/tsTo.
3. Integration test TestBinlogImport updated to verify L1 and L0 segment
positions.

issue: https://github.com/milvus-io/milvus/issues/47275

---------

Signed-off-by: bigsheeper <yihao.dai@zilliz.com>
This commit is contained in:
yihao.dai
2026-01-28 14:13:32 +08:00
committed by GitHub
parent d81e6d3996
commit cc18864e62
10 changed files with 324 additions and 35 deletions
+6 -16
View File
@@ -442,7 +442,7 @@ func (c *importChecker) checkIndexBuildingJob(job ImportJob) {
metrics.ImportJobLatency.WithLabelValues(metrics.ImportStageWaitL0Import).Observe(float64(buildIndexDuration.Milliseconds()))
log.Info("import job l0 import done", zap.Duration("jobTimeCost/l0Import", waitL0ImportDuration))
if c.updateSegmentState(originSegmentIDs, statsSegmentIDs) {
if c.unsetSegmentImporting(originSegmentIDs, statsSegmentIDs) {
return
}
// all finished, update import job state to `Completed`.
@@ -475,7 +475,8 @@ func (c *importChecker) waitL0ImortTaskDone(job ImportJob) bool {
return false
}
func (c *importChecker) updateSegmentState(originSegmentIDs, statsSegmentIDs []int64) bool {
// unsetSegmentImporting unsets the isImporting flag for segments.
func (c *importChecker) unsetSegmentImporting(originSegmentIDs, statsSegmentIDs []int64) bool {
// Here, all segment indexes have been successfully built, try unset isImporting flag for all segments.
isImportingSegments := lo.Filter(append(originSegmentIDs, statsSegmentIDs...), func(segmentID int64, _ int) bool {
segment := c.meta.GetSegment(c.ctx, segmentID)
@@ -485,21 +486,10 @@ func (c *importChecker) updateSegmentState(originSegmentIDs, statsSegmentIDs []i
}
return segment.GetIsImporting()
})
channels, err := c.meta.GetSegmentsChannels(isImportingSegments)
if err != nil {
log.Warn("get segments channels failed", zap.Error(err))
return true
}
for _, segmentID := range isImportingSegments {
channelCP := c.meta.GetChannelCheckpoint(channels[segmentID])
if channelCP == nil {
log.Warn("nil channel checkpoint")
return true
}
op1 := UpdateStartPosition([]*datapb.SegmentStartPosition{{StartPosition: channelCP, SegmentID: segmentID}})
op2 := UpdateDmlPosition(segmentID, channelCP)
op3 := UpdateIsImporting(segmentID, false)
err = c.meta.UpdateSegmentsInfo(c.ctx, op1, op2, op3)
op := UpdateIsImporting(segmentID, false)
err := c.meta.UpdateSegmentsInfo(c.ctx, op)
if err != nil {
log.Warn("update import segment failed", zap.Error(err))
return true
+41 -4
View File
@@ -18,6 +18,7 @@ package datacoord
import (
"context"
"math"
"strconv"
"time"
@@ -29,6 +30,7 @@ import (
"github.com/milvus-io/milvus/internal/datacoord/session"
"github.com/milvus-io/milvus/internal/json"
"github.com/milvus-io/milvus/internal/metastore/kv/binlog"
"github.com/milvus-io/milvus/internal/util/importutilv2"
"github.com/milvus-io/milvus/pkg/v2/log"
"github.com/milvus-io/milvus/pkg/v2/metrics"
"github.com/milvus-io/milvus/pkg/v2/proto/datapb"
@@ -206,6 +208,7 @@ func (t *importTask) QueryTaskOnWorker(cluster session.Cluster) {
}
if resp.GetState() == datapb.ImportTaskStateV2_Completed {
totalRows := int64(0)
job := t.importMeta.GetJob(context.TODO(), t.GetJobID())
for _, info := range resp.GetImportSegmentsInfo() {
// try to parse path and fill logID
err = binlog.CompressBinLogs(info.GetBinlogs(), info.GetDeltalogs(), info.GetStatslogs(), info.GetBm25Logs())
@@ -214,10 +217,21 @@ func (t *importTask) QueryTaskOnWorker(cluster session.Cluster) {
WrapTaskLog(t, zap.Int64("segmentID", info.GetSegmentID()), zap.Error(err))...)
return
}
op1 := UpdateBinlogsOperator(info.GetSegmentID(), info.GetBinlogs(), info.GetStatslogs(), info.GetDeltalogs(), info.GetBm25Logs())
// Extract actual timestamps from binlogs for segment positions
var minTs, maxTs uint64
isL0Import := importutilv2.IsL0Import(job.GetOptions())
if isL0Import {
minTs, maxTs = extractTimestampFromBinlogs(info.GetDeltalogs())
} else {
minTs, maxTs = extractTimestampFromBinlogs(info.GetBinlogs())
}
opBinlog := UpdateBinlogsOperator(info.GetSegmentID(), info.GetBinlogs(), info.GetStatslogs(), info.GetDeltalogs(), info.GetBm25Logs())
opManifest := UpdateManifest(info.GetSegmentID(), info.GetManifestPath())
op2 := UpdateStatusOperator(info.GetSegmentID(), commonpb.SegmentState_Flushed)
err = t.meta.UpdateSegmentsInfo(context.TODO(), op1, opManifest, op2)
opState := UpdateStatusOperator(info.GetSegmentID(), commonpb.SegmentState_Flushed)
opPosition := UpdateImportSegmentPosition(info.GetSegmentID(), minTs, maxTs)
err = t.meta.UpdateSegmentsInfo(context.TODO(), opBinlog, opManifest, opState, opPosition)
if err != nil {
updateErr := t.importMeta.UpdateJob(context.TODO(), t.GetJobID(), UpdateJobState(internalpb.ImportJobState_Failed), UpdateJobReason(err.Error()))
if updateErr != nil {
@@ -226,7 +240,11 @@ func (t *importTask) QueryTaskOnWorker(cluster session.Cluster) {
log.Warn("update import segment binlogs failed", WrapTaskLog(t, zap.String("err", err.Error()))...)
return
}
log.Info("update import segment info done", WrapTaskLog(t, zap.Int64("segmentID", info.GetSegmentID()), zap.Any("segmentInfo", info))...)
log.Info("update import segment info done", WrapTaskLog(t,
zap.Int64("segmentID", info.GetSegmentID()),
zap.Uint64("minTs", minTs),
zap.Uint64("maxTs", maxTs),
zap.Any("segmentInfo", info))...)
totalRows += info.GetImportedRows()
}
completeTime := time.Now().Format("2006-01-02T15:04:05Z07:00")
@@ -286,3 +304,22 @@ func (t *importTask) MarshalJSON() ([]byte, error) {
}
return json.Marshal(importTask)
}
// extractTimestampFromBinlogs extracts min and max timestamps from binlogs.
// The timestamps are stored in Binlog.TimestampFrom and Binlog.TimestampTo
// by BulkPackWriterV2.writeInserts() during import sync.
func extractTimestampFromBinlogs(binlogs []*datapb.FieldBinlog) (minTs, maxTs uint64) {
minTs = math.MaxUint64
maxTs = 0
for _, fieldBinlog := range binlogs {
for _, binlog := range fieldBinlog.GetBinlogs() {
if binlog.GetTimestampFrom() < minTs {
minTs = binlog.GetTimestampFrom()
}
if binlog.GetTimestampTo() > maxTs {
maxTs = binlog.GetTimestampTo()
}
}
}
return minTs, maxTs
}
@@ -18,6 +18,7 @@ package datacoord
import (
"context"
"math"
"testing"
"time"
@@ -413,10 +414,19 @@ func TestImportTask_QueryTaskOnWorker(t *testing.T) {
catalog.EXPECT().ListPreImportTasks(mock.Anything).Return(nil, nil)
catalog.EXPECT().ListImportTasks(mock.Anything).Return(nil, nil)
catalog.EXPECT().SaveImportTask(mock.Anything, mock.Anything).Return(nil)
catalog.EXPECT().SaveImportJob(mock.Anything, mock.Anything).Return(nil)
im, err := NewImportMeta(context.TODO(), catalog, nil, nil)
assert.NoError(t, err)
var job ImportJob = &importJob{
ImportJob: &datapb.ImportJob{
JobID: 1,
},
}
err = im.AddJob(context.TODO(), job)
assert.NoError(t, err)
taskProto := &datapb.ImportTaskV2{
JobID: 1,
TaskID: 2,
@@ -476,6 +486,101 @@ func TestImportTask_QueryTaskOnWorker(t *testing.T) {
})
}
func TestExtractTimestampFromBinlogs(t *testing.T) {
t.Run("empty binlogs", func(t *testing.T) {
minTs, maxTs := extractTimestampFromBinlogs(nil)
assert.Equal(t, uint64(math.MaxUint64), minTs)
assert.Equal(t, uint64(0), maxTs)
})
t.Run("empty field binlogs", func(t *testing.T) {
binlogs := []*datapb.FieldBinlog{}
minTs, maxTs := extractTimestampFromBinlogs(binlogs)
assert.Equal(t, uint64(math.MaxUint64), minTs)
assert.Equal(t, uint64(0), maxTs)
})
t.Run("single binlog", func(t *testing.T) {
binlogs := []*datapb.FieldBinlog{
{
FieldID: 100,
Binlogs: []*datapb.Binlog{
{
TimestampFrom: 1000,
TimestampTo: 2000,
},
},
},
}
minTs, maxTs := extractTimestampFromBinlogs(binlogs)
assert.Equal(t, uint64(1000), minTs)
assert.Equal(t, uint64(2000), maxTs)
})
t.Run("multiple binlogs in single field", func(t *testing.T) {
binlogs := []*datapb.FieldBinlog{
{
FieldID: 100,
Binlogs: []*datapb.Binlog{
{
TimestampFrom: 1000,
TimestampTo: 2000,
},
{
TimestampFrom: 500,
TimestampTo: 3000,
},
},
},
}
minTs, maxTs := extractTimestampFromBinlogs(binlogs)
assert.Equal(t, uint64(500), minTs)
assert.Equal(t, uint64(3000), maxTs)
})
t.Run("multiple fields with multiple binlogs", func(t *testing.T) {
binlogs := []*datapb.FieldBinlog{
{
FieldID: 100,
Binlogs: []*datapb.Binlog{
{
TimestampFrom: 1000,
TimestampTo: 2000,
},
},
},
{
FieldID: 101,
Binlogs: []*datapb.Binlog{
{
TimestampFrom: 500,
TimestampTo: 1500,
},
{
TimestampFrom: 800,
TimestampTo: 3000,
},
},
},
}
minTs, maxTs := extractTimestampFromBinlogs(binlogs)
assert.Equal(t, uint64(500), minTs)
assert.Equal(t, uint64(3000), maxTs)
})
t.Run("field binlog with no binlogs inside", func(t *testing.T) {
binlogs := []*datapb.FieldBinlog{
{
FieldID: 100,
Binlogs: []*datapb.Binlog{},
},
}
minTs, maxTs := extractTimestampFromBinlogs(binlogs)
assert.Equal(t, uint64(math.MaxUint64), minTs)
assert.Equal(t, uint64(0), maxTs)
})
}
func TestImportTask_DropTaskOnWorker(t *testing.T) {
t.Run("DropImport rpc failed", func(t *testing.T) {
catalog := mocks.NewDataCoordCatalog(t)
-7
View File
@@ -246,11 +246,6 @@ func AllocImportSegment(ctx context.Context,
return nil, err
}
}
position := &msgpb.MsgPosition{
ChannelName: channelName,
MsgID: nil,
Timestamp: ts,
}
segmentInfo := &datapb.SegmentInfo{
ID: id,
@@ -262,8 +257,6 @@ func AllocImportSegment(ctx context.Context,
MaxRowNum: 0,
Level: level,
LastExpireTime: math.MaxUint64,
StartPosition: position,
DmlPosition: position,
StorageVersion: storageVersion,
}
segmentInfo.IsImporting = true
+29
View File
@@ -1262,6 +1262,35 @@ func UpdateIsImporting(segmentID int64, isImporting bool) UpdateOperator {
}
}
// UpdateImportSegmentPosition updates the segment's StartPosition and DmlPosition
// for import segments using actual timestamps from the imported data.
// Unlike UpdateStartPosition/UpdateDmlPosition, this operator allows nil MsgID
// since import segments don't have message queue positions.
func UpdateImportSegmentPosition(segmentID int64, minTs, maxTs uint64) UpdateOperator {
return func(modPack *updateSegmentPack) bool {
segment := modPack.Get(segmentID)
if segment == nil {
log.Ctx(context.TODO()).Warn("meta update: update import segment position failed - segment not found",
zap.Int64("segmentID", segmentID))
return false
}
channelName := segment.GetInsertChannel()
// Use actual min timestamp for StartPosition
segment.StartPosition = &msgpb.MsgPosition{
ChannelName: channelName,
MsgID: nil,
Timestamp: minTs,
}
// Use actual max timestamp for DmlPosition
segment.DmlPosition = &msgpb.MsgPosition{
ChannelName: channelName,
MsgID: nil,
Timestamp: maxTs,
}
return true
}
}
// UpdateAsDroppedIfEmptyWhenFlushing updates segment state to Dropped if segment is empty and in Flushing state
// It's used to make a empty flushing segment to be dropped directly.
func UpdateAsDroppedIfEmptyWhenFlushing(segmentID int64) UpdateOperator {
@@ -19,6 +19,7 @@ package datacoord
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/milvus-io/milvus/pkg/v2/proto/datapb"
@@ -47,3 +48,69 @@ func (s *TestSegmentOperatorSuite) TestSetMaxRowCount() {
func TestSegmentOperators(t *testing.T) {
suite.Run(t, new(TestSegmentOperatorSuite))
}
func TestUpdateImportSegmentPosition(t *testing.T) {
t.Run("segment not found", func(t *testing.T) {
// Create a meta with empty segments to properly test the "not found" case
segments := NewSegmentsInfo()
m := &meta{segments: segments}
modPack := &updateSegmentPack{
meta: m,
segments: make(map[int64]*SegmentInfo),
}
op := UpdateImportSegmentPosition(100, 1000, 2000)
result := op(modPack)
assert.False(t, result)
})
t.Run("update position successfully", func(t *testing.T) {
segment := &SegmentInfo{
SegmentInfo: &datapb.SegmentInfo{
ID: 100,
InsertChannel: "test_channel",
},
}
modPack := &updateSegmentPack{
segments: map[int64]*SegmentInfo{
100: segment,
},
}
op := UpdateImportSegmentPosition(100, 1000, 2000)
result := op(modPack)
assert.True(t, result)
// Verify StartPosition
assert.NotNil(t, segment.GetStartPosition())
assert.Equal(t, "test_channel", segment.GetStartPosition().GetChannelName())
assert.Nil(t, segment.GetStartPosition().GetMsgID())
assert.Equal(t, uint64(1000), segment.GetStartPosition().GetTimestamp())
// Verify DmlPosition
assert.NotNil(t, segment.GetDmlPosition())
assert.Equal(t, "test_channel", segment.GetDmlPosition().GetChannelName())
assert.Nil(t, segment.GetDmlPosition().GetMsgID())
assert.Equal(t, uint64(2000), segment.GetDmlPosition().GetTimestamp())
})
t.Run("update position with zero timestamps", func(t *testing.T) {
segment := &SegmentInfo{
SegmentInfo: &datapb.SegmentInfo{
ID: 101,
InsertChannel: "channel_2",
},
}
modPack := &updateSegmentPack{
segments: map[int64]*SegmentInfo{
101: segment,
},
}
op := UpdateImportSegmentPosition(101, 0, 0)
result := op(modPack)
assert.True(t, result)
assert.NotNil(t, segment.GetStartPosition())
assert.Equal(t, uint64(0), segment.GetStartPosition().GetTimestamp())
assert.NotNil(t, segment.GetDmlPosition())
assert.Equal(t, uint64(0), segment.GetDmlPosition().GetTimestamp())
})
}
-1
View File
@@ -102,7 +102,6 @@ func NewSyncTask(ctx context.Context,
WithPartitionID(partitionID).
WithChannelName(vchannel).
WithSegmentID(segmentID).
WithTimeRange(ts, ts).
WithLevel(segmentLevel).
WithDataSource(metrics.BulkinsertDataSourceLabel).
WithBatchRows(int64(insertData.GetRowNum()))
+14 -3
View File
@@ -19,6 +19,7 @@ package syncmgr
import (
"context"
"fmt"
"math"
"path"
"github.com/apache/arrow/go/v17/arrow"
@@ -330,6 +331,9 @@ func (bw *BulkPackWriter) writeDelta(ctx context.Context, pack *SyncPack) (*data
defer pkBuilder.Release()
defer tsBuilder.Release()
// Extract min/max timestamps from delete data
var tsFrom uint64 = math.MaxUint64
var tsTo uint64 = 0
for i := int64(0); i < pack.deltaData.RowCount; i++ {
switch pkField.DataType {
case schemapb.DataType_Int64:
@@ -339,7 +343,14 @@ func (bw *BulkPackWriter) writeDelta(ctx context.Context, pack *SyncPack) (*data
default:
return nil, fmt.Errorf("unexpected pk type %v", pkField.DataType)
}
tsBuilder.(*array.Int64Builder).Append(int64(pack.deltaData.Tss[i]))
ts := pack.deltaData.Tss[i]
tsBuilder.(*array.Int64Builder).Append(int64(ts))
if ts < tsFrom {
tsFrom = ts
}
if ts > tsTo {
tsTo = ts
}
}
pkArray := pkBuilder.NewArray()
@@ -362,8 +373,8 @@ func (bw *BulkPackWriter) writeDelta(ctx context.Context, pack *SyncPack) (*data
deltalog := &datapb.Binlog{
EntriesNum: pack.deltaData.RowCount,
TimestampFrom: pack.tsFrom,
TimestampTo: pack.tsTo,
TimestampFrom: tsFrom,
TimestampTo: tsTo,
LogPath: path,
LogSize: pack.deltaData.Size() / 4, // Not used
MemorySize: pack.deltaData.Size(),
@@ -121,10 +121,12 @@ func TestBulkPackWriter_Write(t *testing.T) {
FieldID: 100,
Binlogs: []*datapb.Binlog{
{
EntriesNum: 10,
LogPath: "files/delta_log/123/456/789/10000",
LogSize: 60,
MemorySize: 240,
EntriesNum: 10,
TimestampFrom: 100,
TimestampTo: 109,
LogPath: "files/delta_log/123/456/789/10000",
LogSize: 60,
MemorySize: 240,
},
},
},
+56
View File
@@ -19,6 +19,7 @@ package importv2
import (
"context"
"fmt"
"math"
"strconv"
"strings"
"time"
@@ -56,6 +57,12 @@ type SourceCollectionInfo struct {
SegmentIDs []int64
insertedIDs *schemapb.IDs
storageVersion int
// Timestamp ranges extracted from source segments' binlogs,
// there's only one L1 segment and one L0 segment after import so we can record the min and max timestamps directly.
l1MinTs uint64 // min timestamp from L1 segments' insert binlogs
l1MaxTs uint64 // max timestamp from L1 segments' insert binlogs
l0MinTs uint64 // min timestamp from L0 segments' delta binlogs
l0MaxTs uint64 // max timestamp from L0 segments' delta binlogs
}
func (s *BulkInsertSuite) PrepareSourceCollection(dim int, dmlGroup *DMLGroup) *SourceCollectionInfo {
@@ -201,6 +208,37 @@ func (s *BulkInsertSuite) PrepareSourceCollection(dim int, dmlGroup *DMLGroup) *
s.True(len(l0Segments) > 0)
}
// Extract timestamp ranges from source segments' binlogs
var l1MinTs uint64 = math.MaxUint64
var l1MaxTs uint64 = 0
for _, segment := range segments {
for _, fieldBinlog := range segment.GetBinlogs() {
for _, binlog := range fieldBinlog.GetBinlogs() {
if binlog.GetTimestampFrom() < l1MinTs {
l1MinTs = binlog.GetTimestampFrom()
}
if binlog.GetTimestampTo() > l1MaxTs {
l1MaxTs = binlog.GetTimestampTo()
}
}
}
}
var l0MinTs uint64 = math.MaxUint64
var l0MaxTs uint64 = 0
for _, segment := range l0Segments {
for _, fieldBinlog := range segment.GetDeltalogs() {
for _, binlog := range fieldBinlog.GetBinlogs() {
if binlog.GetTimestampFrom() < l0MinTs {
l0MinTs = binlog.GetTimestampFrom()
}
if binlog.GetTimestampTo() > l0MaxTs {
l0MaxTs = binlog.GetTimestampTo()
}
}
}
}
// search
expr := fmt.Sprintf("%s > 0", integration.Int64Field)
nq := 10
@@ -264,6 +302,10 @@ func (s *BulkInsertSuite) PrepareSourceCollection(dim int, dmlGroup *DMLGroup) *
}),
insertedIDs: totalInsertedIDs,
storageVersion: int(storage.StorageV2),
l1MinTs: l1MinTs,
l1MaxTs: l1MaxTs,
l0MinTs: l0MinTs,
l0MaxTs: l0MaxTs,
}
}
@@ -366,6 +408,13 @@ func (s *BulkInsertSuite) runBinlogTest(dmlGroup *DMLGroup) {
s.True(len(segment.GetStatslogs()) > 0)
s.NoError(CheckLogID(segment.GetStatslogs()))
// Verify L1 segment positions match actual timestamps from source collection
s.Equal(sourceCollectionInfo.l1MinTs, segment.GetStartPosition().GetTimestamp(),
"L1 segment StartPosition should match actual min timestamp from source binlogs")
s.Equal(sourceCollectionInfo.l1MaxTs, segment.GetDmlPosition().GetTimestamp(),
"L1 segment DmlPosition should match actual max timestamp from source binlogs")
log.Info("L1 segment position verification passed")
// l0 import
if totalDeleteRowNum > 0 {
files = make([]*internalpb.ImportFile, 0)
@@ -403,6 +452,13 @@ func (s *BulkInsertSuite) runBinlogTest(dmlGroup *DMLGroup) {
s.True(len(segment.GetDeltalogs()) > 0)
s.NoError(CheckLogID(segment.GetDeltalogs()))
s.True(len(segment.GetStatslogs()) == 0)
// Verify L0 segment positions match actual timestamps from source collection
s.Equal(sourceCollectionInfo.l0MinTs, segment.GetStartPosition().GetTimestamp(),
"L0 segment StartPosition should match actual min timestamp from source deltalogs")
s.Equal(sourceCollectionInfo.l0MaxTs, segment.GetDmlPosition().GetTimestamp(),
"L0 segment DmlPosition should match actual max timestamp from source deltalogs")
log.Info("L0 segment position verification passed")
}
// load