fix: gate growing-source flush on StorageV3 (#51410)

issue: #51250
Keep TEXT segments on StorageV3 while making growing-source flush an
explicit allowed mode instead of a mandatory TEXT path.

Propagate create-segment storage versions through WAL flusher and write
buffer, reject storage-version mismatches in DataCoord, and keep raw
segcore chunks sticky for segments that may be flushed from growing
source.

Signed-off-by: luzhang <luzhang@zilliz.com>
Co-authored-by: luzhang <luzhang@zilliz.com>
This commit is contained in:
zhagnlu
2026-07-17 10:22:39 +08:00
committed by GitHub
co-authored by luzhang
parent f662cec16d
commit 9ffdaee294
32 changed files with 627 additions and 235 deletions
@@ -393,9 +393,7 @@ SegmentGrowingImpl::mask_with_delete(BitsetTypeView& bitset,
void
SegmentGrowingImpl::try_remove_chunks(FieldId fieldId) {
if (segcore_config_.get_storage_v3_enabled() &&
(SchemaHasTextField(*schema_) ||
segcore_config_.get_enable_growing_source_flush())) {
if (retain_insert_record_chunks_for_flush_) {
// StorageV3 TEXT and growing-source flush persist growing segments
// through milvus-storage. Interim indexes may also contain raw vector
// data, but FlushGrowingSegmentData reads directly from insert_record_
@@ -431,6 +431,9 @@ class SegmentGrowingImpl : public SegmentGrowing {
segcore_config_(segcore_config),
schema_(std::move(schema)),
index_meta_(indexMeta),
retain_insert_record_chunks_for_flush_(
ShouldRetainInsertRecordChunksForFlush(*schema_,
segcore_config_)),
insert_record_(
*schema_, segcore_config.get_chunk_rows(), mmap_descriptor_),
indexing_record_(
@@ -838,10 +841,30 @@ class SegmentGrowingImpl : public SegmentGrowing {
InitializeArrayOffsets();
private:
static bool
ShouldRetainInsertRecordChunksForFlush(
const Schema& schema, const SegcoreConfig& segcore_config) {
if (!segcore_config.get_storage_v3_enabled()) {
return false;
}
if (segcore_config.get_enable_growing_source_flush()) {
return true;
}
return std::any_of(schema.get_fields().begin(),
schema.get_fields().end(),
[](const auto& field) {
return field.second.get_data_type() ==
DataType::TEXT;
});
}
storage::MmapChunkDescriptorPtr mmap_descriptor_ = nullptr;
SegcoreConfig segcore_config_;
SchemaPtr schema_;
IndexMetaPtr index_meta_;
// Segment-level sticky decision. Once a growing segment is created, raw
// chunks must not start/stop being retained because global config changes.
bool retain_insert_record_chunks_for_flush_;
// inserted fields data and row_ids, timestamps
InsertRecord<false> insert_record_;
@@ -369,6 +369,75 @@ TEST_P(GrowingIndexTest, Correctness) {
}
}
TEST(GrowingIndex, RetainInsertRecordChunksForFlushIsSegmentSticky) {
constexpr int64_t dim = 4;
constexpr int64_t row_count = 100;
auto schema = std::make_shared<Schema>();
auto pk = schema->AddDebugField("pk", DataType::INT64);
auto vec = schema->AddDebugField(
"embeddings", DataType::VECTOR_FLOAT, dim, knowhere::metric::L2);
schema->set_primary_field_id(pk);
std::map<std::string, std::string> index_params = {
{"index_type", knowhere::IndexEnum::INDEX_FAISS_IVFFLAT},
{"metric_type", knowhere::metric::L2},
{"nlist", "1"}};
std::map<std::string, std::string> type_params = {
{"dim", std::to_string(dim)}};
FieldIndexMeta field_index_meta(
vec, std::move(index_params), std::move(type_params));
std::map<FieldId, FieldIndexMeta> field_map = {{vec, field_index_meta}};
IndexMetaPtr meta =
std::make_shared<CollectionIndexMeta>(100, std::move(field_map));
auto& config = SegcoreConfig::default_config();
ScopedSegcoreConfigRestore config_restore(config);
InterimIndexConfigForTest interim_config;
interim_config.chunk_rows = 16;
interim_config.nlist = 1;
interim_config.nprobe = 1;
interim_config.dense_vector_interim_index_type =
knowhere::IndexEnum::INDEX_FAISS_IVFFLAT_CC;
interim_config.sub_dim = dim;
interim_config.refine_ratio = 1.0F;
interim_config.refine_quant_type = "NONE";
interim_config.refine_with_quant_flag = false;
ApplyInterimIndexConfigForTest(interim_config, config);
config.set_storage_v3_enabled(true);
config.set_enable_growing_source_flush(true);
auto retained_segment = CreateGrowingSegment(schema, meta, 1, config);
auto* retained_impl =
dynamic_cast<SegmentGrowingImpl*>(retained_segment.get());
ASSERT_NE(retained_impl, nullptr);
config.set_enable_growing_source_flush(false);
auto insert = [&](SegmentGrowing* segment) {
auto dataset = DataGen(schema, row_count);
auto offset = segment->PreInsert(row_count);
segment->Insert(offset,
row_count,
dataset.row_ids_.data(),
dataset.timestamps_.data(),
dataset.raw_);
};
insert(retained_segment.get());
EXPECT_GT(
retained_impl->get_insert_record().get_data_base(vec)->num_chunk(), 0);
auto non_retained_segment = CreateGrowingSegment(schema, meta, 2, config);
auto* non_retained_impl =
dynamic_cast<SegmentGrowingImpl*>(non_retained_segment.get());
ASSERT_NE(non_retained_impl, nullptr);
insert(non_retained_segment.get());
EXPECT_EQ(
non_retained_impl->get_insert_record().get_data_base(vec)->num_chunk(),
0);
}
TEST_P(GrowingIndexTest, AddWithoutBuildPool) {
constexpr int N = 1024;
constexpr int dim = 4;
@@ -46,6 +46,9 @@ class ScopedSegcoreConfigRestore {
nprobe_(config.get_nprobe()),
enable_interim_segment_index_(
config.get_enable_interim_segment_index()),
storage_v3_enabled_(config.get_storage_v3_enabled()),
enable_growing_source_flush_(
config.get_enable_growing_source_flush()),
sub_dim_(config.get_sub_dim()),
refine_ratio_(config.get_refine_ratio()),
dense_vector_interim_index_type_(
@@ -60,6 +63,8 @@ class ScopedSegcoreConfigRestore {
config_.set_nlist(nlist_);
config_.set_nprobe(nprobe_);
config_.set_enable_interim_segment_index(enable_interim_segment_index_);
config_.set_storage_v3_enabled(storage_v3_enabled_);
config_.set_enable_growing_source_flush(enable_growing_source_flush_);
config_.set_sub_dim(sub_dim_);
config_.set_refine_ratio(refine_ratio_);
config_.set_dense_vector_intermin_index_type(
@@ -78,6 +83,8 @@ class ScopedSegcoreConfigRestore {
int64_t nlist_;
int64_t nprobe_;
bool enable_interim_segment_index_;
bool storage_v3_enabled_;
bool enable_growing_source_flush_;
int64_t sub_dim_;
float refine_ratio_;
std::string dense_vector_interim_index_type_;
+19
View File
@@ -1058,6 +1058,25 @@ func SetStorageVersion(segmentID int64, version int64) UpdateOperator {
}
}
func ValidateSaveBinlogStorageVersion(segmentID int64, incoming int64) UpdateOperator {
return func(modPack *updateSegmentPack) bool {
segment := modPack.Get(segmentID)
if segment == nil {
modPack.err = merr.WrapErrSegmentNotFound(segmentID)
return false
}
current := segment.GetStorageVersion()
if incoming != current {
modPack.err = merr.WrapErrDataIntegrityMsg(
"segment %d storage version mismatch, current=%d incoming=%d",
segmentID, current, incoming)
return false
}
return true
}
}
func UpdateCompactedOperator(segmentID int64) UpdateOperator {
return func(modPack *updateSegmentPack) bool {
segment := modPack.Get(segmentID)
+6 -6
View File
@@ -658,13 +658,13 @@ func (s *Server) SaveBinlogPaths(ctx context.Context, req *datapb.SaveBinlogPath
mlog.Warn(context.TODO(), "failed to get segment, the segment not healthy", mlog.Err(err))
return merr.Status(err), nil
}
if err := s.validateTextSegmentStorage(req); err != nil {
incomingStorageVersion := req.GetStorageVersion()
if err := s.validateTextSegmentStorage(req, incomingStorageVersion); err != nil {
mlog.Warn(context.TODO(), "invalid TEXT segment storage format", mlog.Err(err))
return merr.Status(err), nil
}
// Set storage version
operators = append(operators, SetStorageVersion(req.GetSegmentID(), req.GetStorageVersion()))
operators = append(operators, ValidateSaveBinlogStorageVersion(req.GetSegmentID(), incomingStorageVersion))
// Set segment state
if req.GetDropped() {
@@ -753,18 +753,18 @@ func (s *Server) SaveBinlogPaths(ctx context.Context, req *datapb.SaveBinlogPath
return merr.Success(), nil
}
func (s *Server) validateTextSegmentStorage(req *datapb.SaveBinlogPathsRequest) error {
func (s *Server) validateTextSegmentStorage(req *datapb.SaveBinlogPathsRequest, storageVersion int64) error {
if req.GetSegLevel() == datapb.SegmentLevel_L0 || req.GetDropped() {
return nil
}
if !s.meta.collectionHasTextFields(req.GetCollectionID()) {
return nil
}
if req.GetStorageVersion() < storage.StorageV3 {
if storageVersion < storage.StorageV3 {
return merr.WrapErrParameterInvalidMsg(
"TEXT segment %d must be saved with StorageV3 manifest, got storage version %d",
req.GetSegmentID(),
req.GetStorageVersion())
storageVersion)
}
if req.GetManifestPath() == "" {
return merr.WrapErrParameterInvalidMsg(
+59
View File
@@ -275,6 +275,65 @@ func (s *ServerSuite) TestSaveBinlogPath_SaveUnhealthySegment() {
}
}
func (s *ServerSuite) TestSaveBinlogPath_StorageVersionImmutable() {
s.testServer.meta.AddCollection(&collectionInfo{ID: 0})
info := &datapb.SegmentInfo{
ID: 10,
InsertChannel: "ch1",
State: commonpb.SegmentState_Growing,
Level: datapb.SegmentLevel_L1,
StorageVersion: storage.StorageV2,
}
err := s.testServer.meta.AddSegment(context.TODO(), NewSegmentInfo(info))
s.Require().NoError(err)
resp, err := s.testServer.SaveBinlogPaths(context.Background(), &datapb.SaveBinlogPathsRequest{
Base: &commonpb.MsgBase{
Timestamp: uint64(time.Now().Unix()),
},
SegmentID: 10,
Channel: "ch1",
StorageVersion: storage.StorageV3,
})
s.NoError(err)
s.ErrorIs(merr.Error(resp), merr.ErrDataIntegrity)
resp, err = s.testServer.SaveBinlogPaths(context.Background(), &datapb.SaveBinlogPathsRequest{
Base: &commonpb.MsgBase{
Timestamp: uint64(time.Now().Unix()),
},
SegmentID: 10,
Channel: "ch1",
})
s.NoError(err)
s.ErrorIs(merr.Error(resp), merr.ErrDataIntegrity)
segment := s.testServer.meta.GetSegment(context.TODO(), 10)
s.EqualValues(storage.StorageV2, segment.GetStorageVersion())
info = &datapb.SegmentInfo{
ID: 11,
InsertChannel: "ch1",
State: commonpb.SegmentState_Growing,
Level: datapb.SegmentLevel_L1,
StorageVersion: storage.StorageV1,
}
err = s.testServer.meta.AddSegment(context.TODO(), NewSegmentInfo(info))
s.Require().NoError(err)
resp, err = s.testServer.SaveBinlogPaths(context.Background(), &datapb.SaveBinlogPathsRequest{
Base: &commonpb.MsgBase{
Timestamp: uint64(time.Now().Unix()),
},
SegmentID: 11,
Channel: "ch1",
StorageVersion: storage.StorageV2,
})
s.NoError(err)
s.ErrorIs(merr.Error(resp), merr.ErrDataIntegrity)
segment = s.testServer.meta.GetSegment(context.TODO(), 11)
s.EqualValues(storage.StorageV1, segment.GetStorageVersion())
}
func (s *ServerSuite) TestSaveBinlogPath_SaveDroppedSegment() {
s.testServer.meta.AddCollection(&collectionInfo{ID: 0})
@@ -823,6 +823,10 @@ func (t *GrowingSourceSyncTask) schemaBasedPattern(columnGroups []storagecommon.
}
func (t *GrowingSourceSyncTask) buildFlushConfig(segment *metacache.SegmentInfo, columnGroups []storagecommon.ColumnGroup) (*GrowingFlushConfig, error) {
if segment.GetStorageVersion() != storage.StorageV3 {
return nil, merr.WrapErrDataIntegrityMsg("growing source flush requires StorageV3 segment, segmentID=%d storageVersion=%d",
t.segmentID, segment.GetStorageVersion())
}
segmentBasePath := path.Join(t.chunkManager.RootPath(), common.SegmentInsertLogPath,
metautil.JoinIDPath(t.collectionID, t.partitionID, t.segmentID))
partitionBasePath := path.Join(t.chunkManager.RootPath(), common.SegmentInsertLogPath,
@@ -111,9 +111,10 @@ func TestGrowingSourceSyncTaskBuildFlushConfigBM25(t *testing.T) {
cm := mock_storage.NewMockChunkManager(t)
cm.EXPECT().RootPath().Return("/root").Maybe()
segment := metacache.NewSegmentInfo(&datapb.SegmentInfo{
ID: segmentID,
PartitionID: 2,
ManifestPath: `{"ver":7,"base_path":"/root/insert_log/3/2/1"}`,
ID: segmentID,
PartitionID: 2,
StorageVersion: storage.StorageV3,
ManifestPath: `{"ver":7,"base_path":"/root/insert_log/3/2/1"}`,
}, pkoracle.NewBloomFilterSet(), nil)
task := NewGrowingSourceSyncTask().
@@ -145,8 +146,9 @@ func TestGrowingSourceSyncTaskBuildFlushConfigStartsFromEarliestManifest(t *test
cm := mock_storage.NewMockChunkManager(t)
cm.EXPECT().RootPath().Return("/root").Maybe()
segment := metacache.NewSegmentInfo(&datapb.SegmentInfo{
ID: 1,
PartitionID: 2,
ID: 1,
PartitionID: 2,
StorageVersion: storage.StorageV3,
}, pkoracle.NewBloomFilterSet(), nil)
task := NewGrowingSourceSyncTask().
@@ -171,7 +173,7 @@ func TestGrowingSourceSyncTaskBuildFlushConfigBM25AllocatorError(t *testing.T) {
}
cm := mock_storage.NewMockChunkManager(t)
cm.EXPECT().RootPath().Return("/root").Maybe()
segment := metacache.NewSegmentInfo(&datapb.SegmentInfo{ID: 1}, pkoracle.NewBloomFilterSet(), nil)
segment := metacache.NewSegmentInfo(&datapb.SegmentInfo{ID: 1, StorageVersion: storage.StorageV3}, pkoracle.NewBloomFilterSet(), nil)
task := NewGrowingSourceSyncTask().
WithCollectionID(3).
WithPartitionID(2).
@@ -195,7 +197,7 @@ func TestGrowingSourceSyncTaskBuildFlushConfigBM25RequiresAllocator(t *testing.T
}
cm := mock_storage.NewMockChunkManager(t)
cm.EXPECT().RootPath().Return("/root").Maybe()
segment := metacache.NewSegmentInfo(&datapb.SegmentInfo{ID: 1}, pkoracle.NewBloomFilterSet(), nil)
segment := metacache.NewSegmentInfo(&datapb.SegmentInfo{ID: 1, StorageVersion: storage.StorageV3}, pkoracle.NewBloomFilterSet(), nil)
task := NewGrowingSourceSyncTask().
WithCollectionID(3).
WithPartitionID(2).
@@ -455,9 +457,10 @@ func TestGrowingSourceSyncTaskCommitRetainedSourceOnlyOnFinalization(t *testing.
segmentID := int64(1)
mc := metacache.NewMockMetaCache(t)
segment := metacache.NewSegmentInfo(&datapb.SegmentInfo{
ID: segmentID,
PartitionID: 2,
ManifestPath: "manifest",
ID: segmentID,
PartitionID: 2,
StorageVersion: storage.StorageV3,
ManifestPath: "manifest",
}, pkoracle.NewBloomFilterSet(), nil)
metacache.UpdateNumOfRows(10)(segment)
source := &fakeCommitGrowingFlushSource{}
@@ -554,9 +557,10 @@ func TestGrowingSourceSyncTaskMergesReturnedBM25Stats(t *testing.T) {
mc := metacache.NewMockMetaCache(t)
segment := metacache.NewSegmentInfo(&datapb.SegmentInfo{
ID: segmentID,
PartitionID: 2,
State: commonpb.SegmentState_Growing,
ID: segmentID,
PartitionID: 2,
State: commonpb.SegmentState_Growing,
StorageVersion: storage.StorageV3,
}, pkoracle.NewBloomFilterSet(), nil)
mc.EXPECT().GetSegmentByID(segmentID).Return(segment, true)
+5 -1
View File
@@ -167,6 +167,10 @@ func (b *brokerMetaWriter) UpdateGrowingSourceSync(ctx context.Context, task *Gr
if !ok {
return merr.WrapErrSegmentNotFound(task.segmentID)
}
if segment.GetStorageVersion() != storage.StorageV3 {
return merr.WrapErrDataIntegrityMsg("growing source sync requires StorageV3 segment, segmentID=%d storageVersion=%d",
task.segmentID, segment.GetStorageVersion())
}
insertFieldBinlogs := segment.Binlogs()
if len(task.insertBinlogs) > 0 {
@@ -215,7 +219,7 @@ func (b *brokerMetaWriter) UpdateGrowingSourceSync(ctx context.Context, task *Gr
Dropped: task.IsDrop(),
Channel: task.channelName,
SegLevel: task.level,
StorageVersion: storage.StorageV3,
StorageVersion: segment.GetStorageVersion(),
WithFullBinlogs: true,
ManifestPath: task.manifestPath,
}
@@ -315,8 +315,9 @@ func (s *MetaWriterSuite) TestGrowingSourceSyncMetaErrorsReturnError() {
bfs := pkoracle.NewBloomFilterSet()
seg := metacache.NewSegmentInfo(&datapb.SegmentInfo{
ID: 1,
PartitionID: 2,
ID: 1,
PartitionID: 2,
StorageVersion: storage.StorageV3,
}, bfs, nil)
s.metacache.EXPECT().GetSegmentByID(int64(1)).Return(seg, true).Once()
s.metacache.EXPECT().GetSegmentsBy(mock.Anything, mock.Anything, mock.Anything).Return(nil).Once()
@@ -62,11 +62,14 @@ func (wb *l0WriteBuffer) BufferData(insertData []*InsertData, deleteMsgs []*msgs
wb.mut.Lock()
for _, inData := range insertData {
if wb.useGrowingSourceFlush {
if wb.allowGrowingSourceFlush {
targetOffset := wb.growingSourceTargetOffset(inData.segmentID, inData.rowNum)
decision := wb.decideGrowingFlushSource(inData.segmentID, targetOffset, endPos)
if decision.sourceType == metacache.FlushSourceGrowing {
wb.recordGrowingSourceProgress(inData, startPos, endPos, schemaVersion, targetOffset)
if err := wb.recordGrowingSourceProgress(inData, startPos, endPos, schemaVersion, targetOffset); err != nil {
wb.mut.Unlock()
return err
}
continue
}
}
@@ -106,7 +109,14 @@ func (wb *l0WriteBuffer) BufferData(insertData []*InsertData, deleteMsgs []*msgs
// bufferInsert function InsertMsg into bufferred InsertData and returns primary key field data for future usage.
func (wb *l0WriteBuffer) bufferInsert(inData *InsertData, startPos, endPos *msgpb.MsgPosition, schemaVersion int32) error {
wb.CreateNewGrowingSegment(inData.partitionID, inData.segmentID, startPos, schemaVersion)
if err := wb.CreateNewGrowingSegment(CreateGrowingSegmentInfo{
PartitionID: inData.partitionID,
SegmentID: inData.segmentID,
StartPos: startPos,
SchemaVersion: schemaVersion,
}); err != nil {
return err
}
segBuf := wb.getOrCreateBuffer(inData.segmentID, startPos.GetTimestamp())
totalMemSize := segBuf.insertBuffer.Buffer(inData, startPos, endPos)
@@ -410,11 +410,11 @@ func (s *L0WriteBufferSuite) TestBufferDataGrowingSourceMode() {
s.Run("usable_source_records_progress_and_pins_checkpoint", func() {
textSchema := s.textSchema()
metacache := s.newTextMetaCache(textSchema)
mc := s.newTextMetaCache(textSchema)
var resolvedSegmentID int64
var resolvedTargetOffset int64
resolveCalls := 0
wb, err := NewL0WriteBuffer(s.channelName, metacache, s.syncMgr, &writeBufferOption{
wb, err := NewL0WriteBuffer(s.channelName, mc, s.syncMgr, &writeBufferOption{
idAllocator: s.allocator,
growingSourceRetryInterval: time.Hour,
growingSourceResolver: func(segmentID int64, targetOffset int64, _ *msgpb.MsgPosition) (syncmgr.GrowingFlushSource, syncmgr.GrowingSourceState) {
@@ -433,11 +433,24 @@ func (s *L0WriteBufferSuite) TestBufferDataGrowingSourceMode() {
insertData, err := PrepareInsert(textSchema, s.pkSchema, []*msgstream.InsertMsg{msg})
s.NoError(err)
// 3 first-insert path calls plus 1 triggerSync policy check for the
// recorded growing source progress.
metacache.EXPECT().GetSegmentByID(int64(1001)).Return(nil, false).Times(4)
metacache.EXPECT().AddSegment(mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return()
metacache.EXPECT().UpdateSegments(mock.Anything, mock.Anything).Return()
segmentInfo := metacache.NewSegmentInfo(&datapb.SegmentInfo{
ID: 1001,
PartitionID: 10,
CollectionID: s.collID,
InsertChannel: s.channelName,
StartPosition: &msgpb.MsgPosition{Timestamp: 100},
State: commonpb.SegmentState_Growing,
StorageVersion: storage.StorageV3,
SchemaVersion: 100,
}, pkoracle.NewBloomFilterSet(), nil)
// growingSourceBaseOffset and CreateNewGrowingSegment see a new
// segment; recordGrowingSourceProgress and triggerSync see the segment
// created by AddSegment.
mc.EXPECT().GetSegmentByID(int64(1001)).Return(nil, false).Times(3)
mc.EXPECT().GetSegmentByID(int64(1001)).Return(segmentInfo, true).Times(2)
mc.EXPECT().AddSegment(mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return()
mc.EXPECT().UpdateSegments(mock.Anything, mock.Anything).Return()
err = wb.BufferData(insertData, nil, &msgpb.MsgPosition{Timestamp: 100}, &msgpb.MsgPosition{Timestamp: 200}, 100)
s.NoError(err)
@@ -556,8 +569,8 @@ func (s *L0WriteBufferSuite) TestBufferDataGrowingSourceMode() {
s.Run("pending_source_records_progress_instead_of_falling_back_to_writebuffer", func() {
textSchema := s.textSchema()
metacache := s.newTextMetaCache(textSchema)
wb, err := NewL0WriteBuffer(s.channelName, metacache, s.syncMgr, &writeBufferOption{
mc := s.newTextMetaCache(textSchema)
wb, err := NewL0WriteBuffer(s.channelName, mc, s.syncMgr, &writeBufferOption{
idAllocator: s.allocator,
growingSourceRetryInterval: time.Hour,
growingSourceResolver: func(segmentID int64, targetOffset int64, _ *msgpb.MsgPosition) (syncmgr.GrowingFlushSource, syncmgr.GrowingSourceState) {
@@ -570,10 +583,24 @@ func (s *L0WriteBufferSuite) TestBufferDataGrowingSourceMode() {
insertData, err := PrepareInsert(textSchema, s.pkSchema, []*msgstream.InsertMsg{msg})
s.NoError(err)
// 3 first-insert path calls plus 1 triggerSync policy check.
metacache.EXPECT().GetSegmentByID(int64(1002)).Return(nil, false).Times(4)
metacache.EXPECT().AddSegment(mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return()
metacache.EXPECT().UpdateSegments(mock.Anything, mock.Anything).Return()
segmentInfo := metacache.NewSegmentInfo(&datapb.SegmentInfo{
ID: 1002,
PartitionID: 10,
CollectionID: s.collID,
InsertChannel: s.channelName,
StartPosition: &msgpb.MsgPosition{Timestamp: 100},
State: commonpb.SegmentState_Growing,
StorageVersion: storage.StorageV3,
SchemaVersion: 100,
}, pkoracle.NewBloomFilterSet(), nil)
// growingSourceBaseOffset and CreateNewGrowingSegment see a new
// segment; recordGrowingSourceProgress and triggerSync see the segment
// created by AddSegment.
mc.EXPECT().GetSegmentByID(int64(1002)).Return(nil, false).Times(3)
mc.EXPECT().GetSegmentByID(int64(1002)).Return(segmentInfo, true).Times(2)
mc.EXPECT().AddSegment(mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return()
mc.EXPECT().UpdateSegments(mock.Anything, mock.Anything).Return()
err = wb.BufferData(insertData, nil, &msgpb.MsgPosition{Timestamp: 100}, &msgpb.MsgPosition{Timestamp: 200}, 100)
s.NoError(err)
@@ -1666,11 +1693,25 @@ func (s *L0WriteBufferSuite) TestCheckReleaseManualFlushNeed() {
s.False(checker.CheckReleaseManualFlushNeed(nil))
s.True(checker.CheckReleaseManualFlushNeed([]int64{1300}))
wb.CreateNewGrowingSegment(0, 1301, &msgpb.MsgPosition{Timestamp: 100}, 100)
err = wb.CreateNewGrowingSegment(CreateGrowingSegmentInfo{
PartitionID: 0,
SegmentID: 1301,
StartPos: &msgpb.MsgPosition{Timestamp: 100},
SchemaVersion: 100,
StorageVersion: storage.StorageV3,
})
s.NoError(err)
mc.UpdateSegments(metacache.SetFlushSourceMode(metacache.FlushSourceWriteBuffer), metacache.WithSegmentIDs(1301))
s.False(checker.CheckReleaseManualFlushNeed([]int64{1301}))
wb.CreateNewGrowingSegment(0, 1302, &msgpb.MsgPosition{Timestamp: 100}, 100)
err = wb.CreateNewGrowingSegment(CreateGrowingSegmentInfo{
PartitionID: 0,
SegmentID: 1302,
StartPos: &msgpb.MsgPosition{Timestamp: 100},
SchemaVersion: 100,
StorageVersion: storage.StorageV3,
})
s.NoError(err)
mc.UpdateSegments(metacache.SetFlushSourceMode(metacache.FlushSourceGrowing), metacache.WithSegmentIDs(1302))
s.True(checker.CheckReleaseManualFlushNeed([]int64{1302}))
@@ -1678,7 +1719,14 @@ func (s *L0WriteBufferSuite) TestCheckReleaseManualFlushNeed() {
s.False(checker.CheckReleaseManualFlushNeed([]int64{1302}))
s.True(checker.CheckReleaseManualFlushNeed([]int64{1301, 1300}))
wb.CreateNewGrowingSegment(0, 1303, &msgpb.MsgPosition{Timestamp: 100}, 100)
err = wb.CreateNewGrowingSegment(CreateGrowingSegmentInfo{
PartitionID: 0,
SegmentID: 1303,
StartPos: &msgpb.MsgPosition{Timestamp: 100},
SchemaVersion: 100,
StorageVersion: storage.StorageV3,
})
s.NoError(err)
mc.UpdateSegments(metacache.SetFlushSourceMode(metacache.FlushSourceGrowing), metacache.WithSegmentIDs(1303))
s.True(checker.CheckReleaseManualFlushNeed([]int64{1300, 1303}))
}
+9 -10
View File
@@ -27,7 +27,7 @@ type BufferManager interface {
// Register adds a WriteBuffer with provided schema & options.
Register(channel string, metacache metacache.MetaCache, opts ...WriteBufferOption) error
// CreateNewGrowingSegment notifies writeBuffer to create a new growing segment.
CreateNewGrowingSegment(ctx context.Context, channel string, partition int64, segmentID int64, schemaVersion int32) error
CreateNewGrowingSegment(ctx context.Context, channel string, info CreateGrowingSegmentInfo) error
// SealSegments notifies writeBuffer corresponding to provided channel to seal segments.
// which will cause segment start flush procedure.
SealSegments(ctx context.Context, channel string, segmentIDs []int64) error
@@ -47,8 +47,8 @@ type BufferManager interface {
// NotifyCheckpointUpdated notify write buffer checkpoint updated to reset flushTs.
NotifyCheckpointUpdated(channel string, ts uint64)
// UseGrowingSourceFlush returns true if the collection on this channel has growing-source fields.
UseGrowingSourceFlush(channel string) bool
// AllowGrowingSourceFlush returns true if this channel may try growing-source flush.
AllowGrowingSourceFlush(channel string) bool
// GetGrowingFlushProgress returns growing-source progress for the given channel.
// If segmentIDs is empty, all tracked growing-source segments are returned.
// Otherwise, the requested segmentIDs are returned together with all tracked
@@ -187,17 +187,16 @@ func (m *bufferManager) Register(channel string, metacache metacache.MetaCache,
}
// CreateNewGrowingSegment notifies writeBuffer to create a new growing segment.
func (m *bufferManager) CreateNewGrowingSegment(ctx context.Context, channel string, partitionID int64, segmentID int64, schemaVersion int32) error {
func (m *bufferManager) CreateNewGrowingSegment(ctx context.Context, channel string, info CreateGrowingSegmentInfo) error {
buf, loaded := m.buffers.Get(channel)
if !loaded {
mlog.Warn(ctx, "write buffer not found when create new growing segment",
mlog.String("channel", channel),
mlog.FieldPartitionID(partitionID),
mlog.FieldSegmentID(segmentID))
mlog.FieldPartitionID(info.PartitionID),
mlog.FieldSegmentID(info.SegmentID))
return merr.WrapErrChannelNotFound(channel)
}
buf.CreateNewGrowingSegment(partitionID, segmentID, nil, schemaVersion)
return nil
return buf.CreateNewGrowingSegment(info)
}
// SealSegments call sync segment and change segments state to Flushed.
@@ -250,12 +249,12 @@ func (m *bufferManager) BufferData(channel string, insertData []*InsertData, del
return buf.BufferData(insertData, deleteMsgs, startPos, endPos, schemaVersion)
}
func (m *bufferManager) UseGrowingSourceFlush(channel string) bool {
func (m *bufferManager) AllowGrowingSourceFlush(channel string) bool {
buf, loaded := m.buffers.Get(channel)
if !loaded {
return false
}
return buf.UseGrowingSourceFlush()
return buf.AllowGrowingSourceFlush()
}
func (m *bufferManager) CheckReleaseManualFlushNeed(ctx context.Context, channel string, segmentIDs []int64) (bool, error) {
@@ -15,6 +15,7 @@ import (
"github.com/milvus-io/milvus/internal/allocator"
"github.com/milvus-io/milvus/internal/flushcommon/metacache"
"github.com/milvus-io/milvus/internal/flushcommon/syncmgr"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/pkg/v3/common"
"github.com/milvus-io/milvus/pkg/v3/proto/datapb"
"github.com/milvus-io/milvus/pkg/v3/util/hardware"
@@ -143,7 +144,11 @@ func (s *ManagerSuite) TestFlushAllSegments() {
func (s *ManagerSuite) TestCreateNewGrowingSegment() {
manager := s.manager
err := manager.CreateNewGrowingSegment(context.Background(), s.channelName, 1, 1, 0)
err := manager.CreateNewGrowingSegment(context.Background(), s.channelName, CreateGrowingSegmentInfo{
PartitionID: 1,
SegmentID: 1,
StorageVersion: storage.StorageV2,
})
s.Error(err)
s.metacache.EXPECT().GetSegmentByID(mock.Anything).Return(nil, false).Once()
@@ -157,7 +162,12 @@ func (s *ManagerSuite) TestCreateNewGrowingSegment() {
s.NoError(err)
s.manager.buffers.Insert(s.channelName, wb)
err = manager.CreateNewGrowingSegment(context.Background(), s.channelName, 1, 1, 100)
err = manager.CreateNewGrowingSegment(context.Background(), s.channelName, CreateGrowingSegmentInfo{
PartitionID: 1,
SegmentID: 1,
SchemaVersion: 100,
StorageVersion: storage.StorageV2,
})
s.NoError(err)
}
@@ -77,17 +77,17 @@ func (_c *MockBufferManager_BufferData_Call) RunAndReturn(run func(string, []*In
return _c
}
// CreateNewGrowingSegment provides a mock function with given fields: ctx, channel, partition, segmentID, schemaVersion
func (_m *MockBufferManager) CreateNewGrowingSegment(ctx context.Context, channel string, partition int64, segmentID int64, schemaVersion int32) error {
ret := _m.Called(ctx, channel, partition, segmentID, schemaVersion)
// CreateNewGrowingSegment provides a mock function with given fields: ctx, channel, info
func (_m *MockBufferManager) CreateNewGrowingSegment(ctx context.Context, channel string, info CreateGrowingSegmentInfo) error {
ret := _m.Called(ctx, channel, info)
if len(ret) == 0 {
panic("no return value specified for CreateNewGrowingSegment")
}
var r0 error
if rf, ok := ret.Get(0).(func(context.Context, string, int64, int64, int32) error); ok {
r0 = rf(ctx, channel, partition, segmentID, schemaVersion)
if rf, ok := ret.Get(0).(func(context.Context, string, CreateGrowingSegmentInfo) error); ok {
r0 = rf(ctx, channel, info)
} else {
r0 = ret.Error(0)
}
@@ -103,16 +103,14 @@ type MockBufferManager_CreateNewGrowingSegment_Call struct {
// CreateNewGrowingSegment is a helper method to define mock.On call
// - ctx context.Context
// - channel string
// - partition int64
// - segmentID int64
// - schemaVersion int32
func (_e *MockBufferManager_Expecter) CreateNewGrowingSegment(ctx interface{}, channel interface{}, partition interface{}, segmentID interface{}, schemaVersion interface{}) *MockBufferManager_CreateNewGrowingSegment_Call {
return &MockBufferManager_CreateNewGrowingSegment_Call{Call: _e.mock.On("CreateNewGrowingSegment", ctx, channel, partition, segmentID, schemaVersion)}
// - info CreateGrowingSegmentInfo
func (_e *MockBufferManager_Expecter) CreateNewGrowingSegment(ctx interface{}, channel interface{}, info interface{}) *MockBufferManager_CreateNewGrowingSegment_Call {
return &MockBufferManager_CreateNewGrowingSegment_Call{Call: _e.mock.On("CreateNewGrowingSegment", ctx, channel, info)}
}
func (_c *MockBufferManager_CreateNewGrowingSegment_Call) Run(run func(ctx context.Context, channel string, partition int64, segmentID int64, schemaVersion int32)) *MockBufferManager_CreateNewGrowingSegment_Call {
func (_c *MockBufferManager_CreateNewGrowingSegment_Call) Run(run func(ctx context.Context, channel string, info CreateGrowingSegmentInfo)) *MockBufferManager_CreateNewGrowingSegment_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(string), args[2].(int64), args[3].(int64), args[4].(int32))
run(args[0].(context.Context), args[1].(string), args[2].(CreateGrowingSegmentInfo))
})
return _c
}
@@ -122,7 +120,7 @@ func (_c *MockBufferManager_CreateNewGrowingSegment_Call) Return(_a0 error) *Moc
return _c
}
func (_c *MockBufferManager_CreateNewGrowingSegment_Call) RunAndReturn(run func(context.Context, string, int64, int64, int32) error) *MockBufferManager_CreateNewGrowingSegment_Call {
func (_c *MockBufferManager_CreateNewGrowingSegment_Call) RunAndReturn(run func(context.Context, string, CreateGrowingSegmentInfo) error) *MockBufferManager_CreateNewGrowingSegment_Call {
_c.Call.Return(run)
return _c
}
@@ -368,12 +366,12 @@ func (_c *MockBufferManager_GetGrowingFlushProgress_Call) RunAndReturn(run func(
return _c
}
// UseGrowingSourceFlush provides a mock function with given fields: channel
func (_m *MockBufferManager) UseGrowingSourceFlush(channel string) bool {
// AllowGrowingSourceFlush provides a mock function with given fields: channel
func (_m *MockBufferManager) AllowGrowingSourceFlush(channel string) bool {
ret := _m.Called(channel)
if len(ret) == 0 {
panic("no return value specified for UseGrowingSourceFlush")
panic("no return value specified for AllowGrowingSourceFlush")
}
var r0 bool
@@ -386,30 +384,30 @@ func (_m *MockBufferManager) UseGrowingSourceFlush(channel string) bool {
return r0
}
// MockBufferManager_UseGrowingSourceFlush_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UseGrowingSourceFlush'
type MockBufferManager_UseGrowingSourceFlush_Call struct {
// MockBufferManager_AllowGrowingSourceFlush_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AllowGrowingSourceFlush'
type MockBufferManager_AllowGrowingSourceFlush_Call struct {
*mock.Call
}
// UseGrowingSourceFlush is a helper method to define mock.On call
// AllowGrowingSourceFlush is a helper method to define mock.On call
// - channel string
func (_e *MockBufferManager_Expecter) UseGrowingSourceFlush(channel interface{}) *MockBufferManager_UseGrowingSourceFlush_Call {
return &MockBufferManager_UseGrowingSourceFlush_Call{Call: _e.mock.On("UseGrowingSourceFlush", channel)}
func (_e *MockBufferManager_Expecter) AllowGrowingSourceFlush(channel interface{}) *MockBufferManager_AllowGrowingSourceFlush_Call {
return &MockBufferManager_AllowGrowingSourceFlush_Call{Call: _e.mock.On("AllowGrowingSourceFlush", channel)}
}
func (_c *MockBufferManager_UseGrowingSourceFlush_Call) Run(run func(channel string)) *MockBufferManager_UseGrowingSourceFlush_Call {
func (_c *MockBufferManager_AllowGrowingSourceFlush_Call) Run(run func(channel string)) *MockBufferManager_AllowGrowingSourceFlush_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(string))
})
return _c
}
func (_c *MockBufferManager_UseGrowingSourceFlush_Call) Return(_a0 bool) *MockBufferManager_UseGrowingSourceFlush_Call {
func (_c *MockBufferManager_AllowGrowingSourceFlush_Call) Return(_a0 bool) *MockBufferManager_AllowGrowingSourceFlush_Call {
_c.Call.Return(_a0)
return _c
}
func (_c *MockBufferManager_UseGrowingSourceFlush_Call) RunAndReturn(run func(string) bool) *MockBufferManager_UseGrowingSourceFlush_Call {
func (_c *MockBufferManager_AllowGrowingSourceFlush_Call) RunAndReturn(run func(string) bool) *MockBufferManager_AllowGrowingSourceFlush_Call {
_c.Call.Return(run)
return _c
}
@@ -108,9 +108,22 @@ func (_c *MockWriteBuffer_Close_Call) RunAndReturn(run func(context.Context, boo
return _c
}
// CreateNewGrowingSegment provides a mock function with given fields: partitionID, segmentID, startPos, schemaVersion
func (_m *MockWriteBuffer) CreateNewGrowingSegment(partitionID int64, segmentID int64, startPos *msgpb.MsgPosition, schemaVersion int32) {
_m.Called(partitionID, segmentID, startPos, schemaVersion)
// CreateNewGrowingSegment provides a mock function with given fields: info
func (_m *MockWriteBuffer) CreateNewGrowingSegment(info CreateGrowingSegmentInfo) error {
ret := _m.Called(info)
if len(ret) == 0 {
panic("no return value specified for CreateNewGrowingSegment")
}
var r0 error
if rf, ok := ret.Get(0).(func(CreateGrowingSegmentInfo) error); ok {
r0 = rf(info)
} else {
r0 = ret.Error(0)
}
return r0
}
// MockWriteBuffer_CreateNewGrowingSegment_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CreateNewGrowingSegment'
@@ -119,28 +132,25 @@ type MockWriteBuffer_CreateNewGrowingSegment_Call struct {
}
// CreateNewGrowingSegment is a helper method to define mock.On call
// - partitionID int64
// - segmentID int64
// - startPos *msgpb.MsgPosition
// - schemaVersion int32
func (_e *MockWriteBuffer_Expecter) CreateNewGrowingSegment(partitionID interface{}, segmentID interface{}, startPos interface{}, schemaVersion interface{}) *MockWriteBuffer_CreateNewGrowingSegment_Call {
return &MockWriteBuffer_CreateNewGrowingSegment_Call{Call: _e.mock.On("CreateNewGrowingSegment", partitionID, segmentID, startPos, schemaVersion)}
// - info CreateGrowingSegmentInfo
func (_e *MockWriteBuffer_Expecter) CreateNewGrowingSegment(info interface{}) *MockWriteBuffer_CreateNewGrowingSegment_Call {
return &MockWriteBuffer_CreateNewGrowingSegment_Call{Call: _e.mock.On("CreateNewGrowingSegment", info)}
}
func (_c *MockWriteBuffer_CreateNewGrowingSegment_Call) Run(run func(partitionID int64, segmentID int64, startPos *msgpb.MsgPosition, schemaVersion int32)) *MockWriteBuffer_CreateNewGrowingSegment_Call {
func (_c *MockWriteBuffer_CreateNewGrowingSegment_Call) Run(run func(info CreateGrowingSegmentInfo)) *MockWriteBuffer_CreateNewGrowingSegment_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(int64), args[1].(int64), args[2].(*msgpb.MsgPosition), args[3].(int32))
run(args[0].(CreateGrowingSegmentInfo))
})
return _c
}
func (_c *MockWriteBuffer_CreateNewGrowingSegment_Call) Return() *MockWriteBuffer_CreateNewGrowingSegment_Call {
_c.Call.Return()
func (_c *MockWriteBuffer_CreateNewGrowingSegment_Call) Return(_a0 error) *MockWriteBuffer_CreateNewGrowingSegment_Call {
_c.Call.Return(_a0)
return _c
}
func (_c *MockWriteBuffer_CreateNewGrowingSegment_Call) RunAndReturn(run func(int64, int64, *msgpb.MsgPosition, int32)) *MockWriteBuffer_CreateNewGrowingSegment_Call {
_c.Run(run)
func (_c *MockWriteBuffer_CreateNewGrowingSegment_Call) RunAndReturn(run func(CreateGrowingSegmentInfo) error) *MockWriteBuffer_CreateNewGrowingSegment_Call {
_c.Call.Return(run)
return _c
}
@@ -421,12 +431,12 @@ func (_c *MockWriteBuffer_HasSegment_Call) RunAndReturn(run func(int64) bool) *M
return _c
}
// UseGrowingSourceFlush provides a mock function with no fields
func (_m *MockWriteBuffer) UseGrowingSourceFlush() bool {
// AllowGrowingSourceFlush provides a mock function with no fields
func (_m *MockWriteBuffer) AllowGrowingSourceFlush() bool {
ret := _m.Called()
if len(ret) == 0 {
panic("no return value specified for UseGrowingSourceFlush")
panic("no return value specified for AllowGrowingSourceFlush")
}
var r0 bool
@@ -439,29 +449,29 @@ func (_m *MockWriteBuffer) UseGrowingSourceFlush() bool {
return r0
}
// MockWriteBuffer_UseGrowingSourceFlush_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UseGrowingSourceFlush'
type MockWriteBuffer_UseGrowingSourceFlush_Call struct {
// MockWriteBuffer_AllowGrowingSourceFlush_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AllowGrowingSourceFlush'
type MockWriteBuffer_AllowGrowingSourceFlush_Call struct {
*mock.Call
}
// UseGrowingSourceFlush is a helper method to define mock.On call
func (_e *MockWriteBuffer_Expecter) UseGrowingSourceFlush() *MockWriteBuffer_UseGrowingSourceFlush_Call {
return &MockWriteBuffer_UseGrowingSourceFlush_Call{Call: _e.mock.On("UseGrowingSourceFlush")}
// AllowGrowingSourceFlush is a helper method to define mock.On call
func (_e *MockWriteBuffer_Expecter) AllowGrowingSourceFlush() *MockWriteBuffer_AllowGrowingSourceFlush_Call {
return &MockWriteBuffer_AllowGrowingSourceFlush_Call{Call: _e.mock.On("AllowGrowingSourceFlush")}
}
func (_c *MockWriteBuffer_UseGrowingSourceFlush_Call) Run(run func()) *MockWriteBuffer_UseGrowingSourceFlush_Call {
func (_c *MockWriteBuffer_AllowGrowingSourceFlush_Call) Run(run func()) *MockWriteBuffer_AllowGrowingSourceFlush_Call {
_c.Call.Run(func(args mock.Arguments) {
run()
})
return _c
}
func (_c *MockWriteBuffer_UseGrowingSourceFlush_Call) Return(_a0 bool) *MockWriteBuffer_UseGrowingSourceFlush_Call {
func (_c *MockWriteBuffer_AllowGrowingSourceFlush_Call) Return(_a0 bool) *MockWriteBuffer_AllowGrowingSourceFlush_Call {
_c.Call.Return(_a0)
return _c
}
func (_c *MockWriteBuffer_UseGrowingSourceFlush_Call) RunAndReturn(run func() bool) *MockWriteBuffer_UseGrowingSourceFlush_Call {
func (_c *MockWriteBuffer_AllowGrowingSourceFlush_Call) RunAndReturn(run func() bool) *MockWriteBuffer_AllowGrowingSourceFlush_Call {
_c.Call.Return(run)
return _c
}
@@ -22,6 +22,7 @@ import (
"github.com/milvus-io/milvus/internal/flushcommon/syncmgr"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/internal/storagev2/packed"
"github.com/milvus-io/milvus/internal/util/streamingutil"
"github.com/milvus-io/milvus/pkg/v3/common"
"github.com/milvus-io/milvus/pkg/v3/metrics"
"github.com/milvus-io/milvus/pkg/v3/mlog"
@@ -52,7 +53,7 @@ type WriteBuffer interface {
// HasSegment checks whether certain segment exists in this buffer.
HasSegment(segmentID int64) bool
// CreateNewGrowingSegment creates a new growing segment in the buffer.
CreateNewGrowingSegment(partitionID int64, segmentID int64, startPos *msgpb.MsgPosition, schemaVersion int32)
CreateNewGrowingSegment(info CreateGrowingSegmentInfo) error
// BufferData is the method to buffer dml data msgs.
BufferData(insertMsgs []*InsertData, deleteMsgs []*msgstream.DeleteMsg, startPos, endPos *msgpb.MsgPosition, schemaVersion int32) error
// FlushTimestamp set flush timestamp for write buffer
@@ -73,8 +74,8 @@ type WriteBuffer interface {
MemorySize() int64
// EvictBuffer evicts buffer to sync manager which match provided sync policies.
EvictBuffer(policies ...SyncPolicy)
// UseGrowingSourceFlush returns true if the collection on this channel has growing-source fields.
UseGrowingSourceFlush() bool
// AllowGrowingSourceFlush returns true if this write buffer may try growing-source flush.
AllowGrowingSourceFlush() bool
// GetGrowingFlushProgress returns growing-source progress for the given
// segments after this write buffer has processed up to fenceTs. If segmentIDs
// is empty, all tracked growing-source segments are returned. Otherwise,
@@ -84,6 +85,14 @@ type WriteBuffer interface {
Close(ctx context.Context, drop bool)
}
type CreateGrowingSegmentInfo struct {
PartitionID int64
SegmentID int64
StartPos *msgpb.MsgPosition
SchemaVersion int32
StorageVersion int64
}
type GrowingFlushSegmentProgress struct {
SegmentID int64
TargetOffset int64
@@ -269,9 +278,9 @@ type writeBufferBase struct {
errHandler func(err error)
taskObserverCallback func(t syncmgr.Task, err error) // execute when a sync task finished, should be concurrent safe.
// growing-source collection flag. growing-source can be flushed either from an optional growing
// segment source or from WriteBuffer payload when no growing source is usable.
useGrowingSourceFlush bool
// Channel-level admission flag for trying growing-source flush. Actual segment
// source selection remains sticky in metacache.
allowGrowingSourceFlush bool
growingSourceResolver GrowingSourceResolver
@@ -302,7 +311,7 @@ func newWriteBufferBase(channel string, metacache metacache.MetaCache, syncMgr s
return nil, err
}
useGrowingSourceFlush := typeutil.UseGrowingSourceFlush(schema,
allowGrowingSourceFlush := typeutil.AllowGrowingSourceFlush(schema,
paramtable.Get().CommonCfg.UseLoonFFI.GetAsBool(),
paramtable.Get().CommonCfg.EnableGrowingSourceFlush.GetAsBool())
growingSourceResolver := option.growingSourceResolver
@@ -332,7 +341,7 @@ func newWriteBufferBase(channel string, metacache metacache.MetaCache, syncMgr s
flushTimestamp: flushTs,
errHandler: option.errorHandler,
taskObserverCallback: option.taskObserverCallback,
useGrowingSourceFlush: useGrowingSourceFlush,
allowGrowingSourceFlush: allowGrowingSourceFlush,
growingSourceResolver: growingSourceResolver,
growingSourceProgress: make(map[int64]*growingSourceProgress),
growingSourceRetryInterval: growingSourceRetryInterval,
@@ -399,8 +408,8 @@ func (wb *writeBufferBase) GetFlushTimestamp() uint64 {
return wb.flushTimestamp.Load()
}
func (wb *writeBufferBase) UseGrowingSourceFlush() bool {
return wb.useGrowingSourceFlush
func (wb *writeBufferBase) AllowGrowingSourceFlush() bool {
return wb.allowGrowingSourceFlush
}
func (wb *writeBufferBase) CheckReleaseManualFlushNeed(segmentIDs []int64) bool {
@@ -612,6 +621,9 @@ func (wb *writeBufferBase) decideGrowingFlushSource(segmentID int64, targetOffse
// must return the same kind so that progress / payload tracking stays
// consistent for the segment's lifetime.
if seg, ok := wb.metaCache.GetSegmentByID(segmentID); ok {
if seg.GetStorageVersion() != storage.StorageV3 {
return growingFlushSourceDecision{sourceType: metacache.FlushSourceWriteBuffer}
}
switch seg.FlushSourceMode() {
case metacache.FlushSourceGrowing:
state := wb.getGrowingSourceState(segmentID, targetOffset, endPos)
@@ -666,7 +678,7 @@ func (wb *writeBufferBase) getGrowingSourceState(segmentID int64, targetOffset i
}
func (wb *writeBufferBase) warnGrowingSourceFallback(segmentID int64, targetOffset int64, endPos *msgpb.MsgPosition) {
if !wb.useGrowingSourceFlush {
if !wb.allowGrowingSourceFlush {
return
}
wb.growingSourceRatedLogger.RatedWarn(context.TODO(), rate.Limit(1), "growing-source source is unavailable, fallback to WriteBuffer",
@@ -787,8 +799,24 @@ func (wb *writeBufferBase) getGrowingSourceSegmentsToRetry() ([]int64, bool) {
return segments, retryNeeded
}
func (wb *writeBufferBase) recordGrowingSourceProgress(inData *InsertData, startPos, endPos *msgpb.MsgPosition, schemaVersion int32, targetOffset int64) {
wb.CreateNewGrowingSegment(inData.partitionID, inData.segmentID, startPos, schemaVersion)
func (wb *writeBufferBase) recordGrowingSourceProgress(inData *InsertData, startPos, endPos *msgpb.MsgPosition, schemaVersion int32, targetOffset int64) error {
err := wb.CreateNewGrowingSegment(CreateGrowingSegmentInfo{
PartitionID: inData.partitionID,
SegmentID: inData.segmentID,
StartPos: startPos,
SchemaVersion: schemaVersion,
})
if err != nil {
return err
}
segment, ok := wb.metaCache.GetSegmentByID(inData.segmentID)
if !ok {
return merr.WrapErrSegmentNotFound(inData.segmentID)
}
if segment.GetStorageVersion() != storage.StorageV3 {
return merr.WrapErrServiceInternalMsg("growing-source flush requires StorageV3 segment, segmentID=%d storageVersion=%d",
inData.segmentID, segment.GetStorageVersion())
}
progress, ok := wb.growingSourceProgress[inData.segmentID]
if !ok {
progress = &growingSourceProgress{
@@ -813,6 +841,7 @@ func (wb *writeBufferBase) recordGrowingSourceProgress(inData *InsertData, start
wb.updateGrowingSourceBufferedRows(progress),
), metacache.WithSegmentIDs(inData.segmentID))
wb.notifyFlushSourceMode(inData.segmentID)
return nil
}
func (wb *writeBufferBase) growingSourceTargetOffset(segmentID int64, rows int64) int64 {
@@ -853,7 +882,7 @@ func (wb *writeBufferBase) sealSegments(ctx context.Context, segmentIDs []int64)
for _, segmentID := range segmentIDs {
_, ok := wb.metaCache.GetSegmentByID(segmentID)
if !ok {
if !wb.useGrowingSourceFlush {
if !wb.allowGrowingSourceFlush {
mlog.Warn(ctx, "cannot find segment when sealSegments",
mlog.Int64("segmentID", segmentID),
mlog.String("channel", wb.channelName))
@@ -1160,7 +1189,7 @@ func (wb *writeBufferBase) getOrCreateBuffer(segmentID int64, timetick uint64) *
panic(err)
}
wb.buffers[segmentID] = buffer
if wb.useGrowingSourceFlush {
if wb.allowGrowingSourceFlush {
wb.metaCache.UpdateSegments(
metacache.SetFlushSourceMode(metacache.FlushSourceWriteBuffer),
metacache.WithSegmentIDs(segmentID),
@@ -1314,36 +1343,71 @@ func (id *InsertData) batchPkExists(pks []storage.PrimaryKey, tss []uint64, hits
return hits
}
func (wb *writeBufferBase) CreateNewGrowingSegment(partitionID int64, segmentID int64, startPos *msgpb.MsgPosition, schemaVersion int32) {
_, ok := wb.metaCache.GetSegmentByID(segmentID)
func (wb *writeBufferBase) CreateNewGrowingSegment(info CreateGrowingSegmentInfo) error {
_, ok := wb.metaCache.GetSegmentByID(info.SegmentID)
// new segment
if !ok {
storageVersion := storage.StorageV2
manifestPath := ""
if paramtable.Get().CommonCfg.UseLoonFFI.GetAsBool() {
storageVersion = storage.StorageV3
// set manifest path when creating segment
k := metautil.JoinIDPath(wb.collectionID, partitionID, segmentID)
basePath := path.Join(paramtable.Get().MinioCfg.RootPath.GetValue(), common.SegmentInsertLogPath, k)
// ManifestEarliest for first write
manifestPath = packed.MarshalManifestPath(basePath, packed.ManifestEarliest)
storageVersion, err := wb.resolveNewGrowingSegmentStorageVersion(info)
if err != nil {
return err
}
manifestPath := wb.newGrowingSegmentManifestPath(info.PartitionID, info.SegmentID, storageVersion)
segmentInfo := &datapb.SegmentInfo{
ID: segmentID,
PartitionID: partitionID,
ID: info.SegmentID,
PartitionID: info.PartitionID,
CollectionID: wb.collectionID,
InsertChannel: wb.channelName,
StartPosition: startPos,
StartPosition: info.StartPos,
State: commonpb.SegmentState_Growing,
StorageVersion: storageVersion,
ManifestPath: manifestPath,
SchemaVersion: schemaVersion,
SchemaVersion: info.SchemaVersion,
}
wb.metaCache.AddSegment(segmentInfo, func(_ *datapb.SegmentInfo) pkoracle.PkStat {
return pkoracle.NewBloomFilterSetWithBatchSize(wb.getEstBatchSize())
}, metacache.NewBM25StatsFactory, metacache.SetStartPosRecorded(false))
mlog.Info(context.TODO(), "add growing segment", mlog.FieldSegmentID(segmentID), mlog.String("channel", wb.channelName), mlog.Int64("storage version", storageVersion))
mlog.Info(context.TODO(), "add growing segment", mlog.FieldSegmentID(info.SegmentID), mlog.String("channel", wb.channelName), mlog.Int64("storage version", storageVersion))
}
return nil
}
func (wb *writeBufferBase) resolveNewGrowingSegmentStorageVersion(info CreateGrowingSegmentInfo) (int64, error) {
switch info.StorageVersion {
case storage.StorageV2, storage.StorageV3:
return info.StorageVersion, nil
case storage.StorageV1:
if streamingutil.IsStreamingServiceEnabled() {
return 0, merr.WrapErrServiceInternalMsg("missing storage version for streaming growing segment, segmentID=%d", info.SegmentID)
}
inferred := storage.StorageV2
reason := "default non-streaming storage version"
if typeutil.HasTextField(wb.metaCache.GetSchema(0)) {
inferred = storage.StorageV3
reason = "TEXT field requires StorageV3"
} else if paramtable.Get().CommonCfg.UseLoonFFI.GetAsBool() {
inferred = storage.StorageV3
reason = "common.storage.useLoonFFI enabled"
}
mlog.Warn(context.TODO(), "infer missing storage version for non-streaming growing segment",
mlog.FieldSegmentID(info.SegmentID),
mlog.Int64("collectionID", wb.collectionID),
mlog.String("channel", wb.channelName),
mlog.Int64("inferredStorageVersion", inferred),
mlog.String("reason", reason))
return inferred, nil
default:
return 0, merr.WrapErrServiceInternalMsg("unsupported storage version for growing segment, segmentID=%d storageVersion=%d",
info.SegmentID, info.StorageVersion)
}
}
func (wb *writeBufferBase) newGrowingSegmentManifestPath(partitionID int64, segmentID int64, storageVersion int64) string {
if storageVersion != storage.StorageV3 {
return ""
}
k := metautil.JoinIDPath(wb.collectionID, partitionID, segmentID)
basePath := path.Join(paramtable.Get().MinioCfg.RootPath.GetValue(), common.SegmentInsertLogPath, k)
return packed.MarshalManifestPath(basePath, packed.ManifestEarliest)
}
// bufferDelete buffers DeleteMsg into DeleteData.
@@ -1436,6 +1500,10 @@ func (wb *writeBufferBase) getSyncTask(ctx context.Context, segmentID int64) (sy
}
func (wb *writeBufferBase) getGrowingSourceSyncTask(ctx context.Context, segmentInfo *metacache.SegmentInfo, progress *growingSourceProgress) (syncmgr.Task, error) {
if segmentInfo.GetStorageVersion() != storage.StorageV3 {
return nil, merr.WrapErrServiceInternalMsg("growing-source sync requires StorageV3 segment, segmentID=%d storageVersion=%d",
segmentInfo.SegmentID(), segmentInfo.GetStorageVersion())
}
targetOffset := progress.targetOffset
pendingCommitted := progress.pendingCommitted
if pendingCommitted != nil {
@@ -20,6 +20,7 @@ import (
"github.com/milvus-io/milvus/internal/flushcommon/syncmgr"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/internal/util/function"
"github.com/milvus-io/milvus/internal/util/streamingutil"
"github.com/milvus-io/milvus/pkg/v3/common"
"github.com/milvus-io/milvus/pkg/v3/mq/msgstream"
"github.com/milvus-io/milvus/pkg/v3/proto/datapb"
@@ -71,7 +72,7 @@ func (s *WriteBufferSuite) SetupTest() {
func (s *WriteBufferSuite) TestHasSegment() {
segmentID := int64(1001)
s.wb.useGrowingSourceFlush = false
s.wb.allowGrowingSourceFlush = false
s.False(s.wb.HasSegment(segmentID))
s.wb.getOrCreateBuffer(segmentID, 0)
@@ -96,7 +97,7 @@ func (s *WriteBufferSuite) TestFlushSourceModeNotifier() {
}).Return().Once()
s.metacache.EXPECT().GetSegmentByID(segmentID).Return(segment, true).Once()
s.wb.useGrowingSourceFlush = true
s.wb.allowGrowingSourceFlush = true
s.wb.getOrCreateBuffer(segmentID, 0)
s.Equal(segmentID, notifiedSegmentID)
s.Equal(metacache.FlushSourceWriteBuffer, notifiedMode)
@@ -104,21 +105,22 @@ func (s *WriteBufferSuite) TestFlushSourceModeNotifier() {
s.Run("growing_mode", func() {
segmentID := int64(1002)
segment := metacache.NewSegmentInfo(&datapb.SegmentInfo{ID: segmentID}, nil, nil)
segment := metacache.NewSegmentInfo(&datapb.SegmentInfo{ID: segmentID, StorageVersion: storage.StorageV3}, nil, nil)
notifiedSegmentID = 0
notifiedMode = metacache.FlushSourceUnknown
s.metacache.EXPECT().GetSegmentByID(segmentID).Return(nil, false).Once()
s.metacache.EXPECT().AddSegment(mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return().Once()
s.metacache.EXPECT().GetSegmentByID(segmentID).Return(segment, true).Once()
s.metacache.EXPECT().GetSegmentByID(segmentID).Return(segment, true).Once()
s.metacache.EXPECT().UpdateSegments(mock.Anything, mock.Anything).Run(func(action metacache.SegmentAction, _ ...metacache.SegmentFilter) {
action(segment)
}).Return().Once()
s.metacache.EXPECT().GetSegmentByID(segmentID).Return(segment, true).Once()
s.wb.recordGrowingSourceProgress(&InsertData{
err := s.wb.recordGrowingSourceProgress(&InsertData{
segmentID: segmentID,
partitionID: 10,
rowNum: 3,
}, &msgpb.MsgPosition{Timestamp: 100}, &msgpb.MsgPosition{Timestamp: 200}, 1, 3)
s.NoError(err)
s.Equal(segmentID, notifiedSegmentID)
s.Equal(metacache.FlushSourceGrowing, notifiedMode)
})
@@ -132,8 +134,8 @@ func (s *WriteBufferSuite) TestCreateNewGrowingSegmentStorageVersion() {
defer param.Reset(param.CommonCfg.EnableGrowingSourceFlush.Key)
s.Run("non_text_uses_v2_when_ffi_disabled", func() {
s.wb.useGrowingSourceFlush = false
s.False(s.wb.UseGrowingSourceFlush())
s.wb.allowGrowingSourceFlush = false
s.False(s.wb.AllowGrowingSourceFlush())
s.metacache.EXPECT().GetSegmentByID(int64(2001)).Return(nil, false).Once()
s.metacache.EXPECT().AddSegment(mock.MatchedBy(func(info *datapb.SegmentInfo) bool {
return info.GetStorageVersion() == storage.StorageV2 &&
@@ -141,11 +143,17 @@ func (s *WriteBufferSuite) TestCreateNewGrowingSegmentStorageVersion() {
info.GetSchemaVersion() == 11
}), mock.Anything, mock.Anything, mock.Anything).Return().Once()
s.wb.CreateNewGrowingSegment(10, 2001, nil, 11)
err := s.wb.CreateNewGrowingSegment(CreateGrowingSegmentInfo{
PartitionID: 10,
SegmentID: 2001,
SchemaVersion: 11,
StorageVersion: storage.StorageV2,
})
s.NoError(err)
})
s.Run("growing_source_does_not_force_v3_manifest_when_ffi_disabled", func() {
s.wb.useGrowingSourceFlush = true
s.wb.allowGrowingSourceFlush = true
s.metacache.EXPECT().GetSegmentByID(int64(2002)).Return(nil, false).Once()
s.metacache.EXPECT().AddSegment(mock.MatchedBy(func(info *datapb.SegmentInfo) bool {
return info.GetStorageVersion() == storage.StorageV2 &&
@@ -153,10 +161,16 @@ func (s *WriteBufferSuite) TestCreateNewGrowingSegmentStorageVersion() {
info.GetSchemaVersion() == 12
}), mock.Anything, mock.Anything, mock.Anything).Return().Once()
s.wb.CreateNewGrowingSegment(10, 2002, nil, 12)
err := s.wb.CreateNewGrowingSegment(CreateGrowingSegmentInfo{
PartitionID: 10,
SegmentID: 2002,
SchemaVersion: 12,
StorageVersion: storage.StorageV2,
})
s.NoError(err)
})
s.Run("text_schema_does_not_enable_growing_source_when_ffi_disabled", func() {
s.Run("text_schema_uses_v3_manifest_without_enabling_growing_source_when_ffi_disabled", func() {
textSchema := &schemapb.CollectionSchema{
Name: "wb_text_collection",
Fields: []*schemapb.FieldSchema{
@@ -173,16 +187,21 @@ func (s *WriteBufferSuite) TestCreateNewGrowingSegmentStorageVersion() {
wb, err := newWriteBufferBase(s.channelName, mc, s.syncMgr, &writeBufferOption{})
s.Require().NoError(err)
s.False(wb.UseGrowingSourceFlush())
s.False(wb.AllowGrowingSourceFlush())
mc.EXPECT().GetSegmentByID(int64(2003)).Return(nil, false).Once()
mc.EXPECT().AddSegment(mock.MatchedBy(func(info *datapb.SegmentInfo) bool {
return info.GetStorageVersion() == storage.StorageV2 &&
info.GetManifestPath() == "" &&
return info.GetStorageVersion() == storage.StorageV3 &&
info.GetManifestPath() != "" &&
info.GetSchemaVersion() == 13
}), mock.Anything, mock.Anything, mock.Anything).Return().Once()
wb.CreateNewGrowingSegment(10, 2003, nil, 13)
err = wb.CreateNewGrowingSegment(CreateGrowingSegmentInfo{
PartitionID: 10,
SegmentID: 2003,
SchemaVersion: 13,
})
s.NoError(err)
})
s.Run("text_schema_uses_v3_manifest_when_ffi_enabled", func() {
@@ -205,7 +224,7 @@ func (s *WriteBufferSuite) TestCreateNewGrowingSegmentStorageVersion() {
wb, err := newWriteBufferBase(s.channelName, mc, s.syncMgr, &writeBufferOption{})
s.Require().NoError(err)
s.True(wb.UseGrowingSourceFlush())
s.True(wb.AllowGrowingSourceFlush())
mc.EXPECT().GetSegmentByID(int64(2004)).Return(nil, false).Once()
mc.EXPECT().AddSegment(mock.MatchedBy(func(info *datapb.SegmentInfo) bool {
@@ -214,7 +233,26 @@ func (s *WriteBufferSuite) TestCreateNewGrowingSegmentStorageVersion() {
info.GetSchemaVersion() == 14
}), mock.Anything, mock.Anything, mock.Anything).Return().Once()
wb.CreateNewGrowingSegment(10, 2004, nil, 14)
err = wb.CreateNewGrowingSegment(CreateGrowingSegmentInfo{
PartitionID: 10,
SegmentID: 2004,
SchemaVersion: 14,
StorageVersion: storage.StorageV3,
})
s.NoError(err)
})
s.Run("streaming_requires_create_segment_storage_version", func() {
s.T().Setenv(streamingutil.MilvusStreamingServiceEnabled, "1")
s.metacache.EXPECT().GetSegmentByID(int64(2005)).Return(nil, false).Once()
err := s.wb.CreateNewGrowingSegment(CreateGrowingSegmentInfo{
PartitionID: 10,
SegmentID: 2005,
SchemaVersion: 15,
})
s.ErrorIs(err, merr.ErrServiceInternal)
})
}
@@ -234,7 +272,7 @@ func (s *WriteBufferSuite) TestSealSegmentsMissingSegment() {
segmentID := int64(1001)
s.Run("non_text_returns_error", func() {
s.wb.useGrowingSourceFlush = false
s.wb.allowGrowingSourceFlush = false
s.metacache.EXPECT().GetSegmentByID(segmentID).Return(nil, false).Once()
err := s.wb.SealSegments(context.Background(), []int64{segmentID})
@@ -242,9 +280,9 @@ func (s *WriteBufferSuite) TestSealSegmentsMissingSegment() {
})
s.Run("text_skips_missing_segment", func() {
s.wb.useGrowingSourceFlush = true
s.wb.allowGrowingSourceFlush = true
defer func() {
s.wb.useGrowingSourceFlush = false
s.wb.allowGrowingSourceFlush = false
}()
s.metacache.EXPECT().GetSegmentByID(segmentID).Return(nil, false).Once()
+4 -4
View File
@@ -1572,7 +1572,7 @@ func NewShardDelegator(ctx context.Context, collectionID UniqueID, replicaID Uni
// Register growing-source segments as optional local flush sources. Metadata
// commit is still owned by WAL flusher / WriteBuffer.
if sd.useGrowingSourceFlush() {
if sd.allowGrowingSourceFlush() {
sd.growingSourceProvider = newDelegatorGrowingSourceProvider(manager.Segment, func(ctx context.Context, fenceTs uint64) error {
_, err := sd.waitTSafe(ctx, fenceTs)
return err
@@ -1589,12 +1589,12 @@ func NewShardDelegator(ctx context.Context, collectionID UniqueID, replicaID Uni
return sd, nil
}
// useGrowingSourceFlush returns true when the collection should expose growing segments as a flush source.
func (sd *shardDelegator) useGrowingSourceFlush() bool {
// allowGrowingSourceFlush returns true when the collection may expose growing segments as a flush source.
func (sd *shardDelegator) allowGrowingSourceFlush() bool {
if sd == nil || sd.collection == nil {
return false
}
return typeutil.UseGrowingSourceFlush(sd.collection.Schema(),
return typeutil.AllowGrowingSourceFlush(sd.collection.Schema(),
paramtable.Get().CommonCfg.UseLoonFFI.GetAsBool(),
paramtable.Get().CommonCfg.EnableGrowingSourceFlush.GetAsBool())
}
@@ -49,7 +49,12 @@ func (impl *msgHandlerImpl) HandleCreateSegment(ctx context.Context, createSegme
}
logger := mlog.With(mlog.FieldMessage(createSegmentMsg))
if err := impl.wbMgr.CreateNewGrowingSegment(ctx, vchannel, h.PartitionId, h.SegmentId, h.SchemaVersion); err != nil {
if err := impl.wbMgr.CreateNewGrowingSegment(ctx, vchannel, writebuffer.CreateGrowingSegmentInfo{
PartitionID: h.PartitionId,
SegmentID: h.SegmentId,
SchemaVersion: h.SchemaVersion,
StorageVersion: h.StorageVersion,
}); err != nil {
logger.Warn(ctx, "fail to create new growing segment")
return err
}
@@ -132,7 +132,9 @@ func TestFlushMsgHandler_HandleCreateSegment(t *testing.T) {
t.Run("growing-source collection creates WriteBuffer shell", func(t *testing.T) {
wbMgr := writebuffer.NewMockBufferManager(t)
wbMgr.EXPECT().CreateNewGrowingSegment(mock.Anything, vchannel, int64(10), int64(1001), mock.Anything).Return(nil)
wbMgr.EXPECT().CreateNewGrowingSegment(mock.Anything, vchannel, mock.MatchedBy(func(info writebuffer.CreateGrowingSegmentInfo) bool {
return info.PartitionID == 10 && info.SegmentID == 1001 && info.StorageVersion == 0
})).Return(nil)
handler := newMsgHandler(wbMgr)
err := handler.HandleCreateSegment(context.Background(), im)
@@ -141,7 +143,9 @@ func TestFlushMsgHandler_HandleCreateSegment(t *testing.T) {
t.Run("non-growing-source collection calls CreateNewGrowingSegment", func(t *testing.T) {
wbMgr := writebuffer.NewMockBufferManager(t)
wbMgr.EXPECT().CreateNewGrowingSegment(mock.Anything, vchannel, int64(10), int64(1001), mock.Anything).Return(nil)
wbMgr.EXPECT().CreateNewGrowingSegment(mock.Anything, vchannel, mock.MatchedBy(func(info writebuffer.CreateGrowingSegmentInfo) bool {
return info.PartitionID == 10 && info.SegmentID == 1001 && info.StorageVersion == 0
})).Return(nil)
handler := newMsgHandler(wbMgr)
err := handler.HandleCreateSegment(context.Background(), im)
@@ -101,7 +101,7 @@ func TestReleaseManualFlushPreparer(t *testing.T) {
manager.EXPECT().GetAvailableWAL(mock.Anything).Return(wal, nil)
wbManager := writebuffer.NewMockBufferManager(t)
wbManager.EXPECT().UseGrowingSourceFlush("vchannel").Return(true)
wbManager.EXPECT().AllowGrowingSourceFlush("vchannel").Return(true)
wbManager.EXPECT().
GetGrowingFlushProgress(mock.Anything, "vchannel", []int64{1001, 1002}, uint64(200)).
Return([]writebuffer.GrowingFlushSegmentProgress{
@@ -140,7 +140,7 @@ func TestReleaseManualFlushPreparerNoGrowingProgress(t *testing.T) {
manager.EXPECT().GetAvailableWAL(mock.Anything).Return(wal, nil)
wbManager := writebuffer.NewMockBufferManager(t)
wbManager.EXPECT().UseGrowingSourceFlush("vchannel").Return(true)
wbManager.EXPECT().AllowGrowingSourceFlush("vchannel").Return(true)
wbManager.EXPECT().
GetGrowingFlushProgress(mock.Anything, "vchannel", releaseSegmentIDs, uint64(200)).
Return([]writebuffer.GrowingFlushSegmentProgress{
@@ -169,7 +169,7 @@ func TestReleaseManualFlushPreparerSkipsManualFlushWhenCurrentSegmentsDoNotNeedH
MockBufferManager: writebuffer.NewMockBufferManager(t),
needManualFlush: false,
}
wbManager.EXPECT().UseGrowingSourceFlush("vchannel").Return(true)
wbManager.EXPECT().AllowGrowingSourceFlush("vchannel").Return(true)
wbManager.EXPECT().
GetGrowingFlushProgress(mock.Anything, "vchannel", releaseSegmentIDs, uint64(0)).
Return([]writebuffer.GrowingFlushSegmentProgress{
@@ -198,7 +198,7 @@ func TestReleaseManualFlushPreparerPreparesExistingProgressWithoutManualFlush(t
MockBufferManager: writebuffer.NewMockBufferManager(t),
needManualFlush: false,
}
wbManager.EXPECT().UseGrowingSourceFlush("vchannel").Return(true)
wbManager.EXPECT().AllowGrowingSourceFlush("vchannel").Return(true)
wbManager.EXPECT().
GetGrowingFlushProgress(mock.Anything, "vchannel", releaseSegmentIDs, uint64(0)).
Return([]writebuffer.GrowingFlushSegmentProgress{
@@ -227,7 +227,7 @@ func TestReleaseManualFlushPreparerSkipsManualFlushForEmptyInitialSegments(t *te
MockBufferManager: writebuffer.NewMockBufferManager(t),
needManualFlush: false,
}
wbManager.EXPECT().UseGrowingSourceFlush("vchannel").Return(true)
wbManager.EXPECT().AllowGrowingSourceFlush("vchannel").Return(true)
wbManager.EXPECT().
GetGrowingFlushProgress(mock.Anything, "vchannel", []int64(nil), uint64(0)).
Return(nil, nil)
@@ -259,7 +259,7 @@ func TestReleaseManualFlushPreparerFencesEmptyInitialSegments(t *testing.T) {
manager.EXPECT().GetAvailableWAL(mock.Anything).Return(wal, nil)
wbManager := writebuffer.NewMockBufferManager(t)
wbManager.EXPECT().UseGrowingSourceFlush("vchannel").Return(true)
wbManager.EXPECT().AllowGrowingSourceFlush("vchannel").Return(true)
wbManager.EXPECT().
GetGrowingFlushProgress(mock.Anything, "vchannel", affectedSegmentIDs, uint64(200)).
Return([]writebuffer.GrowingFlushSegmentProgress{
@@ -283,7 +283,7 @@ func TestReleaseManualFlushPreparerFencesEmptyInitialSegments(t *testing.T) {
func TestReleaseManualFlushPreparerSkipNonGrowingSource(t *testing.T) {
ctx := context.Background()
wbManager := writebuffer.NewMockBufferManager(t)
wbManager.EXPECT().UseGrowingSourceFlush("vchannel").Return(false)
wbManager.EXPECT().AllowGrowingSourceFlush("vchannel").Return(false)
preparer := NewReleaseManualFlushPreparer(mock_walmanager.NewMockManager(t), wbManager)
prepared, err := preparer.PrepareReleaseManualFlush(ctx, types.PChannelInfo{Name: "pchannel", Term: 1}, 10, "vchannel", []int64{1001})
@@ -37,7 +37,7 @@ func (p *releaseManualFlushPreparer) PrepareReleaseManualFlush(ctx context.Conte
if collectionID == 0 {
return false, status.NewInvalidArgument("collection id is empty")
}
if !p.writeBufferManager.UseGrowingSourceFlush(vchannel) {
if !p.writeBufferManager.AllowGrowingSourceFlush(vchannel) {
mlog.Info(ctx, "skip release manual flush prepare because channel does not use growing-source flush",
mlog.String("vchannel", vchannel),
mlog.Int64("collectionID", collectionID),
@@ -211,6 +211,6 @@ func (m *partitionManager) assignSegment(req *AssignSegmentRequest) (*AssignSegm
// There is no segment can be allocated for the insert request.
// Ask a new pending segment to insert.
m.asyncAllocSegment(req.SchemaVersion, req.UseGrowingSourceFlush)
m.asyncAllocSegment(req.SchemaVersion, req.RequiresStorageV3)
return nil, ErrWaitForNewSegment
}
@@ -17,7 +17,7 @@ import (
)
// asyncAllocSegment allocates a new growing segment asynchronously.
func (m *partitionManager) asyncAllocSegment(schemaVersion int32, useGrowingSourceFlush bool) {
func (m *partitionManager) asyncAllocSegment(schemaVersion int32, requiresStorageV3 bool) {
if m.onAllocating != nil {
m.Logger().Debug(context.TODO(), "segment alloc worker is already on allocating")
// manager is already on allocating.
@@ -26,13 +26,13 @@ func (m *partitionManager) asyncAllocSegment(schemaVersion int32, useGrowingSour
// Create a notifier to notify the waiter when the allocation is done.
m.onAllocating = make(chan struct{})
w := &segmentAllocWorker{
ctx: m.ctx,
collectionID: m.collectionID,
partitionID: m.partitionID,
vchannel: m.vchannel,
wal: m.wal.Get(),
schemaVersion: schemaVersion,
useGrowingSourceFlush: useGrowingSourceFlush,
ctx: m.ctx,
collectionID: m.collectionID,
partitionID: m.partitionID,
vchannel: m.vchannel,
wal: m.wal.Get(),
schemaVersion: schemaVersion,
requiresStorageV3: requiresStorageV3,
}
w.SetLogger(m.Logger())
// It should always done asynchronously.
@@ -50,11 +50,11 @@ type segmentAllocWorker struct {
wal wal.WAL
// The following fields are preserved across retries to ensure the same segment
// configuration is used when rebuilding the message after a failed append.
segmentID uint64 // allocated segment ID
storageVersion int64 // storage version determined at first attempt
limitation segmentLimitation // segment limitation determined at first attempt
schemaVersion int32
useGrowingSourceFlush bool
segmentID uint64 // allocated segment ID
storageVersion int64 // storage version determined at first attempt
limitation segmentLimitation // segment limitation determined at first attempt
schemaVersion int32
requiresStorageV3 bool
}
// do is the main loop of the segment allocation worker.
@@ -147,7 +147,7 @@ func (w *segmentAllocWorker) initSegmentConfig() error {
// Determine storage version.
w.storageVersion = storage.StorageV2
if paramtable.Get().CommonCfg.UseLoonFFI.GetAsBool() {
if w.requiresStorageV3 || paramtable.Get().CommonCfg.UseLoonFFI.GetAsBool() {
w.storageVersion = storage.StorageV3
}
@@ -296,31 +296,30 @@ func TestSegmentAllocWorker_InitSegmentConfig(t *testing.T) {
assert.Equal(t, firstLimitation, w.limitation)
}
func TestSegmentAllocWorkerStorageVersionFollowsUseLoonFFI(t *testing.T) {
func TestSegmentAllocWorkerStorageVersionFollowsRequirements(t *testing.T) {
paramtable.Init()
resource.InitForTest(t)
param := paramtable.Get()
defer param.Reset(param.CommonCfg.UseLoonFFI.Key)
for name, tc := range map[string]struct {
useLoonFFI string
useGrowingSourceFlush bool
expected int64
useLoonFFI string
requiresStorageV3 bool
expected int64
}{
"v2_without_growing_source": {useLoonFFI: "false", useGrowingSourceFlush: false, expected: storage.StorageV2},
"v2_with_growing_source": {useLoonFFI: "false", useGrowingSourceFlush: true, expected: storage.StorageV2},
"v3_without_growing_source": {useLoonFFI: "true", useGrowingSourceFlush: false, expected: storage.StorageV3},
"v3_with_growing_source": {useLoonFFI: "true", useGrowingSourceFlush: true, expected: storage.StorageV3},
"v2_without_requirement": {useLoonFFI: "false", expected: storage.StorageV2},
"v3_required_by_schema": {useLoonFFI: "false", requiresStorageV3: true, expected: storage.StorageV3},
"v3_with_ffi": {useLoonFFI: "true", expected: storage.StorageV3},
} {
t.Run(name, func(t *testing.T) {
param.Save(param.CommonCfg.UseLoonFFI.Key, tc.useLoonFFI)
w := &segmentAllocWorker{
ctx: context.Background(),
collectionID: 1,
partitionID: 2,
vchannel: "v1",
wal: mock_wal.NewMockWAL(t),
useGrowingSourceFlush: tc.useGrowingSourceFlush,
ctx: context.Background(),
collectionID: 1,
partitionID: 2,
vchannel: "v1",
wal: mock_wal.NewMockWAL(t),
requiresStorageV3: tc.requiresStorageV3,
}
w.SetLogger(mlog.With())
@@ -233,15 +233,19 @@ func (c *CollectionInfo) SchemaVersion() int32 {
return s.GetVersion()
}
func (c *CollectionInfo) UseGrowingSourceFlush() bool {
func (c *CollectionInfo) AllowGrowingSourceFlush() bool {
if c == nil || c.Schema == nil {
return false
}
return typeutil.UseGrowingSourceFlush(c.Schema.GetSchema(),
return typeutil.AllowGrowingSourceFlush(c.Schema.GetSchema(),
paramtable.Get().CommonCfg.UseLoonFFI.GetAsBool(),
paramtable.Get().CommonCfg.EnableGrowingSourceFlush.GetAsBool())
}
func (c *CollectionInfo) RequiresStorageV3() bool {
return c.HasTextField()
}
func (c *CollectionInfo) HasTextField() bool {
if c == nil || c.Schema == nil || c.Schema.GetSchema() == nil {
return false
@@ -266,7 +270,7 @@ func (c *CollectionInfo) RuntimeFlushSize(modified stats.ModifiedMetrics) uint64
}
func (c *CollectionInfo) shouldEstimateInterimIndexExtra() bool {
if c == nil || c.Schema == nil || c.Schema.GetSchema() == nil || !c.UseGrowingSourceFlush() {
if c == nil || c.Schema == nil || c.Schema.GetSchema() == nil || !c.AllowGrowingSourceFlush() {
return false
}
params := paramtable.Get()
@@ -12,14 +12,14 @@ import (
// AssignSegmentRequest is a request to allocate segment.
type AssignSegmentRequest struct {
CollectionID int64
PartitionID int64
ModifiedMetrics stats.ModifiedMetrics
RuntimeFlushSize uint64
TimeTick uint64
TxnSession TxnSession
SchemaVersion int32
UseGrowingSourceFlush bool
CollectionID int64
PartitionID int64
ModifiedMetrics stats.ModifiedMetrics
RuntimeFlushSize uint64
TimeTick uint64
TxnSession TxnSession
SchemaVersion int32
RequiresStorageV3 bool
}
// AssignSegmentResult is a result of segment allocation.
@@ -143,7 +143,7 @@ func (m *shardManagerImpl) AssignSegment(req *AssignSegmentRequest) (*AssignSegm
// single place that resolves and stamps it before forwarding to partitionManager.
if info := m.collections[req.CollectionID]; info != nil {
req.SchemaVersion = info.SchemaVersion()
req.UseGrowingSourceFlush = info.UseGrowingSourceFlush()
req.RequiresStorageV3 = info.RequiresStorageV3()
req.RuntimeFlushSize = info.RuntimeFlushSize(req.ModifiedMetrics)
}
@@ -494,7 +494,7 @@ func TestShardManagerSchemaVersionCheck(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, int32(4), ver)
assert.True(t, m.collections[104].HasTextField())
assert.True(t, m.collections[104].UseGrowingSourceFlush())
assert.True(t, m.collections[104].AllowGrowingSourceFlush())
// Test 3: Create collection without schema (legacy), then check version
createMsgNoSchema := message.NewCreateCollectionMessageBuilderV1().
@@ -835,17 +835,17 @@ func TestCollectionInfoSchemaVersion(t *testing.T) {
assert.Equal(t, int32(3), ci.SchemaVersion())
}
func TestCollectionInfoUseGrowingSourceFlush(t *testing.T) {
func TestCollectionInfoAllowGrowingSourceFlush(t *testing.T) {
paramtable.Init()
paramtable.Get().Save(paramtable.Get().CommonCfg.UseLoonFFI.Key, "false")
paramtable.Get().Save(paramtable.Get().CommonCfg.EnableGrowingSourceFlush.Key, "false")
defer paramtable.Get().Reset(paramtable.Get().CommonCfg.UseLoonFFI.Key)
defer paramtable.Get().Reset(paramtable.Get().CommonCfg.EnableGrowingSourceFlush.Key)
assert.False(t, (*CollectionInfo)(nil).UseGrowingSourceFlush())
assert.False(t, (*CollectionInfo)(nil).AllowGrowingSourceFlush())
ci := &CollectionInfo{}
assert.False(t, ci.UseGrowingSourceFlush())
assert.False(t, ci.AllowGrowingSourceFlush())
ci.Schema = &streamingpb.CollectionSchemaOfVChannel{}
assert.False(t, ci.UseGrowingSourceFlush())
assert.False(t, ci.AllowGrowingSourceFlush())
ci.Schema = &streamingpb.CollectionSchemaOfVChannel{
Schema: &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
@@ -854,12 +854,12 @@ func TestCollectionInfoUseGrowingSourceFlush(t *testing.T) {
},
}
paramtable.Get().Save(paramtable.Get().CommonCfg.EnableGrowingSourceFlush.Key, "true")
assert.False(t, ci.UseGrowingSourceFlush())
assert.False(t, ci.AllowGrowingSourceFlush())
paramtable.Get().Save(paramtable.Get().CommonCfg.UseLoonFFI.Key, "true")
assert.True(t, ci.UseGrowingSourceFlush())
assert.True(t, ci.AllowGrowingSourceFlush())
}
func TestCollectionInfoUseGrowingSourceFlush_TextField(t *testing.T) {
func TestCollectionInfoAllowGrowingSourceFlush_TextField(t *testing.T) {
paramtable.Init()
paramtable.Get().Save(paramtable.Get().CommonCfg.UseLoonFFI.Key, "false")
paramtable.Get().Save(paramtable.Get().CommonCfg.EnableGrowingSourceFlush.Key, "false")
@@ -877,9 +877,10 @@ func TestCollectionInfoUseGrowingSourceFlush_TextField(t *testing.T) {
},
}
assert.True(t, ci.HasTextField())
assert.False(t, ci.UseGrowingSourceFlush())
assert.False(t, ci.AllowGrowingSourceFlush())
assert.True(t, ci.RequiresStorageV3())
paramtable.Get().Save(paramtable.Get().CommonCfg.UseLoonFFI.Key, "true")
assert.True(t, ci.UseGrowingSourceFlush())
assert.True(t, ci.AllowGrowingSourceFlush())
}
func TestCollectionInfoRuntimeFlushSize(t *testing.T) {
+10 -3
View File
@@ -114,15 +114,22 @@ func ValidateTextRequiresStorageV3(schema *schemapb.CollectionSchema, storageV3E
return nil
}
// UseGrowingSourceFlush returns whether insert payload for the schema should
// be flushed from QueryNode growing source when available.
func UseGrowingSourceFlush(schema *schemapb.CollectionSchema, storageV3Enabled bool, enableGrowingSourceFlush bool) bool {
// AllowGrowingSourceFlush returns whether insert payload for the schema may try
// flushing from QueryNode growing source when available.
func AllowGrowingSourceFlush(schema *schemapb.CollectionSchema, storageV3Enabled bool, enableGrowingSourceFlush bool) bool {
if !storageV3Enabled {
return false
}
return HasTextField(schema) || enableGrowingSourceFlush
}
// UseGrowingSourceFlush is kept for compatibility. Prefer
// AllowGrowingSourceFlush for new code to avoid implying the source choice is
// mandatory.
func UseGrowingSourceFlush(schema *schemapb.CollectionSchema, storageV3Enabled bool, enableGrowingSourceFlush bool) bool {
return AllowGrowingSourceFlush(schema, storageV3Enabled, enableGrowingSourceFlush)
}
// EstimateSizePerRecord returns the estimate size of a record in a collection
func EstimateSizePerRecord(schema *schemapb.CollectionSchema) (int, error) {
return estimateSizeBy(schema, custom)
+11 -8
View File
@@ -336,7 +336,7 @@ func TestValidateTextRequiresStorageV3(t *testing.T) {
assert.NoError(t, ValidateTextRequiresStorageV3(textSchema, true))
}
func TestUseGrowingSourceFlush(t *testing.T) {
func TestAllowGrowingSourceFlush(t *testing.T) {
ordinarySchema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{FieldID: 100, DataType: schemapb.DataType_Int64},
@@ -349,13 +349,16 @@ func TestUseGrowingSourceFlush(t *testing.T) {
},
}
assert.False(t, UseGrowingSourceFlush(ordinarySchema, false, true))
assert.False(t, UseGrowingSourceFlush(textSchema, false, true))
assert.False(t, UseGrowingSourceFlush(ordinarySchema, true, false))
assert.True(t, UseGrowingSourceFlush(ordinarySchema, true, true))
assert.True(t, UseGrowingSourceFlush(textSchema, true, false))
assert.False(t, UseGrowingSourceFlush(nil, true, false))
assert.True(t, UseGrowingSourceFlush(nil, true, true))
assert.False(t, AllowGrowingSourceFlush(ordinarySchema, false, true))
assert.False(t, AllowGrowingSourceFlush(textSchema, false, true))
assert.False(t, AllowGrowingSourceFlush(ordinarySchema, true, false))
assert.True(t, AllowGrowingSourceFlush(ordinarySchema, true, true))
assert.True(t, AllowGrowingSourceFlush(textSchema, true, false))
assert.False(t, AllowGrowingSourceFlush(nil, true, false))
assert.True(t, AllowGrowingSourceFlush(nil, true, true))
assert.Equal(t,
AllowGrowingSourceFlush(textSchema, true, false),
UseGrowingSourceFlush(textSchema, true, false))
}
func TestSchema_GetVectorFieldSchemas(t *testing.T) {