mirror of
https://github.com/milvus-io/milvus.git
synced 2026-07-21 02:05:41 +00:00
fix: gate growing-source flush on StorageV3 (#51410)
issue: #51250 Keep TEXT segments on StorageV3 while making growing-source flush an explicit allowed mode instead of a mandatory TEXT path. Propagate create-segment storage versions through WAL flusher and write buffer, reject storage-version mismatches in DataCoord, and keep raw segcore chunks sticky for segments that may be flushed from growing source. Signed-off-by: luzhang <luzhang@zilliz.com> Co-authored-by: luzhang <luzhang@zilliz.com>
This commit is contained in:
@@ -393,9 +393,7 @@ SegmentGrowingImpl::mask_with_delete(BitsetTypeView& bitset,
|
||||
|
||||
void
|
||||
SegmentGrowingImpl::try_remove_chunks(FieldId fieldId) {
|
||||
if (segcore_config_.get_storage_v3_enabled() &&
|
||||
(SchemaHasTextField(*schema_) ||
|
||||
segcore_config_.get_enable_growing_source_flush())) {
|
||||
if (retain_insert_record_chunks_for_flush_) {
|
||||
// StorageV3 TEXT and growing-source flush persist growing segments
|
||||
// through milvus-storage. Interim indexes may also contain raw vector
|
||||
// data, but FlushGrowingSegmentData reads directly from insert_record_
|
||||
|
||||
@@ -431,6 +431,9 @@ class SegmentGrowingImpl : public SegmentGrowing {
|
||||
segcore_config_(segcore_config),
|
||||
schema_(std::move(schema)),
|
||||
index_meta_(indexMeta),
|
||||
retain_insert_record_chunks_for_flush_(
|
||||
ShouldRetainInsertRecordChunksForFlush(*schema_,
|
||||
segcore_config_)),
|
||||
insert_record_(
|
||||
*schema_, segcore_config.get_chunk_rows(), mmap_descriptor_),
|
||||
indexing_record_(
|
||||
@@ -838,10 +841,30 @@ class SegmentGrowingImpl : public SegmentGrowing {
|
||||
InitializeArrayOffsets();
|
||||
|
||||
private:
|
||||
static bool
|
||||
ShouldRetainInsertRecordChunksForFlush(
|
||||
const Schema& schema, const SegcoreConfig& segcore_config) {
|
||||
if (!segcore_config.get_storage_v3_enabled()) {
|
||||
return false;
|
||||
}
|
||||
if (segcore_config.get_enable_growing_source_flush()) {
|
||||
return true;
|
||||
}
|
||||
return std::any_of(schema.get_fields().begin(),
|
||||
schema.get_fields().end(),
|
||||
[](const auto& field) {
|
||||
return field.second.get_data_type() ==
|
||||
DataType::TEXT;
|
||||
});
|
||||
}
|
||||
|
||||
storage::MmapChunkDescriptorPtr mmap_descriptor_ = nullptr;
|
||||
SegcoreConfig segcore_config_;
|
||||
SchemaPtr schema_;
|
||||
IndexMetaPtr index_meta_;
|
||||
// Segment-level sticky decision. Once a growing segment is created, raw
|
||||
// chunks must not start/stop being retained because global config changes.
|
||||
bool retain_insert_record_chunks_for_flush_;
|
||||
|
||||
// inserted fields data and row_ids, timestamps
|
||||
InsertRecord<false> insert_record_;
|
||||
|
||||
@@ -369,6 +369,75 @@ TEST_P(GrowingIndexTest, Correctness) {
|
||||
}
|
||||
}
|
||||
|
||||
TEST(GrowingIndex, RetainInsertRecordChunksForFlushIsSegmentSticky) {
|
||||
constexpr int64_t dim = 4;
|
||||
constexpr int64_t row_count = 100;
|
||||
|
||||
auto schema = std::make_shared<Schema>();
|
||||
auto pk = schema->AddDebugField("pk", DataType::INT64);
|
||||
auto vec = schema->AddDebugField(
|
||||
"embeddings", DataType::VECTOR_FLOAT, dim, knowhere::metric::L2);
|
||||
schema->set_primary_field_id(pk);
|
||||
|
||||
std::map<std::string, std::string> index_params = {
|
||||
{"index_type", knowhere::IndexEnum::INDEX_FAISS_IVFFLAT},
|
||||
{"metric_type", knowhere::metric::L2},
|
||||
{"nlist", "1"}};
|
||||
std::map<std::string, std::string> type_params = {
|
||||
{"dim", std::to_string(dim)}};
|
||||
FieldIndexMeta field_index_meta(
|
||||
vec, std::move(index_params), std::move(type_params));
|
||||
std::map<FieldId, FieldIndexMeta> field_map = {{vec, field_index_meta}};
|
||||
IndexMetaPtr meta =
|
||||
std::make_shared<CollectionIndexMeta>(100, std::move(field_map));
|
||||
|
||||
auto& config = SegcoreConfig::default_config();
|
||||
ScopedSegcoreConfigRestore config_restore(config);
|
||||
InterimIndexConfigForTest interim_config;
|
||||
interim_config.chunk_rows = 16;
|
||||
interim_config.nlist = 1;
|
||||
interim_config.nprobe = 1;
|
||||
interim_config.dense_vector_interim_index_type =
|
||||
knowhere::IndexEnum::INDEX_FAISS_IVFFLAT_CC;
|
||||
interim_config.sub_dim = dim;
|
||||
interim_config.refine_ratio = 1.0F;
|
||||
interim_config.refine_quant_type = "NONE";
|
||||
interim_config.refine_with_quant_flag = false;
|
||||
ApplyInterimIndexConfigForTest(interim_config, config);
|
||||
config.set_storage_v3_enabled(true);
|
||||
config.set_enable_growing_source_flush(true);
|
||||
|
||||
auto retained_segment = CreateGrowingSegment(schema, meta, 1, config);
|
||||
auto* retained_impl =
|
||||
dynamic_cast<SegmentGrowingImpl*>(retained_segment.get());
|
||||
ASSERT_NE(retained_impl, nullptr);
|
||||
|
||||
config.set_enable_growing_source_flush(false);
|
||||
|
||||
auto insert = [&](SegmentGrowing* segment) {
|
||||
auto dataset = DataGen(schema, row_count);
|
||||
auto offset = segment->PreInsert(row_count);
|
||||
segment->Insert(offset,
|
||||
row_count,
|
||||
dataset.row_ids_.data(),
|
||||
dataset.timestamps_.data(),
|
||||
dataset.raw_);
|
||||
};
|
||||
|
||||
insert(retained_segment.get());
|
||||
EXPECT_GT(
|
||||
retained_impl->get_insert_record().get_data_base(vec)->num_chunk(), 0);
|
||||
|
||||
auto non_retained_segment = CreateGrowingSegment(schema, meta, 2, config);
|
||||
auto* non_retained_impl =
|
||||
dynamic_cast<SegmentGrowingImpl*>(non_retained_segment.get());
|
||||
ASSERT_NE(non_retained_impl, nullptr);
|
||||
insert(non_retained_segment.get());
|
||||
EXPECT_EQ(
|
||||
non_retained_impl->get_insert_record().get_data_base(vec)->num_chunk(),
|
||||
0);
|
||||
}
|
||||
|
||||
TEST_P(GrowingIndexTest, AddWithoutBuildPool) {
|
||||
constexpr int N = 1024;
|
||||
constexpr int dim = 4;
|
||||
|
||||
@@ -46,6 +46,9 @@ class ScopedSegcoreConfigRestore {
|
||||
nprobe_(config.get_nprobe()),
|
||||
enable_interim_segment_index_(
|
||||
config.get_enable_interim_segment_index()),
|
||||
storage_v3_enabled_(config.get_storage_v3_enabled()),
|
||||
enable_growing_source_flush_(
|
||||
config.get_enable_growing_source_flush()),
|
||||
sub_dim_(config.get_sub_dim()),
|
||||
refine_ratio_(config.get_refine_ratio()),
|
||||
dense_vector_interim_index_type_(
|
||||
@@ -60,6 +63,8 @@ class ScopedSegcoreConfigRestore {
|
||||
config_.set_nlist(nlist_);
|
||||
config_.set_nprobe(nprobe_);
|
||||
config_.set_enable_interim_segment_index(enable_interim_segment_index_);
|
||||
config_.set_storage_v3_enabled(storage_v3_enabled_);
|
||||
config_.set_enable_growing_source_flush(enable_growing_source_flush_);
|
||||
config_.set_sub_dim(sub_dim_);
|
||||
config_.set_refine_ratio(refine_ratio_);
|
||||
config_.set_dense_vector_intermin_index_type(
|
||||
@@ -78,6 +83,8 @@ class ScopedSegcoreConfigRestore {
|
||||
int64_t nlist_;
|
||||
int64_t nprobe_;
|
||||
bool enable_interim_segment_index_;
|
||||
bool storage_v3_enabled_;
|
||||
bool enable_growing_source_flush_;
|
||||
int64_t sub_dim_;
|
||||
float refine_ratio_;
|
||||
std::string dense_vector_interim_index_type_;
|
||||
|
||||
@@ -1058,6 +1058,25 @@ func SetStorageVersion(segmentID int64, version int64) UpdateOperator {
|
||||
}
|
||||
}
|
||||
|
||||
func ValidateSaveBinlogStorageVersion(segmentID int64, incoming int64) UpdateOperator {
|
||||
return func(modPack *updateSegmentPack) bool {
|
||||
segment := modPack.Get(segmentID)
|
||||
if segment == nil {
|
||||
modPack.err = merr.WrapErrSegmentNotFound(segmentID)
|
||||
return false
|
||||
}
|
||||
|
||||
current := segment.GetStorageVersion()
|
||||
if incoming != current {
|
||||
modPack.err = merr.WrapErrDataIntegrityMsg(
|
||||
"segment %d storage version mismatch, current=%d incoming=%d",
|
||||
segmentID, current, incoming)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
func UpdateCompactedOperator(segmentID int64) UpdateOperator {
|
||||
return func(modPack *updateSegmentPack) bool {
|
||||
segment := modPack.Get(segmentID)
|
||||
|
||||
@@ -658,13 +658,13 @@ func (s *Server) SaveBinlogPaths(ctx context.Context, req *datapb.SaveBinlogPath
|
||||
mlog.Warn(context.TODO(), "failed to get segment, the segment not healthy", mlog.Err(err))
|
||||
return merr.Status(err), nil
|
||||
}
|
||||
if err := s.validateTextSegmentStorage(req); err != nil {
|
||||
incomingStorageVersion := req.GetStorageVersion()
|
||||
if err := s.validateTextSegmentStorage(req, incomingStorageVersion); err != nil {
|
||||
mlog.Warn(context.TODO(), "invalid TEXT segment storage format", mlog.Err(err))
|
||||
return merr.Status(err), nil
|
||||
}
|
||||
|
||||
// Set storage version
|
||||
operators = append(operators, SetStorageVersion(req.GetSegmentID(), req.GetStorageVersion()))
|
||||
operators = append(operators, ValidateSaveBinlogStorageVersion(req.GetSegmentID(), incomingStorageVersion))
|
||||
|
||||
// Set segment state
|
||||
if req.GetDropped() {
|
||||
@@ -753,18 +753,18 @@ func (s *Server) SaveBinlogPaths(ctx context.Context, req *datapb.SaveBinlogPath
|
||||
return merr.Success(), nil
|
||||
}
|
||||
|
||||
func (s *Server) validateTextSegmentStorage(req *datapb.SaveBinlogPathsRequest) error {
|
||||
func (s *Server) validateTextSegmentStorage(req *datapb.SaveBinlogPathsRequest, storageVersion int64) error {
|
||||
if req.GetSegLevel() == datapb.SegmentLevel_L0 || req.GetDropped() {
|
||||
return nil
|
||||
}
|
||||
if !s.meta.collectionHasTextFields(req.GetCollectionID()) {
|
||||
return nil
|
||||
}
|
||||
if req.GetStorageVersion() < storage.StorageV3 {
|
||||
if storageVersion < storage.StorageV3 {
|
||||
return merr.WrapErrParameterInvalidMsg(
|
||||
"TEXT segment %d must be saved with StorageV3 manifest, got storage version %d",
|
||||
req.GetSegmentID(),
|
||||
req.GetStorageVersion())
|
||||
storageVersion)
|
||||
}
|
||||
if req.GetManifestPath() == "" {
|
||||
return merr.WrapErrParameterInvalidMsg(
|
||||
|
||||
@@ -275,6 +275,65 @@ func (s *ServerSuite) TestSaveBinlogPath_SaveUnhealthySegment() {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *ServerSuite) TestSaveBinlogPath_StorageVersionImmutable() {
|
||||
s.testServer.meta.AddCollection(&collectionInfo{ID: 0})
|
||||
info := &datapb.SegmentInfo{
|
||||
ID: 10,
|
||||
InsertChannel: "ch1",
|
||||
State: commonpb.SegmentState_Growing,
|
||||
Level: datapb.SegmentLevel_L1,
|
||||
StorageVersion: storage.StorageV2,
|
||||
}
|
||||
err := s.testServer.meta.AddSegment(context.TODO(), NewSegmentInfo(info))
|
||||
s.Require().NoError(err)
|
||||
|
||||
resp, err := s.testServer.SaveBinlogPaths(context.Background(), &datapb.SaveBinlogPathsRequest{
|
||||
Base: &commonpb.MsgBase{
|
||||
Timestamp: uint64(time.Now().Unix()),
|
||||
},
|
||||
SegmentID: 10,
|
||||
Channel: "ch1",
|
||||
StorageVersion: storage.StorageV3,
|
||||
})
|
||||
s.NoError(err)
|
||||
s.ErrorIs(merr.Error(resp), merr.ErrDataIntegrity)
|
||||
|
||||
resp, err = s.testServer.SaveBinlogPaths(context.Background(), &datapb.SaveBinlogPathsRequest{
|
||||
Base: &commonpb.MsgBase{
|
||||
Timestamp: uint64(time.Now().Unix()),
|
||||
},
|
||||
SegmentID: 10,
|
||||
Channel: "ch1",
|
||||
})
|
||||
s.NoError(err)
|
||||
s.ErrorIs(merr.Error(resp), merr.ErrDataIntegrity)
|
||||
segment := s.testServer.meta.GetSegment(context.TODO(), 10)
|
||||
s.EqualValues(storage.StorageV2, segment.GetStorageVersion())
|
||||
|
||||
info = &datapb.SegmentInfo{
|
||||
ID: 11,
|
||||
InsertChannel: "ch1",
|
||||
State: commonpb.SegmentState_Growing,
|
||||
Level: datapb.SegmentLevel_L1,
|
||||
StorageVersion: storage.StorageV1,
|
||||
}
|
||||
err = s.testServer.meta.AddSegment(context.TODO(), NewSegmentInfo(info))
|
||||
s.Require().NoError(err)
|
||||
|
||||
resp, err = s.testServer.SaveBinlogPaths(context.Background(), &datapb.SaveBinlogPathsRequest{
|
||||
Base: &commonpb.MsgBase{
|
||||
Timestamp: uint64(time.Now().Unix()),
|
||||
},
|
||||
SegmentID: 11,
|
||||
Channel: "ch1",
|
||||
StorageVersion: storage.StorageV2,
|
||||
})
|
||||
s.NoError(err)
|
||||
s.ErrorIs(merr.Error(resp), merr.ErrDataIntegrity)
|
||||
segment = s.testServer.meta.GetSegment(context.TODO(), 11)
|
||||
s.EqualValues(storage.StorageV1, segment.GetStorageVersion())
|
||||
}
|
||||
|
||||
func (s *ServerSuite) TestSaveBinlogPath_SaveDroppedSegment() {
|
||||
s.testServer.meta.AddCollection(&collectionInfo{ID: 0})
|
||||
|
||||
|
||||
@@ -823,6 +823,10 @@ func (t *GrowingSourceSyncTask) schemaBasedPattern(columnGroups []storagecommon.
|
||||
}
|
||||
|
||||
func (t *GrowingSourceSyncTask) buildFlushConfig(segment *metacache.SegmentInfo, columnGroups []storagecommon.ColumnGroup) (*GrowingFlushConfig, error) {
|
||||
if segment.GetStorageVersion() != storage.StorageV3 {
|
||||
return nil, merr.WrapErrDataIntegrityMsg("growing source flush requires StorageV3 segment, segmentID=%d storageVersion=%d",
|
||||
t.segmentID, segment.GetStorageVersion())
|
||||
}
|
||||
segmentBasePath := path.Join(t.chunkManager.RootPath(), common.SegmentInsertLogPath,
|
||||
metautil.JoinIDPath(t.collectionID, t.partitionID, t.segmentID))
|
||||
partitionBasePath := path.Join(t.chunkManager.RootPath(), common.SegmentInsertLogPath,
|
||||
|
||||
@@ -111,9 +111,10 @@ func TestGrowingSourceSyncTaskBuildFlushConfigBM25(t *testing.T) {
|
||||
cm := mock_storage.NewMockChunkManager(t)
|
||||
cm.EXPECT().RootPath().Return("/root").Maybe()
|
||||
segment := metacache.NewSegmentInfo(&datapb.SegmentInfo{
|
||||
ID: segmentID,
|
||||
PartitionID: 2,
|
||||
ManifestPath: `{"ver":7,"base_path":"/root/insert_log/3/2/1"}`,
|
||||
ID: segmentID,
|
||||
PartitionID: 2,
|
||||
StorageVersion: storage.StorageV3,
|
||||
ManifestPath: `{"ver":7,"base_path":"/root/insert_log/3/2/1"}`,
|
||||
}, pkoracle.NewBloomFilterSet(), nil)
|
||||
|
||||
task := NewGrowingSourceSyncTask().
|
||||
@@ -145,8 +146,9 @@ func TestGrowingSourceSyncTaskBuildFlushConfigStartsFromEarliestManifest(t *test
|
||||
cm := mock_storage.NewMockChunkManager(t)
|
||||
cm.EXPECT().RootPath().Return("/root").Maybe()
|
||||
segment := metacache.NewSegmentInfo(&datapb.SegmentInfo{
|
||||
ID: 1,
|
||||
PartitionID: 2,
|
||||
ID: 1,
|
||||
PartitionID: 2,
|
||||
StorageVersion: storage.StorageV3,
|
||||
}, pkoracle.NewBloomFilterSet(), nil)
|
||||
|
||||
task := NewGrowingSourceSyncTask().
|
||||
@@ -171,7 +173,7 @@ func TestGrowingSourceSyncTaskBuildFlushConfigBM25AllocatorError(t *testing.T) {
|
||||
}
|
||||
cm := mock_storage.NewMockChunkManager(t)
|
||||
cm.EXPECT().RootPath().Return("/root").Maybe()
|
||||
segment := metacache.NewSegmentInfo(&datapb.SegmentInfo{ID: 1}, pkoracle.NewBloomFilterSet(), nil)
|
||||
segment := metacache.NewSegmentInfo(&datapb.SegmentInfo{ID: 1, StorageVersion: storage.StorageV3}, pkoracle.NewBloomFilterSet(), nil)
|
||||
task := NewGrowingSourceSyncTask().
|
||||
WithCollectionID(3).
|
||||
WithPartitionID(2).
|
||||
@@ -195,7 +197,7 @@ func TestGrowingSourceSyncTaskBuildFlushConfigBM25RequiresAllocator(t *testing.T
|
||||
}
|
||||
cm := mock_storage.NewMockChunkManager(t)
|
||||
cm.EXPECT().RootPath().Return("/root").Maybe()
|
||||
segment := metacache.NewSegmentInfo(&datapb.SegmentInfo{ID: 1}, pkoracle.NewBloomFilterSet(), nil)
|
||||
segment := metacache.NewSegmentInfo(&datapb.SegmentInfo{ID: 1, StorageVersion: storage.StorageV3}, pkoracle.NewBloomFilterSet(), nil)
|
||||
task := NewGrowingSourceSyncTask().
|
||||
WithCollectionID(3).
|
||||
WithPartitionID(2).
|
||||
@@ -455,9 +457,10 @@ func TestGrowingSourceSyncTaskCommitRetainedSourceOnlyOnFinalization(t *testing.
|
||||
segmentID := int64(1)
|
||||
mc := metacache.NewMockMetaCache(t)
|
||||
segment := metacache.NewSegmentInfo(&datapb.SegmentInfo{
|
||||
ID: segmentID,
|
||||
PartitionID: 2,
|
||||
ManifestPath: "manifest",
|
||||
ID: segmentID,
|
||||
PartitionID: 2,
|
||||
StorageVersion: storage.StorageV3,
|
||||
ManifestPath: "manifest",
|
||||
}, pkoracle.NewBloomFilterSet(), nil)
|
||||
metacache.UpdateNumOfRows(10)(segment)
|
||||
source := &fakeCommitGrowingFlushSource{}
|
||||
@@ -554,9 +557,10 @@ func TestGrowingSourceSyncTaskMergesReturnedBM25Stats(t *testing.T) {
|
||||
|
||||
mc := metacache.NewMockMetaCache(t)
|
||||
segment := metacache.NewSegmentInfo(&datapb.SegmentInfo{
|
||||
ID: segmentID,
|
||||
PartitionID: 2,
|
||||
State: commonpb.SegmentState_Growing,
|
||||
ID: segmentID,
|
||||
PartitionID: 2,
|
||||
State: commonpb.SegmentState_Growing,
|
||||
StorageVersion: storage.StorageV3,
|
||||
}, pkoracle.NewBloomFilterSet(), nil)
|
||||
|
||||
mc.EXPECT().GetSegmentByID(segmentID).Return(segment, true)
|
||||
|
||||
@@ -167,6 +167,10 @@ func (b *brokerMetaWriter) UpdateGrowingSourceSync(ctx context.Context, task *Gr
|
||||
if !ok {
|
||||
return merr.WrapErrSegmentNotFound(task.segmentID)
|
||||
}
|
||||
if segment.GetStorageVersion() != storage.StorageV3 {
|
||||
return merr.WrapErrDataIntegrityMsg("growing source sync requires StorageV3 segment, segmentID=%d storageVersion=%d",
|
||||
task.segmentID, segment.GetStorageVersion())
|
||||
}
|
||||
|
||||
insertFieldBinlogs := segment.Binlogs()
|
||||
if len(task.insertBinlogs) > 0 {
|
||||
@@ -215,7 +219,7 @@ func (b *brokerMetaWriter) UpdateGrowingSourceSync(ctx context.Context, task *Gr
|
||||
Dropped: task.IsDrop(),
|
||||
Channel: task.channelName,
|
||||
SegLevel: task.level,
|
||||
StorageVersion: storage.StorageV3,
|
||||
StorageVersion: segment.GetStorageVersion(),
|
||||
WithFullBinlogs: true,
|
||||
ManifestPath: task.manifestPath,
|
||||
}
|
||||
|
||||
@@ -315,8 +315,9 @@ func (s *MetaWriterSuite) TestGrowingSourceSyncMetaErrorsReturnError() {
|
||||
|
||||
bfs := pkoracle.NewBloomFilterSet()
|
||||
seg := metacache.NewSegmentInfo(&datapb.SegmentInfo{
|
||||
ID: 1,
|
||||
PartitionID: 2,
|
||||
ID: 1,
|
||||
PartitionID: 2,
|
||||
StorageVersion: storage.StorageV3,
|
||||
}, bfs, nil)
|
||||
s.metacache.EXPECT().GetSegmentByID(int64(1)).Return(seg, true).Once()
|
||||
s.metacache.EXPECT().GetSegmentsBy(mock.Anything, mock.Anything, mock.Anything).Return(nil).Once()
|
||||
|
||||
@@ -62,11 +62,14 @@ func (wb *l0WriteBuffer) BufferData(insertData []*InsertData, deleteMsgs []*msgs
|
||||
wb.mut.Lock()
|
||||
|
||||
for _, inData := range insertData {
|
||||
if wb.useGrowingSourceFlush {
|
||||
if wb.allowGrowingSourceFlush {
|
||||
targetOffset := wb.growingSourceTargetOffset(inData.segmentID, inData.rowNum)
|
||||
decision := wb.decideGrowingFlushSource(inData.segmentID, targetOffset, endPos)
|
||||
if decision.sourceType == metacache.FlushSourceGrowing {
|
||||
wb.recordGrowingSourceProgress(inData, startPos, endPos, schemaVersion, targetOffset)
|
||||
if err := wb.recordGrowingSourceProgress(inData, startPos, endPos, schemaVersion, targetOffset); err != nil {
|
||||
wb.mut.Unlock()
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
@@ -106,7 +109,14 @@ func (wb *l0WriteBuffer) BufferData(insertData []*InsertData, deleteMsgs []*msgs
|
||||
|
||||
// bufferInsert function InsertMsg into bufferred InsertData and returns primary key field data for future usage.
|
||||
func (wb *l0WriteBuffer) bufferInsert(inData *InsertData, startPos, endPos *msgpb.MsgPosition, schemaVersion int32) error {
|
||||
wb.CreateNewGrowingSegment(inData.partitionID, inData.segmentID, startPos, schemaVersion)
|
||||
if err := wb.CreateNewGrowingSegment(CreateGrowingSegmentInfo{
|
||||
PartitionID: inData.partitionID,
|
||||
SegmentID: inData.segmentID,
|
||||
StartPos: startPos,
|
||||
SchemaVersion: schemaVersion,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
segBuf := wb.getOrCreateBuffer(inData.segmentID, startPos.GetTimestamp())
|
||||
|
||||
totalMemSize := segBuf.insertBuffer.Buffer(inData, startPos, endPos)
|
||||
|
||||
@@ -410,11 +410,11 @@ func (s *L0WriteBufferSuite) TestBufferDataGrowingSourceMode() {
|
||||
|
||||
s.Run("usable_source_records_progress_and_pins_checkpoint", func() {
|
||||
textSchema := s.textSchema()
|
||||
metacache := s.newTextMetaCache(textSchema)
|
||||
mc := s.newTextMetaCache(textSchema)
|
||||
var resolvedSegmentID int64
|
||||
var resolvedTargetOffset int64
|
||||
resolveCalls := 0
|
||||
wb, err := NewL0WriteBuffer(s.channelName, metacache, s.syncMgr, &writeBufferOption{
|
||||
wb, err := NewL0WriteBuffer(s.channelName, mc, s.syncMgr, &writeBufferOption{
|
||||
idAllocator: s.allocator,
|
||||
growingSourceRetryInterval: time.Hour,
|
||||
growingSourceResolver: func(segmentID int64, targetOffset int64, _ *msgpb.MsgPosition) (syncmgr.GrowingFlushSource, syncmgr.GrowingSourceState) {
|
||||
@@ -433,11 +433,24 @@ func (s *L0WriteBufferSuite) TestBufferDataGrowingSourceMode() {
|
||||
insertData, err := PrepareInsert(textSchema, s.pkSchema, []*msgstream.InsertMsg{msg})
|
||||
s.NoError(err)
|
||||
|
||||
// 3 first-insert path calls plus 1 triggerSync policy check for the
|
||||
// recorded growing source progress.
|
||||
metacache.EXPECT().GetSegmentByID(int64(1001)).Return(nil, false).Times(4)
|
||||
metacache.EXPECT().AddSegment(mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return()
|
||||
metacache.EXPECT().UpdateSegments(mock.Anything, mock.Anything).Return()
|
||||
segmentInfo := metacache.NewSegmentInfo(&datapb.SegmentInfo{
|
||||
ID: 1001,
|
||||
PartitionID: 10,
|
||||
CollectionID: s.collID,
|
||||
InsertChannel: s.channelName,
|
||||
StartPosition: &msgpb.MsgPosition{Timestamp: 100},
|
||||
State: commonpb.SegmentState_Growing,
|
||||
StorageVersion: storage.StorageV3,
|
||||
SchemaVersion: 100,
|
||||
}, pkoracle.NewBloomFilterSet(), nil)
|
||||
|
||||
// growingSourceBaseOffset and CreateNewGrowingSegment see a new
|
||||
// segment; recordGrowingSourceProgress and triggerSync see the segment
|
||||
// created by AddSegment.
|
||||
mc.EXPECT().GetSegmentByID(int64(1001)).Return(nil, false).Times(3)
|
||||
mc.EXPECT().GetSegmentByID(int64(1001)).Return(segmentInfo, true).Times(2)
|
||||
mc.EXPECT().AddSegment(mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return()
|
||||
mc.EXPECT().UpdateSegments(mock.Anything, mock.Anything).Return()
|
||||
|
||||
err = wb.BufferData(insertData, nil, &msgpb.MsgPosition{Timestamp: 100}, &msgpb.MsgPosition{Timestamp: 200}, 100)
|
||||
s.NoError(err)
|
||||
@@ -556,8 +569,8 @@ func (s *L0WriteBufferSuite) TestBufferDataGrowingSourceMode() {
|
||||
|
||||
s.Run("pending_source_records_progress_instead_of_falling_back_to_writebuffer", func() {
|
||||
textSchema := s.textSchema()
|
||||
metacache := s.newTextMetaCache(textSchema)
|
||||
wb, err := NewL0WriteBuffer(s.channelName, metacache, s.syncMgr, &writeBufferOption{
|
||||
mc := s.newTextMetaCache(textSchema)
|
||||
wb, err := NewL0WriteBuffer(s.channelName, mc, s.syncMgr, &writeBufferOption{
|
||||
idAllocator: s.allocator,
|
||||
growingSourceRetryInterval: time.Hour,
|
||||
growingSourceResolver: func(segmentID int64, targetOffset int64, _ *msgpb.MsgPosition) (syncmgr.GrowingFlushSource, syncmgr.GrowingSourceState) {
|
||||
@@ -570,10 +583,24 @@ func (s *L0WriteBufferSuite) TestBufferDataGrowingSourceMode() {
|
||||
insertData, err := PrepareInsert(textSchema, s.pkSchema, []*msgstream.InsertMsg{msg})
|
||||
s.NoError(err)
|
||||
|
||||
// 3 first-insert path calls plus 1 triggerSync policy check.
|
||||
metacache.EXPECT().GetSegmentByID(int64(1002)).Return(nil, false).Times(4)
|
||||
metacache.EXPECT().AddSegment(mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return()
|
||||
metacache.EXPECT().UpdateSegments(mock.Anything, mock.Anything).Return()
|
||||
segmentInfo := metacache.NewSegmentInfo(&datapb.SegmentInfo{
|
||||
ID: 1002,
|
||||
PartitionID: 10,
|
||||
CollectionID: s.collID,
|
||||
InsertChannel: s.channelName,
|
||||
StartPosition: &msgpb.MsgPosition{Timestamp: 100},
|
||||
State: commonpb.SegmentState_Growing,
|
||||
StorageVersion: storage.StorageV3,
|
||||
SchemaVersion: 100,
|
||||
}, pkoracle.NewBloomFilterSet(), nil)
|
||||
|
||||
// growingSourceBaseOffset and CreateNewGrowingSegment see a new
|
||||
// segment; recordGrowingSourceProgress and triggerSync see the segment
|
||||
// created by AddSegment.
|
||||
mc.EXPECT().GetSegmentByID(int64(1002)).Return(nil, false).Times(3)
|
||||
mc.EXPECT().GetSegmentByID(int64(1002)).Return(segmentInfo, true).Times(2)
|
||||
mc.EXPECT().AddSegment(mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return()
|
||||
mc.EXPECT().UpdateSegments(mock.Anything, mock.Anything).Return()
|
||||
|
||||
err = wb.BufferData(insertData, nil, &msgpb.MsgPosition{Timestamp: 100}, &msgpb.MsgPosition{Timestamp: 200}, 100)
|
||||
s.NoError(err)
|
||||
@@ -1666,11 +1693,25 @@ func (s *L0WriteBufferSuite) TestCheckReleaseManualFlushNeed() {
|
||||
s.False(checker.CheckReleaseManualFlushNeed(nil))
|
||||
s.True(checker.CheckReleaseManualFlushNeed([]int64{1300}))
|
||||
|
||||
wb.CreateNewGrowingSegment(0, 1301, &msgpb.MsgPosition{Timestamp: 100}, 100)
|
||||
err = wb.CreateNewGrowingSegment(CreateGrowingSegmentInfo{
|
||||
PartitionID: 0,
|
||||
SegmentID: 1301,
|
||||
StartPos: &msgpb.MsgPosition{Timestamp: 100},
|
||||
SchemaVersion: 100,
|
||||
StorageVersion: storage.StorageV3,
|
||||
})
|
||||
s.NoError(err)
|
||||
mc.UpdateSegments(metacache.SetFlushSourceMode(metacache.FlushSourceWriteBuffer), metacache.WithSegmentIDs(1301))
|
||||
s.False(checker.CheckReleaseManualFlushNeed([]int64{1301}))
|
||||
|
||||
wb.CreateNewGrowingSegment(0, 1302, &msgpb.MsgPosition{Timestamp: 100}, 100)
|
||||
err = wb.CreateNewGrowingSegment(CreateGrowingSegmentInfo{
|
||||
PartitionID: 0,
|
||||
SegmentID: 1302,
|
||||
StartPos: &msgpb.MsgPosition{Timestamp: 100},
|
||||
SchemaVersion: 100,
|
||||
StorageVersion: storage.StorageV3,
|
||||
})
|
||||
s.NoError(err)
|
||||
mc.UpdateSegments(metacache.SetFlushSourceMode(metacache.FlushSourceGrowing), metacache.WithSegmentIDs(1302))
|
||||
s.True(checker.CheckReleaseManualFlushNeed([]int64{1302}))
|
||||
|
||||
@@ -1678,7 +1719,14 @@ func (s *L0WriteBufferSuite) TestCheckReleaseManualFlushNeed() {
|
||||
s.False(checker.CheckReleaseManualFlushNeed([]int64{1302}))
|
||||
s.True(checker.CheckReleaseManualFlushNeed([]int64{1301, 1300}))
|
||||
|
||||
wb.CreateNewGrowingSegment(0, 1303, &msgpb.MsgPosition{Timestamp: 100}, 100)
|
||||
err = wb.CreateNewGrowingSegment(CreateGrowingSegmentInfo{
|
||||
PartitionID: 0,
|
||||
SegmentID: 1303,
|
||||
StartPos: &msgpb.MsgPosition{Timestamp: 100},
|
||||
SchemaVersion: 100,
|
||||
StorageVersion: storage.StorageV3,
|
||||
})
|
||||
s.NoError(err)
|
||||
mc.UpdateSegments(metacache.SetFlushSourceMode(metacache.FlushSourceGrowing), metacache.WithSegmentIDs(1303))
|
||||
s.True(checker.CheckReleaseManualFlushNeed([]int64{1300, 1303}))
|
||||
}
|
||||
|
||||
@@ -27,7 +27,7 @@ type BufferManager interface {
|
||||
// Register adds a WriteBuffer with provided schema & options.
|
||||
Register(channel string, metacache metacache.MetaCache, opts ...WriteBufferOption) error
|
||||
// CreateNewGrowingSegment notifies writeBuffer to create a new growing segment.
|
||||
CreateNewGrowingSegment(ctx context.Context, channel string, partition int64, segmentID int64, schemaVersion int32) error
|
||||
CreateNewGrowingSegment(ctx context.Context, channel string, info CreateGrowingSegmentInfo) error
|
||||
// SealSegments notifies writeBuffer corresponding to provided channel to seal segments.
|
||||
// which will cause segment start flush procedure.
|
||||
SealSegments(ctx context.Context, channel string, segmentIDs []int64) error
|
||||
@@ -47,8 +47,8 @@ type BufferManager interface {
|
||||
// NotifyCheckpointUpdated notify write buffer checkpoint updated to reset flushTs.
|
||||
NotifyCheckpointUpdated(channel string, ts uint64)
|
||||
|
||||
// UseGrowingSourceFlush returns true if the collection on this channel has growing-source fields.
|
||||
UseGrowingSourceFlush(channel string) bool
|
||||
// AllowGrowingSourceFlush returns true if this channel may try growing-source flush.
|
||||
AllowGrowingSourceFlush(channel string) bool
|
||||
// GetGrowingFlushProgress returns growing-source progress for the given channel.
|
||||
// If segmentIDs is empty, all tracked growing-source segments are returned.
|
||||
// Otherwise, the requested segmentIDs are returned together with all tracked
|
||||
@@ -187,17 +187,16 @@ func (m *bufferManager) Register(channel string, metacache metacache.MetaCache,
|
||||
}
|
||||
|
||||
// CreateNewGrowingSegment notifies writeBuffer to create a new growing segment.
|
||||
func (m *bufferManager) CreateNewGrowingSegment(ctx context.Context, channel string, partitionID int64, segmentID int64, schemaVersion int32) error {
|
||||
func (m *bufferManager) CreateNewGrowingSegment(ctx context.Context, channel string, info CreateGrowingSegmentInfo) error {
|
||||
buf, loaded := m.buffers.Get(channel)
|
||||
if !loaded {
|
||||
mlog.Warn(ctx, "write buffer not found when create new growing segment",
|
||||
mlog.String("channel", channel),
|
||||
mlog.FieldPartitionID(partitionID),
|
||||
mlog.FieldSegmentID(segmentID))
|
||||
mlog.FieldPartitionID(info.PartitionID),
|
||||
mlog.FieldSegmentID(info.SegmentID))
|
||||
return merr.WrapErrChannelNotFound(channel)
|
||||
}
|
||||
buf.CreateNewGrowingSegment(partitionID, segmentID, nil, schemaVersion)
|
||||
return nil
|
||||
return buf.CreateNewGrowingSegment(info)
|
||||
}
|
||||
|
||||
// SealSegments call sync segment and change segments state to Flushed.
|
||||
@@ -250,12 +249,12 @@ func (m *bufferManager) BufferData(channel string, insertData []*InsertData, del
|
||||
return buf.BufferData(insertData, deleteMsgs, startPos, endPos, schemaVersion)
|
||||
}
|
||||
|
||||
func (m *bufferManager) UseGrowingSourceFlush(channel string) bool {
|
||||
func (m *bufferManager) AllowGrowingSourceFlush(channel string) bool {
|
||||
buf, loaded := m.buffers.Get(channel)
|
||||
if !loaded {
|
||||
return false
|
||||
}
|
||||
return buf.UseGrowingSourceFlush()
|
||||
return buf.AllowGrowingSourceFlush()
|
||||
}
|
||||
|
||||
func (m *bufferManager) CheckReleaseManualFlushNeed(ctx context.Context, channel string, segmentIDs []int64) (bool, error) {
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"github.com/milvus-io/milvus/internal/allocator"
|
||||
"github.com/milvus-io/milvus/internal/flushcommon/metacache"
|
||||
"github.com/milvus-io/milvus/internal/flushcommon/syncmgr"
|
||||
"github.com/milvus-io/milvus/internal/storage"
|
||||
"github.com/milvus-io/milvus/pkg/v3/common"
|
||||
"github.com/milvus-io/milvus/pkg/v3/proto/datapb"
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/hardware"
|
||||
@@ -143,7 +144,11 @@ func (s *ManagerSuite) TestFlushAllSegments() {
|
||||
|
||||
func (s *ManagerSuite) TestCreateNewGrowingSegment() {
|
||||
manager := s.manager
|
||||
err := manager.CreateNewGrowingSegment(context.Background(), s.channelName, 1, 1, 0)
|
||||
err := manager.CreateNewGrowingSegment(context.Background(), s.channelName, CreateGrowingSegmentInfo{
|
||||
PartitionID: 1,
|
||||
SegmentID: 1,
|
||||
StorageVersion: storage.StorageV2,
|
||||
})
|
||||
s.Error(err)
|
||||
|
||||
s.metacache.EXPECT().GetSegmentByID(mock.Anything).Return(nil, false).Once()
|
||||
@@ -157,7 +162,12 @@ func (s *ManagerSuite) TestCreateNewGrowingSegment() {
|
||||
s.NoError(err)
|
||||
|
||||
s.manager.buffers.Insert(s.channelName, wb)
|
||||
err = manager.CreateNewGrowingSegment(context.Background(), s.channelName, 1, 1, 100)
|
||||
err = manager.CreateNewGrowingSegment(context.Background(), s.channelName, CreateGrowingSegmentInfo{
|
||||
PartitionID: 1,
|
||||
SegmentID: 1,
|
||||
SchemaVersion: 100,
|
||||
StorageVersion: storage.StorageV2,
|
||||
})
|
||||
s.NoError(err)
|
||||
}
|
||||
|
||||
|
||||
@@ -77,17 +77,17 @@ func (_c *MockBufferManager_BufferData_Call) RunAndReturn(run func(string, []*In
|
||||
return _c
|
||||
}
|
||||
|
||||
// CreateNewGrowingSegment provides a mock function with given fields: ctx, channel, partition, segmentID, schemaVersion
|
||||
func (_m *MockBufferManager) CreateNewGrowingSegment(ctx context.Context, channel string, partition int64, segmentID int64, schemaVersion int32) error {
|
||||
ret := _m.Called(ctx, channel, partition, segmentID, schemaVersion)
|
||||
// CreateNewGrowingSegment provides a mock function with given fields: ctx, channel, info
|
||||
func (_m *MockBufferManager) CreateNewGrowingSegment(ctx context.Context, channel string, info CreateGrowingSegmentInfo) error {
|
||||
ret := _m.Called(ctx, channel, info)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for CreateNewGrowingSegment")
|
||||
}
|
||||
|
||||
var r0 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, string, int64, int64, int32) error); ok {
|
||||
r0 = rf(ctx, channel, partition, segmentID, schemaVersion)
|
||||
if rf, ok := ret.Get(0).(func(context.Context, string, CreateGrowingSegmentInfo) error); ok {
|
||||
r0 = rf(ctx, channel, info)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
@@ -103,16 +103,14 @@ type MockBufferManager_CreateNewGrowingSegment_Call struct {
|
||||
// CreateNewGrowingSegment is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - channel string
|
||||
// - partition int64
|
||||
// - segmentID int64
|
||||
// - schemaVersion int32
|
||||
func (_e *MockBufferManager_Expecter) CreateNewGrowingSegment(ctx interface{}, channel interface{}, partition interface{}, segmentID interface{}, schemaVersion interface{}) *MockBufferManager_CreateNewGrowingSegment_Call {
|
||||
return &MockBufferManager_CreateNewGrowingSegment_Call{Call: _e.mock.On("CreateNewGrowingSegment", ctx, channel, partition, segmentID, schemaVersion)}
|
||||
// - info CreateGrowingSegmentInfo
|
||||
func (_e *MockBufferManager_Expecter) CreateNewGrowingSegment(ctx interface{}, channel interface{}, info interface{}) *MockBufferManager_CreateNewGrowingSegment_Call {
|
||||
return &MockBufferManager_CreateNewGrowingSegment_Call{Call: _e.mock.On("CreateNewGrowingSegment", ctx, channel, info)}
|
||||
}
|
||||
|
||||
func (_c *MockBufferManager_CreateNewGrowingSegment_Call) Run(run func(ctx context.Context, channel string, partition int64, segmentID int64, schemaVersion int32)) *MockBufferManager_CreateNewGrowingSegment_Call {
|
||||
func (_c *MockBufferManager_CreateNewGrowingSegment_Call) Run(run func(ctx context.Context, channel string, info CreateGrowingSegmentInfo)) *MockBufferManager_CreateNewGrowingSegment_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context), args[1].(string), args[2].(int64), args[3].(int64), args[4].(int32))
|
||||
run(args[0].(context.Context), args[1].(string), args[2].(CreateGrowingSegmentInfo))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
@@ -122,7 +120,7 @@ func (_c *MockBufferManager_CreateNewGrowingSegment_Call) Return(_a0 error) *Moc
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockBufferManager_CreateNewGrowingSegment_Call) RunAndReturn(run func(context.Context, string, int64, int64, int32) error) *MockBufferManager_CreateNewGrowingSegment_Call {
|
||||
func (_c *MockBufferManager_CreateNewGrowingSegment_Call) RunAndReturn(run func(context.Context, string, CreateGrowingSegmentInfo) error) *MockBufferManager_CreateNewGrowingSegment_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
@@ -368,12 +366,12 @@ func (_c *MockBufferManager_GetGrowingFlushProgress_Call) RunAndReturn(run func(
|
||||
return _c
|
||||
}
|
||||
|
||||
// UseGrowingSourceFlush provides a mock function with given fields: channel
|
||||
func (_m *MockBufferManager) UseGrowingSourceFlush(channel string) bool {
|
||||
// AllowGrowingSourceFlush provides a mock function with given fields: channel
|
||||
func (_m *MockBufferManager) AllowGrowingSourceFlush(channel string) bool {
|
||||
ret := _m.Called(channel)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for UseGrowingSourceFlush")
|
||||
panic("no return value specified for AllowGrowingSourceFlush")
|
||||
}
|
||||
|
||||
var r0 bool
|
||||
@@ -386,30 +384,30 @@ func (_m *MockBufferManager) UseGrowingSourceFlush(channel string) bool {
|
||||
return r0
|
||||
}
|
||||
|
||||
// MockBufferManager_UseGrowingSourceFlush_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UseGrowingSourceFlush'
|
||||
type MockBufferManager_UseGrowingSourceFlush_Call struct {
|
||||
// MockBufferManager_AllowGrowingSourceFlush_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AllowGrowingSourceFlush'
|
||||
type MockBufferManager_AllowGrowingSourceFlush_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// UseGrowingSourceFlush is a helper method to define mock.On call
|
||||
// AllowGrowingSourceFlush is a helper method to define mock.On call
|
||||
// - channel string
|
||||
func (_e *MockBufferManager_Expecter) UseGrowingSourceFlush(channel interface{}) *MockBufferManager_UseGrowingSourceFlush_Call {
|
||||
return &MockBufferManager_UseGrowingSourceFlush_Call{Call: _e.mock.On("UseGrowingSourceFlush", channel)}
|
||||
func (_e *MockBufferManager_Expecter) AllowGrowingSourceFlush(channel interface{}) *MockBufferManager_AllowGrowingSourceFlush_Call {
|
||||
return &MockBufferManager_AllowGrowingSourceFlush_Call{Call: _e.mock.On("AllowGrowingSourceFlush", channel)}
|
||||
}
|
||||
|
||||
func (_c *MockBufferManager_UseGrowingSourceFlush_Call) Run(run func(channel string)) *MockBufferManager_UseGrowingSourceFlush_Call {
|
||||
func (_c *MockBufferManager_AllowGrowingSourceFlush_Call) Run(run func(channel string)) *MockBufferManager_AllowGrowingSourceFlush_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(string))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockBufferManager_UseGrowingSourceFlush_Call) Return(_a0 bool) *MockBufferManager_UseGrowingSourceFlush_Call {
|
||||
func (_c *MockBufferManager_AllowGrowingSourceFlush_Call) Return(_a0 bool) *MockBufferManager_AllowGrowingSourceFlush_Call {
|
||||
_c.Call.Return(_a0)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockBufferManager_UseGrowingSourceFlush_Call) RunAndReturn(run func(string) bool) *MockBufferManager_UseGrowingSourceFlush_Call {
|
||||
func (_c *MockBufferManager_AllowGrowingSourceFlush_Call) RunAndReturn(run func(string) bool) *MockBufferManager_AllowGrowingSourceFlush_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
@@ -108,9 +108,22 @@ func (_c *MockWriteBuffer_Close_Call) RunAndReturn(run func(context.Context, boo
|
||||
return _c
|
||||
}
|
||||
|
||||
// CreateNewGrowingSegment provides a mock function with given fields: partitionID, segmentID, startPos, schemaVersion
|
||||
func (_m *MockWriteBuffer) CreateNewGrowingSegment(partitionID int64, segmentID int64, startPos *msgpb.MsgPosition, schemaVersion int32) {
|
||||
_m.Called(partitionID, segmentID, startPos, schemaVersion)
|
||||
// CreateNewGrowingSegment provides a mock function with given fields: info
|
||||
func (_m *MockWriteBuffer) CreateNewGrowingSegment(info CreateGrowingSegmentInfo) error {
|
||||
ret := _m.Called(info)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for CreateNewGrowingSegment")
|
||||
}
|
||||
|
||||
var r0 error
|
||||
if rf, ok := ret.Get(0).(func(CreateGrowingSegmentInfo) error); ok {
|
||||
r0 = rf(info)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// MockWriteBuffer_CreateNewGrowingSegment_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CreateNewGrowingSegment'
|
||||
@@ -119,28 +132,25 @@ type MockWriteBuffer_CreateNewGrowingSegment_Call struct {
|
||||
}
|
||||
|
||||
// CreateNewGrowingSegment is a helper method to define mock.On call
|
||||
// - partitionID int64
|
||||
// - segmentID int64
|
||||
// - startPos *msgpb.MsgPosition
|
||||
// - schemaVersion int32
|
||||
func (_e *MockWriteBuffer_Expecter) CreateNewGrowingSegment(partitionID interface{}, segmentID interface{}, startPos interface{}, schemaVersion interface{}) *MockWriteBuffer_CreateNewGrowingSegment_Call {
|
||||
return &MockWriteBuffer_CreateNewGrowingSegment_Call{Call: _e.mock.On("CreateNewGrowingSegment", partitionID, segmentID, startPos, schemaVersion)}
|
||||
// - info CreateGrowingSegmentInfo
|
||||
func (_e *MockWriteBuffer_Expecter) CreateNewGrowingSegment(info interface{}) *MockWriteBuffer_CreateNewGrowingSegment_Call {
|
||||
return &MockWriteBuffer_CreateNewGrowingSegment_Call{Call: _e.mock.On("CreateNewGrowingSegment", info)}
|
||||
}
|
||||
|
||||
func (_c *MockWriteBuffer_CreateNewGrowingSegment_Call) Run(run func(partitionID int64, segmentID int64, startPos *msgpb.MsgPosition, schemaVersion int32)) *MockWriteBuffer_CreateNewGrowingSegment_Call {
|
||||
func (_c *MockWriteBuffer_CreateNewGrowingSegment_Call) Run(run func(info CreateGrowingSegmentInfo)) *MockWriteBuffer_CreateNewGrowingSegment_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(int64), args[1].(int64), args[2].(*msgpb.MsgPosition), args[3].(int32))
|
||||
run(args[0].(CreateGrowingSegmentInfo))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockWriteBuffer_CreateNewGrowingSegment_Call) Return() *MockWriteBuffer_CreateNewGrowingSegment_Call {
|
||||
_c.Call.Return()
|
||||
func (_c *MockWriteBuffer_CreateNewGrowingSegment_Call) Return(_a0 error) *MockWriteBuffer_CreateNewGrowingSegment_Call {
|
||||
_c.Call.Return(_a0)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockWriteBuffer_CreateNewGrowingSegment_Call) RunAndReturn(run func(int64, int64, *msgpb.MsgPosition, int32)) *MockWriteBuffer_CreateNewGrowingSegment_Call {
|
||||
_c.Run(run)
|
||||
func (_c *MockWriteBuffer_CreateNewGrowingSegment_Call) RunAndReturn(run func(CreateGrowingSegmentInfo) error) *MockWriteBuffer_CreateNewGrowingSegment_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
@@ -421,12 +431,12 @@ func (_c *MockWriteBuffer_HasSegment_Call) RunAndReturn(run func(int64) bool) *M
|
||||
return _c
|
||||
}
|
||||
|
||||
// UseGrowingSourceFlush provides a mock function with no fields
|
||||
func (_m *MockWriteBuffer) UseGrowingSourceFlush() bool {
|
||||
// AllowGrowingSourceFlush provides a mock function with no fields
|
||||
func (_m *MockWriteBuffer) AllowGrowingSourceFlush() bool {
|
||||
ret := _m.Called()
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for UseGrowingSourceFlush")
|
||||
panic("no return value specified for AllowGrowingSourceFlush")
|
||||
}
|
||||
|
||||
var r0 bool
|
||||
@@ -439,29 +449,29 @@ func (_m *MockWriteBuffer) UseGrowingSourceFlush() bool {
|
||||
return r0
|
||||
}
|
||||
|
||||
// MockWriteBuffer_UseGrowingSourceFlush_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UseGrowingSourceFlush'
|
||||
type MockWriteBuffer_UseGrowingSourceFlush_Call struct {
|
||||
// MockWriteBuffer_AllowGrowingSourceFlush_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AllowGrowingSourceFlush'
|
||||
type MockWriteBuffer_AllowGrowingSourceFlush_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// UseGrowingSourceFlush is a helper method to define mock.On call
|
||||
func (_e *MockWriteBuffer_Expecter) UseGrowingSourceFlush() *MockWriteBuffer_UseGrowingSourceFlush_Call {
|
||||
return &MockWriteBuffer_UseGrowingSourceFlush_Call{Call: _e.mock.On("UseGrowingSourceFlush")}
|
||||
// AllowGrowingSourceFlush is a helper method to define mock.On call
|
||||
func (_e *MockWriteBuffer_Expecter) AllowGrowingSourceFlush() *MockWriteBuffer_AllowGrowingSourceFlush_Call {
|
||||
return &MockWriteBuffer_AllowGrowingSourceFlush_Call{Call: _e.mock.On("AllowGrowingSourceFlush")}
|
||||
}
|
||||
|
||||
func (_c *MockWriteBuffer_UseGrowingSourceFlush_Call) Run(run func()) *MockWriteBuffer_UseGrowingSourceFlush_Call {
|
||||
func (_c *MockWriteBuffer_AllowGrowingSourceFlush_Call) Run(run func()) *MockWriteBuffer_AllowGrowingSourceFlush_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run()
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockWriteBuffer_UseGrowingSourceFlush_Call) Return(_a0 bool) *MockWriteBuffer_UseGrowingSourceFlush_Call {
|
||||
func (_c *MockWriteBuffer_AllowGrowingSourceFlush_Call) Return(_a0 bool) *MockWriteBuffer_AllowGrowingSourceFlush_Call {
|
||||
_c.Call.Return(_a0)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockWriteBuffer_UseGrowingSourceFlush_Call) RunAndReturn(run func() bool) *MockWriteBuffer_UseGrowingSourceFlush_Call {
|
||||
func (_c *MockWriteBuffer_AllowGrowingSourceFlush_Call) RunAndReturn(run func() bool) *MockWriteBuffer_AllowGrowingSourceFlush_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"github.com/milvus-io/milvus/internal/flushcommon/syncmgr"
|
||||
"github.com/milvus-io/milvus/internal/storage"
|
||||
"github.com/milvus-io/milvus/internal/storagev2/packed"
|
||||
"github.com/milvus-io/milvus/internal/util/streamingutil"
|
||||
"github.com/milvus-io/milvus/pkg/v3/common"
|
||||
"github.com/milvus-io/milvus/pkg/v3/metrics"
|
||||
"github.com/milvus-io/milvus/pkg/v3/mlog"
|
||||
@@ -52,7 +53,7 @@ type WriteBuffer interface {
|
||||
// HasSegment checks whether certain segment exists in this buffer.
|
||||
HasSegment(segmentID int64) bool
|
||||
// CreateNewGrowingSegment creates a new growing segment in the buffer.
|
||||
CreateNewGrowingSegment(partitionID int64, segmentID int64, startPos *msgpb.MsgPosition, schemaVersion int32)
|
||||
CreateNewGrowingSegment(info CreateGrowingSegmentInfo) error
|
||||
// BufferData is the method to buffer dml data msgs.
|
||||
BufferData(insertMsgs []*InsertData, deleteMsgs []*msgstream.DeleteMsg, startPos, endPos *msgpb.MsgPosition, schemaVersion int32) error
|
||||
// FlushTimestamp set flush timestamp for write buffer
|
||||
@@ -73,8 +74,8 @@ type WriteBuffer interface {
|
||||
MemorySize() int64
|
||||
// EvictBuffer evicts buffer to sync manager which match provided sync policies.
|
||||
EvictBuffer(policies ...SyncPolicy)
|
||||
// UseGrowingSourceFlush returns true if the collection on this channel has growing-source fields.
|
||||
UseGrowingSourceFlush() bool
|
||||
// AllowGrowingSourceFlush returns true if this write buffer may try growing-source flush.
|
||||
AllowGrowingSourceFlush() bool
|
||||
// GetGrowingFlushProgress returns growing-source progress for the given
|
||||
// segments after this write buffer has processed up to fenceTs. If segmentIDs
|
||||
// is empty, all tracked growing-source segments are returned. Otherwise,
|
||||
@@ -84,6 +85,14 @@ type WriteBuffer interface {
|
||||
Close(ctx context.Context, drop bool)
|
||||
}
|
||||
|
||||
type CreateGrowingSegmentInfo struct {
|
||||
PartitionID int64
|
||||
SegmentID int64
|
||||
StartPos *msgpb.MsgPosition
|
||||
SchemaVersion int32
|
||||
StorageVersion int64
|
||||
}
|
||||
|
||||
type GrowingFlushSegmentProgress struct {
|
||||
SegmentID int64
|
||||
TargetOffset int64
|
||||
@@ -269,9 +278,9 @@ type writeBufferBase struct {
|
||||
errHandler func(err error)
|
||||
taskObserverCallback func(t syncmgr.Task, err error) // execute when a sync task finished, should be concurrent safe.
|
||||
|
||||
// growing-source collection flag. growing-source can be flushed either from an optional growing
|
||||
// segment source or from WriteBuffer payload when no growing source is usable.
|
||||
useGrowingSourceFlush bool
|
||||
// Channel-level admission flag for trying growing-source flush. Actual segment
|
||||
// source selection remains sticky in metacache.
|
||||
allowGrowingSourceFlush bool
|
||||
|
||||
growingSourceResolver GrowingSourceResolver
|
||||
|
||||
@@ -302,7 +311,7 @@ func newWriteBufferBase(channel string, metacache metacache.MetaCache, syncMgr s
|
||||
return nil, err
|
||||
}
|
||||
|
||||
useGrowingSourceFlush := typeutil.UseGrowingSourceFlush(schema,
|
||||
allowGrowingSourceFlush := typeutil.AllowGrowingSourceFlush(schema,
|
||||
paramtable.Get().CommonCfg.UseLoonFFI.GetAsBool(),
|
||||
paramtable.Get().CommonCfg.EnableGrowingSourceFlush.GetAsBool())
|
||||
growingSourceResolver := option.growingSourceResolver
|
||||
@@ -332,7 +341,7 @@ func newWriteBufferBase(channel string, metacache metacache.MetaCache, syncMgr s
|
||||
flushTimestamp: flushTs,
|
||||
errHandler: option.errorHandler,
|
||||
taskObserverCallback: option.taskObserverCallback,
|
||||
useGrowingSourceFlush: useGrowingSourceFlush,
|
||||
allowGrowingSourceFlush: allowGrowingSourceFlush,
|
||||
growingSourceResolver: growingSourceResolver,
|
||||
growingSourceProgress: make(map[int64]*growingSourceProgress),
|
||||
growingSourceRetryInterval: growingSourceRetryInterval,
|
||||
@@ -399,8 +408,8 @@ func (wb *writeBufferBase) GetFlushTimestamp() uint64 {
|
||||
return wb.flushTimestamp.Load()
|
||||
}
|
||||
|
||||
func (wb *writeBufferBase) UseGrowingSourceFlush() bool {
|
||||
return wb.useGrowingSourceFlush
|
||||
func (wb *writeBufferBase) AllowGrowingSourceFlush() bool {
|
||||
return wb.allowGrowingSourceFlush
|
||||
}
|
||||
|
||||
func (wb *writeBufferBase) CheckReleaseManualFlushNeed(segmentIDs []int64) bool {
|
||||
@@ -612,6 +621,9 @@ func (wb *writeBufferBase) decideGrowingFlushSource(segmentID int64, targetOffse
|
||||
// must return the same kind so that progress / payload tracking stays
|
||||
// consistent for the segment's lifetime.
|
||||
if seg, ok := wb.metaCache.GetSegmentByID(segmentID); ok {
|
||||
if seg.GetStorageVersion() != storage.StorageV3 {
|
||||
return growingFlushSourceDecision{sourceType: metacache.FlushSourceWriteBuffer}
|
||||
}
|
||||
switch seg.FlushSourceMode() {
|
||||
case metacache.FlushSourceGrowing:
|
||||
state := wb.getGrowingSourceState(segmentID, targetOffset, endPos)
|
||||
@@ -666,7 +678,7 @@ func (wb *writeBufferBase) getGrowingSourceState(segmentID int64, targetOffset i
|
||||
}
|
||||
|
||||
func (wb *writeBufferBase) warnGrowingSourceFallback(segmentID int64, targetOffset int64, endPos *msgpb.MsgPosition) {
|
||||
if !wb.useGrowingSourceFlush {
|
||||
if !wb.allowGrowingSourceFlush {
|
||||
return
|
||||
}
|
||||
wb.growingSourceRatedLogger.RatedWarn(context.TODO(), rate.Limit(1), "growing-source source is unavailable, fallback to WriteBuffer",
|
||||
@@ -787,8 +799,24 @@ func (wb *writeBufferBase) getGrowingSourceSegmentsToRetry() ([]int64, bool) {
|
||||
return segments, retryNeeded
|
||||
}
|
||||
|
||||
func (wb *writeBufferBase) recordGrowingSourceProgress(inData *InsertData, startPos, endPos *msgpb.MsgPosition, schemaVersion int32, targetOffset int64) {
|
||||
wb.CreateNewGrowingSegment(inData.partitionID, inData.segmentID, startPos, schemaVersion)
|
||||
func (wb *writeBufferBase) recordGrowingSourceProgress(inData *InsertData, startPos, endPos *msgpb.MsgPosition, schemaVersion int32, targetOffset int64) error {
|
||||
err := wb.CreateNewGrowingSegment(CreateGrowingSegmentInfo{
|
||||
PartitionID: inData.partitionID,
|
||||
SegmentID: inData.segmentID,
|
||||
StartPos: startPos,
|
||||
SchemaVersion: schemaVersion,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
segment, ok := wb.metaCache.GetSegmentByID(inData.segmentID)
|
||||
if !ok {
|
||||
return merr.WrapErrSegmentNotFound(inData.segmentID)
|
||||
}
|
||||
if segment.GetStorageVersion() != storage.StorageV3 {
|
||||
return merr.WrapErrServiceInternalMsg("growing-source flush requires StorageV3 segment, segmentID=%d storageVersion=%d",
|
||||
inData.segmentID, segment.GetStorageVersion())
|
||||
}
|
||||
progress, ok := wb.growingSourceProgress[inData.segmentID]
|
||||
if !ok {
|
||||
progress = &growingSourceProgress{
|
||||
@@ -813,6 +841,7 @@ func (wb *writeBufferBase) recordGrowingSourceProgress(inData *InsertData, start
|
||||
wb.updateGrowingSourceBufferedRows(progress),
|
||||
), metacache.WithSegmentIDs(inData.segmentID))
|
||||
wb.notifyFlushSourceMode(inData.segmentID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (wb *writeBufferBase) growingSourceTargetOffset(segmentID int64, rows int64) int64 {
|
||||
@@ -853,7 +882,7 @@ func (wb *writeBufferBase) sealSegments(ctx context.Context, segmentIDs []int64)
|
||||
for _, segmentID := range segmentIDs {
|
||||
_, ok := wb.metaCache.GetSegmentByID(segmentID)
|
||||
if !ok {
|
||||
if !wb.useGrowingSourceFlush {
|
||||
if !wb.allowGrowingSourceFlush {
|
||||
mlog.Warn(ctx, "cannot find segment when sealSegments",
|
||||
mlog.Int64("segmentID", segmentID),
|
||||
mlog.String("channel", wb.channelName))
|
||||
@@ -1160,7 +1189,7 @@ func (wb *writeBufferBase) getOrCreateBuffer(segmentID int64, timetick uint64) *
|
||||
panic(err)
|
||||
}
|
||||
wb.buffers[segmentID] = buffer
|
||||
if wb.useGrowingSourceFlush {
|
||||
if wb.allowGrowingSourceFlush {
|
||||
wb.metaCache.UpdateSegments(
|
||||
metacache.SetFlushSourceMode(metacache.FlushSourceWriteBuffer),
|
||||
metacache.WithSegmentIDs(segmentID),
|
||||
@@ -1314,36 +1343,71 @@ func (id *InsertData) batchPkExists(pks []storage.PrimaryKey, tss []uint64, hits
|
||||
return hits
|
||||
}
|
||||
|
||||
func (wb *writeBufferBase) CreateNewGrowingSegment(partitionID int64, segmentID int64, startPos *msgpb.MsgPosition, schemaVersion int32) {
|
||||
_, ok := wb.metaCache.GetSegmentByID(segmentID)
|
||||
func (wb *writeBufferBase) CreateNewGrowingSegment(info CreateGrowingSegmentInfo) error {
|
||||
_, ok := wb.metaCache.GetSegmentByID(info.SegmentID)
|
||||
// new segment
|
||||
if !ok {
|
||||
storageVersion := storage.StorageV2
|
||||
manifestPath := ""
|
||||
if paramtable.Get().CommonCfg.UseLoonFFI.GetAsBool() {
|
||||
storageVersion = storage.StorageV3
|
||||
// set manifest path when creating segment
|
||||
k := metautil.JoinIDPath(wb.collectionID, partitionID, segmentID)
|
||||
basePath := path.Join(paramtable.Get().MinioCfg.RootPath.GetValue(), common.SegmentInsertLogPath, k)
|
||||
// ManifestEarliest for first write
|
||||
manifestPath = packed.MarshalManifestPath(basePath, packed.ManifestEarliest)
|
||||
storageVersion, err := wb.resolveNewGrowingSegmentStorageVersion(info)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
manifestPath := wb.newGrowingSegmentManifestPath(info.PartitionID, info.SegmentID, storageVersion)
|
||||
segmentInfo := &datapb.SegmentInfo{
|
||||
ID: segmentID,
|
||||
PartitionID: partitionID,
|
||||
ID: info.SegmentID,
|
||||
PartitionID: info.PartitionID,
|
||||
CollectionID: wb.collectionID,
|
||||
InsertChannel: wb.channelName,
|
||||
StartPosition: startPos,
|
||||
StartPosition: info.StartPos,
|
||||
State: commonpb.SegmentState_Growing,
|
||||
StorageVersion: storageVersion,
|
||||
ManifestPath: manifestPath,
|
||||
SchemaVersion: schemaVersion,
|
||||
SchemaVersion: info.SchemaVersion,
|
||||
}
|
||||
wb.metaCache.AddSegment(segmentInfo, func(_ *datapb.SegmentInfo) pkoracle.PkStat {
|
||||
return pkoracle.NewBloomFilterSetWithBatchSize(wb.getEstBatchSize())
|
||||
}, metacache.NewBM25StatsFactory, metacache.SetStartPosRecorded(false))
|
||||
mlog.Info(context.TODO(), "add growing segment", mlog.FieldSegmentID(segmentID), mlog.String("channel", wb.channelName), mlog.Int64("storage version", storageVersion))
|
||||
mlog.Info(context.TODO(), "add growing segment", mlog.FieldSegmentID(info.SegmentID), mlog.String("channel", wb.channelName), mlog.Int64("storage version", storageVersion))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (wb *writeBufferBase) resolveNewGrowingSegmentStorageVersion(info CreateGrowingSegmentInfo) (int64, error) {
|
||||
switch info.StorageVersion {
|
||||
case storage.StorageV2, storage.StorageV3:
|
||||
return info.StorageVersion, nil
|
||||
case storage.StorageV1:
|
||||
if streamingutil.IsStreamingServiceEnabled() {
|
||||
return 0, merr.WrapErrServiceInternalMsg("missing storage version for streaming growing segment, segmentID=%d", info.SegmentID)
|
||||
}
|
||||
inferred := storage.StorageV2
|
||||
reason := "default non-streaming storage version"
|
||||
if typeutil.HasTextField(wb.metaCache.GetSchema(0)) {
|
||||
inferred = storage.StorageV3
|
||||
reason = "TEXT field requires StorageV3"
|
||||
} else if paramtable.Get().CommonCfg.UseLoonFFI.GetAsBool() {
|
||||
inferred = storage.StorageV3
|
||||
reason = "common.storage.useLoonFFI enabled"
|
||||
}
|
||||
mlog.Warn(context.TODO(), "infer missing storage version for non-streaming growing segment",
|
||||
mlog.FieldSegmentID(info.SegmentID),
|
||||
mlog.Int64("collectionID", wb.collectionID),
|
||||
mlog.String("channel", wb.channelName),
|
||||
mlog.Int64("inferredStorageVersion", inferred),
|
||||
mlog.String("reason", reason))
|
||||
return inferred, nil
|
||||
default:
|
||||
return 0, merr.WrapErrServiceInternalMsg("unsupported storage version for growing segment, segmentID=%d storageVersion=%d",
|
||||
info.SegmentID, info.StorageVersion)
|
||||
}
|
||||
}
|
||||
|
||||
func (wb *writeBufferBase) newGrowingSegmentManifestPath(partitionID int64, segmentID int64, storageVersion int64) string {
|
||||
if storageVersion != storage.StorageV3 {
|
||||
return ""
|
||||
}
|
||||
k := metautil.JoinIDPath(wb.collectionID, partitionID, segmentID)
|
||||
basePath := path.Join(paramtable.Get().MinioCfg.RootPath.GetValue(), common.SegmentInsertLogPath, k)
|
||||
return packed.MarshalManifestPath(basePath, packed.ManifestEarliest)
|
||||
}
|
||||
|
||||
// bufferDelete buffers DeleteMsg into DeleteData.
|
||||
@@ -1436,6 +1500,10 @@ func (wb *writeBufferBase) getSyncTask(ctx context.Context, segmentID int64) (sy
|
||||
}
|
||||
|
||||
func (wb *writeBufferBase) getGrowingSourceSyncTask(ctx context.Context, segmentInfo *metacache.SegmentInfo, progress *growingSourceProgress) (syncmgr.Task, error) {
|
||||
if segmentInfo.GetStorageVersion() != storage.StorageV3 {
|
||||
return nil, merr.WrapErrServiceInternalMsg("growing-source sync requires StorageV3 segment, segmentID=%d storageVersion=%d",
|
||||
segmentInfo.SegmentID(), segmentInfo.GetStorageVersion())
|
||||
}
|
||||
targetOffset := progress.targetOffset
|
||||
pendingCommitted := progress.pendingCommitted
|
||||
if pendingCommitted != nil {
|
||||
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
"github.com/milvus-io/milvus/internal/flushcommon/syncmgr"
|
||||
"github.com/milvus-io/milvus/internal/storage"
|
||||
"github.com/milvus-io/milvus/internal/util/function"
|
||||
"github.com/milvus-io/milvus/internal/util/streamingutil"
|
||||
"github.com/milvus-io/milvus/pkg/v3/common"
|
||||
"github.com/milvus-io/milvus/pkg/v3/mq/msgstream"
|
||||
"github.com/milvus-io/milvus/pkg/v3/proto/datapb"
|
||||
@@ -71,7 +72,7 @@ func (s *WriteBufferSuite) SetupTest() {
|
||||
func (s *WriteBufferSuite) TestHasSegment() {
|
||||
segmentID := int64(1001)
|
||||
|
||||
s.wb.useGrowingSourceFlush = false
|
||||
s.wb.allowGrowingSourceFlush = false
|
||||
s.False(s.wb.HasSegment(segmentID))
|
||||
|
||||
s.wb.getOrCreateBuffer(segmentID, 0)
|
||||
@@ -96,7 +97,7 @@ func (s *WriteBufferSuite) TestFlushSourceModeNotifier() {
|
||||
}).Return().Once()
|
||||
s.metacache.EXPECT().GetSegmentByID(segmentID).Return(segment, true).Once()
|
||||
|
||||
s.wb.useGrowingSourceFlush = true
|
||||
s.wb.allowGrowingSourceFlush = true
|
||||
s.wb.getOrCreateBuffer(segmentID, 0)
|
||||
s.Equal(segmentID, notifiedSegmentID)
|
||||
s.Equal(metacache.FlushSourceWriteBuffer, notifiedMode)
|
||||
@@ -104,21 +105,22 @@ func (s *WriteBufferSuite) TestFlushSourceModeNotifier() {
|
||||
|
||||
s.Run("growing_mode", func() {
|
||||
segmentID := int64(1002)
|
||||
segment := metacache.NewSegmentInfo(&datapb.SegmentInfo{ID: segmentID}, nil, nil)
|
||||
segment := metacache.NewSegmentInfo(&datapb.SegmentInfo{ID: segmentID, StorageVersion: storage.StorageV3}, nil, nil)
|
||||
notifiedSegmentID = 0
|
||||
notifiedMode = metacache.FlushSourceUnknown
|
||||
s.metacache.EXPECT().GetSegmentByID(segmentID).Return(nil, false).Once()
|
||||
s.metacache.EXPECT().AddSegment(mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return().Once()
|
||||
s.metacache.EXPECT().GetSegmentByID(segmentID).Return(segment, true).Once()
|
||||
s.metacache.EXPECT().GetSegmentByID(segmentID).Return(segment, true).Once()
|
||||
s.metacache.EXPECT().UpdateSegments(mock.Anything, mock.Anything).Run(func(action metacache.SegmentAction, _ ...metacache.SegmentFilter) {
|
||||
action(segment)
|
||||
}).Return().Once()
|
||||
s.metacache.EXPECT().GetSegmentByID(segmentID).Return(segment, true).Once()
|
||||
|
||||
s.wb.recordGrowingSourceProgress(&InsertData{
|
||||
err := s.wb.recordGrowingSourceProgress(&InsertData{
|
||||
segmentID: segmentID,
|
||||
partitionID: 10,
|
||||
rowNum: 3,
|
||||
}, &msgpb.MsgPosition{Timestamp: 100}, &msgpb.MsgPosition{Timestamp: 200}, 1, 3)
|
||||
s.NoError(err)
|
||||
s.Equal(segmentID, notifiedSegmentID)
|
||||
s.Equal(metacache.FlushSourceGrowing, notifiedMode)
|
||||
})
|
||||
@@ -132,8 +134,8 @@ func (s *WriteBufferSuite) TestCreateNewGrowingSegmentStorageVersion() {
|
||||
defer param.Reset(param.CommonCfg.EnableGrowingSourceFlush.Key)
|
||||
|
||||
s.Run("non_text_uses_v2_when_ffi_disabled", func() {
|
||||
s.wb.useGrowingSourceFlush = false
|
||||
s.False(s.wb.UseGrowingSourceFlush())
|
||||
s.wb.allowGrowingSourceFlush = false
|
||||
s.False(s.wb.AllowGrowingSourceFlush())
|
||||
s.metacache.EXPECT().GetSegmentByID(int64(2001)).Return(nil, false).Once()
|
||||
s.metacache.EXPECT().AddSegment(mock.MatchedBy(func(info *datapb.SegmentInfo) bool {
|
||||
return info.GetStorageVersion() == storage.StorageV2 &&
|
||||
@@ -141,11 +143,17 @@ func (s *WriteBufferSuite) TestCreateNewGrowingSegmentStorageVersion() {
|
||||
info.GetSchemaVersion() == 11
|
||||
}), mock.Anything, mock.Anything, mock.Anything).Return().Once()
|
||||
|
||||
s.wb.CreateNewGrowingSegment(10, 2001, nil, 11)
|
||||
err := s.wb.CreateNewGrowingSegment(CreateGrowingSegmentInfo{
|
||||
PartitionID: 10,
|
||||
SegmentID: 2001,
|
||||
SchemaVersion: 11,
|
||||
StorageVersion: storage.StorageV2,
|
||||
})
|
||||
s.NoError(err)
|
||||
})
|
||||
|
||||
s.Run("growing_source_does_not_force_v3_manifest_when_ffi_disabled", func() {
|
||||
s.wb.useGrowingSourceFlush = true
|
||||
s.wb.allowGrowingSourceFlush = true
|
||||
s.metacache.EXPECT().GetSegmentByID(int64(2002)).Return(nil, false).Once()
|
||||
s.metacache.EXPECT().AddSegment(mock.MatchedBy(func(info *datapb.SegmentInfo) bool {
|
||||
return info.GetStorageVersion() == storage.StorageV2 &&
|
||||
@@ -153,10 +161,16 @@ func (s *WriteBufferSuite) TestCreateNewGrowingSegmentStorageVersion() {
|
||||
info.GetSchemaVersion() == 12
|
||||
}), mock.Anything, mock.Anything, mock.Anything).Return().Once()
|
||||
|
||||
s.wb.CreateNewGrowingSegment(10, 2002, nil, 12)
|
||||
err := s.wb.CreateNewGrowingSegment(CreateGrowingSegmentInfo{
|
||||
PartitionID: 10,
|
||||
SegmentID: 2002,
|
||||
SchemaVersion: 12,
|
||||
StorageVersion: storage.StorageV2,
|
||||
})
|
||||
s.NoError(err)
|
||||
})
|
||||
|
||||
s.Run("text_schema_does_not_enable_growing_source_when_ffi_disabled", func() {
|
||||
s.Run("text_schema_uses_v3_manifest_without_enabling_growing_source_when_ffi_disabled", func() {
|
||||
textSchema := &schemapb.CollectionSchema{
|
||||
Name: "wb_text_collection",
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
@@ -173,16 +187,21 @@ func (s *WriteBufferSuite) TestCreateNewGrowingSegmentStorageVersion() {
|
||||
|
||||
wb, err := newWriteBufferBase(s.channelName, mc, s.syncMgr, &writeBufferOption{})
|
||||
s.Require().NoError(err)
|
||||
s.False(wb.UseGrowingSourceFlush())
|
||||
s.False(wb.AllowGrowingSourceFlush())
|
||||
|
||||
mc.EXPECT().GetSegmentByID(int64(2003)).Return(nil, false).Once()
|
||||
mc.EXPECT().AddSegment(mock.MatchedBy(func(info *datapb.SegmentInfo) bool {
|
||||
return info.GetStorageVersion() == storage.StorageV2 &&
|
||||
info.GetManifestPath() == "" &&
|
||||
return info.GetStorageVersion() == storage.StorageV3 &&
|
||||
info.GetManifestPath() != "" &&
|
||||
info.GetSchemaVersion() == 13
|
||||
}), mock.Anything, mock.Anything, mock.Anything).Return().Once()
|
||||
|
||||
wb.CreateNewGrowingSegment(10, 2003, nil, 13)
|
||||
err = wb.CreateNewGrowingSegment(CreateGrowingSegmentInfo{
|
||||
PartitionID: 10,
|
||||
SegmentID: 2003,
|
||||
SchemaVersion: 13,
|
||||
})
|
||||
s.NoError(err)
|
||||
})
|
||||
|
||||
s.Run("text_schema_uses_v3_manifest_when_ffi_enabled", func() {
|
||||
@@ -205,7 +224,7 @@ func (s *WriteBufferSuite) TestCreateNewGrowingSegmentStorageVersion() {
|
||||
|
||||
wb, err := newWriteBufferBase(s.channelName, mc, s.syncMgr, &writeBufferOption{})
|
||||
s.Require().NoError(err)
|
||||
s.True(wb.UseGrowingSourceFlush())
|
||||
s.True(wb.AllowGrowingSourceFlush())
|
||||
|
||||
mc.EXPECT().GetSegmentByID(int64(2004)).Return(nil, false).Once()
|
||||
mc.EXPECT().AddSegment(mock.MatchedBy(func(info *datapb.SegmentInfo) bool {
|
||||
@@ -214,7 +233,26 @@ func (s *WriteBufferSuite) TestCreateNewGrowingSegmentStorageVersion() {
|
||||
info.GetSchemaVersion() == 14
|
||||
}), mock.Anything, mock.Anything, mock.Anything).Return().Once()
|
||||
|
||||
wb.CreateNewGrowingSegment(10, 2004, nil, 14)
|
||||
err = wb.CreateNewGrowingSegment(CreateGrowingSegmentInfo{
|
||||
PartitionID: 10,
|
||||
SegmentID: 2004,
|
||||
SchemaVersion: 14,
|
||||
StorageVersion: storage.StorageV3,
|
||||
})
|
||||
s.NoError(err)
|
||||
})
|
||||
|
||||
s.Run("streaming_requires_create_segment_storage_version", func() {
|
||||
s.T().Setenv(streamingutil.MilvusStreamingServiceEnabled, "1")
|
||||
|
||||
s.metacache.EXPECT().GetSegmentByID(int64(2005)).Return(nil, false).Once()
|
||||
|
||||
err := s.wb.CreateNewGrowingSegment(CreateGrowingSegmentInfo{
|
||||
PartitionID: 10,
|
||||
SegmentID: 2005,
|
||||
SchemaVersion: 15,
|
||||
})
|
||||
s.ErrorIs(err, merr.ErrServiceInternal)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -234,7 +272,7 @@ func (s *WriteBufferSuite) TestSealSegmentsMissingSegment() {
|
||||
segmentID := int64(1001)
|
||||
|
||||
s.Run("non_text_returns_error", func() {
|
||||
s.wb.useGrowingSourceFlush = false
|
||||
s.wb.allowGrowingSourceFlush = false
|
||||
s.metacache.EXPECT().GetSegmentByID(segmentID).Return(nil, false).Once()
|
||||
|
||||
err := s.wb.SealSegments(context.Background(), []int64{segmentID})
|
||||
@@ -242,9 +280,9 @@ func (s *WriteBufferSuite) TestSealSegmentsMissingSegment() {
|
||||
})
|
||||
|
||||
s.Run("text_skips_missing_segment", func() {
|
||||
s.wb.useGrowingSourceFlush = true
|
||||
s.wb.allowGrowingSourceFlush = true
|
||||
defer func() {
|
||||
s.wb.useGrowingSourceFlush = false
|
||||
s.wb.allowGrowingSourceFlush = false
|
||||
}()
|
||||
s.metacache.EXPECT().GetSegmentByID(segmentID).Return(nil, false).Once()
|
||||
|
||||
|
||||
@@ -1572,7 +1572,7 @@ func NewShardDelegator(ctx context.Context, collectionID UniqueID, replicaID Uni
|
||||
|
||||
// Register growing-source segments as optional local flush sources. Metadata
|
||||
// commit is still owned by WAL flusher / WriteBuffer.
|
||||
if sd.useGrowingSourceFlush() {
|
||||
if sd.allowGrowingSourceFlush() {
|
||||
sd.growingSourceProvider = newDelegatorGrowingSourceProvider(manager.Segment, func(ctx context.Context, fenceTs uint64) error {
|
||||
_, err := sd.waitTSafe(ctx, fenceTs)
|
||||
return err
|
||||
@@ -1589,12 +1589,12 @@ func NewShardDelegator(ctx context.Context, collectionID UniqueID, replicaID Uni
|
||||
return sd, nil
|
||||
}
|
||||
|
||||
// useGrowingSourceFlush returns true when the collection should expose growing segments as a flush source.
|
||||
func (sd *shardDelegator) useGrowingSourceFlush() bool {
|
||||
// allowGrowingSourceFlush returns true when the collection may expose growing segments as a flush source.
|
||||
func (sd *shardDelegator) allowGrowingSourceFlush() bool {
|
||||
if sd == nil || sd.collection == nil {
|
||||
return false
|
||||
}
|
||||
return typeutil.UseGrowingSourceFlush(sd.collection.Schema(),
|
||||
return typeutil.AllowGrowingSourceFlush(sd.collection.Schema(),
|
||||
paramtable.Get().CommonCfg.UseLoonFFI.GetAsBool(),
|
||||
paramtable.Get().CommonCfg.EnableGrowingSourceFlush.GetAsBool())
|
||||
}
|
||||
|
||||
@@ -49,7 +49,12 @@ func (impl *msgHandlerImpl) HandleCreateSegment(ctx context.Context, createSegme
|
||||
}
|
||||
logger := mlog.With(mlog.FieldMessage(createSegmentMsg))
|
||||
|
||||
if err := impl.wbMgr.CreateNewGrowingSegment(ctx, vchannel, h.PartitionId, h.SegmentId, h.SchemaVersion); err != nil {
|
||||
if err := impl.wbMgr.CreateNewGrowingSegment(ctx, vchannel, writebuffer.CreateGrowingSegmentInfo{
|
||||
PartitionID: h.PartitionId,
|
||||
SegmentID: h.SegmentId,
|
||||
SchemaVersion: h.SchemaVersion,
|
||||
StorageVersion: h.StorageVersion,
|
||||
}); err != nil {
|
||||
logger.Warn(ctx, "fail to create new growing segment")
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -132,7 +132,9 @@ func TestFlushMsgHandler_HandleCreateSegment(t *testing.T) {
|
||||
|
||||
t.Run("growing-source collection creates WriteBuffer shell", func(t *testing.T) {
|
||||
wbMgr := writebuffer.NewMockBufferManager(t)
|
||||
wbMgr.EXPECT().CreateNewGrowingSegment(mock.Anything, vchannel, int64(10), int64(1001), mock.Anything).Return(nil)
|
||||
wbMgr.EXPECT().CreateNewGrowingSegment(mock.Anything, vchannel, mock.MatchedBy(func(info writebuffer.CreateGrowingSegmentInfo) bool {
|
||||
return info.PartitionID == 10 && info.SegmentID == 1001 && info.StorageVersion == 0
|
||||
})).Return(nil)
|
||||
|
||||
handler := newMsgHandler(wbMgr)
|
||||
err := handler.HandleCreateSegment(context.Background(), im)
|
||||
@@ -141,7 +143,9 @@ func TestFlushMsgHandler_HandleCreateSegment(t *testing.T) {
|
||||
|
||||
t.Run("non-growing-source collection calls CreateNewGrowingSegment", func(t *testing.T) {
|
||||
wbMgr := writebuffer.NewMockBufferManager(t)
|
||||
wbMgr.EXPECT().CreateNewGrowingSegment(mock.Anything, vchannel, int64(10), int64(1001), mock.Anything).Return(nil)
|
||||
wbMgr.EXPECT().CreateNewGrowingSegment(mock.Anything, vchannel, mock.MatchedBy(func(info writebuffer.CreateGrowingSegmentInfo) bool {
|
||||
return info.PartitionID == 10 && info.SegmentID == 1001 && info.StorageVersion == 0
|
||||
})).Return(nil)
|
||||
|
||||
handler := newMsgHandler(wbMgr)
|
||||
err := handler.HandleCreateSegment(context.Background(), im)
|
||||
|
||||
@@ -101,7 +101,7 @@ func TestReleaseManualFlushPreparer(t *testing.T) {
|
||||
manager.EXPECT().GetAvailableWAL(mock.Anything).Return(wal, nil)
|
||||
|
||||
wbManager := writebuffer.NewMockBufferManager(t)
|
||||
wbManager.EXPECT().UseGrowingSourceFlush("vchannel").Return(true)
|
||||
wbManager.EXPECT().AllowGrowingSourceFlush("vchannel").Return(true)
|
||||
wbManager.EXPECT().
|
||||
GetGrowingFlushProgress(mock.Anything, "vchannel", []int64{1001, 1002}, uint64(200)).
|
||||
Return([]writebuffer.GrowingFlushSegmentProgress{
|
||||
@@ -140,7 +140,7 @@ func TestReleaseManualFlushPreparerNoGrowingProgress(t *testing.T) {
|
||||
manager.EXPECT().GetAvailableWAL(mock.Anything).Return(wal, nil)
|
||||
|
||||
wbManager := writebuffer.NewMockBufferManager(t)
|
||||
wbManager.EXPECT().UseGrowingSourceFlush("vchannel").Return(true)
|
||||
wbManager.EXPECT().AllowGrowingSourceFlush("vchannel").Return(true)
|
||||
wbManager.EXPECT().
|
||||
GetGrowingFlushProgress(mock.Anything, "vchannel", releaseSegmentIDs, uint64(200)).
|
||||
Return([]writebuffer.GrowingFlushSegmentProgress{
|
||||
@@ -169,7 +169,7 @@ func TestReleaseManualFlushPreparerSkipsManualFlushWhenCurrentSegmentsDoNotNeedH
|
||||
MockBufferManager: writebuffer.NewMockBufferManager(t),
|
||||
needManualFlush: false,
|
||||
}
|
||||
wbManager.EXPECT().UseGrowingSourceFlush("vchannel").Return(true)
|
||||
wbManager.EXPECT().AllowGrowingSourceFlush("vchannel").Return(true)
|
||||
wbManager.EXPECT().
|
||||
GetGrowingFlushProgress(mock.Anything, "vchannel", releaseSegmentIDs, uint64(0)).
|
||||
Return([]writebuffer.GrowingFlushSegmentProgress{
|
||||
@@ -198,7 +198,7 @@ func TestReleaseManualFlushPreparerPreparesExistingProgressWithoutManualFlush(t
|
||||
MockBufferManager: writebuffer.NewMockBufferManager(t),
|
||||
needManualFlush: false,
|
||||
}
|
||||
wbManager.EXPECT().UseGrowingSourceFlush("vchannel").Return(true)
|
||||
wbManager.EXPECT().AllowGrowingSourceFlush("vchannel").Return(true)
|
||||
wbManager.EXPECT().
|
||||
GetGrowingFlushProgress(mock.Anything, "vchannel", releaseSegmentIDs, uint64(0)).
|
||||
Return([]writebuffer.GrowingFlushSegmentProgress{
|
||||
@@ -227,7 +227,7 @@ func TestReleaseManualFlushPreparerSkipsManualFlushForEmptyInitialSegments(t *te
|
||||
MockBufferManager: writebuffer.NewMockBufferManager(t),
|
||||
needManualFlush: false,
|
||||
}
|
||||
wbManager.EXPECT().UseGrowingSourceFlush("vchannel").Return(true)
|
||||
wbManager.EXPECT().AllowGrowingSourceFlush("vchannel").Return(true)
|
||||
wbManager.EXPECT().
|
||||
GetGrowingFlushProgress(mock.Anything, "vchannel", []int64(nil), uint64(0)).
|
||||
Return(nil, nil)
|
||||
@@ -259,7 +259,7 @@ func TestReleaseManualFlushPreparerFencesEmptyInitialSegments(t *testing.T) {
|
||||
manager.EXPECT().GetAvailableWAL(mock.Anything).Return(wal, nil)
|
||||
|
||||
wbManager := writebuffer.NewMockBufferManager(t)
|
||||
wbManager.EXPECT().UseGrowingSourceFlush("vchannel").Return(true)
|
||||
wbManager.EXPECT().AllowGrowingSourceFlush("vchannel").Return(true)
|
||||
wbManager.EXPECT().
|
||||
GetGrowingFlushProgress(mock.Anything, "vchannel", affectedSegmentIDs, uint64(200)).
|
||||
Return([]writebuffer.GrowingFlushSegmentProgress{
|
||||
@@ -283,7 +283,7 @@ func TestReleaseManualFlushPreparerFencesEmptyInitialSegments(t *testing.T) {
|
||||
func TestReleaseManualFlushPreparerSkipNonGrowingSource(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
wbManager := writebuffer.NewMockBufferManager(t)
|
||||
wbManager.EXPECT().UseGrowingSourceFlush("vchannel").Return(false)
|
||||
wbManager.EXPECT().AllowGrowingSourceFlush("vchannel").Return(false)
|
||||
|
||||
preparer := NewReleaseManualFlushPreparer(mock_walmanager.NewMockManager(t), wbManager)
|
||||
prepared, err := preparer.PrepareReleaseManualFlush(ctx, types.PChannelInfo{Name: "pchannel", Term: 1}, 10, "vchannel", []int64{1001})
|
||||
|
||||
@@ -37,7 +37,7 @@ func (p *releaseManualFlushPreparer) PrepareReleaseManualFlush(ctx context.Conte
|
||||
if collectionID == 0 {
|
||||
return false, status.NewInvalidArgument("collection id is empty")
|
||||
}
|
||||
if !p.writeBufferManager.UseGrowingSourceFlush(vchannel) {
|
||||
if !p.writeBufferManager.AllowGrowingSourceFlush(vchannel) {
|
||||
mlog.Info(ctx, "skip release manual flush prepare because channel does not use growing-source flush",
|
||||
mlog.String("vchannel", vchannel),
|
||||
mlog.Int64("collectionID", collectionID),
|
||||
|
||||
@@ -211,6 +211,6 @@ func (m *partitionManager) assignSegment(req *AssignSegmentRequest) (*AssignSegm
|
||||
|
||||
// There is no segment can be allocated for the insert request.
|
||||
// Ask a new pending segment to insert.
|
||||
m.asyncAllocSegment(req.SchemaVersion, req.UseGrowingSourceFlush)
|
||||
m.asyncAllocSegment(req.SchemaVersion, req.RequiresStorageV3)
|
||||
return nil, ErrWaitForNewSegment
|
||||
}
|
||||
|
||||
+14
-14
@@ -17,7 +17,7 @@ import (
|
||||
)
|
||||
|
||||
// asyncAllocSegment allocates a new growing segment asynchronously.
|
||||
func (m *partitionManager) asyncAllocSegment(schemaVersion int32, useGrowingSourceFlush bool) {
|
||||
func (m *partitionManager) asyncAllocSegment(schemaVersion int32, requiresStorageV3 bool) {
|
||||
if m.onAllocating != nil {
|
||||
m.Logger().Debug(context.TODO(), "segment alloc worker is already on allocating")
|
||||
// manager is already on allocating.
|
||||
@@ -26,13 +26,13 @@ func (m *partitionManager) asyncAllocSegment(schemaVersion int32, useGrowingSour
|
||||
// Create a notifier to notify the waiter when the allocation is done.
|
||||
m.onAllocating = make(chan struct{})
|
||||
w := &segmentAllocWorker{
|
||||
ctx: m.ctx,
|
||||
collectionID: m.collectionID,
|
||||
partitionID: m.partitionID,
|
||||
vchannel: m.vchannel,
|
||||
wal: m.wal.Get(),
|
||||
schemaVersion: schemaVersion,
|
||||
useGrowingSourceFlush: useGrowingSourceFlush,
|
||||
ctx: m.ctx,
|
||||
collectionID: m.collectionID,
|
||||
partitionID: m.partitionID,
|
||||
vchannel: m.vchannel,
|
||||
wal: m.wal.Get(),
|
||||
schemaVersion: schemaVersion,
|
||||
requiresStorageV3: requiresStorageV3,
|
||||
}
|
||||
w.SetLogger(m.Logger())
|
||||
// It should always done asynchronously.
|
||||
@@ -50,11 +50,11 @@ type segmentAllocWorker struct {
|
||||
wal wal.WAL
|
||||
// The following fields are preserved across retries to ensure the same segment
|
||||
// configuration is used when rebuilding the message after a failed append.
|
||||
segmentID uint64 // allocated segment ID
|
||||
storageVersion int64 // storage version determined at first attempt
|
||||
limitation segmentLimitation // segment limitation determined at first attempt
|
||||
schemaVersion int32
|
||||
useGrowingSourceFlush bool
|
||||
segmentID uint64 // allocated segment ID
|
||||
storageVersion int64 // storage version determined at first attempt
|
||||
limitation segmentLimitation // segment limitation determined at first attempt
|
||||
schemaVersion int32
|
||||
requiresStorageV3 bool
|
||||
}
|
||||
|
||||
// do is the main loop of the segment allocation worker.
|
||||
@@ -147,7 +147,7 @@ func (w *segmentAllocWorker) initSegmentConfig() error {
|
||||
|
||||
// Determine storage version.
|
||||
w.storageVersion = storage.StorageV2
|
||||
if paramtable.Get().CommonCfg.UseLoonFFI.GetAsBool() {
|
||||
if w.requiresStorageV3 || paramtable.Get().CommonCfg.UseLoonFFI.GetAsBool() {
|
||||
w.storageVersion = storage.StorageV3
|
||||
}
|
||||
|
||||
|
||||
+13
-14
@@ -296,31 +296,30 @@ func TestSegmentAllocWorker_InitSegmentConfig(t *testing.T) {
|
||||
assert.Equal(t, firstLimitation, w.limitation)
|
||||
}
|
||||
|
||||
func TestSegmentAllocWorkerStorageVersionFollowsUseLoonFFI(t *testing.T) {
|
||||
func TestSegmentAllocWorkerStorageVersionFollowsRequirements(t *testing.T) {
|
||||
paramtable.Init()
|
||||
resource.InitForTest(t)
|
||||
param := paramtable.Get()
|
||||
defer param.Reset(param.CommonCfg.UseLoonFFI.Key)
|
||||
|
||||
for name, tc := range map[string]struct {
|
||||
useLoonFFI string
|
||||
useGrowingSourceFlush bool
|
||||
expected int64
|
||||
useLoonFFI string
|
||||
requiresStorageV3 bool
|
||||
expected int64
|
||||
}{
|
||||
"v2_without_growing_source": {useLoonFFI: "false", useGrowingSourceFlush: false, expected: storage.StorageV2},
|
||||
"v2_with_growing_source": {useLoonFFI: "false", useGrowingSourceFlush: true, expected: storage.StorageV2},
|
||||
"v3_without_growing_source": {useLoonFFI: "true", useGrowingSourceFlush: false, expected: storage.StorageV3},
|
||||
"v3_with_growing_source": {useLoonFFI: "true", useGrowingSourceFlush: true, expected: storage.StorageV3},
|
||||
"v2_without_requirement": {useLoonFFI: "false", expected: storage.StorageV2},
|
||||
"v3_required_by_schema": {useLoonFFI: "false", requiresStorageV3: true, expected: storage.StorageV3},
|
||||
"v3_with_ffi": {useLoonFFI: "true", expected: storage.StorageV3},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
param.Save(param.CommonCfg.UseLoonFFI.Key, tc.useLoonFFI)
|
||||
w := &segmentAllocWorker{
|
||||
ctx: context.Background(),
|
||||
collectionID: 1,
|
||||
partitionID: 2,
|
||||
vchannel: "v1",
|
||||
wal: mock_wal.NewMockWAL(t),
|
||||
useGrowingSourceFlush: tc.useGrowingSourceFlush,
|
||||
ctx: context.Background(),
|
||||
collectionID: 1,
|
||||
partitionID: 2,
|
||||
vchannel: "v1",
|
||||
wal: mock_wal.NewMockWAL(t),
|
||||
requiresStorageV3: tc.requiresStorageV3,
|
||||
}
|
||||
w.SetLogger(mlog.With())
|
||||
|
||||
|
||||
@@ -233,15 +233,19 @@ func (c *CollectionInfo) SchemaVersion() int32 {
|
||||
return s.GetVersion()
|
||||
}
|
||||
|
||||
func (c *CollectionInfo) UseGrowingSourceFlush() bool {
|
||||
func (c *CollectionInfo) AllowGrowingSourceFlush() bool {
|
||||
if c == nil || c.Schema == nil {
|
||||
return false
|
||||
}
|
||||
return typeutil.UseGrowingSourceFlush(c.Schema.GetSchema(),
|
||||
return typeutil.AllowGrowingSourceFlush(c.Schema.GetSchema(),
|
||||
paramtable.Get().CommonCfg.UseLoonFFI.GetAsBool(),
|
||||
paramtable.Get().CommonCfg.EnableGrowingSourceFlush.GetAsBool())
|
||||
}
|
||||
|
||||
func (c *CollectionInfo) RequiresStorageV3() bool {
|
||||
return c.HasTextField()
|
||||
}
|
||||
|
||||
func (c *CollectionInfo) HasTextField() bool {
|
||||
if c == nil || c.Schema == nil || c.Schema.GetSchema() == nil {
|
||||
return false
|
||||
@@ -266,7 +270,7 @@ func (c *CollectionInfo) RuntimeFlushSize(modified stats.ModifiedMetrics) uint64
|
||||
}
|
||||
|
||||
func (c *CollectionInfo) shouldEstimateInterimIndexExtra() bool {
|
||||
if c == nil || c.Schema == nil || c.Schema.GetSchema() == nil || !c.UseGrowingSourceFlush() {
|
||||
if c == nil || c.Schema == nil || c.Schema.GetSchema() == nil || !c.AllowGrowingSourceFlush() {
|
||||
return false
|
||||
}
|
||||
params := paramtable.Get()
|
||||
|
||||
+9
-9
@@ -12,14 +12,14 @@ import (
|
||||
|
||||
// AssignSegmentRequest is a request to allocate segment.
|
||||
type AssignSegmentRequest struct {
|
||||
CollectionID int64
|
||||
PartitionID int64
|
||||
ModifiedMetrics stats.ModifiedMetrics
|
||||
RuntimeFlushSize uint64
|
||||
TimeTick uint64
|
||||
TxnSession TxnSession
|
||||
SchemaVersion int32
|
||||
UseGrowingSourceFlush bool
|
||||
CollectionID int64
|
||||
PartitionID int64
|
||||
ModifiedMetrics stats.ModifiedMetrics
|
||||
RuntimeFlushSize uint64
|
||||
TimeTick uint64
|
||||
TxnSession TxnSession
|
||||
SchemaVersion int32
|
||||
RequiresStorageV3 bool
|
||||
}
|
||||
|
||||
// AssignSegmentResult is a result of segment allocation.
|
||||
@@ -143,7 +143,7 @@ func (m *shardManagerImpl) AssignSegment(req *AssignSegmentRequest) (*AssignSegm
|
||||
// single place that resolves and stamps it before forwarding to partitionManager.
|
||||
if info := m.collections[req.CollectionID]; info != nil {
|
||||
req.SchemaVersion = info.SchemaVersion()
|
||||
req.UseGrowingSourceFlush = info.UseGrowingSourceFlush()
|
||||
req.RequiresStorageV3 = info.RequiresStorageV3()
|
||||
req.RuntimeFlushSize = info.RuntimeFlushSize(req.ModifiedMetrics)
|
||||
}
|
||||
|
||||
|
||||
@@ -494,7 +494,7 @@ func TestShardManagerSchemaVersionCheck(t *testing.T) {
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, int32(4), ver)
|
||||
assert.True(t, m.collections[104].HasTextField())
|
||||
assert.True(t, m.collections[104].UseGrowingSourceFlush())
|
||||
assert.True(t, m.collections[104].AllowGrowingSourceFlush())
|
||||
|
||||
// Test 3: Create collection without schema (legacy), then check version
|
||||
createMsgNoSchema := message.NewCreateCollectionMessageBuilderV1().
|
||||
@@ -835,17 +835,17 @@ func TestCollectionInfoSchemaVersion(t *testing.T) {
|
||||
assert.Equal(t, int32(3), ci.SchemaVersion())
|
||||
}
|
||||
|
||||
func TestCollectionInfoUseGrowingSourceFlush(t *testing.T) {
|
||||
func TestCollectionInfoAllowGrowingSourceFlush(t *testing.T) {
|
||||
paramtable.Init()
|
||||
paramtable.Get().Save(paramtable.Get().CommonCfg.UseLoonFFI.Key, "false")
|
||||
paramtable.Get().Save(paramtable.Get().CommonCfg.EnableGrowingSourceFlush.Key, "false")
|
||||
defer paramtable.Get().Reset(paramtable.Get().CommonCfg.UseLoonFFI.Key)
|
||||
defer paramtable.Get().Reset(paramtable.Get().CommonCfg.EnableGrowingSourceFlush.Key)
|
||||
assert.False(t, (*CollectionInfo)(nil).UseGrowingSourceFlush())
|
||||
assert.False(t, (*CollectionInfo)(nil).AllowGrowingSourceFlush())
|
||||
ci := &CollectionInfo{}
|
||||
assert.False(t, ci.UseGrowingSourceFlush())
|
||||
assert.False(t, ci.AllowGrowingSourceFlush())
|
||||
ci.Schema = &streamingpb.CollectionSchemaOfVChannel{}
|
||||
assert.False(t, ci.UseGrowingSourceFlush())
|
||||
assert.False(t, ci.AllowGrowingSourceFlush())
|
||||
ci.Schema = &streamingpb.CollectionSchemaOfVChannel{
|
||||
Schema: &schemapb.CollectionSchema{
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
@@ -854,12 +854,12 @@ func TestCollectionInfoUseGrowingSourceFlush(t *testing.T) {
|
||||
},
|
||||
}
|
||||
paramtable.Get().Save(paramtable.Get().CommonCfg.EnableGrowingSourceFlush.Key, "true")
|
||||
assert.False(t, ci.UseGrowingSourceFlush())
|
||||
assert.False(t, ci.AllowGrowingSourceFlush())
|
||||
paramtable.Get().Save(paramtable.Get().CommonCfg.UseLoonFFI.Key, "true")
|
||||
assert.True(t, ci.UseGrowingSourceFlush())
|
||||
assert.True(t, ci.AllowGrowingSourceFlush())
|
||||
}
|
||||
|
||||
func TestCollectionInfoUseGrowingSourceFlush_TextField(t *testing.T) {
|
||||
func TestCollectionInfoAllowGrowingSourceFlush_TextField(t *testing.T) {
|
||||
paramtable.Init()
|
||||
paramtable.Get().Save(paramtable.Get().CommonCfg.UseLoonFFI.Key, "false")
|
||||
paramtable.Get().Save(paramtable.Get().CommonCfg.EnableGrowingSourceFlush.Key, "false")
|
||||
@@ -877,9 +877,10 @@ func TestCollectionInfoUseGrowingSourceFlush_TextField(t *testing.T) {
|
||||
},
|
||||
}
|
||||
assert.True(t, ci.HasTextField())
|
||||
assert.False(t, ci.UseGrowingSourceFlush())
|
||||
assert.False(t, ci.AllowGrowingSourceFlush())
|
||||
assert.True(t, ci.RequiresStorageV3())
|
||||
paramtable.Get().Save(paramtable.Get().CommonCfg.UseLoonFFI.Key, "true")
|
||||
assert.True(t, ci.UseGrowingSourceFlush())
|
||||
assert.True(t, ci.AllowGrowingSourceFlush())
|
||||
}
|
||||
|
||||
func TestCollectionInfoRuntimeFlushSize(t *testing.T) {
|
||||
|
||||
@@ -114,15 +114,22 @@ func ValidateTextRequiresStorageV3(schema *schemapb.CollectionSchema, storageV3E
|
||||
return nil
|
||||
}
|
||||
|
||||
// UseGrowingSourceFlush returns whether insert payload for the schema should
|
||||
// be flushed from QueryNode growing source when available.
|
||||
func UseGrowingSourceFlush(schema *schemapb.CollectionSchema, storageV3Enabled bool, enableGrowingSourceFlush bool) bool {
|
||||
// AllowGrowingSourceFlush returns whether insert payload for the schema may try
|
||||
// flushing from QueryNode growing source when available.
|
||||
func AllowGrowingSourceFlush(schema *schemapb.CollectionSchema, storageV3Enabled bool, enableGrowingSourceFlush bool) bool {
|
||||
if !storageV3Enabled {
|
||||
return false
|
||||
}
|
||||
return HasTextField(schema) || enableGrowingSourceFlush
|
||||
}
|
||||
|
||||
// UseGrowingSourceFlush is kept for compatibility. Prefer
|
||||
// AllowGrowingSourceFlush for new code to avoid implying the source choice is
|
||||
// mandatory.
|
||||
func UseGrowingSourceFlush(schema *schemapb.CollectionSchema, storageV3Enabled bool, enableGrowingSourceFlush bool) bool {
|
||||
return AllowGrowingSourceFlush(schema, storageV3Enabled, enableGrowingSourceFlush)
|
||||
}
|
||||
|
||||
// EstimateSizePerRecord returns the estimate size of a record in a collection
|
||||
func EstimateSizePerRecord(schema *schemapb.CollectionSchema) (int, error) {
|
||||
return estimateSizeBy(schema, custom)
|
||||
|
||||
@@ -336,7 +336,7 @@ func TestValidateTextRequiresStorageV3(t *testing.T) {
|
||||
assert.NoError(t, ValidateTextRequiresStorageV3(textSchema, true))
|
||||
}
|
||||
|
||||
func TestUseGrowingSourceFlush(t *testing.T) {
|
||||
func TestAllowGrowingSourceFlush(t *testing.T) {
|
||||
ordinarySchema := &schemapb.CollectionSchema{
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
{FieldID: 100, DataType: schemapb.DataType_Int64},
|
||||
@@ -349,13 +349,16 @@ func TestUseGrowingSourceFlush(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
assert.False(t, UseGrowingSourceFlush(ordinarySchema, false, true))
|
||||
assert.False(t, UseGrowingSourceFlush(textSchema, false, true))
|
||||
assert.False(t, UseGrowingSourceFlush(ordinarySchema, true, false))
|
||||
assert.True(t, UseGrowingSourceFlush(ordinarySchema, true, true))
|
||||
assert.True(t, UseGrowingSourceFlush(textSchema, true, false))
|
||||
assert.False(t, UseGrowingSourceFlush(nil, true, false))
|
||||
assert.True(t, UseGrowingSourceFlush(nil, true, true))
|
||||
assert.False(t, AllowGrowingSourceFlush(ordinarySchema, false, true))
|
||||
assert.False(t, AllowGrowingSourceFlush(textSchema, false, true))
|
||||
assert.False(t, AllowGrowingSourceFlush(ordinarySchema, true, false))
|
||||
assert.True(t, AllowGrowingSourceFlush(ordinarySchema, true, true))
|
||||
assert.True(t, AllowGrowingSourceFlush(textSchema, true, false))
|
||||
assert.False(t, AllowGrowingSourceFlush(nil, true, false))
|
||||
assert.True(t, AllowGrowingSourceFlush(nil, true, true))
|
||||
assert.Equal(t,
|
||||
AllowGrowingSourceFlush(textSchema, true, false),
|
||||
UseGrowingSourceFlush(textSchema, true, false))
|
||||
}
|
||||
|
||||
func TestSchema_GetVectorFieldSchemas(t *testing.T) {
|
||||
|
||||
Reference in New Issue
Block a user