mirror of
https://github.com/milvus-io/milvus.git
synced 2026-07-21 10:15:43 +00:00
sijie-ni-0214andGitHub
a7db8873e1
## Problem During schema evolution, SQN can convert an insert payload with the old schema and then race with a schema update before ProcessInsert creates and writes the growing segment. If the old payload still contains a dropped field, the native collection may already use the new schema and segcore rejects the write with unexpected new field. ## Solution - Add a collection-local schema-transition RW lock. - Insert holds the reader lock from schema snapshot use through payload conversion and delegator.ProcessInsert, so a growing segment is created and written in the same schema epoch as its payload. - Schema update and existing-collection load refresh paths take the writer lock around native schema mutation and Go schema snapshot publication. - Acquire a temporary collection lease before releasing collectionManager.mut. This keeps the collection alive while the writer waits, without serializing unrelated collection-manager operations behind a schema transition. - Preserve dropped-field replay handling: when DDL has already completed, payload conversion filters fields absent from the current schema before inserting into a new-schema growing segment. ## Tests - Add a deterministic native regression: pause after a v950 payload containing field 580 is converted, queue the v951 drop-field update, then resume a real NewSegment plus growing.Insert and verify both insert and DDL succeed. - Assert the transition reader covers both payload conversion and ProcessInsert. - Cover the DDL-first replay path, where field 580 is filtered before native insertion. - Cover both schema writer entry points: UpdateSchema and existing-collection PutOrRef. ## Validation - make static-check - make build-go - Native pipeline regression tests repeated 20 times with dynamic,test - Collection schema-transition and lease tests repeated 20 times - internal/querynodev2/pipeline and internal/querynodev2/segments package tests - gofumpt -l and git diff --check Fixes #51436 Signed-off-by: sijie-ni-0214 <sijie.ni@zilliz.com>
54 lines
1.8 KiB
Go
54 lines
1.8 KiB
Go
// Licensed to the LF AI & Data foundation under one
|
|
// or more contributor license agreements. See the NOTICE file
|
|
// distributed with this work for additional information
|
|
// regarding copyright ownership. The ASF licenses this file
|
|
// to you under the Apache License, Version 2.0 (the
|
|
// "License"); you may not use this file except in compliance
|
|
// with the License. You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
//go:build test
|
|
|
|
package segments
|
|
|
|
import (
|
|
"runtime"
|
|
"time"
|
|
)
|
|
|
|
// WaitForSchemaTransitionWriterForTest waits until a schema writer is queued
|
|
// behind an active insert transition reader. It is available only in test
|
|
// builds so cross-package regression tests can synchronize on the writer
|
|
// queue instead of using a timer as the success condition.
|
|
func (c *Collection) WaitForSchemaTransitionWriterForTest(timeout time.Duration) bool {
|
|
deadline := time.Now().Add(timeout)
|
|
for {
|
|
if !c.schemaTransitionMu.TryRLock() {
|
|
return true
|
|
}
|
|
c.schemaTransitionMu.RUnlock()
|
|
if time.Now().After(deadline) {
|
|
return false
|
|
}
|
|
runtime.Gosched()
|
|
}
|
|
}
|
|
|
|
// HasInsertSchemaTransitionReaderForTest reports whether an insert reader is
|
|
// holding the transition mutex. Callers must invoke it before starting a schema
|
|
// writer, so a failed TryLock unambiguously means a reader is active.
|
|
func (c *Collection) HasInsertSchemaTransitionReaderForTest() bool {
|
|
if c.schemaTransitionMu.TryLock() {
|
|
c.schemaTransitionMu.Unlock()
|
|
return false
|
|
}
|
|
return true
|
|
}
|