mirror of
https://github.com/milvus-io/milvus.git
synced 2026-07-21 10:15:43 +00:00
enhance: add S3-based backfill result commit endpoint (#49265)
issue: zilliztech/spark-milvus#74 Add a new proxy management HTTP endpoint /management/datacoord/backfill/commit that takes an object-storage path to a BackfillResult JSON produced by the Spark offline backfill job, lets DataCoord download and parse it, then dispatches per-segment metadata updates through the existing BatchUpdateManifest broadcast pipeline. V2 (StorageV2 packed-parquet) and V3 (Loon manifest) segments share the broadcast, so the broadcaster's SharedDBName + SharedCollectionName resource keys serialise the commit against compaction and other DDL-like operations on the same collection. The ack callback classifies each item and applies either UpdateManifestVersion (V3) or the ported UpdateSegmentColumnGroupsOperator (V2) in one UpdateSegmentsInfo batch. The V2 path trusts the group-level row_count in the result JSON rather than reading parquet footers; row_count is split evenly across the group's binlog files (remainder added to the last file). Changes: - proto: new CommitBackfillResult RPC on DataCoord; extend BatchUpdateManifestItem with a v2_column_groups payload + new BatchUpdateManifestV2ColumnGroups message. - datacoord: new services_commit_backfill.go handler; new backfill_result.go (JSON decoding, object-URI normaliser, bucket validation, V2 group constructor). Port UpdateSegmentColumnGroupsOperator from stash/update_columngroup_restful into meta.go. Extend the BatchUpdateManifest broadcast callback to dispatch V2/V3 items. - proxy: new /management/datacoord/backfill/commit HTTP handler. - mixcoord: grpc server/client/wrapper delegators. - storage: expose RemoteChunkManager.BucketName() for bucket-mismatch rejection in the URI normaliser. Tests: - backfill_result_test.go: JSON parsing, normaliser, V2 group builder. - services_test.go: TestServer_CommitBackfillResult (V3 happy, V2 happy, mixed/partial failure, success=false, bad JSON, pre-validation-all-fail, unhealthy). Extend BatchUpdateManifest callback tests with V2 and mixed-item scenarios. - meta_test.go: TestUpdateSegmentColumnGroupsOperator (add, replace in-place, child-field stripping, not-found, DataVersion monotonicity). - management_test.go: proxy handler contract (missing param, success, rpc error, downstream status error). --------- Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>
This commit is contained in:
congqixia
@@ -1314,6 +1314,10 @@ func (s *mixCoordImpl) BatchUpdateManifest(ctx context.Context, req *datapb.Batc
|
||||
return s.datacoordServer.BatchUpdateManifest(ctx, req)
|
||||
}
|
||||
|
||||
func (s *mixCoordImpl) CommitBackfillResult(ctx context.Context, req *datapb.CommitBackfillResultRequest) (*datapb.CommitBackfillResultResponse, error) {
|
||||
return s.datacoordServer.CommitBackfillResult(ctx, req)
|
||||
}
|
||||
|
||||
// Client Telemetry methods - forwarded to rootcoord
|
||||
|
||||
func (s *mixCoordImpl) ClientHeartbeat(ctx context.Context, req *milvuspb.ClientHeartbeatRequest) (*milvuspb.ClientHeartbeatResponse, error) {
|
||||
|
||||
@@ -0,0 +1,218 @@
|
||||
// Licensed to the LF AI & Data foundation under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package datacoord
|
||||
|
||||
import (
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/cockroachdb/errors"
|
||||
|
||||
"github.com/milvus-io/milvus/internal/storage"
|
||||
"github.com/milvus-io/milvus/pkg/v2/proto/datapb"
|
||||
)
|
||||
|
||||
// BackfillResult is the decoded form of the JSON produced by the Spark
|
||||
// backfill job. Only fields required for commit are modeled; other diagnostic
|
||||
// fields (executionTimeMs, usedSourceByField, etc.) are ignored.
|
||||
//
|
||||
// Reference: spark-milvus/docs/backfill-result-json-format.md
|
||||
type BackfillResult struct {
|
||||
Success bool `json:"success"`
|
||||
CollectionID int64 `json:"collectionId"`
|
||||
PartitionID int64 `json:"partitionId"`
|
||||
NewFieldNames []string `json:"newFieldNames"`
|
||||
Segments map[string]BackfillSegment `json:"segments"` // key = segmentID as decimal string
|
||||
}
|
||||
|
||||
// BackfillSegment is one entry in BackfillResult.Segments.
|
||||
//
|
||||
// V3 entries carry {version, manifestPaths} and omit storage_version/column_groups.
|
||||
// V2 entries carry {storage_version: 2, column_groups: [...]} and have version == -1.
|
||||
type BackfillSegment struct {
|
||||
Version int64 `json:"version"` // V3: committedVersion (>0); V2: -1
|
||||
RowCount int64 `json:"rowCount"`
|
||||
OutputPath string `json:"outputPath"`
|
||||
ManifestPaths []string `json:"manifestPaths"`
|
||||
StorageVersion *int64 `json:"storage_version,omitempty"` // ptr to distinguish absent vs 0
|
||||
ColumnGroups []BackfillV2ColumnGroup `json:"column_groups,omitempty"`
|
||||
}
|
||||
|
||||
// BackfillV2ColumnGroup describes one V2 column group produced by backfill.
|
||||
// Invariant from backfill: field_ids has exactly one element (single-field groups).
|
||||
type BackfillV2ColumnGroup struct {
|
||||
FieldIDs []int64 `json:"field_ids"`
|
||||
BinlogFiles []string `json:"binlog_files"` // ascending by log_id
|
||||
RowCount int64 `json:"row_count"` // group-level; trusted as segment NumOfRows
|
||||
}
|
||||
|
||||
// IsV2 reports whether this entry represents a StorageV2 segment.
|
||||
func (s *BackfillSegment) IsV2() bool {
|
||||
return s.StorageVersion != nil && *s.StorageVersion == storage.StorageV2 && len(s.ColumnGroups) > 0
|
||||
}
|
||||
|
||||
// knownObjectSchemes lists URI schemes recognized by normalizeObjectKey.
|
||||
// All map to "treat host as bucket, path as object key".
|
||||
var knownObjectSchemes = map[string]struct{}{
|
||||
"s3": {},
|
||||
"s3a": {},
|
||||
"s3n": {},
|
||||
"gs": {},
|
||||
"oss": {},
|
||||
"minio": {},
|
||||
}
|
||||
|
||||
// normalizeObjectKey converts a spark-style URI into a chunk-manager object key.
|
||||
//
|
||||
// Rules:
|
||||
// 1. No scheme -> trim leading '/', return as-is.
|
||||
// 2. Known scheme -> parse "<scheme>://<host>/<path>"; if expectedBucket != "" and
|
||||
// host != expectedBucket, return ErrBucketMismatch. Otherwise return <path>.
|
||||
// 3. Unknown scheme -> return ErrUnsupportedScheme.
|
||||
func normalizeObjectKey(raw, expectedBucket string) (string, error) {
|
||||
if raw == "" {
|
||||
return "", errors.New("empty object path")
|
||||
}
|
||||
if !strings.Contains(raw, "://") {
|
||||
key := strings.TrimPrefix(raw, "/")
|
||||
if key == "" {
|
||||
return "", errors.Newf("empty object key in %q", raw)
|
||||
}
|
||||
return key, nil
|
||||
}
|
||||
idx := strings.Index(raw, "://")
|
||||
scheme := strings.ToLower(raw[:idx])
|
||||
if _, ok := knownObjectSchemes[scheme]; !ok {
|
||||
return "", errors.Newf("unsupported object URI scheme %q in %q", scheme, raw)
|
||||
}
|
||||
rest := raw[idx+3:]
|
||||
slash := strings.Index(rest, "/")
|
||||
if slash < 0 {
|
||||
return "", errors.Newf("malformed object URI %q: missing object key", raw)
|
||||
}
|
||||
bucket := rest[:slash]
|
||||
key := rest[slash+1:]
|
||||
if expectedBucket != "" && bucket != expectedBucket {
|
||||
return "", errors.Newf("object URI bucket %q differs from datacoord bucket %q (path=%s)", bucket, expectedBucket, raw)
|
||||
}
|
||||
// Reject inputs like "s3a://bucket/" that parse to an empty key -- passing
|
||||
// an empty key to chunkManager.Read has undefined behavior across SDKs.
|
||||
if key == "" {
|
||||
return "", errors.Newf("empty object key in %q", raw)
|
||||
}
|
||||
return key, nil
|
||||
}
|
||||
|
||||
// parseLogIDFromKey extracts the trailing path segment as int64. Returns 0 with
|
||||
// ok=false if the trailing segment is not a decimal integer (e.g. when it has a
|
||||
// file extension). Callers should proceed even on ok=false -- LogID is
|
||||
// informational for this particular flow.
|
||||
func parseLogIDFromKey(key string) (int64, bool) {
|
||||
base := path.Base(key)
|
||||
if base == "" || base == "." || base == "/" {
|
||||
return 0, false
|
||||
}
|
||||
id, err := strconv.ParseInt(base, 10, 64)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
return id, true
|
||||
}
|
||||
|
||||
// buildV2Groups constructs the datapb.FieldBinlog map used by the V2 update
|
||||
// operator. EntriesNum is distributed across BinlogFiles by integer division
|
||||
// (remainder added to the last file) -- the sum equals g.RowCount, which the
|
||||
// backfill contract guarantees to equal segment.NumOfRows.
|
||||
//
|
||||
// This function does NOT read parquet footers. It trusts the row counts in the
|
||||
// result JSON per the backfill contract (see
|
||||
// spark-milvus/docs/backfill-result-json-format.md).
|
||||
func buildV2Groups(bucket string, entry *BackfillSegment) (map[int64]*datapb.FieldBinlog, error) {
|
||||
out := make(map[int64]*datapb.FieldBinlog, len(entry.ColumnGroups))
|
||||
for i := range entry.ColumnGroups {
|
||||
g := &entry.ColumnGroups[i]
|
||||
if len(g.FieldIDs) != 1 {
|
||||
return nil, errors.Newf("backfill invariant violated: column group has %d field_ids (expected 1)", len(g.FieldIDs))
|
||||
}
|
||||
n := int64(len(g.BinlogFiles))
|
||||
if n == 0 {
|
||||
return nil, errors.Newf("column group for field %d has no binlog files", g.FieldIDs[0])
|
||||
}
|
||||
fid := g.FieldIDs[0]
|
||||
if _, dup := out[fid]; dup {
|
||||
return nil, errors.Newf("duplicate column group for field %d", fid)
|
||||
}
|
||||
// row_count flows into EntriesNum; non-positive values are undefined
|
||||
// (zero collapses presence markers, negatives break accounting).
|
||||
if g.RowCount <= 0 {
|
||||
return nil, errors.Newf("column group for field %d has non-positive row_count %d", fid, g.RowCount)
|
||||
}
|
||||
avg := g.RowCount / n
|
||||
rem := g.RowCount - avg*n
|
||||
binlogs := make([]*datapb.Binlog, 0, n)
|
||||
for idx, p := range g.BinlogFiles {
|
||||
key, err := normalizeObjectKey(p, bucket)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows := avg
|
||||
if int64(idx) == n-1 {
|
||||
rows += rem
|
||||
}
|
||||
logID, ok := parseLogIDFromKey(key)
|
||||
if !ok {
|
||||
return nil, errors.Newf("column group for field %d has binlog file %q with non-numeric trailing segment", fid, p)
|
||||
}
|
||||
// LogPath must be empty at persistence time -- catalog.checkLogID
|
||||
// rejects any Binlog with LogPath != "" (see
|
||||
// internal/metastore/kv/datacoord/util.go). Canonical on-disk
|
||||
// form is {LogID, LogPath:""}; DecompressBinLog reconstructs the
|
||||
// path from LogID + (collection, partition, segment, field) at
|
||||
// load time.
|
||||
binlogs = append(binlogs, &datapb.Binlog{
|
||||
EntriesNum: rows,
|
||||
LogID: logID,
|
||||
})
|
||||
}
|
||||
out[fid] = &datapb.FieldBinlog{
|
||||
FieldID: fid,
|
||||
// ChildFields carries the real field IDs that index creation
|
||||
// (getSegmentBinlogFields) and backfill-compaction detection
|
||||
// (getMissingFunctions) rely on. Without it the new group is
|
||||
// invisible to both paths; it also lets the operator's strip
|
||||
// logic reference-count the field out of the old groups.
|
||||
// The backfill invariant guarantees len(g.FieldIDs) == 1.
|
||||
ChildFields: []int64{fid},
|
||||
Binlogs: binlogs,
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// bucketFromChunkManager returns the bucket name of the given chunk manager if
|
||||
// it exposes BucketName(); otherwise returns an empty string (bucket check will
|
||||
// be skipped). Used to avoid a hard dependency on *RemoteChunkManager.
|
||||
func bucketFromChunkManager(cm storage.ChunkManager) string {
|
||||
type bucketNameProvider interface {
|
||||
BucketName() string
|
||||
}
|
||||
if p, ok := cm.(bucketNameProvider); ok {
|
||||
return p.BucketName()
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,324 @@
|
||||
// Licensed to the LF AI & Data foundation under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package datacoord
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/milvus-io/milvus/internal/json"
|
||||
"github.com/milvus-io/milvus/internal/storage"
|
||||
)
|
||||
|
||||
// mixedBackfillResultJSON mirrors the fixture shape in
|
||||
// spark-milvus/docs/backfill-result-json-format.md.
|
||||
const mixedBackfillResultJSON = `{
|
||||
"success": true,
|
||||
"collectionId": 450000000001,
|
||||
"partitionId": 450000000002,
|
||||
"segmentsProcessed": 2,
|
||||
"newFieldNames": ["embedding_v2", "score"],
|
||||
"segments": {
|
||||
"450000000100": {
|
||||
"version": 42,
|
||||
"rowCount": 333334,
|
||||
"outputPath": "s3a://bucket/seg/100",
|
||||
"manifestPaths": ["s3a://bucket/seg/100/manifest/0042"]
|
||||
},
|
||||
"450000000101": {
|
||||
"version": -1,
|
||||
"rowCount": 333333,
|
||||
"outputPath": "s3a://bucket/seg/101",
|
||||
"manifestPaths": [
|
||||
"s3a://bucket/seg/101/100/7",
|
||||
"s3a://bucket/seg/101/101/9"
|
||||
],
|
||||
"storage_version": 2,
|
||||
"column_groups": [
|
||||
{
|
||||
"field_ids": [100],
|
||||
"binlog_files": ["s3a://bucket/seg/101/100/7"],
|
||||
"row_count": 333333
|
||||
},
|
||||
{
|
||||
"field_ids": [101],
|
||||
"binlog_files": ["s3a://bucket/seg/101/101/9"],
|
||||
"row_count": 333333
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}`
|
||||
|
||||
func TestBackfillResult_ParseMixedV2V3(t *testing.T) {
|
||||
var r BackfillResult
|
||||
err := json.Unmarshal([]byte(mixedBackfillResultJSON), &r)
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, r.Success)
|
||||
assert.Equal(t, int64(450000000001), r.CollectionID)
|
||||
assert.Equal(t, int64(450000000002), r.PartitionID)
|
||||
assert.ElementsMatch(t, []string{"embedding_v2", "score"}, r.NewFieldNames)
|
||||
assert.Len(t, r.Segments, 2)
|
||||
|
||||
v3 := r.Segments["450000000100"]
|
||||
assert.False(t, v3.IsV2())
|
||||
assert.Equal(t, int64(42), v3.Version)
|
||||
|
||||
v2 := r.Segments["450000000101"]
|
||||
assert.True(t, v2.IsV2())
|
||||
assert.Equal(t, int64(-1), v2.Version)
|
||||
assert.Len(t, v2.ColumnGroups, 2)
|
||||
assert.Equal(t, int64(333333), v2.ColumnGroups[0].RowCount)
|
||||
assert.Equal(t, []int64{100}, v2.ColumnGroups[0].FieldIDs)
|
||||
}
|
||||
|
||||
func TestBackfillResult_IsV2(t *testing.T) {
|
||||
v2 := storage.StorageV2
|
||||
v3 := storage.StorageV3
|
||||
zero := int64(0)
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
seg BackfillSegment
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "absent storage version -> V3",
|
||||
seg: BackfillSegment{},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "storage_version=2 with groups -> V2",
|
||||
seg: BackfillSegment{
|
||||
StorageVersion: &v2,
|
||||
ColumnGroups: []BackfillV2ColumnGroup{{FieldIDs: []int64{100}, BinlogFiles: []string{"a"}, RowCount: 1}},
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "storage_version=2 without groups -> V3 (defensive)",
|
||||
seg: BackfillSegment{StorageVersion: &v2},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "storage_version=3 -> V3",
|
||||
seg: BackfillSegment{StorageVersion: &v3},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "storage_version=0 ptr -> V3",
|
||||
seg: BackfillSegment{StorageVersion: &zero},
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
assert.Equal(t, tc.want, tc.seg.IsV2())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeObjectKey(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
raw string
|
||||
bucket string
|
||||
wantKey string
|
||||
wantErr bool
|
||||
errMatch string
|
||||
}{
|
||||
{"s3a with matching bucket", "s3a://bkt/path/foo", "bkt", "path/foo", false, ""},
|
||||
{"s3 scheme", "s3://bkt/path/foo", "bkt", "path/foo", false, ""},
|
||||
{"s3n scheme", "s3n://bkt/deep/path/f", "bkt", "deep/path/f", false, ""},
|
||||
{"gs scheme", "gs://mybkt/a/b", "mybkt", "a/b", false, ""},
|
||||
{"minio scheme", "minio://bkt/a", "bkt", "a", false, ""},
|
||||
{"no scheme, leading slash", "/data/foo", "", "data/foo", false, ""},
|
||||
{"no scheme, no slash", "data/foo", "", "data/foo", false, ""},
|
||||
{"no scheme, ignore bucket arg", "data/foo", "bkt", "data/foo", false, ""},
|
||||
{"empty path", "", "", "", true, "empty object path"},
|
||||
{"bucket mismatch", "s3a://other/foo", "bkt", "", true, "differs from datacoord bucket"},
|
||||
{"empty expected bucket skips check", "s3a://anything/foo", "", "foo", false, ""},
|
||||
{"unsupported scheme", "hdfs://host/path", "", "", true, "unsupported object URI scheme"},
|
||||
{"missing key", "s3a://bkt", "bkt", "", true, "missing object key"},
|
||||
// trailing slash produces an empty key -- chunkManager.Read with an
|
||||
// empty key has undefined behavior, must be rejected.
|
||||
{"empty key after scheme", "s3a://bkt/", "bkt", "", true, "empty object key"},
|
||||
{"empty key no scheme", "/", "", "", true, "empty object key"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got, err := normalizeObjectKey(tc.raw, tc.bucket)
|
||||
if tc.wantErr {
|
||||
assert.Error(t, err)
|
||||
if tc.errMatch != "" {
|
||||
assert.Contains(t, err.Error(), tc.errMatch)
|
||||
}
|
||||
return
|
||||
}
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, tc.wantKey, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseLogIDFromKey(t *testing.T) {
|
||||
cases := []struct {
|
||||
key string
|
||||
wantID int64
|
||||
ok bool
|
||||
}{
|
||||
{"seg/101/100/7", 7, true},
|
||||
{"7", 7, true},
|
||||
{"seg/101/100/12345", 12345, true},
|
||||
{"seg/101/100/foo", 0, false},
|
||||
{"seg/101/100/7.parquet", 0, false},
|
||||
{"", 0, false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
got, ok := parseLogIDFromKey(tc.key)
|
||||
assert.Equal(t, tc.ok, ok, tc.key)
|
||||
assert.Equal(t, tc.wantID, got, tc.key)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildV2Groups(t *testing.T) {
|
||||
v2 := storage.StorageV2
|
||||
|
||||
t.Run("single file per group", func(t *testing.T) {
|
||||
seg := &BackfillSegment{
|
||||
StorageVersion: &v2,
|
||||
ColumnGroups: []BackfillV2ColumnGroup{
|
||||
{
|
||||
FieldIDs: []int64{100},
|
||||
BinlogFiles: []string{"s3a://bkt/seg/1/100/7"},
|
||||
RowCount: 1000,
|
||||
},
|
||||
},
|
||||
}
|
||||
groups, err := buildV2Groups("bkt", seg)
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, groups, 1)
|
||||
fb := groups[100]
|
||||
assert.Equal(t, int64(100), fb.GetFieldID())
|
||||
// ChildFields must be populated — index creation and backfill-
|
||||
// compaction detection look up fields via ChildFields, not FieldID.
|
||||
assert.Equal(t, []int64{100}, fb.GetChildFields())
|
||||
assert.Len(t, fb.GetBinlogs(), 1)
|
||||
assert.Equal(t, int64(1000), fb.GetBinlogs()[0].GetEntriesNum())
|
||||
assert.Equal(t, int64(7), fb.GetBinlogs()[0].GetLogID())
|
||||
// LogPath must be empty at persistence -- catalog.checkLogID
|
||||
// rejects non-empty LogPath. DecompressBinLog rebuilds it on load.
|
||||
assert.Equal(t, "", fb.GetBinlogs()[0].GetLogPath())
|
||||
})
|
||||
|
||||
t.Run("rejects binlog file with non-numeric trailing segment", func(t *testing.T) {
|
||||
seg := &BackfillSegment{
|
||||
ColumnGroups: []BackfillV2ColumnGroup{
|
||||
{FieldIDs: []int64{100}, BinlogFiles: []string{"s3a://bkt/seg/1/100/7.parquet"}, RowCount: 1},
|
||||
},
|
||||
}
|
||||
_, err := buildV2Groups("bkt", seg)
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "non-numeric trailing segment")
|
||||
})
|
||||
|
||||
t.Run("multi file splits row_count evenly with remainder on last", func(t *testing.T) {
|
||||
seg := &BackfillSegment{
|
||||
StorageVersion: &v2,
|
||||
ColumnGroups: []BackfillV2ColumnGroup{
|
||||
{
|
||||
FieldIDs: []int64{200},
|
||||
BinlogFiles: []string{"s3a://bkt/seg/1/200/1", "s3a://bkt/seg/1/200/2", "s3a://bkt/seg/1/200/3"},
|
||||
RowCount: 10,
|
||||
},
|
||||
},
|
||||
}
|
||||
groups, err := buildV2Groups("bkt", seg)
|
||||
assert.NoError(t, err)
|
||||
fb := groups[200]
|
||||
binlogs := fb.GetBinlogs()
|
||||
assert.Len(t, binlogs, 3)
|
||||
assert.Equal(t, int64(3), binlogs[0].GetEntriesNum())
|
||||
assert.Equal(t, int64(3), binlogs[1].GetEntriesNum())
|
||||
assert.Equal(t, int64(4), binlogs[2].GetEntriesNum())
|
||||
// sum preserved
|
||||
sum := int64(0)
|
||||
for _, b := range binlogs {
|
||||
sum += b.GetEntriesNum()
|
||||
}
|
||||
assert.Equal(t, int64(10), sum)
|
||||
})
|
||||
|
||||
t.Run("rejects multi field_ids", func(t *testing.T) {
|
||||
seg := &BackfillSegment{
|
||||
ColumnGroups: []BackfillV2ColumnGroup{
|
||||
{FieldIDs: []int64{100, 200}, BinlogFiles: []string{"x"}, RowCount: 1},
|
||||
},
|
||||
}
|
||||
_, err := buildV2Groups("", seg)
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "field_ids")
|
||||
})
|
||||
|
||||
t.Run("rejects empty binlog files", func(t *testing.T) {
|
||||
seg := &BackfillSegment{
|
||||
ColumnGroups: []BackfillV2ColumnGroup{
|
||||
{FieldIDs: []int64{100}, BinlogFiles: nil, RowCount: 1},
|
||||
},
|
||||
}
|
||||
_, err := buildV2Groups("", seg)
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "no binlog files")
|
||||
})
|
||||
|
||||
t.Run("rejects non-positive row_count", func(t *testing.T) {
|
||||
for _, rc := range []int64{0, -1} {
|
||||
seg := &BackfillSegment{
|
||||
ColumnGroups: []BackfillV2ColumnGroup{
|
||||
{FieldIDs: []int64{100}, BinlogFiles: []string{"x"}, RowCount: rc},
|
||||
},
|
||||
}
|
||||
_, err := buildV2Groups("", seg)
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "non-positive row_count")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("rejects duplicate field id groups", func(t *testing.T) {
|
||||
seg := &BackfillSegment{
|
||||
ColumnGroups: []BackfillV2ColumnGroup{
|
||||
{FieldIDs: []int64{100}, BinlogFiles: []string{"path/100/1"}, RowCount: 1},
|
||||
{FieldIDs: []int64{100}, BinlogFiles: []string{"path/100/2"}, RowCount: 1},
|
||||
},
|
||||
}
|
||||
_, err := buildV2Groups("", seg)
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "duplicate column group")
|
||||
})
|
||||
|
||||
t.Run("bucket mismatch bubbles up", func(t *testing.T) {
|
||||
seg := &BackfillSegment{
|
||||
ColumnGroups: []BackfillV2ColumnGroup{
|
||||
{FieldIDs: []int64{100}, BinlogFiles: []string{"s3a://other/foo/7"}, RowCount: 1},
|
||||
},
|
||||
}
|
||||
_, err := buildV2Groups("expected", seg)
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "differs from datacoord bucket")
|
||||
})
|
||||
}
|
||||
@@ -27,16 +27,41 @@ import (
|
||||
|
||||
func (c *DDLCallbacks) batchUpdateManifestV2AckCallback(ctx context.Context, result message.BroadcastResultBatchUpdateManifestMessageV2) error {
|
||||
body := result.Message.MustBody()
|
||||
var operators []UpdateOperator
|
||||
var (
|
||||
operators []UpdateOperator
|
||||
v2Count int
|
||||
v3Count int
|
||||
)
|
||||
for _, item := range body.GetItems() {
|
||||
operators = append(operators, UpdateManifestVersion(item.GetSegmentId(), item.GetManifestVersion()))
|
||||
segID := item.GetSegmentId()
|
||||
cg := item.GetV2ColumnGroups()
|
||||
hasV3 := item.GetManifestVersion() > 0
|
||||
hasV2 := cg != nil && len(cg.GetColumnGroups()) > 0
|
||||
switch {
|
||||
case hasV2 && hasV3:
|
||||
log.Ctx(ctx).Warn("batch update manifest item has both V2 and V3 payload; skipping",
|
||||
zap.Int64("segmentID", segID))
|
||||
continue
|
||||
case hasV2:
|
||||
operators = append(operators, UpdateSegmentColumnGroupsOperator(segID, cg.GetColumnGroups()))
|
||||
v2Count++
|
||||
case hasV3:
|
||||
operators = append(operators, UpdateManifestVersion(segID, item.GetManifestVersion()))
|
||||
v3Count++
|
||||
default:
|
||||
log.Ctx(ctx).Warn("batch update manifest item has no payload; skipping",
|
||||
zap.Int64("segmentID", segID))
|
||||
}
|
||||
}
|
||||
if len(operators) > 0 {
|
||||
if err := c.meta.UpdateSegmentsInfo(ctx, operators...); err != nil {
|
||||
log.Ctx(ctx).Warn("batch update manifest version failed", zap.Error(err))
|
||||
log.Ctx(ctx).Warn("batch update manifest failed", zap.Error(err))
|
||||
return err
|
||||
}
|
||||
}
|
||||
log.Ctx(ctx).Info("batch update manifest version handled", zap.Int("itemCount", len(body.GetItems())))
|
||||
log.Ctx(ctx).Info("batch update manifest handled",
|
||||
zap.Int("itemCount", len(body.GetItems())),
|
||||
zap.Int("v3Count", v3Count),
|
||||
zap.Int("v2Count", v2Count))
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1167,6 +1167,77 @@ func UpdateBinlogsFromSaveBinlogPathsOperator(segmentID int64, binlogs, statslog
|
||||
}
|
||||
}
|
||||
|
||||
// UpdateSegmentColumnGroupsOperator upserts storage-v2 column groups on a
|
||||
// segment's FieldBinlogs and removes the listed child fields from any other
|
||||
// pre-existing group whose child_fields contained them, so that every field
|
||||
// lives in exactly one column group. Idempotent: if a group with the same
|
||||
// top-level fieldID already exists, it is replaced in place.
|
||||
//
|
||||
// The caller must validate up front that:
|
||||
// - the segment exists,
|
||||
// - its storage_version is 2.
|
||||
func UpdateSegmentColumnGroupsOperator(segmentID int64, groups map[int64]*datapb.FieldBinlog) UpdateOperator {
|
||||
return func(modPack *updateSegmentPack) bool {
|
||||
segment := modPack.Get(segmentID)
|
||||
if segment == nil {
|
||||
log.Ctx(context.TODO()).Warn("meta update: update column groups failed - segment not found",
|
||||
zap.Int64("segmentID", segmentID))
|
||||
return false
|
||||
}
|
||||
|
||||
incomingChildFields := typeutil.NewSet[int64]()
|
||||
for _, g := range groups {
|
||||
incomingChildFields.Insert(g.GetChildFields()...)
|
||||
}
|
||||
|
||||
// Strip incoming child fields from any other existing group, then drop
|
||||
// in-place groups that the request is replacing. Also drop groups that
|
||||
// become empty (all ChildFields claimed by incoming groups) and record
|
||||
// their FieldIDs so the catalog removes the orphan etcd KV -- otherwise
|
||||
// listBinlogs' prefix scan will resurrect the zombie on restart.
|
||||
var droppedFieldIDs []int64
|
||||
kept := segment.Binlogs[:0]
|
||||
for _, existing := range segment.Binlogs {
|
||||
if _, replaced := groups[existing.GetFieldID()]; replaced {
|
||||
continue
|
||||
}
|
||||
if len(existing.GetChildFields()) > 0 {
|
||||
existing.ChildFields = lo.Filter(existing.GetChildFields(), func(fid int64, _ int) bool {
|
||||
return !incomingChildFields.Contain(fid)
|
||||
})
|
||||
if len(existing.ChildFields) == 0 {
|
||||
droppedFieldIDs = append(droppedFieldIDs, existing.GetFieldID())
|
||||
continue
|
||||
}
|
||||
}
|
||||
kept = append(kept, existing)
|
||||
}
|
||||
segment.Binlogs = kept
|
||||
|
||||
for _, g := range groups {
|
||||
segment.Binlogs = append(segment.Binlogs, g)
|
||||
}
|
||||
|
||||
// Bump DataVersion so querynodes with the segment already loaded will Reopen;
|
||||
// ManifestPath is intentionally not moved here (see segment_checker.isSegmentUpdate).
|
||||
segment.DataVersion++
|
||||
|
||||
// Backfill column-group commit only mutates segment.Binlogs; skipping
|
||||
// Deltalogs / Statslogs / Bm25Statslogs avoids rewriting their KVs on
|
||||
// every call and the write amplification that comes with it.
|
||||
modPack.increments[segmentID] = metastore.BinlogsIncrement{
|
||||
Segment: segment.SegmentInfo,
|
||||
UpdateMask: metastore.BinlogsUpdateMask{
|
||||
WithoutDeltalogs: true,
|
||||
WithoutStatslogs: true,
|
||||
WithoutBm25Statslogs: true,
|
||||
},
|
||||
DroppedBinlogFieldIDs: droppedFieldIDs,
|
||||
}
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// update startPosition
|
||||
func UpdateStartPosition(startPositions []*datapb.SegmentStartPosition) UpdateOperator {
|
||||
return func(modPack *updateSegmentPack) bool {
|
||||
@@ -1295,7 +1366,18 @@ func UpdateManifestVersion(segmentID int64, manifestVersion int64) UpdateOperato
|
||||
zap.Int64("segmentID", segmentID), zap.Error(err))
|
||||
return false
|
||||
}
|
||||
if currentVer == manifestVersion {
|
||||
// Guard against version rollback. classifyBackfillSegments pre-checks
|
||||
// monotonicity at broadcast time, but a concurrent compaction may
|
||||
// advance ManifestPath between pre-check and this apply (compaction
|
||||
// commits use a different serialization path than this broadcaster).
|
||||
// Only accept strictly forward motion; equality is a no-op.
|
||||
if currentVer >= manifestVersion {
|
||||
if currentVer > manifestVersion {
|
||||
log.Ctx(context.TODO()).Warn("meta update: update manifest version rejected - would regress",
|
||||
zap.Int64("segmentID", segmentID),
|
||||
zap.Int64("currentVer", currentVer),
|
||||
zap.Int64("incomingVer", manifestVersion))
|
||||
}
|
||||
return false
|
||||
}
|
||||
segment.ManifestPath = packed.MarshalManifestPath(basePath, manifestVersion)
|
||||
|
||||
@@ -39,6 +39,7 @@ import (
|
||||
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
|
||||
"github.com/milvus-io/milvus/internal/datacoord/broker"
|
||||
mockkv "github.com/milvus-io/milvus/internal/kv/mocks"
|
||||
"github.com/milvus-io/milvus/internal/metastore"
|
||||
"github.com/milvus-io/milvus/internal/metastore/kv/datacoord"
|
||||
mocks2 "github.com/milvus-io/milvus/internal/metastore/mocks"
|
||||
"github.com/milvus-io/milvus/internal/metastore/model"
|
||||
@@ -2579,6 +2580,248 @@ func TestUpdateManifestVersion(t *testing.T) {
|
||||
assert.Equal(t, "/data/segments/1", basePath)
|
||||
assert.Equal(t, int64(5), version)
|
||||
})
|
||||
|
||||
t.Run("rollback rejected - currentVer > incomingVer is a no-op", func(t *testing.T) {
|
||||
meta, err := newMemoryMeta(t)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Current version = 10. A stale broadcast carrying version = 5 must
|
||||
// not regress the pointer. classifyBackfillSegments pre-checks
|
||||
// monotonicity at broadcast time, but concurrent compaction may have
|
||||
// advanced ManifestPath between pre-check and this apply -- the
|
||||
// operator-level guard is the last line of defense.
|
||||
manifestPath := packed.MarshalManifestPath("/data/segments/1", 10)
|
||||
meta.AddSegment(context.Background(), &SegmentInfo{
|
||||
SegmentInfo: &datapb.SegmentInfo{
|
||||
ID: 1,
|
||||
State: commonpb.SegmentState_Flushed,
|
||||
ManifestPath: manifestPath,
|
||||
},
|
||||
})
|
||||
|
||||
operator := UpdateManifestVersion(1, 5)
|
||||
pack := &updateSegmentPack{
|
||||
meta: meta,
|
||||
segments: make(map[int64]*SegmentInfo),
|
||||
}
|
||||
assert.False(t, operator(pack))
|
||||
|
||||
// Confirm the stored manifest path was not mutated in the pack.
|
||||
got := pack.Get(1)
|
||||
_, currentVer, err := packed.UnmarshalManifestPath(got.ManifestPath)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, int64(10), currentVer)
|
||||
})
|
||||
}
|
||||
|
||||
func TestUpdateSegmentColumnGroupsOperator(t *testing.T) {
|
||||
// Helper: build a segment with two pre-existing column groups, where the
|
||||
// first group owns top-level fieldID=100 and child_fields=[200,201], and
|
||||
// the second group owns top-level fieldID=300.
|
||||
newSegmentWithExistingGroups := func() *SegmentInfo {
|
||||
return &SegmentInfo{
|
||||
SegmentInfo: &datapb.SegmentInfo{
|
||||
ID: 1,
|
||||
CollectionID: 1000,
|
||||
State: commonpb.SegmentState_Flushed,
|
||||
StorageVersion: storage.StorageV2,
|
||||
DataVersion: int32(5),
|
||||
Binlogs: []*datapb.FieldBinlog{
|
||||
{
|
||||
FieldID: 100,
|
||||
ChildFields: []int64{200, 201},
|
||||
Binlogs: []*datapb.Binlog{{LogID: 1}},
|
||||
},
|
||||
{
|
||||
FieldID: 300,
|
||||
Binlogs: []*datapb.Binlog{{LogID: 2}},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
t.Run("segment not found returns false", func(t *testing.T) {
|
||||
m, err := newMemoryMeta(t)
|
||||
assert.NoError(t, err)
|
||||
op := UpdateSegmentColumnGroupsOperator(999, map[int64]*datapb.FieldBinlog{
|
||||
400: {FieldID: 400},
|
||||
})
|
||||
pack := &updateSegmentPack{
|
||||
meta: m,
|
||||
segments: make(map[int64]*SegmentInfo),
|
||||
increments: make(map[int64]metastore.BinlogsIncrement),
|
||||
}
|
||||
assert.False(t, op(pack))
|
||||
})
|
||||
|
||||
t.Run("append new group bumps DataVersion", func(t *testing.T) {
|
||||
m, err := newMemoryMeta(t)
|
||||
assert.NoError(t, err)
|
||||
seg := newSegmentWithExistingGroups()
|
||||
m.AddSegment(context.Background(), seg)
|
||||
|
||||
op := UpdateSegmentColumnGroupsOperator(1, map[int64]*datapb.FieldBinlog{
|
||||
400: {FieldID: 400, Binlogs: []*datapb.Binlog{{LogID: 10, EntriesNum: 100}}},
|
||||
})
|
||||
pack := &updateSegmentPack{
|
||||
meta: m,
|
||||
segments: make(map[int64]*SegmentInfo),
|
||||
increments: make(map[int64]metastore.BinlogsIncrement),
|
||||
}
|
||||
assert.True(t, op(pack))
|
||||
|
||||
got := pack.Get(1)
|
||||
assert.NotNil(t, got)
|
||||
assert.Equal(t, int32(6), got.DataVersion)
|
||||
// Two pre-existing + one new.
|
||||
assert.Len(t, got.Binlogs, 3)
|
||||
var fids []int64
|
||||
for _, fb := range got.Binlogs {
|
||||
fids = append(fids, fb.GetFieldID())
|
||||
}
|
||||
assert.ElementsMatch(t, []int64{100, 300, 400}, fids)
|
||||
// child_fields on 100 untouched because no child collision.
|
||||
for _, fb := range got.Binlogs {
|
||||
if fb.GetFieldID() == 100 {
|
||||
assert.ElementsMatch(t, []int64{200, 201}, fb.GetChildFields())
|
||||
}
|
||||
}
|
||||
_, ok := pack.increments[1]
|
||||
assert.True(t, ok)
|
||||
})
|
||||
|
||||
t.Run("strips child fields from existing group", func(t *testing.T) {
|
||||
m, err := newMemoryMeta(t)
|
||||
assert.NoError(t, err)
|
||||
seg := newSegmentWithExistingGroups()
|
||||
m.AddSegment(context.Background(), seg)
|
||||
|
||||
// new group 500 owns child 200 which was held by group 100.
|
||||
op := UpdateSegmentColumnGroupsOperator(1, map[int64]*datapb.FieldBinlog{
|
||||
500: {FieldID: 500, ChildFields: []int64{200}},
|
||||
})
|
||||
pack := &updateSegmentPack{
|
||||
meta: m,
|
||||
segments: make(map[int64]*SegmentInfo),
|
||||
increments: make(map[int64]metastore.BinlogsIncrement),
|
||||
}
|
||||
assert.True(t, op(pack))
|
||||
|
||||
got := pack.Get(1)
|
||||
for _, fb := range got.Binlogs {
|
||||
if fb.GetFieldID() == 100 {
|
||||
assert.ElementsMatch(t, []int64{201}, fb.GetChildFields(),
|
||||
"child 200 should have been stripped from old group")
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("replace same fieldID in place", func(t *testing.T) {
|
||||
m, err := newMemoryMeta(t)
|
||||
assert.NoError(t, err)
|
||||
seg := newSegmentWithExistingGroups()
|
||||
m.AddSegment(context.Background(), seg)
|
||||
|
||||
op := UpdateSegmentColumnGroupsOperator(1, map[int64]*datapb.FieldBinlog{
|
||||
100: {FieldID: 100, Binlogs: []*datapb.Binlog{{LogID: 999, EntriesNum: 7}}},
|
||||
})
|
||||
pack := &updateSegmentPack{
|
||||
meta: m,
|
||||
segments: make(map[int64]*SegmentInfo),
|
||||
increments: make(map[int64]metastore.BinlogsIncrement),
|
||||
}
|
||||
assert.True(t, op(pack))
|
||||
|
||||
got := pack.Get(1)
|
||||
// 100 replaced, 300 preserved => still 2 groups.
|
||||
assert.Len(t, got.Binlogs, 2)
|
||||
for _, fb := range got.Binlogs {
|
||||
if fb.GetFieldID() == 100 {
|
||||
assert.Len(t, fb.GetBinlogs(), 1)
|
||||
assert.Equal(t, int64(999), fb.GetBinlogs()[0].GetLogID())
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("drops empty-children existing group and records DroppedBinlogFieldIDs", func(t *testing.T) {
|
||||
// Pre-existing single-child group (fieldID=100 owns child 200) whose
|
||||
// only child is claimed by a new backfill group (fieldID=200). After
|
||||
// stripping, group 100's ChildFields is empty -- the operator must
|
||||
// drop it from segment.Binlogs AND record 100 in DroppedBinlogFieldIDs
|
||||
// so the catalog removes the orphan etcd KV (without it, listBinlogs'
|
||||
// prefix scan would resurrect the zombie on restart).
|
||||
m, err := newMemoryMeta(t)
|
||||
assert.NoError(t, err)
|
||||
m.AddSegment(context.Background(), &SegmentInfo{
|
||||
SegmentInfo: &datapb.SegmentInfo{
|
||||
ID: 1,
|
||||
CollectionID: 1000,
|
||||
State: commonpb.SegmentState_Flushed,
|
||||
StorageVersion: storage.StorageV2,
|
||||
Binlogs: []*datapb.FieldBinlog{
|
||||
{
|
||||
FieldID: 100,
|
||||
ChildFields: []int64{200}, // single child
|
||||
Binlogs: []*datapb.Binlog{{LogID: 1}},
|
||||
},
|
||||
{
|
||||
FieldID: 300,
|
||||
ChildFields: []int64{301, 302},
|
||||
Binlogs: []*datapb.Binlog{{LogID: 2}},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
op := UpdateSegmentColumnGroupsOperator(1, map[int64]*datapb.FieldBinlog{
|
||||
200: {FieldID: 200, ChildFields: []int64{200}, Binlogs: []*datapb.Binlog{{LogID: 99}}},
|
||||
})
|
||||
pack := &updateSegmentPack{
|
||||
meta: m,
|
||||
segments: make(map[int64]*SegmentInfo),
|
||||
increments: make(map[int64]metastore.BinlogsIncrement),
|
||||
}
|
||||
assert.True(t, op(pack))
|
||||
|
||||
got := pack.Get(1)
|
||||
// Group 100 must be gone from in-memory binlogs; 300 (unaffected) plus
|
||||
// new 200 remain.
|
||||
assert.Len(t, got.Binlogs, 2)
|
||||
fids := lo.Map(got.Binlogs, func(fb *datapb.FieldBinlog, _ int) int64 { return fb.GetFieldID() })
|
||||
assert.ElementsMatch(t, []int64{200, 300}, fids)
|
||||
|
||||
// Increment carries the orphan FieldID so AlterSegments can remove
|
||||
// the persisted KV.
|
||||
inc, ok := pack.increments[1]
|
||||
assert.True(t, ok)
|
||||
assert.ElementsMatch(t, []int64{100}, inc.DroppedBinlogFieldIDs)
|
||||
})
|
||||
|
||||
t.Run("DataVersion monotonic across reruns", func(t *testing.T) {
|
||||
m, err := newMemoryMeta(t)
|
||||
assert.NoError(t, err)
|
||||
seg := newSegmentWithExistingGroups()
|
||||
m.AddSegment(context.Background(), seg)
|
||||
|
||||
err = m.UpdateSegmentsInfo(context.Background(),
|
||||
UpdateSegmentColumnGroupsOperator(1, map[int64]*datapb.FieldBinlog{
|
||||
400: {FieldID: 400, Binlogs: []*datapb.Binlog{{LogID: 10}}},
|
||||
}),
|
||||
)
|
||||
assert.NoError(t, err)
|
||||
err = m.UpdateSegmentsInfo(context.Background(),
|
||||
UpdateSegmentColumnGroupsOperator(1, map[int64]*datapb.FieldBinlog{
|
||||
400: {FieldID: 400, Binlogs: []*datapb.Binlog{{LogID: 11}}},
|
||||
}),
|
||||
)
|
||||
assert.NoError(t, err)
|
||||
|
||||
got := m.GetHealthySegment(context.Background(), 1)
|
||||
assert.NotNil(t, got)
|
||||
// Started at 5, two bumps => 7.
|
||||
assert.Equal(t, int32(7), got.DataVersion)
|
||||
})
|
||||
}
|
||||
|
||||
func Test_meta_SetSegmentsCompacting(t *testing.T) {
|
||||
|
||||
@@ -169,6 +169,10 @@ func (m *mockMixCoord) BatchUpdateManifest(context.Context, *datapb.BatchUpdateM
|
||||
return merr.Success(), nil
|
||||
}
|
||||
|
||||
func (m *mockMixCoord) CommitBackfillResult(context.Context, *datapb.CommitBackfillResultRequest) (*datapb.CommitBackfillResultResponse, error) {
|
||||
return &datapb.CommitBackfillResultResponse{Status: merr.Success()}, nil
|
||||
}
|
||||
|
||||
func (m *mockMixCoord) DescribeDatabase(ctx context.Context, in *rootcoordpb.DescribeDatabaseRequest) (*rootcoordpb.DescribeDatabaseResponse, error) {
|
||||
return &rootcoordpb.DescribeDatabaseResponse{
|
||||
Status: merr.Success(),
|
||||
|
||||
@@ -0,0 +1,384 @@
|
||||
// Licensed to the LF AI & Data foundation under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package datacoord
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
"strconv"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
|
||||
"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
|
||||
"github.com/milvus-io/milvus/internal/distributed/streaming"
|
||||
"github.com/milvus-io/milvus/internal/json"
|
||||
"github.com/milvus-io/milvus/internal/storage"
|
||||
"github.com/milvus-io/milvus/internal/storagev2/packed"
|
||||
"github.com/milvus-io/milvus/internal/streamingcoord/server/broadcaster/broadcast"
|
||||
"github.com/milvus-io/milvus/pkg/v2/log"
|
||||
"github.com/milvus-io/milvus/pkg/v2/proto/datapb"
|
||||
"github.com/milvus-io/milvus/pkg/v2/proto/messagespb"
|
||||
"github.com/milvus-io/milvus/pkg/v2/streaming/util/message"
|
||||
"github.com/milvus-io/milvus/pkg/v2/util/merr"
|
||||
)
|
||||
|
||||
// CommitBackfillResult fetches the Spark-produced BackfillResult JSON from
|
||||
// object storage, classifies each segment entry (V2 vs V3), and dispatches the
|
||||
// updates through the BatchUpdateManifest broadcast pipeline. V2 segments carry
|
||||
// a column-group upsert; V3 segments carry a manifest version bump. The
|
||||
// broadcaster ensures serialization against compaction and other DDL-like
|
||||
// operations on the same collection.
|
||||
func (s *Server) CommitBackfillResult(ctx context.Context, req *datapb.CommitBackfillResultRequest) (*datapb.CommitBackfillResultResponse, error) {
|
||||
log := log.Ctx(ctx).With(zap.String("resultPath", req.GetResultPath()))
|
||||
if err := merr.CheckHealthy(s.GetStateCode()); err != nil {
|
||||
return &datapb.CommitBackfillResultResponse{Status: merr.Status(err)}, nil
|
||||
}
|
||||
|
||||
result, err := s.loadBackfillResult(ctx, req.GetResultPath())
|
||||
if err != nil {
|
||||
log.Warn("CommitBackfillResult failed to load result JSON", zap.Error(err))
|
||||
return &datapb.CommitBackfillResultResponse{Status: merr.Status(err)}, nil
|
||||
}
|
||||
|
||||
items, statuses := s.classifyBackfillSegments(ctx, result)
|
||||
total := int32(len(result.Segments))
|
||||
|
||||
// Nothing passed pre-validation: surface as top-level failure so the caller
|
||||
// knows no broadcast happened. segment_statuses still carry per-segment
|
||||
// diagnostics.
|
||||
if len(items) == 0 {
|
||||
return &datapb.CommitBackfillResultResponse{
|
||||
Status: merr.Status(merr.WrapErrParameterInvalidMsg("no backfill segments passed pre-validation")),
|
||||
TotalSegments: total,
|
||||
SegmentStatuses: statuses,
|
||||
FailedSegments: int32(len(statuses)),
|
||||
}, nil
|
||||
}
|
||||
|
||||
coll, err := s.broker.DescribeCollectionInternal(ctx, result.CollectionID)
|
||||
if err != nil {
|
||||
log.Warn("CommitBackfillResult failed to describe collection", zap.Error(err), zap.Int64("collectionID", result.CollectionID))
|
||||
return &datapb.CommitBackfillResultResponse{Status: merr.Status(err)}, nil
|
||||
}
|
||||
|
||||
// Split items across multiple broadcast messages so a single
|
||||
// BatchUpdateManifestMessageBody never exceeds the broker's message size
|
||||
// limit (Pulsar defaults to 5MiB). Each batch acquires its own broadcaster
|
||||
// because broadcasterWithRK consumes its resource-key guards on the first
|
||||
// Broadcast call and would panic on any subsequent call. Failure of one
|
||||
// batch does not cancel subsequent batches; per-segment statuses reflect
|
||||
// batch-level outcomes.
|
||||
channels := []string{streaming.WAL().ControlChannel()}
|
||||
var lastErr error
|
||||
for start := 0; start < len(items); start += maxItemsPerBroadcast {
|
||||
end := start + maxItemsPerBroadcast
|
||||
if end > len(items) {
|
||||
end = len(items)
|
||||
}
|
||||
batch := items[start:end]
|
||||
if err := broadcastBackfillBatch(ctx, coll, result.CollectionID, channels, batch); err != nil {
|
||||
log.Error("CommitBackfillResult broadcast batch failed",
|
||||
zap.Error(err), zap.Int("batchStart", start), zap.Int("batchEnd", end))
|
||||
lastErr = err
|
||||
appendItemStatuses(&statuses, batch, false, err.Error())
|
||||
continue
|
||||
}
|
||||
appendItemStatuses(&statuses, batch, true, "")
|
||||
}
|
||||
|
||||
committed, failed := countStatuses(statuses)
|
||||
log.Info("CommitBackfillResult broadcast completed",
|
||||
zap.Int32("total", total),
|
||||
zap.Int32("committed", committed),
|
||||
zap.Int32("failed", failed))
|
||||
|
||||
// Top-level Success unless every broadcast failed -- partial failures are
|
||||
// surfaced through per-segment statuses.
|
||||
respStatus := merr.Success()
|
||||
if committed == 0 && lastErr != nil {
|
||||
respStatus = merr.Status(lastErr)
|
||||
}
|
||||
|
||||
return &datapb.CommitBackfillResultResponse{
|
||||
Status: respStatus,
|
||||
TotalSegments: total,
|
||||
CommittedSegments: committed,
|
||||
FailedSegments: failed,
|
||||
SegmentStatuses: sortStatuses(statuses),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// maxItemsPerBroadcast caps the number of BatchUpdateManifestItem entries
|
||||
// packed into a single broadcast message. With item payloads in the
|
||||
// ~1-2KiB range for V2 column groups this stays well under Pulsar's default
|
||||
// 5MiB maxMessageSize while keeping broadcast overhead low.
|
||||
const maxItemsPerBroadcast = 512
|
||||
|
||||
// broadcastBackfillBatch acquires a fresh broadcaster bound to the
|
||||
// collection's shared resource keys and issues exactly one broadcast for the
|
||||
// given items. broadcasterWithRK nils out its lock guards on the first
|
||||
// Broadcast call, so each batch needs its own broadcaster.
|
||||
func broadcastBackfillBatch(
|
||||
ctx context.Context,
|
||||
coll *milvuspb.DescribeCollectionResponse,
|
||||
collectionID int64,
|
||||
channels []string,
|
||||
items []*messagespb.BatchUpdateManifestItem,
|
||||
) error {
|
||||
broadcaster, err := broadcast.StartBroadcastWithResourceKeys(ctx,
|
||||
message.NewSharedDBNameResourceKey(coll.GetDbName()),
|
||||
message.NewSharedCollectionNameResourceKey(coll.GetDbName(), coll.GetCollectionName()),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer broadcaster.Close()
|
||||
|
||||
_, err = broadcaster.Broadcast(ctx, message.NewBatchUpdateManifestMessageBuilderV2().
|
||||
WithHeader(&message.BatchUpdateManifestMessageHeader{
|
||||
CollectionId: collectionID,
|
||||
}).
|
||||
WithBody(&message.BatchUpdateManifestMessageBody{
|
||||
Items: items,
|
||||
}).
|
||||
WithBroadcast(channels).
|
||||
MustBuildBroadcast(),
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
func appendItemStatuses(out *[]*datapb.CommitBackfillResultSegmentStatus, batch []*messagespb.BatchUpdateManifestItem, ok bool, reason string) {
|
||||
for _, it := range batch {
|
||||
kind := "v3"
|
||||
if it.GetV2ColumnGroups() != nil {
|
||||
kind = "v2"
|
||||
}
|
||||
*out = append(*out, &datapb.CommitBackfillResultSegmentStatus{
|
||||
SegmentId: it.GetSegmentId(), Ok: ok, Kind: kind, Reason: reason,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// maxBackfillResultBytes caps the size of the result JSON read from object
|
||||
// storage. The JSON is produced by an external system (Spark) and loaded into
|
||||
// memory in one shot; a hard cap protects DataCoord from OOM on an oversized
|
||||
// or malicious input. Real-world backfill results for collections in the
|
||||
// hundreds of thousands of segments comfortably fit within this limit.
|
||||
const maxBackfillResultBytes int64 = 64 * 1024 * 1024 // 64MiB
|
||||
|
||||
// loadBackfillResult reads and decodes the result JSON. The bucket is inferred
|
||||
// from the configured chunk manager (if it exposes BucketName()) and used to
|
||||
// reject s3a://<other-bucket>/... paths early.
|
||||
func (s *Server) loadBackfillResult(ctx context.Context, rawPath string) (*BackfillResult, error) {
|
||||
if rawPath == "" {
|
||||
return nil, merr.WrapErrParameterInvalidMsg("result_path is required")
|
||||
}
|
||||
bucket := bucketFromChunkManager(s.meta.chunkManager)
|
||||
key, err := normalizeObjectKey(rawPath, bucket)
|
||||
if err != nil {
|
||||
return nil, merr.WrapErrParameterInvalidMsg(err.Error())
|
||||
}
|
||||
// Pre-check the object size so an untrusted external caller cannot force
|
||||
// an unbounded in-memory Read.
|
||||
size, err := s.meta.chunkManager.Size(ctx, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if size > maxBackfillResultBytes {
|
||||
return nil, merr.WrapErrParameterInvalidMsg(
|
||||
"backfill result JSON " + strconv.FormatInt(size, 10) +
|
||||
" bytes exceeds limit " + strconv.FormatInt(maxBackfillResultBytes, 10))
|
||||
}
|
||||
raw, err := s.meta.chunkManager.Read(ctx, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var result BackfillResult
|
||||
if err := json.Unmarshal(raw, &result); err != nil {
|
||||
return nil, merr.WrapErrParameterInvalidMsg("failed to decode backfill result JSON: " + err.Error())
|
||||
}
|
||||
if !result.Success {
|
||||
return nil, merr.WrapErrParameterInvalidMsg("backfill reported success=false; refusing to commit")
|
||||
}
|
||||
if result.CollectionID == 0 {
|
||||
return nil, merr.WrapErrParameterInvalidMsg("backfill result missing collectionId")
|
||||
}
|
||||
if len(result.Segments) == 0 {
|
||||
return nil, merr.WrapErrParameterInvalidMsg("backfill result has no segments")
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// classifyBackfillSegments validates each segment entry and constructs the
|
||||
// broadcast items. Returns the items to broadcast plus per-segment failure
|
||||
// statuses recorded during pre-validation (so callers can surface them to the
|
||||
// client even when a segment never reached broadcast).
|
||||
func (s *Server) classifyBackfillSegments(ctx context.Context, result *BackfillResult) ([]*messagespb.BatchUpdateManifestItem, []*datapb.CommitBackfillResultSegmentStatus) {
|
||||
bucket := bucketFromChunkManager(s.meta.chunkManager)
|
||||
items := make([]*messagespb.BatchUpdateManifestItem, 0, len(result.Segments))
|
||||
statuses := make([]*datapb.CommitBackfillResultSegmentStatus, 0)
|
||||
|
||||
// Deterministic order so broadcast items & diagnostic output are stable.
|
||||
segIDs := make([]string, 0, len(result.Segments))
|
||||
for k := range result.Segments {
|
||||
segIDs = append(segIDs, k)
|
||||
}
|
||||
sort.Strings(segIDs)
|
||||
|
||||
for _, segIDStr := range segIDs {
|
||||
entry := result.Segments[segIDStr]
|
||||
segID, perr := strconv.ParseInt(segIDStr, 10, 64)
|
||||
if perr != nil {
|
||||
statuses = append(statuses, &datapb.CommitBackfillResultSegmentStatus{
|
||||
SegmentId: 0, Ok: false, Kind: "", Reason: "invalid segment id " + segIDStr,
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
segInfo := s.meta.GetSegment(ctx, segID)
|
||||
if segInfo == nil {
|
||||
statuses = append(statuses, &datapb.CommitBackfillResultSegmentStatus{
|
||||
SegmentId: segID, Ok: false, Kind: inferKind(&entry), Reason: "segment not found in meta",
|
||||
})
|
||||
continue
|
||||
}
|
||||
if segInfo.GetCollectionID() != result.CollectionID {
|
||||
statuses = append(statuses, &datapb.CommitBackfillResultSegmentStatus{
|
||||
SegmentId: segID, Ok: false, Kind: inferKind(&entry),
|
||||
Reason: "segment does not belong to the result's collection",
|
||||
})
|
||||
continue
|
||||
}
|
||||
// Partition-scoped backfills set PartitionID to the target partition;
|
||||
// collection-wide backfills leave it at 0 (no check). When non-zero,
|
||||
// rejecting a mismatching segment prevents writing metadata against
|
||||
// the wrong partition.
|
||||
if result.PartitionID != 0 && segInfo.GetPartitionID() != result.PartitionID {
|
||||
statuses = append(statuses, &datapb.CommitBackfillResultSegmentStatus{
|
||||
SegmentId: segID, Ok: false, Kind: inferKind(&entry),
|
||||
Reason: "segment does not belong to the result's partition",
|
||||
})
|
||||
continue
|
||||
}
|
||||
if segInfo.GetState() != commonpb.SegmentState_Flushed {
|
||||
statuses = append(statuses, &datapb.CommitBackfillResultSegmentStatus{
|
||||
SegmentId: segID, Ok: false, Kind: inferKind(&entry),
|
||||
Reason: "segment state is not Flushed: " + segInfo.GetState().String(),
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
if entry.IsV2() {
|
||||
if segInfo.GetStorageVersion() != storage.StorageV2 {
|
||||
statuses = append(statuses, &datapb.CommitBackfillResultSegmentStatus{
|
||||
SegmentId: segID, Ok: false, Kind: "v2",
|
||||
Reason: "segment storage version is not V2",
|
||||
})
|
||||
continue
|
||||
}
|
||||
groups, err := buildV2Groups(bucket, &entry)
|
||||
if err != nil {
|
||||
statuses = append(statuses, &datapb.CommitBackfillResultSegmentStatus{
|
||||
SegmentId: segID, Ok: false, Kind: "v2", Reason: err.Error(),
|
||||
})
|
||||
continue
|
||||
}
|
||||
items = append(items, &messagespb.BatchUpdateManifestItem{
|
||||
SegmentId: segID,
|
||||
V2ColumnGroups: &messagespb.BatchUpdateManifestV2ColumnGroups{
|
||||
ColumnGroups: groups,
|
||||
},
|
||||
})
|
||||
} else {
|
||||
// V3 path
|
||||
if segInfo.GetStorageVersion() != storage.StorageV3 {
|
||||
statuses = append(statuses, &datapb.CommitBackfillResultSegmentStatus{
|
||||
SegmentId: segID, Ok: false, Kind: "v3",
|
||||
Reason: "segment storage version is not V3",
|
||||
})
|
||||
continue
|
||||
}
|
||||
// UpdateManifestVersion no-ops when ManifestPath is empty. Reject
|
||||
// here so the caller never sees a fake committed=true.
|
||||
if segInfo.GetManifestPath() == "" {
|
||||
statuses = append(statuses, &datapb.CommitBackfillResultSegmentStatus{
|
||||
SegmentId: segID, Ok: false, Kind: "v3",
|
||||
Reason: "segment has no existing manifest path",
|
||||
})
|
||||
continue
|
||||
}
|
||||
if entry.Version <= 0 {
|
||||
statuses = append(statuses, &datapb.CommitBackfillResultSegmentStatus{
|
||||
SegmentId: segID, Ok: false, Kind: "v3",
|
||||
Reason: "missing or invalid manifest version",
|
||||
})
|
||||
continue
|
||||
}
|
||||
// Reject stale results (e.g. Spark retry) that would move the
|
||||
// manifest pointer backwards. UpdateManifestVersion short-circuits
|
||||
// only on equality, so enforcing strict monotonicity here is the
|
||||
// correct guard against silent rollback.
|
||||
_, currentVer, verErr := packed.UnmarshalManifestPath(segInfo.GetManifestPath())
|
||||
if verErr != nil {
|
||||
statuses = append(statuses, &datapb.CommitBackfillResultSegmentStatus{
|
||||
SegmentId: segID, Ok: false, Kind: "v3",
|
||||
Reason: "failed to parse current manifest path: " + verErr.Error(),
|
||||
})
|
||||
continue
|
||||
}
|
||||
if entry.Version <= currentVer {
|
||||
statuses = append(statuses, &datapb.CommitBackfillResultSegmentStatus{
|
||||
SegmentId: segID, Ok: false, Kind: "v3",
|
||||
Reason: "incoming manifest version " + strconv.FormatInt(entry.Version, 10) +
|
||||
" is not greater than current " + strconv.FormatInt(currentVer, 10),
|
||||
})
|
||||
continue
|
||||
}
|
||||
items = append(items, &messagespb.BatchUpdateManifestItem{
|
||||
SegmentId: segID,
|
||||
ManifestVersion: entry.Version,
|
||||
})
|
||||
}
|
||||
}
|
||||
return items, statuses
|
||||
}
|
||||
|
||||
func inferKind(entry *BackfillSegment) string {
|
||||
if entry.IsV2() {
|
||||
return "v2"
|
||||
}
|
||||
return "v3"
|
||||
}
|
||||
|
||||
func countStatuses(statuses []*datapb.CommitBackfillResultSegmentStatus) (committed, failed int32) {
|
||||
for _, st := range statuses {
|
||||
if st.GetOk() {
|
||||
committed++
|
||||
} else {
|
||||
failed++
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func sortStatuses(statuses []*datapb.CommitBackfillResultSegmentStatus) []*datapb.CommitBackfillResultSegmentStatus {
|
||||
sort.SliceStable(statuses, func(i, j int) bool {
|
||||
return statuses[i].GetSegmentId() < statuses[j].GetSegmentId()
|
||||
})
|
||||
return statuses
|
||||
}
|
||||
@@ -4,6 +4,8 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -29,9 +31,11 @@ import (
|
||||
"github.com/milvus-io/milvus/internal/metastore/model"
|
||||
mocks2 "github.com/milvus-io/milvus/internal/mocks"
|
||||
"github.com/milvus-io/milvus/internal/mocks/distributed/mock_streaming"
|
||||
"github.com/milvus-io/milvus/internal/mocks/mock_storage"
|
||||
"github.com/milvus-io/milvus/internal/mocks/streamingcoord/server/mock_balancer"
|
||||
"github.com/milvus-io/milvus/internal/mocks/streamingcoord/server/mock_broadcaster"
|
||||
"github.com/milvus-io/milvus/internal/storage"
|
||||
"github.com/milvus-io/milvus/internal/storagev2/packed"
|
||||
"github.com/milvus-io/milvus/internal/streamingcoord/server/balancer"
|
||||
"github.com/milvus-io/milvus/internal/streamingcoord/server/balancer/balance"
|
||||
"github.com/milvus-io/milvus/internal/streamingcoord/server/balancer/channel"
|
||||
@@ -3482,6 +3486,571 @@ func TestServer_PinSnapshotData_AcquiresResourceKeyLock(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
// --- Test CommitBackfillResult ---
|
||||
|
||||
func TestServer_CommitBackfillResult(t *testing.T) {
|
||||
// Small helper to build a minimal Server with a healthy state and a
|
||||
// chunk manager that always returns the supplied JSON bytes.
|
||||
newServerForCommit := func(t *testing.T, m *meta, b broker.Broker, jsonBytes []byte) *Server {
|
||||
cm := mock_storage.NewMockChunkManager(t)
|
||||
cm.EXPECT().Size(mock.Anything, mock.Anything).Return(int64(len(jsonBytes)), nil).Maybe()
|
||||
cm.EXPECT().Read(mock.Anything, mock.Anything).Return(jsonBytes, nil).Maybe()
|
||||
m.chunkManager = cm
|
||||
s := &Server{
|
||||
meta: m,
|
||||
broker: b,
|
||||
}
|
||||
s.stateCode.Store(commonpb.StateCode_Healthy)
|
||||
return s
|
||||
}
|
||||
|
||||
// V3 happy path: 2 V3 segments both belong to collection 100, broadcast is
|
||||
// captured and inspected.
|
||||
t.Run("v3_happy_path", func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
m, err := newMemoryMeta(t)
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, id := range []int64{1001, 1002} {
|
||||
m.AddSegment(ctx, &SegmentInfo{
|
||||
SegmentInfo: &datapb.SegmentInfo{
|
||||
ID: id,
|
||||
CollectionID: 100,
|
||||
State: commonpb.SegmentState_Flushed,
|
||||
StorageVersion: storage.StorageV3,
|
||||
ManifestPath: packed.MarshalManifestPath("/seg/"+strconv.FormatInt(id, 10), 1),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
jsonStr := `{
|
||||
"success": true,
|
||||
"collectionId": 100,
|
||||
"segments": {
|
||||
"1001": {"version": 10, "rowCount": 5, "outputPath": "s3a://bkt/seg/1001", "manifestPaths": []},
|
||||
"1002": {"version": 20, "rowCount": 7, "outputPath": "s3a://bkt/seg/1002", "manifestPaths": []}
|
||||
}
|
||||
}`
|
||||
|
||||
mockBroker := broker.NewMockBroker(t)
|
||||
mockBroker.EXPECT().DescribeCollectionInternal(mock.Anything, mock.Anything).
|
||||
Return(&milvuspb.DescribeCollectionResponse{
|
||||
Status: merr.Success(),
|
||||
DbName: "default",
|
||||
CollectionName: "test_collection",
|
||||
}, nil)
|
||||
|
||||
server := newServerForCommit(t, m, mockBroker, []byte(jsonStr))
|
||||
|
||||
wal := mock_streaming.NewMockWALAccesser(t)
|
||||
wal.EXPECT().ControlChannel().Return("by-dev-rootcoord-dml_0").Maybe()
|
||||
streaming.SetWALForTest(wal)
|
||||
|
||||
bapi := mock_broadcaster.NewMockBroadcastAPI(t)
|
||||
var captured message.BroadcastMutableMessage
|
||||
bapi.EXPECT().Broadcast(mock.Anything, mock.Anything).RunAndReturn(
|
||||
func(ctx context.Context, msg message.BroadcastMutableMessage) (*types2.BroadcastAppendResult, error) {
|
||||
captured = msg
|
||||
return &types2.BroadcastAppendResult{
|
||||
BroadcastID: 1,
|
||||
AppendResults: map[string]*types2.AppendResult{
|
||||
"by-dev-rootcoord-dml_0": {
|
||||
MessageID: rmq.NewRmqID(1),
|
||||
TimeTick: tsoutil.ComposeTSByTime(time.Now(), 0),
|
||||
LastConfirmedMessageID: rmq.NewRmqID(1),
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
})
|
||||
bapi.EXPECT().Close().Return()
|
||||
patch := mockey.Mock(broadcast.StartBroadcastWithResourceKeys).To(
|
||||
func(ctx context.Context, keys ...message.ResourceKey) (broadcaster.BroadcastAPI, error) {
|
||||
return bapi, nil
|
||||
}).Build()
|
||||
defer patch.UnPatch()
|
||||
|
||||
resp, err := server.CommitBackfillResult(ctx, &datapb.CommitBackfillResultRequest{
|
||||
ResultPath: "s3a://bucket/path/to/result.json",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, merr.Ok(resp.GetStatus()))
|
||||
assert.Equal(t, int32(2), resp.GetTotalSegments())
|
||||
assert.Equal(t, int32(2), resp.GetCommittedSegments())
|
||||
assert.Equal(t, int32(0), resp.GetFailedSegments())
|
||||
// Ensure the broadcast carries exactly two V3 items.
|
||||
assert.NotNil(t, captured)
|
||||
// Access the message body through the specialized wrapper.
|
||||
specialized := message.MustAsMutableBatchUpdateManifestMessageV2(captured)
|
||||
body := specialized.MustBody()
|
||||
assert.Len(t, body.GetItems(), 2)
|
||||
for _, it := range body.GetItems() {
|
||||
assert.Nil(t, it.GetV2ColumnGroups())
|
||||
assert.Greater(t, it.GetManifestVersion(), int64(0))
|
||||
}
|
||||
})
|
||||
|
||||
// Unhealthy state: reject without reading anything.
|
||||
t.Run("server_not_healthy", func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
server := &Server{}
|
||||
server.stateCode.Store(commonpb.StateCode_Abnormal)
|
||||
resp, err := server.CommitBackfillResult(ctx, &datapb.CommitBackfillResultRequest{
|
||||
ResultPath: "s3a://bkt/foo",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.Error(t, merr.Error(resp.GetStatus()))
|
||||
})
|
||||
|
||||
t.Run("success_false_rejected", func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
m, err := newMemoryMeta(t)
|
||||
require.NoError(t, err)
|
||||
jsonStr := `{"success": false, "collectionId": 1, "segments": {"1": {"version": 1}}}`
|
||||
server := newServerForCommit(t, m, nil, []byte(jsonStr))
|
||||
resp, err := server.CommitBackfillResult(ctx, &datapb.CommitBackfillResultRequest{
|
||||
ResultPath: "s3a://bkt/foo",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.Error(t, merr.Error(resp.GetStatus()))
|
||||
})
|
||||
|
||||
t.Run("bad_json_rejected", func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
m, err := newMemoryMeta(t)
|
||||
require.NoError(t, err)
|
||||
server := newServerForCommit(t, m, nil, []byte("not json"))
|
||||
resp, err := server.CommitBackfillResult(ctx, &datapb.CommitBackfillResultRequest{
|
||||
ResultPath: "s3a://bkt/foo",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.Error(t, merr.Error(resp.GetStatus()))
|
||||
})
|
||||
|
||||
// All segments fail pre-validation -> no broadcast, top-level error.
|
||||
t.Run("all_segments_prevalidation_fail", func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
m, err := newMemoryMeta(t)
|
||||
require.NoError(t, err)
|
||||
// Seg 1 does NOT exist in meta, so pre-validation rejects it.
|
||||
jsonStr := `{
|
||||
"success": true,
|
||||
"collectionId": 100,
|
||||
"segments": {
|
||||
"1": {"version": 10, "rowCount": 1, "outputPath": "x", "manifestPaths": []}
|
||||
}
|
||||
}`
|
||||
server := newServerForCommit(t, m, nil, []byte(jsonStr))
|
||||
resp, err := server.CommitBackfillResult(ctx, &datapb.CommitBackfillResultRequest{
|
||||
ResultPath: "s3a://bkt/foo",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.Error(t, merr.Error(resp.GetStatus()))
|
||||
assert.Equal(t, int32(1), resp.GetTotalSegments())
|
||||
assert.Equal(t, int32(1), resp.GetFailedSegments())
|
||||
assert.Equal(t, int32(0), resp.GetCommittedSegments())
|
||||
require.Len(t, resp.GetSegmentStatuses(), 1)
|
||||
assert.False(t, resp.GetSegmentStatuses()[0].GetOk())
|
||||
assert.Contains(t, resp.GetSegmentStatuses()[0].GetReason(), "not found")
|
||||
})
|
||||
|
||||
// Mixed: one segment passes pre-validation (goes to broadcast), one fails
|
||||
// (wrong collection).
|
||||
t.Run("partial_failure_reported", func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
m, err := newMemoryMeta(t)
|
||||
require.NoError(t, err)
|
||||
m.AddSegment(ctx, &SegmentInfo{SegmentInfo: &datapb.SegmentInfo{
|
||||
ID: 101, CollectionID: 100, State: commonpb.SegmentState_Flushed,
|
||||
StorageVersion: storage.StorageV3,
|
||||
ManifestPath: packed.MarshalManifestPath("/seg/101", 1),
|
||||
}})
|
||||
// 102 belongs to a different collection
|
||||
m.AddSegment(ctx, &SegmentInfo{SegmentInfo: &datapb.SegmentInfo{
|
||||
ID: 102, CollectionID: 999, State: commonpb.SegmentState_Flushed,
|
||||
StorageVersion: storage.StorageV3,
|
||||
ManifestPath: packed.MarshalManifestPath("/seg/102", 1),
|
||||
}})
|
||||
jsonStr := `{
|
||||
"success": true,
|
||||
"collectionId": 100,
|
||||
"segments": {
|
||||
"101": {"version": 10, "rowCount": 1, "outputPath": "x", "manifestPaths": []},
|
||||
"102": {"version": 20, "rowCount": 1, "outputPath": "x", "manifestPaths": []}
|
||||
}
|
||||
}`
|
||||
mockBroker := broker.NewMockBroker(t)
|
||||
mockBroker.EXPECT().DescribeCollectionInternal(mock.Anything, mock.Anything).
|
||||
Return(&milvuspb.DescribeCollectionResponse{
|
||||
Status: merr.Success(),
|
||||
DbName: "default",
|
||||
CollectionName: "c",
|
||||
}, nil).Maybe()
|
||||
server := newServerForCommit(t, m, mockBroker, []byte(jsonStr))
|
||||
|
||||
wal := mock_streaming.NewMockWALAccesser(t)
|
||||
wal.EXPECT().ControlChannel().Return("by-dev-rootcoord-dml_0").Maybe()
|
||||
streaming.SetWALForTest(wal)
|
||||
|
||||
bapi := mock_broadcaster.NewMockBroadcastAPI(t)
|
||||
bapi.EXPECT().Broadcast(mock.Anything, mock.Anything).Return(&types2.BroadcastAppendResult{
|
||||
BroadcastID: 1,
|
||||
AppendResults: map[string]*types2.AppendResult{
|
||||
"by-dev-rootcoord-dml_0": {
|
||||
MessageID: rmq.NewRmqID(1),
|
||||
TimeTick: tsoutil.ComposeTSByTime(time.Now(), 0),
|
||||
LastConfirmedMessageID: rmq.NewRmqID(1),
|
||||
},
|
||||
},
|
||||
}, nil)
|
||||
bapi.EXPECT().Close().Return()
|
||||
patch := mockey.Mock(broadcast.StartBroadcastWithResourceKeys).To(
|
||||
func(ctx context.Context, keys ...message.ResourceKey) (broadcaster.BroadcastAPI, error) {
|
||||
return bapi, nil
|
||||
}).Build()
|
||||
defer patch.UnPatch()
|
||||
|
||||
resp, err := server.CommitBackfillResult(ctx, &datapb.CommitBackfillResultRequest{
|
||||
ResultPath: "s3a://bucket/result.json",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, merr.Ok(resp.GetStatus()))
|
||||
assert.Equal(t, int32(2), resp.GetTotalSegments())
|
||||
assert.Equal(t, int32(1), resp.GetCommittedSegments())
|
||||
assert.Equal(t, int32(1), resp.GetFailedSegments())
|
||||
})
|
||||
|
||||
// V2 happy path: one V2 column-group entry reaches broadcast with V2
|
||||
// payload populated.
|
||||
t.Run("v2_happy_path", func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
m, err := newMemoryMeta(t)
|
||||
require.NoError(t, err)
|
||||
m.AddSegment(ctx, &SegmentInfo{SegmentInfo: &datapb.SegmentInfo{
|
||||
ID: 201, CollectionID: 100, State: commonpb.SegmentState_Flushed,
|
||||
StorageVersion: storage.StorageV2,
|
||||
}})
|
||||
jsonStr := `{
|
||||
"success": true,
|
||||
"collectionId": 100,
|
||||
"segments": {
|
||||
"201": {
|
||||
"version": -1,
|
||||
"rowCount": 100,
|
||||
"outputPath": "s3a://bkt/seg/201",
|
||||
"manifestPaths": ["s3a://bkt/seg/201/100/7"],
|
||||
"storage_version": 2,
|
||||
"column_groups": [
|
||||
{"field_ids":[100], "binlog_files":["s3a://bkt/seg/201/100/7"], "row_count": 100}
|
||||
]
|
||||
}
|
||||
}
|
||||
}`
|
||||
mockBroker := broker.NewMockBroker(t)
|
||||
mockBroker.EXPECT().DescribeCollectionInternal(mock.Anything, mock.Anything).
|
||||
Return(&milvuspb.DescribeCollectionResponse{
|
||||
Status: merr.Success(), DbName: "default", CollectionName: "c",
|
||||
}, nil)
|
||||
server := newServerForCommit(t, m, mockBroker, []byte(jsonStr))
|
||||
|
||||
wal := mock_streaming.NewMockWALAccesser(t)
|
||||
wal.EXPECT().ControlChannel().Return("by-dev-rootcoord-dml_0").Maybe()
|
||||
streaming.SetWALForTest(wal)
|
||||
|
||||
bapi := mock_broadcaster.NewMockBroadcastAPI(t)
|
||||
var captured message.BroadcastMutableMessage
|
||||
bapi.EXPECT().Broadcast(mock.Anything, mock.Anything).RunAndReturn(
|
||||
func(ctx context.Context, msg message.BroadcastMutableMessage) (*types2.BroadcastAppendResult, error) {
|
||||
captured = msg
|
||||
return &types2.BroadcastAppendResult{
|
||||
BroadcastID: 1,
|
||||
AppendResults: map[string]*types2.AppendResult{
|
||||
"by-dev-rootcoord-dml_0": {
|
||||
MessageID: rmq.NewRmqID(1),
|
||||
TimeTick: tsoutil.ComposeTSByTime(time.Now(), 0),
|
||||
LastConfirmedMessageID: rmq.NewRmqID(1),
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
})
|
||||
bapi.EXPECT().Close().Return()
|
||||
patch := mockey.Mock(broadcast.StartBroadcastWithResourceKeys).To(
|
||||
func(ctx context.Context, keys ...message.ResourceKey) (broadcaster.BroadcastAPI, error) {
|
||||
return bapi, nil
|
||||
}).Build()
|
||||
defer patch.UnPatch()
|
||||
|
||||
resp, err := server.CommitBackfillResult(ctx, &datapb.CommitBackfillResultRequest{
|
||||
ResultPath: "s3a://bucket/result.json",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, merr.Ok(resp.GetStatus()))
|
||||
assert.Equal(t, int32(1), resp.GetCommittedSegments())
|
||||
|
||||
specialized := message.MustAsMutableBatchUpdateManifestMessageV2(captured)
|
||||
body := specialized.MustBody()
|
||||
require.Len(t, body.GetItems(), 1)
|
||||
it := body.GetItems()[0]
|
||||
assert.Equal(t, int64(201), it.GetSegmentId())
|
||||
assert.Equal(t, int64(0), it.GetManifestVersion())
|
||||
require.NotNil(t, it.GetV2ColumnGroups())
|
||||
require.Contains(t, it.GetV2ColumnGroups().GetColumnGroups(), int64(100))
|
||||
fb := it.GetV2ColumnGroups().GetColumnGroups()[100]
|
||||
require.Len(t, fb.GetBinlogs(), 1)
|
||||
assert.Equal(t, int64(100), fb.GetBinlogs()[0].GetEntriesNum())
|
||||
})
|
||||
|
||||
// Partition-scoped backfill: result.PartitionID != 0 and segment belongs
|
||||
// to a different partition -> rejected.
|
||||
t.Run("wrong_partition_rejected", func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
m, err := newMemoryMeta(t)
|
||||
require.NoError(t, err)
|
||||
m.AddSegment(ctx, &SegmentInfo{SegmentInfo: &datapb.SegmentInfo{
|
||||
ID: 401, CollectionID: 100, PartitionID: 999,
|
||||
State: commonpb.SegmentState_Flushed,
|
||||
StorageVersion: storage.StorageV3,
|
||||
ManifestPath: packed.MarshalManifestPath("/seg/401", 1),
|
||||
}})
|
||||
jsonStr := `{
|
||||
"success": true,
|
||||
"collectionId": 100,
|
||||
"partitionId": 42,
|
||||
"segments": {
|
||||
"401": {"version": 10, "rowCount": 1, "outputPath": "x", "manifestPaths": []}
|
||||
}
|
||||
}`
|
||||
server := newServerForCommit(t, m, nil, []byte(jsonStr))
|
||||
resp, err := server.CommitBackfillResult(ctx, &datapb.CommitBackfillResultRequest{
|
||||
ResultPath: "s3a://bkt/foo",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.Error(t, merr.Error(resp.GetStatus()))
|
||||
assert.Equal(t, int32(1), resp.GetFailedSegments())
|
||||
require.Len(t, resp.GetSegmentStatuses(), 1)
|
||||
assert.False(t, resp.GetSegmentStatuses()[0].GetOk())
|
||||
assert.Contains(t, resp.GetSegmentStatuses()[0].GetReason(), "does not belong to the result's partition")
|
||||
})
|
||||
|
||||
// V3 entry pointing at a segment whose actual storage version is V2 must
|
||||
// be rejected: UpdateManifestVersion would no-op and the caller would see
|
||||
// a fake committed=true.
|
||||
t.Run("v3_rejected_on_non_v3_segment", func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
m, err := newMemoryMeta(t)
|
||||
require.NoError(t, err)
|
||||
m.AddSegment(ctx, &SegmentInfo{SegmentInfo: &datapb.SegmentInfo{
|
||||
ID: 301, CollectionID: 100, State: commonpb.SegmentState_Flushed,
|
||||
StorageVersion: storage.StorageV2, // V2 segment, JSON misroutes it as V3
|
||||
ManifestPath: packed.MarshalManifestPath("/seg/301", 1),
|
||||
}})
|
||||
jsonStr := `{
|
||||
"success": true,
|
||||
"collectionId": 100,
|
||||
"segments": {
|
||||
"301": {"version": 10, "rowCount": 1, "outputPath": "x", "manifestPaths": []}
|
||||
}
|
||||
}`
|
||||
server := newServerForCommit(t, m, nil, []byte(jsonStr))
|
||||
resp, err := server.CommitBackfillResult(ctx, &datapb.CommitBackfillResultRequest{
|
||||
ResultPath: "s3a://bkt/foo",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.Error(t, merr.Error(resp.GetStatus()))
|
||||
assert.Equal(t, int32(1), resp.GetFailedSegments())
|
||||
require.Len(t, resp.GetSegmentStatuses(), 1)
|
||||
assert.False(t, resp.GetSegmentStatuses()[0].GetOk())
|
||||
assert.Contains(t, resp.GetSegmentStatuses()[0].GetReason(), "storage version is not V3")
|
||||
})
|
||||
|
||||
// V3 entry whose version is not strictly greater than the segment's
|
||||
// current manifest version must be rejected. This prevents a stale
|
||||
// Spark retry from silently rolling the manifest pointer backwards.
|
||||
t.Run("v3_rejected_on_stale_version", func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
m, err := newMemoryMeta(t)
|
||||
require.NoError(t, err)
|
||||
m.AddSegment(ctx, &SegmentInfo{SegmentInfo: &datapb.SegmentInfo{
|
||||
ID: 401, CollectionID: 100, State: commonpb.SegmentState_Flushed,
|
||||
StorageVersion: storage.StorageV3,
|
||||
// current manifest version = 10
|
||||
ManifestPath: packed.MarshalManifestPath("/seg/401", 10),
|
||||
}})
|
||||
// JSON reports version=5, which is less than current 10.
|
||||
jsonStr := `{
|
||||
"success": true,
|
||||
"collectionId": 100,
|
||||
"segments": {
|
||||
"401": {"version": 5, "rowCount": 1, "outputPath": "x", "manifestPaths": []}
|
||||
}
|
||||
}`
|
||||
server := newServerForCommit(t, m, nil, []byte(jsonStr))
|
||||
resp, err := server.CommitBackfillResult(ctx, &datapb.CommitBackfillResultRequest{
|
||||
ResultPath: "s3a://bkt/foo",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.Error(t, merr.Error(resp.GetStatus()))
|
||||
assert.Equal(t, int32(1), resp.GetFailedSegments())
|
||||
require.Len(t, resp.GetSegmentStatuses(), 1)
|
||||
assert.False(t, resp.GetSegmentStatuses()[0].GetOk())
|
||||
assert.Contains(t, resp.GetSegmentStatuses()[0].GetReason(), "not greater than current")
|
||||
})
|
||||
|
||||
// Same-version retry must also be rejected -- UpdateManifestVersion
|
||||
// short-circuits on equality so the item would otherwise broadcast as a
|
||||
// no-op while we report committed=true.
|
||||
t.Run("v3_rejected_on_equal_version", func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
m, err := newMemoryMeta(t)
|
||||
require.NoError(t, err)
|
||||
m.AddSegment(ctx, &SegmentInfo{SegmentInfo: &datapb.SegmentInfo{
|
||||
ID: 402, CollectionID: 100, State: commonpb.SegmentState_Flushed,
|
||||
StorageVersion: storage.StorageV3,
|
||||
ManifestPath: packed.MarshalManifestPath("/seg/402", 7),
|
||||
}})
|
||||
jsonStr := `{
|
||||
"success": true,
|
||||
"collectionId": 100,
|
||||
"segments": {
|
||||
"402": {"version": 7, "rowCount": 1, "outputPath": "x", "manifestPaths": []}
|
||||
}
|
||||
}`
|
||||
server := newServerForCommit(t, m, nil, []byte(jsonStr))
|
||||
resp, err := server.CommitBackfillResult(ctx, &datapb.CommitBackfillResultRequest{
|
||||
ResultPath: "s3a://bkt/foo",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.Error(t, merr.Error(resp.GetStatus()))
|
||||
require.Len(t, resp.GetSegmentStatuses(), 1)
|
||||
assert.Contains(t, resp.GetSegmentStatuses()[0].GetReason(), "not greater than current")
|
||||
})
|
||||
|
||||
// V3 entry pointing at a V3 segment that has never had a manifest written
|
||||
// (ManifestPath == "") must be rejected at pre-validation.
|
||||
t.Run("v3_rejected_on_empty_manifest_path", func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
m, err := newMemoryMeta(t)
|
||||
require.NoError(t, err)
|
||||
m.AddSegment(ctx, &SegmentInfo{SegmentInfo: &datapb.SegmentInfo{
|
||||
ID: 302, CollectionID: 100, State: commonpb.SegmentState_Flushed,
|
||||
StorageVersion: storage.StorageV3,
|
||||
// ManifestPath intentionally empty.
|
||||
}})
|
||||
jsonStr := `{
|
||||
"success": true,
|
||||
"collectionId": 100,
|
||||
"segments": {
|
||||
"302": {"version": 10, "rowCount": 1, "outputPath": "x", "manifestPaths": []}
|
||||
}
|
||||
}`
|
||||
server := newServerForCommit(t, m, nil, []byte(jsonStr))
|
||||
resp, err := server.CommitBackfillResult(ctx, &datapb.CommitBackfillResultRequest{
|
||||
ResultPath: "s3a://bkt/foo",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.Error(t, merr.Error(resp.GetStatus()))
|
||||
assert.Equal(t, int32(1), resp.GetFailedSegments())
|
||||
require.Len(t, resp.GetSegmentStatuses(), 1)
|
||||
assert.False(t, resp.GetSegmentStatuses()[0].GetOk())
|
||||
assert.Contains(t, resp.GetSegmentStatuses()[0].GetReason(), "no existing manifest path")
|
||||
})
|
||||
|
||||
// A result JSON that exceeds the hard-cap must be rejected before Read
|
||||
// so an oversized or malicious file cannot OOM DataCoord.
|
||||
t.Run("oversized_result_rejected", func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
m, err := newMemoryMeta(t)
|
||||
require.NoError(t, err)
|
||||
|
||||
cm := mock_storage.NewMockChunkManager(t)
|
||||
cm.EXPECT().Size(mock.Anything, mock.Anything).Return(maxBackfillResultBytes+1, nil)
|
||||
// Read must not be called -- assert by omitting the expectation.
|
||||
m.chunkManager = cm
|
||||
|
||||
s := &Server{meta: m}
|
||||
s.stateCode.Store(commonpb.StateCode_Healthy)
|
||||
|
||||
resp, err := s.CommitBackfillResult(ctx, &datapb.CommitBackfillResultRequest{
|
||||
ResultPath: "s3a://bkt/foo",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.Error(t, merr.Error(resp.GetStatus()))
|
||||
assert.Contains(t, resp.GetStatus().GetReason(), "exceeds limit")
|
||||
})
|
||||
|
||||
// More items than maxItemsPerBroadcast must be split across several
|
||||
// broadcast messages so the payload stays under typical MQ limits.
|
||||
t.Run("items_split_across_broadcast_batches", func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
m, err := newMemoryMeta(t)
|
||||
require.NoError(t, err)
|
||||
|
||||
segIDs := make([]int64, 0, maxItemsPerBroadcast+5)
|
||||
for i := int64(1); i <= int64(maxItemsPerBroadcast+5); i++ {
|
||||
segIDs = append(segIDs, 10000+i)
|
||||
m.AddSegment(ctx, &SegmentInfo{SegmentInfo: &datapb.SegmentInfo{
|
||||
ID: 10000 + i, CollectionID: 100, State: commonpb.SegmentState_Flushed,
|
||||
StorageVersion: storage.StorageV3,
|
||||
ManifestPath: packed.MarshalManifestPath("/seg/"+strconv.FormatInt(10000+i, 10), 1),
|
||||
}})
|
||||
}
|
||||
// Build a JSON result referencing every segment.
|
||||
var b strings.Builder
|
||||
b.WriteString(`{"success": true, "collectionId": 100, "segments": {`)
|
||||
for i, id := range segIDs {
|
||||
if i > 0 {
|
||||
b.WriteByte(',')
|
||||
}
|
||||
b.WriteString(`"` + strconv.FormatInt(id, 10) + `": {"version": 10, "rowCount": 1, "outputPath": "x", "manifestPaths": []}`)
|
||||
}
|
||||
b.WriteString(`}}`)
|
||||
|
||||
mockBroker := broker.NewMockBroker(t)
|
||||
mockBroker.EXPECT().DescribeCollectionInternal(mock.Anything, mock.Anything).
|
||||
Return(&milvuspb.DescribeCollectionResponse{
|
||||
Status: merr.Success(), DbName: "default", CollectionName: "c",
|
||||
}, nil)
|
||||
|
||||
server := newServerForCommit(t, m, mockBroker, []byte(b.String()))
|
||||
|
||||
wal := mock_streaming.NewMockWALAccesser(t)
|
||||
wal.EXPECT().ControlChannel().Return("by-dev-rootcoord-dml_0").Maybe()
|
||||
streaming.SetWALForTest(wal)
|
||||
|
||||
bapi := mock_broadcaster.NewMockBroadcastAPI(t)
|
||||
var broadcastCalls int
|
||||
bapi.EXPECT().Broadcast(mock.Anything, mock.Anything).RunAndReturn(
|
||||
func(ctx context.Context, msg message.BroadcastMutableMessage) (*types2.BroadcastAppendResult, error) {
|
||||
broadcastCalls++
|
||||
return &types2.BroadcastAppendResult{
|
||||
BroadcastID: uint64(broadcastCalls),
|
||||
AppendResults: map[string]*types2.AppendResult{
|
||||
"by-dev-rootcoord-dml_0": {
|
||||
MessageID: rmq.NewRmqID(1),
|
||||
TimeTick: tsoutil.ComposeTSByTime(time.Now(), 0),
|
||||
LastConfirmedMessageID: rmq.NewRmqID(1),
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
})
|
||||
bapi.EXPECT().Close().Return()
|
||||
patch := mockey.Mock(broadcast.StartBroadcastWithResourceKeys).To(
|
||||
func(ctx context.Context, keys ...message.ResourceKey) (broadcaster.BroadcastAPI, error) {
|
||||
return bapi, nil
|
||||
}).Build()
|
||||
defer patch.UnPatch()
|
||||
|
||||
resp, err := server.CommitBackfillResult(ctx, &datapb.CommitBackfillResultRequest{
|
||||
ResultPath: "s3a://bucket/result.json",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, merr.Ok(resp.GetStatus()))
|
||||
assert.Equal(t, int32(len(segIDs)), resp.GetCommittedSegments())
|
||||
assert.Equal(t, int32(0), resp.GetFailedSegments())
|
||||
// 517 items split across batches of 512 = 2 broadcasts.
|
||||
assert.Equal(t, 2, broadcastCalls)
|
||||
})
|
||||
}
|
||||
|
||||
// --- Test BatchUpdateManifest ---
|
||||
|
||||
func TestServer_BatchUpdateManifest(t *testing.T) {
|
||||
@@ -3766,6 +4335,113 @@ func TestServer_BatchUpdateManifest_Callback(t *testing.T) {
|
||||
})
|
||||
assert.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("v2_column_groups_dispatches_operator", func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
registry.ResetRegistration()
|
||||
|
||||
var capturedOps int
|
||||
mockUpdate := mockey.Mock((*meta).UpdateSegmentsInfo).To(
|
||||
func(m *meta, ctx context.Context, operators ...UpdateOperator) error {
|
||||
capturedOps = len(operators)
|
||||
return nil
|
||||
}).Build()
|
||||
defer mockUpdate.UnPatch()
|
||||
|
||||
server := &Server{
|
||||
ctx: ctx,
|
||||
meta: &meta{segments: NewSegmentsInfo()},
|
||||
}
|
||||
server.stateCode.Store(commonpb.StateCode_Healthy)
|
||||
RegisterDDLCallbacks(server)
|
||||
|
||||
msg := message.NewBatchUpdateManifestMessageBuilderV2().
|
||||
WithHeader(&message.BatchUpdateManifestMessageHeader{
|
||||
CollectionId: 100,
|
||||
}).
|
||||
WithBody(&message.BatchUpdateManifestMessageBody{
|
||||
Items: []*messagespb.BatchUpdateManifestItem{
|
||||
{SegmentId: 1, ManifestVersion: 15}, // V3
|
||||
{
|
||||
SegmentId: 2, // V2
|
||||
V2ColumnGroups: &messagespb.BatchUpdateManifestV2ColumnGroups{
|
||||
ColumnGroups: map[int64]*datapb.FieldBinlog{
|
||||
200: {FieldID: 200, Binlogs: []*datapb.Binlog{{LogID: 7}}},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}).
|
||||
WithBroadcast([]string{"control_channel"}).
|
||||
MustBuildBroadcast()
|
||||
|
||||
err := registry.CallMessageAckCallback(ctx, msg, map[string]*message.AppendResult{
|
||||
"control_channel": {
|
||||
MessageID: rmq.NewRmqID(1),
|
||||
LastConfirmedMessageID: rmq.NewRmqID(1),
|
||||
TimeTick: 1,
|
||||
},
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
// Two operators dispatched: one V3 UpdateManifestVersion, one V2
|
||||
// UpdateSegmentColumnGroupsOperator. Both flow through the single
|
||||
// UpdateSegmentsInfo batch call.
|
||||
assert.Equal(t, 2, capturedOps)
|
||||
})
|
||||
|
||||
t.Run("item_with_both_v2_and_v3_is_skipped", func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
registry.ResetRegistration()
|
||||
|
||||
var capturedOps int
|
||||
mockUpdate := mockey.Mock((*meta).UpdateSegmentsInfo).To(
|
||||
func(m *meta, ctx context.Context, operators ...UpdateOperator) error {
|
||||
capturedOps = len(operators)
|
||||
return nil
|
||||
}).Build()
|
||||
defer mockUpdate.UnPatch()
|
||||
|
||||
server := &Server{
|
||||
ctx: ctx,
|
||||
meta: &meta{segments: NewSegmentsInfo()},
|
||||
}
|
||||
server.stateCode.Store(commonpb.StateCode_Healthy)
|
||||
RegisterDDLCallbacks(server)
|
||||
|
||||
msg := message.NewBatchUpdateManifestMessageBuilderV2().
|
||||
WithHeader(&message.BatchUpdateManifestMessageHeader{
|
||||
CollectionId: 100,
|
||||
}).
|
||||
WithBody(&message.BatchUpdateManifestMessageBody{
|
||||
Items: []*messagespb.BatchUpdateManifestItem{
|
||||
{
|
||||
SegmentId: 1,
|
||||
ManifestVersion: 15,
|
||||
V2ColumnGroups: &messagespb.BatchUpdateManifestV2ColumnGroups{
|
||||
ColumnGroups: map[int64]*datapb.FieldBinlog{
|
||||
200: {FieldID: 200, Binlogs: []*datapb.Binlog{{LogID: 7}}},
|
||||
},
|
||||
},
|
||||
},
|
||||
{SegmentId: 2, ManifestVersion: 25}, // valid V3
|
||||
},
|
||||
}).
|
||||
WithBroadcast([]string{"control_channel"}).
|
||||
MustBuildBroadcast()
|
||||
|
||||
err := registry.CallMessageAckCallback(ctx, msg, map[string]*message.AppendResult{
|
||||
"control_channel": {
|
||||
MessageID: rmq.NewRmqID(1),
|
||||
LastConfirmedMessageID: rmq.NewRmqID(1),
|
||||
TimeTick: 1,
|
||||
},
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
// ambiguous item is skipped, only the valid V3 item becomes an operator
|
||||
assert.Equal(t, 1, capturedOps)
|
||||
})
|
||||
}
|
||||
|
||||
// --- Test RefreshExternalCollection ---
|
||||
|
||||
@@ -2139,6 +2139,17 @@ func (c *Client) BatchUpdateManifest(ctx context.Context, req *datapb.BatchUpdat
|
||||
})
|
||||
}
|
||||
|
||||
func (c *Client) CommitBackfillResult(ctx context.Context, req *datapb.CommitBackfillResultRequest, opts ...grpc.CallOption) (*datapb.CommitBackfillResultResponse, error) {
|
||||
req = typeutil.Clone(req)
|
||||
commonpbutil.UpdateMsgBase(
|
||||
req.GetBase(),
|
||||
commonpbutil.FillMsgBaseFromClient(paramtable.GetNodeID(), commonpbutil.WithTargetID(c.grpcClient.GetNodeID())),
|
||||
)
|
||||
return wrapGrpcCall(ctx, c, func(client MixCoordClient) (*datapb.CommitBackfillResultResponse, error) {
|
||||
return client.CommitBackfillResult(ctx, req, opts...)
|
||||
})
|
||||
}
|
||||
|
||||
// ClientHeartbeat handles client telemetry heartbeat requests
|
||||
func (c *Client) ClientHeartbeat(ctx context.Context, req *milvuspb.ClientHeartbeatRequest, opts ...grpc.CallOption) (*milvuspb.ClientHeartbeatResponse, error) {
|
||||
return wrapGrpcCall(ctx, c, func(client MixCoordClient) (*milvuspb.ClientHeartbeatResponse, error) {
|
||||
|
||||
@@ -1027,6 +1027,10 @@ func (s *Server) BatchUpdateManifest(ctx context.Context, req *datapb.BatchUpdat
|
||||
return s.mixCoord.BatchUpdateManifest(ctx, req)
|
||||
}
|
||||
|
||||
func (s *Server) CommitBackfillResult(ctx context.Context, req *datapb.CommitBackfillResultRequest) (*datapb.CommitBackfillResultResponse, error) {
|
||||
return s.mixCoord.CommitBackfillResult(ctx, req)
|
||||
}
|
||||
|
||||
// ClientHeartbeat handles client telemetry heartbeat requests
|
||||
func (s *Server) ClientHeartbeat(ctx context.Context, req *milvuspb.ClientHeartbeatRequest) (*milvuspb.ClientHeartbeatResponse, error) {
|
||||
return s.mixCoord.ClientHeartbeat(ctx, req)
|
||||
|
||||
@@ -56,6 +56,8 @@ const (
|
||||
RouteGcPause = "/management/datacoord/garbage_collection/pause"
|
||||
RouteGcResume = "/management/datacoord/garbage_collection/resume"
|
||||
|
||||
RouteCommitBackfill = "/management/datacoord/backfill/commit"
|
||||
|
||||
RouteSuspendQueryCoordBalance = "/management/querycoord/balance/suspend"
|
||||
RouteResumeQueryCoordBalance = "/management/querycoord/balance/resume"
|
||||
RouteQueryCoordBalanceStatus = "/management/querycoord/balance/status"
|
||||
|
||||
@@ -130,6 +130,12 @@ func (t AlterType) String() string {
|
||||
type BinlogsIncrement struct {
|
||||
Segment *datapb.SegmentInfo
|
||||
UpdateMask BinlogsUpdateMask
|
||||
// DroppedBinlogFieldIDs lists FieldBinlog.FieldID values whose insert-binlog
|
||||
// KV entries must be removed from etcd. AlterSegments only upserts; without
|
||||
// explicit removal, a prefix scan in listBinlogs will resurrect the zombie
|
||||
// entry on restart. Used by operators (e.g. V2 column-group backfill commit)
|
||||
// that structurally drop a FieldBinlog from segment.Binlogs.
|
||||
DroppedBinlogFieldIDs []int64
|
||||
}
|
||||
|
||||
type BinlogsUpdateMask struct {
|
||||
|
||||
@@ -316,6 +316,7 @@ func (kc *Catalog) AlterSegments(ctx context.Context, segments []*datapb.Segment
|
||||
kvs[k] = v
|
||||
}
|
||||
|
||||
var removals []string
|
||||
for _, b := range binlogs {
|
||||
segment := b.Segment
|
||||
|
||||
@@ -332,9 +333,32 @@ func (kc *Catalog) AlterSegments(ctx context.Context, segments []*datapb.Segment
|
||||
}
|
||||
|
||||
maps.Copy(kvs, binlogKvs)
|
||||
|
||||
for _, fid := range b.DroppedBinlogFieldIDs {
|
||||
removals = append(removals,
|
||||
buildFieldBinlogPath(
|
||||
segment.GetCollectionID(),
|
||||
segment.GetPartitionID(),
|
||||
segment.GetID(),
|
||||
fid))
|
||||
}
|
||||
}
|
||||
|
||||
return kc.SaveByBatch(ctx, kvs)
|
||||
if err := kc.SaveByBatch(ctx, kvs); err != nil {
|
||||
return err
|
||||
}
|
||||
// Explicit removal is required: AlterSegments persists binlogs as
|
||||
// independent per-FieldID KVs and listBinlogs rebuilds them via a prefix
|
||||
// scan on restart. An operator that structurally drops a FieldBinlog
|
||||
// from segment.Binlogs (e.g. when all ChildFields of a column group are
|
||||
// claimed by a backfill commit) must also delete the orphan KV, otherwise
|
||||
// the stripped group resurrects on the next datacoord start.
|
||||
if len(removals) > 0 {
|
||||
if err := kc.MetaKv.MultiSaveAndRemove(ctx, nil, removals); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (kc *Catalog) handleDroppedSegment(ctx context.Context, segment *datapb.SegmentInfo) (kvs map[string]string, err error) {
|
||||
|
||||
@@ -390,6 +390,65 @@ func (_c *MockDataCoord_CheckHealth_Call) RunAndReturn(run func(context.Context,
|
||||
return _c
|
||||
}
|
||||
|
||||
// CommitBackfillResult provides a mock function with given fields: _a0, _a1
|
||||
func (_m *MockDataCoord) CommitBackfillResult(_a0 context.Context, _a1 *datapb.CommitBackfillResultRequest) (*datapb.CommitBackfillResultResponse, error) {
|
||||
ret := _m.Called(_a0, _a1)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for CommitBackfillResult")
|
||||
}
|
||||
|
||||
var r0 *datapb.CommitBackfillResultResponse
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *datapb.CommitBackfillResultRequest) (*datapb.CommitBackfillResultResponse, error)); ok {
|
||||
return rf(_a0, _a1)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *datapb.CommitBackfillResultRequest) *datapb.CommitBackfillResultResponse); ok {
|
||||
r0 = rf(_a0, _a1)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*datapb.CommitBackfillResultResponse)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, *datapb.CommitBackfillResultRequest) error); ok {
|
||||
r1 = rf(_a0, _a1)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockDataCoord_CommitBackfillResult_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CommitBackfillResult'
|
||||
type MockDataCoord_CommitBackfillResult_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// CommitBackfillResult is a helper method to define mock.On call
|
||||
// - _a0 context.Context
|
||||
// - _a1 *datapb.CommitBackfillResultRequest
|
||||
func (_e *MockDataCoord_Expecter) CommitBackfillResult(_a0 interface{}, _a1 interface{}) *MockDataCoord_CommitBackfillResult_Call {
|
||||
return &MockDataCoord_CommitBackfillResult_Call{Call: _e.mock.On("CommitBackfillResult", _a0, _a1)}
|
||||
}
|
||||
|
||||
func (_c *MockDataCoord_CommitBackfillResult_Call) Run(run func(_a0 context.Context, _a1 *datapb.CommitBackfillResultRequest)) *MockDataCoord_CommitBackfillResult_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context), args[1].(*datapb.CommitBackfillResultRequest))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockDataCoord_CommitBackfillResult_Call) Return(_a0 *datapb.CommitBackfillResultResponse, _a1 error) *MockDataCoord_CommitBackfillResult_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockDataCoord_CommitBackfillResult_Call) RunAndReturn(run func(context.Context, *datapb.CommitBackfillResultRequest) (*datapb.CommitBackfillResultResponse, error)) *MockDataCoord_CommitBackfillResult_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// CreateIndex provides a mock function with given fields: _a0, _a1
|
||||
func (_m *MockDataCoord) CreateIndex(_a0 context.Context, _a1 *indexpb.CreateIndexRequest) (*commonpb.Status, error) {
|
||||
ret := _m.Called(_a0, _a1)
|
||||
|
||||
@@ -522,6 +522,80 @@ func (_c *MockDataCoordClient_Close_Call) RunAndReturn(run func() error) *MockDa
|
||||
return _c
|
||||
}
|
||||
|
||||
// CommitBackfillResult provides a mock function with given fields: ctx, in, opts
|
||||
func (_m *MockDataCoordClient) CommitBackfillResult(ctx context.Context, in *datapb.CommitBackfillResultRequest, opts ...grpc.CallOption) (*datapb.CommitBackfillResultResponse, error) {
|
||||
_va := make([]interface{}, len(opts))
|
||||
for _i := range opts {
|
||||
_va[_i] = opts[_i]
|
||||
}
|
||||
var _ca []interface{}
|
||||
_ca = append(_ca, ctx, in)
|
||||
_ca = append(_ca, _va...)
|
||||
ret := _m.Called(_ca...)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for CommitBackfillResult")
|
||||
}
|
||||
|
||||
var r0 *datapb.CommitBackfillResultResponse
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *datapb.CommitBackfillResultRequest, ...grpc.CallOption) (*datapb.CommitBackfillResultResponse, error)); ok {
|
||||
return rf(ctx, in, opts...)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *datapb.CommitBackfillResultRequest, ...grpc.CallOption) *datapb.CommitBackfillResultResponse); ok {
|
||||
r0 = rf(ctx, in, opts...)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*datapb.CommitBackfillResultResponse)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, *datapb.CommitBackfillResultRequest, ...grpc.CallOption) error); ok {
|
||||
r1 = rf(ctx, in, opts...)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockDataCoordClient_CommitBackfillResult_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CommitBackfillResult'
|
||||
type MockDataCoordClient_CommitBackfillResult_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// CommitBackfillResult is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - in *datapb.CommitBackfillResultRequest
|
||||
// - opts ...grpc.CallOption
|
||||
func (_e *MockDataCoordClient_Expecter) CommitBackfillResult(ctx interface{}, in interface{}, opts ...interface{}) *MockDataCoordClient_CommitBackfillResult_Call {
|
||||
return &MockDataCoordClient_CommitBackfillResult_Call{Call: _e.mock.On("CommitBackfillResult",
|
||||
append([]interface{}{ctx, in}, opts...)...)}
|
||||
}
|
||||
|
||||
func (_c *MockDataCoordClient_CommitBackfillResult_Call) Run(run func(ctx context.Context, in *datapb.CommitBackfillResultRequest, opts ...grpc.CallOption)) *MockDataCoordClient_CommitBackfillResult_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
variadicArgs := make([]grpc.CallOption, len(args)-2)
|
||||
for i, a := range args[2:] {
|
||||
if a != nil {
|
||||
variadicArgs[i] = a.(grpc.CallOption)
|
||||
}
|
||||
}
|
||||
run(args[0].(context.Context), args[1].(*datapb.CommitBackfillResultRequest), variadicArgs...)
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockDataCoordClient_CommitBackfillResult_Call) Return(_a0 *datapb.CommitBackfillResultResponse, _a1 error) *MockDataCoordClient_CommitBackfillResult_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockDataCoordClient_CommitBackfillResult_Call) RunAndReturn(run func(context.Context, *datapb.CommitBackfillResultRequest, ...grpc.CallOption) (*datapb.CommitBackfillResultResponse, error)) *MockDataCoordClient_CommitBackfillResult_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// CreateIndex provides a mock function with given fields: ctx, in, opts
|
||||
func (_m *MockDataCoordClient) CreateIndex(ctx context.Context, in *indexpb.CreateIndexRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
|
||||
_va := make([]interface{}, len(opts))
|
||||
|
||||
@@ -1403,6 +1403,65 @@ func (_c *MixCoord_ClientHeartbeat_Call) RunAndReturn(run func(context.Context,
|
||||
return _c
|
||||
}
|
||||
|
||||
// CommitBackfillResult provides a mock function with given fields: _a0, _a1
|
||||
func (_m *MixCoord) CommitBackfillResult(_a0 context.Context, _a1 *datapb.CommitBackfillResultRequest) (*datapb.CommitBackfillResultResponse, error) {
|
||||
ret := _m.Called(_a0, _a1)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for CommitBackfillResult")
|
||||
}
|
||||
|
||||
var r0 *datapb.CommitBackfillResultResponse
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *datapb.CommitBackfillResultRequest) (*datapb.CommitBackfillResultResponse, error)); ok {
|
||||
return rf(_a0, _a1)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *datapb.CommitBackfillResultRequest) *datapb.CommitBackfillResultResponse); ok {
|
||||
r0 = rf(_a0, _a1)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*datapb.CommitBackfillResultResponse)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, *datapb.CommitBackfillResultRequest) error); ok {
|
||||
r1 = rf(_a0, _a1)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MixCoord_CommitBackfillResult_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CommitBackfillResult'
|
||||
type MixCoord_CommitBackfillResult_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// CommitBackfillResult is a helper method to define mock.On call
|
||||
// - _a0 context.Context
|
||||
// - _a1 *datapb.CommitBackfillResultRequest
|
||||
func (_e *MixCoord_Expecter) CommitBackfillResult(_a0 interface{}, _a1 interface{}) *MixCoord_CommitBackfillResult_Call {
|
||||
return &MixCoord_CommitBackfillResult_Call{Call: _e.mock.On("CommitBackfillResult", _a0, _a1)}
|
||||
}
|
||||
|
||||
func (_c *MixCoord_CommitBackfillResult_Call) Run(run func(_a0 context.Context, _a1 *datapb.CommitBackfillResultRequest)) *MixCoord_CommitBackfillResult_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context), args[1].(*datapb.CommitBackfillResultRequest))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MixCoord_CommitBackfillResult_Call) Return(_a0 *datapb.CommitBackfillResultResponse, _a1 error) *MixCoord_CommitBackfillResult_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MixCoord_CommitBackfillResult_Call) RunAndReturn(run func(context.Context, *datapb.CommitBackfillResultRequest) (*datapb.CommitBackfillResultResponse, error)) *MixCoord_CommitBackfillResult_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// ComputePhraseMatchSlop provides a mock function with given fields: _a0, _a1
|
||||
func (_m *MixCoord) ComputePhraseMatchSlop(_a0 context.Context, _a1 *querypb.ComputePhraseMatchSlopRequest) (*querypb.ComputePhraseMatchSlopResponse, error) {
|
||||
ret := _m.Called(_a0, _a1)
|
||||
|
||||
@@ -1786,6 +1786,80 @@ func (_c *MockMixCoordClient_Close_Call) RunAndReturn(run func() error) *MockMix
|
||||
return _c
|
||||
}
|
||||
|
||||
// CommitBackfillResult provides a mock function with given fields: ctx, in, opts
|
||||
func (_m *MockMixCoordClient) CommitBackfillResult(ctx context.Context, in *datapb.CommitBackfillResultRequest, opts ...grpc.CallOption) (*datapb.CommitBackfillResultResponse, error) {
|
||||
_va := make([]interface{}, len(opts))
|
||||
for _i := range opts {
|
||||
_va[_i] = opts[_i]
|
||||
}
|
||||
var _ca []interface{}
|
||||
_ca = append(_ca, ctx, in)
|
||||
_ca = append(_ca, _va...)
|
||||
ret := _m.Called(_ca...)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for CommitBackfillResult")
|
||||
}
|
||||
|
||||
var r0 *datapb.CommitBackfillResultResponse
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *datapb.CommitBackfillResultRequest, ...grpc.CallOption) (*datapb.CommitBackfillResultResponse, error)); ok {
|
||||
return rf(ctx, in, opts...)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *datapb.CommitBackfillResultRequest, ...grpc.CallOption) *datapb.CommitBackfillResultResponse); ok {
|
||||
r0 = rf(ctx, in, opts...)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*datapb.CommitBackfillResultResponse)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, *datapb.CommitBackfillResultRequest, ...grpc.CallOption) error); ok {
|
||||
r1 = rf(ctx, in, opts...)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockMixCoordClient_CommitBackfillResult_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CommitBackfillResult'
|
||||
type MockMixCoordClient_CommitBackfillResult_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// CommitBackfillResult is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - in *datapb.CommitBackfillResultRequest
|
||||
// - opts ...grpc.CallOption
|
||||
func (_e *MockMixCoordClient_Expecter) CommitBackfillResult(ctx interface{}, in interface{}, opts ...interface{}) *MockMixCoordClient_CommitBackfillResult_Call {
|
||||
return &MockMixCoordClient_CommitBackfillResult_Call{Call: _e.mock.On("CommitBackfillResult",
|
||||
append([]interface{}{ctx, in}, opts...)...)}
|
||||
}
|
||||
|
||||
func (_c *MockMixCoordClient_CommitBackfillResult_Call) Run(run func(ctx context.Context, in *datapb.CommitBackfillResultRequest, opts ...grpc.CallOption)) *MockMixCoordClient_CommitBackfillResult_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
variadicArgs := make([]grpc.CallOption, len(args)-2)
|
||||
for i, a := range args[2:] {
|
||||
if a != nil {
|
||||
variadicArgs[i] = a.(grpc.CallOption)
|
||||
}
|
||||
}
|
||||
run(args[0].(context.Context), args[1].(*datapb.CommitBackfillResultRequest), variadicArgs...)
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockMixCoordClient_CommitBackfillResult_Call) Return(_a0 *datapb.CommitBackfillResultResponse, _a1 error) *MockMixCoordClient_CommitBackfillResult_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockMixCoordClient_CommitBackfillResult_Call) RunAndReturn(run func(context.Context, *datapb.CommitBackfillResultRequest, ...grpc.CallOption) (*datapb.CommitBackfillResultResponse, error)) *MockMixCoordClient_CommitBackfillResult_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// ComputePhraseMatchSlop provides a mock function with given fields: ctx, in, opts
|
||||
func (_m *MockMixCoordClient) ComputePhraseMatchSlop(ctx context.Context, in *querypb.ComputePhraseMatchSlopRequest, opts ...grpc.CallOption) (*querypb.ComputePhraseMatchSlopResponse, error) {
|
||||
_va := make([]interface{}, len(opts))
|
||||
|
||||
@@ -49,6 +49,10 @@ func RegisterMgrRoute(proxy *Proxy) {
|
||||
Path: management.RouteGcResume,
|
||||
HandlerFunc: proxy.ResumeDatacoordGC,
|
||||
})
|
||||
management.Register(&management.Handler{
|
||||
Path: management.RouteCommitBackfill,
|
||||
HandlerFunc: proxy.CommitBackfillResult,
|
||||
})
|
||||
management.Register(&management.Handler{
|
||||
Path: management.RouteListQueryNode,
|
||||
HandlerFunc: proxy.ListQueryNode,
|
||||
@@ -159,6 +163,58 @@ func (node *Proxy) PauseDatacoordGC(w http.ResponseWriter, req *http.Request) {
|
||||
fmt.Fprintf(w, `{"msg": "OK", "ticket": "%s"}`, ticket)
|
||||
}
|
||||
|
||||
// CommitBackfillResult is the proxy-side handler for the
|
||||
// /management/datacoord/backfill/commit endpoint. It forwards the S3 result
|
||||
// path to DataCoord.CommitBackfillResult and returns the aggregated
|
||||
// per-segment commit status as JSON.
|
||||
func (node *Proxy) CommitBackfillResult(w http.ResponseWriter, req *http.Request) {
|
||||
writeJSON := func(status int, payload map[string]interface{}) {
|
||||
w.WriteHeader(status)
|
||||
bs, _ := json.Marshal(payload)
|
||||
w.Write(bs)
|
||||
}
|
||||
|
||||
resultPath := req.URL.Query().Get("result_path")
|
||||
if resultPath == "" {
|
||||
writeJSON(http.StatusBadRequest, map[string]interface{}{
|
||||
"msg": "result_path query parameter is required",
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
resp, err := node.mixCoord.CommitBackfillResult(req.Context(), &datapb.CommitBackfillResultRequest{
|
||||
Base: commonpbutil.NewMsgBase(),
|
||||
ResultPath: resultPath,
|
||||
})
|
||||
if err != nil {
|
||||
// Use json.Marshal so an err.Error() containing quotes or control
|
||||
// characters can't break the JSON response envelope.
|
||||
writeJSON(http.StatusInternalServerError, map[string]interface{}{
|
||||
"msg": fmt.Sprintf("failed to commit backfill result, %s", err.Error()),
|
||||
})
|
||||
return
|
||||
}
|
||||
if !merr.Ok(resp.GetStatus()) {
|
||||
// Even on failure we include the per-segment diagnostics so callers can
|
||||
// see which segments tripped pre-validation.
|
||||
writeJSON(http.StatusInternalServerError, map[string]interface{}{
|
||||
"msg": fmt.Sprintf("failed to commit backfill result, %s", resp.GetStatus().GetReason()),
|
||||
"total_segments": resp.GetTotalSegments(),
|
||||
"committed_segments": resp.GetCommittedSegments(),
|
||||
"failed_segments": resp.GetFailedSegments(),
|
||||
"segment_statuses": resp.GetSegmentStatuses(),
|
||||
})
|
||||
return
|
||||
}
|
||||
writeJSON(http.StatusOK, map[string]interface{}{
|
||||
"msg": "OK",
|
||||
"total_segments": resp.GetTotalSegments(),
|
||||
"committed_segments": resp.GetCommittedSegments(),
|
||||
"failed_segments": resp.GetFailedSegments(),
|
||||
"segment_statuses": resp.GetSegmentStatuses(),
|
||||
})
|
||||
}
|
||||
|
||||
func (node *Proxy) ResumeDatacoordGC(w http.ResponseWriter, req *http.Request) {
|
||||
ticket := req.URL.Query().Get("ticket")
|
||||
var collectionID string
|
||||
|
||||
@@ -18,6 +18,7 @@ package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
gojson "encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
@@ -745,6 +746,125 @@ func (s *ProxyManagementSuite) TestCheckQueryNodeDistribution() {
|
||||
})
|
||||
}
|
||||
|
||||
func (s *ProxyManagementSuite) TestCommitBackfillResult() {
|
||||
s.Run("missing_result_path", func() {
|
||||
s.SetupTest()
|
||||
defer s.TearDownTest()
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, management.RouteCommitBackfill, nil)
|
||||
s.Require().NoError(err)
|
||||
|
||||
recorder := httptest.NewRecorder()
|
||||
s.proxy.CommitBackfillResult(recorder, req)
|
||||
|
||||
s.Equal(http.StatusBadRequest, recorder.Code)
|
||||
s.Contains(recorder.Body.String(), "result_path query parameter is required")
|
||||
})
|
||||
|
||||
s.Run("success", func() {
|
||||
s.SetupTest()
|
||||
defer s.TearDownTest()
|
||||
s.mixcoord.EXPECT().CommitBackfillResult(mock.Anything, mock.Anything).RunAndReturn(
|
||||
func(ctx context.Context, req *datapb.CommitBackfillResultRequest, options ...grpc.CallOption) (*datapb.CommitBackfillResultResponse, error) {
|
||||
s.Equal("s3a://bkt/foo.json", req.GetResultPath())
|
||||
return &datapb.CommitBackfillResultResponse{
|
||||
Status: merr.Success(),
|
||||
TotalSegments: 2,
|
||||
CommittedSegments: 2,
|
||||
SegmentStatuses: []*datapb.CommitBackfillResultSegmentStatus{
|
||||
{SegmentId: 1, Ok: true, Kind: "v3"},
|
||||
{SegmentId: 2, Ok: true, Kind: "v2"},
|
||||
},
|
||||
}, nil
|
||||
})
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet,
|
||||
management.RouteCommitBackfill+"?result_path=s3a%3A%2F%2Fbkt%2Ffoo.json", nil)
|
||||
s.Require().NoError(err)
|
||||
|
||||
recorder := httptest.NewRecorder()
|
||||
s.proxy.CommitBackfillResult(recorder, req)
|
||||
|
||||
s.Equal(http.StatusOK, recorder.Code)
|
||||
body := recorder.Body.String()
|
||||
s.Contains(body, `"msg":"OK"`)
|
||||
s.Contains(body, `"total_segments":2`)
|
||||
s.Contains(body, `"committed_segments":2`)
|
||||
})
|
||||
|
||||
s.Run("downstream_rpc_error", func() {
|
||||
s.SetupTest()
|
||||
defer s.TearDownTest()
|
||||
s.mixcoord.EXPECT().CommitBackfillResult(mock.Anything, mock.Anything).RunAndReturn(
|
||||
func(ctx context.Context, req *datapb.CommitBackfillResultRequest, options ...grpc.CallOption) (*datapb.CommitBackfillResultResponse, error) {
|
||||
return nil, errors.New("network broken")
|
||||
})
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet,
|
||||
management.RouteCommitBackfill+"?result_path=s3a%3A%2F%2Fbkt%2Ffoo.json", nil)
|
||||
s.Require().NoError(err)
|
||||
|
||||
recorder := httptest.NewRecorder()
|
||||
s.proxy.CommitBackfillResult(recorder, req)
|
||||
|
||||
s.Equal(http.StatusInternalServerError, recorder.Code)
|
||||
s.Contains(recorder.Body.String(), "network broken")
|
||||
})
|
||||
|
||||
// Error strings carrying JSON-special characters (quotes, backslashes)
|
||||
// must not break the response envelope — the body must remain valid JSON.
|
||||
s.Run("error_with_json_special_chars_stays_valid_json", func() {
|
||||
s.SetupTest()
|
||||
defer s.TearDownTest()
|
||||
s.mixcoord.EXPECT().CommitBackfillResult(mock.Anything, mock.Anything).RunAndReturn(
|
||||
func(ctx context.Context, req *datapb.CommitBackfillResultRequest, options ...grpc.CallOption) (*datapb.CommitBackfillResultResponse, error) {
|
||||
return nil, errors.New(`boom "quoted" \and\ slashed`)
|
||||
})
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet,
|
||||
management.RouteCommitBackfill+"?result_path=s3a%3A%2F%2Fbkt%2Ffoo.json", nil)
|
||||
s.Require().NoError(err)
|
||||
|
||||
recorder := httptest.NewRecorder()
|
||||
s.proxy.CommitBackfillResult(recorder, req)
|
||||
|
||||
s.Equal(http.StatusInternalServerError, recorder.Code)
|
||||
var payload map[string]interface{}
|
||||
s.Require().NoError(gojson.Unmarshal(recorder.Body.Bytes(), &payload),
|
||||
"response body must remain valid JSON, got: %s", recorder.Body.String())
|
||||
s.Contains(payload["msg"].(string), `boom "quoted" \and\ slashed`)
|
||||
})
|
||||
|
||||
s.Run("downstream_status_error", func() {
|
||||
s.SetupTest()
|
||||
defer s.TearDownTest()
|
||||
s.mixcoord.EXPECT().CommitBackfillResult(mock.Anything, mock.Anything).RunAndReturn(
|
||||
func(ctx context.Context, req *datapb.CommitBackfillResultRequest, options ...grpc.CallOption) (*datapb.CommitBackfillResultResponse, error) {
|
||||
return &datapb.CommitBackfillResultResponse{
|
||||
Status: merr.Status(merr.WrapErrParameterInvalidMsg("bad json")),
|
||||
TotalSegments: 1,
|
||||
FailedSegments: 1,
|
||||
SegmentStatuses: []*datapb.CommitBackfillResultSegmentStatus{
|
||||
{SegmentId: 1, Ok: false, Kind: "v3", Reason: "bad json"},
|
||||
},
|
||||
}, nil
|
||||
})
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet,
|
||||
management.RouteCommitBackfill+"?result_path=s3a%3A%2F%2Fbkt%2Ffoo.json", nil)
|
||||
s.Require().NoError(err)
|
||||
|
||||
recorder := httptest.NewRecorder()
|
||||
s.proxy.CommitBackfillResult(recorder, req)
|
||||
|
||||
s.Equal(http.StatusInternalServerError, recorder.Code)
|
||||
body := recorder.Body.String()
|
||||
s.Contains(body, "bad json")
|
||||
s.Contains(body, `"failed_segments":1`)
|
||||
s.Contains(body, `"segment_statuses"`)
|
||||
})
|
||||
}
|
||||
|
||||
func TestProxyManagement(t *testing.T) {
|
||||
suite.Run(t, new(ProxyManagementSuite))
|
||||
}
|
||||
|
||||
@@ -1789,6 +1789,10 @@ func (coord *MixCoordMock) BatchUpdateManifest(ctx context.Context, req *datapb.
|
||||
return merr.Success(), nil
|
||||
}
|
||||
|
||||
func (coord *MixCoordMock) CommitBackfillResult(ctx context.Context, req *datapb.CommitBackfillResultRequest, opts ...grpc.CallOption) (*datapb.CommitBackfillResultResponse, error) {
|
||||
return &datapb.CommitBackfillResultResponse{Status: merr.Success()}, nil
|
||||
}
|
||||
|
||||
type DescribeCollectionFunc func(ctx context.Context, request *milvuspb.DescribeCollectionRequest, opts ...grpc.CallOption) (*milvuspb.DescribeCollectionResponse, error)
|
||||
|
||||
type ShowPartitionsFunc func(ctx context.Context, request *milvuspb.ShowPartitionsRequest, opts ...grpc.CallOption) (*milvuspb.ShowPartitionsResponse, error)
|
||||
|
||||
@@ -112,6 +112,11 @@ func (mcm *RemoteChunkManager) RootPath() string {
|
||||
return mcm.rootPath
|
||||
}
|
||||
|
||||
// BucketName returns the bucket name this chunk manager is configured with.
|
||||
func (mcm *RemoteChunkManager) BucketName() string {
|
||||
return mcm.bucketName
|
||||
}
|
||||
|
||||
// UnderlyingObjectStorage returns the underlying object storage.
|
||||
func (mcm *RemoteChunkManager) UnderlyingObjectStorage() ObjectStorage {
|
||||
return mcm.client
|
||||
|
||||
@@ -117,6 +117,9 @@ service DataCoord {
|
||||
rpc UnpinSnapshotData(UnpinSnapshotDataRequest) returns(common.Status){}
|
||||
// batch update manifest
|
||||
rpc BatchUpdateManifest(BatchUpdateManifestRequest) returns(common.Status){}
|
||||
// commit backfill result (reads result JSON from object storage and dispatches
|
||||
// V2/V3 segment updates through the broadcaster)
|
||||
rpc CommitBackfillResult(CommitBackfillResultRequest) returns(CommitBackfillResultResponse){}
|
||||
|
||||
// External Table Refresh APIs
|
||||
rpc RefreshExternalCollection(RefreshExternalCollectionRequest) returns(RefreshExternalCollectionResponse){}
|
||||
@@ -1660,6 +1663,34 @@ message BatchUpdateManifestItem {
|
||||
int64 manifest_version = 2;
|
||||
}
|
||||
|
||||
// CommitBackfillResultRequest is sent by the proxy management HTTP endpoint to
|
||||
// hand off the path of a BackfillResult JSON file on object storage. DataCoord
|
||||
// downloads the JSON, classifies each segment entry (V2/V3), and dispatches
|
||||
// the resulting operators through the BatchUpdateManifest broadcast pipeline.
|
||||
message CommitBackfillResultRequest {
|
||||
common.MsgBase base = 1;
|
||||
// Accepted forms:
|
||||
// s3a://<bucket>/<key> (canonical form produced by Spark)
|
||||
// s3://<bucket>/<key>
|
||||
// /<key> (bucket implied)
|
||||
string result_path = 2;
|
||||
}
|
||||
|
||||
message CommitBackfillResultSegmentStatus {
|
||||
int64 segment_id = 1;
|
||||
bool ok = 2;
|
||||
string reason = 3;
|
||||
string kind = 4; // "v2" | "v3"
|
||||
}
|
||||
|
||||
message CommitBackfillResultResponse {
|
||||
common.Status status = 1;
|
||||
int32 total_segments = 2;
|
||||
int32 committed_segments = 3;
|
||||
int32 failed_segments = 4;
|
||||
repeated CommitBackfillResultSegmentStatus segment_statuses = 5;
|
||||
}
|
||||
|
||||
message PinSnapshotDataRequest {
|
||||
common.MsgBase base = 1;
|
||||
string name = 2; // snapshot name
|
||||
|
||||
+1427
-1127
File diff suppressed because it is too large
Load Diff
@@ -80,6 +80,7 @@ const (
|
||||
DataCoord_PinSnapshotData_FullMethodName = "/milvus.proto.data.DataCoord/PinSnapshotData"
|
||||
DataCoord_UnpinSnapshotData_FullMethodName = "/milvus.proto.data.DataCoord/UnpinSnapshotData"
|
||||
DataCoord_BatchUpdateManifest_FullMethodName = "/milvus.proto.data.DataCoord/BatchUpdateManifest"
|
||||
DataCoord_CommitBackfillResult_FullMethodName = "/milvus.proto.data.DataCoord/CommitBackfillResult"
|
||||
DataCoord_RefreshExternalCollection_FullMethodName = "/milvus.proto.data.DataCoord/RefreshExternalCollection"
|
||||
DataCoord_GetRefreshExternalCollectionProgress_FullMethodName = "/milvus.proto.data.DataCoord/GetRefreshExternalCollectionProgress"
|
||||
DataCoord_ListRefreshExternalCollectionJobs_FullMethodName = "/milvus.proto.data.DataCoord/ListRefreshExternalCollectionJobs"
|
||||
@@ -157,6 +158,9 @@ type DataCoordClient interface {
|
||||
UnpinSnapshotData(ctx context.Context, in *UnpinSnapshotDataRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
|
||||
// batch update manifest
|
||||
BatchUpdateManifest(ctx context.Context, in *BatchUpdateManifestRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
|
||||
// commit backfill result (reads result JSON from object storage and dispatches
|
||||
// V2/V3 segment updates through the broadcaster)
|
||||
CommitBackfillResult(ctx context.Context, in *CommitBackfillResultRequest, opts ...grpc.CallOption) (*CommitBackfillResultResponse, error)
|
||||
// External Table Refresh APIs
|
||||
RefreshExternalCollection(ctx context.Context, in *RefreshExternalCollectionRequest, opts ...grpc.CallOption) (*RefreshExternalCollectionResponse, error)
|
||||
GetRefreshExternalCollectionProgress(ctx context.Context, in *GetRefreshExternalCollectionProgressRequest, opts ...grpc.CallOption) (*GetRefreshExternalCollectionProgressResponse, error)
|
||||
@@ -686,6 +690,15 @@ func (c *dataCoordClient) BatchUpdateManifest(ctx context.Context, in *BatchUpda
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *dataCoordClient) CommitBackfillResult(ctx context.Context, in *CommitBackfillResultRequest, opts ...grpc.CallOption) (*CommitBackfillResultResponse, error) {
|
||||
out := new(CommitBackfillResultResponse)
|
||||
err := c.cc.Invoke(ctx, DataCoord_CommitBackfillResult_FullMethodName, in, out, opts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *dataCoordClient) RefreshExternalCollection(ctx context.Context, in *RefreshExternalCollectionRequest, opts ...grpc.CallOption) (*RefreshExternalCollectionResponse, error) {
|
||||
out := new(RefreshExternalCollectionResponse)
|
||||
err := c.cc.Invoke(ctx, DataCoord_RefreshExternalCollection_FullMethodName, in, out, opts...)
|
||||
@@ -785,6 +798,9 @@ type DataCoordServer interface {
|
||||
UnpinSnapshotData(context.Context, *UnpinSnapshotDataRequest) (*commonpb.Status, error)
|
||||
// batch update manifest
|
||||
BatchUpdateManifest(context.Context, *BatchUpdateManifestRequest) (*commonpb.Status, error)
|
||||
// commit backfill result (reads result JSON from object storage and dispatches
|
||||
// V2/V3 segment updates through the broadcaster)
|
||||
CommitBackfillResult(context.Context, *CommitBackfillResultRequest) (*CommitBackfillResultResponse, error)
|
||||
// External Table Refresh APIs
|
||||
RefreshExternalCollection(context.Context, *RefreshExternalCollectionRequest) (*RefreshExternalCollectionResponse, error)
|
||||
GetRefreshExternalCollectionProgress(context.Context, *GetRefreshExternalCollectionProgressRequest) (*GetRefreshExternalCollectionProgressResponse, error)
|
||||
@@ -966,6 +982,9 @@ func (UnimplementedDataCoordServer) UnpinSnapshotData(context.Context, *UnpinSna
|
||||
func (UnimplementedDataCoordServer) BatchUpdateManifest(context.Context, *BatchUpdateManifestRequest) (*commonpb.Status, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method BatchUpdateManifest not implemented")
|
||||
}
|
||||
func (UnimplementedDataCoordServer) CommitBackfillResult(context.Context, *CommitBackfillResultRequest) (*CommitBackfillResultResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method CommitBackfillResult not implemented")
|
||||
}
|
||||
func (UnimplementedDataCoordServer) RefreshExternalCollection(context.Context, *RefreshExternalCollectionRequest) (*RefreshExternalCollectionResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method RefreshExternalCollection not implemented")
|
||||
}
|
||||
@@ -2013,6 +2032,24 @@ func _DataCoord_BatchUpdateManifest_Handler(srv interface{}, ctx context.Context
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func _DataCoord_CommitBackfillResult_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(CommitBackfillResultRequest)
|
||||
if err := dec(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if interceptor == nil {
|
||||
return srv.(DataCoordServer).CommitBackfillResult(ctx, in)
|
||||
}
|
||||
info := &grpc.UnaryServerInfo{
|
||||
Server: srv,
|
||||
FullMethod: DataCoord_CommitBackfillResult_FullMethodName,
|
||||
}
|
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||
return srv.(DataCoordServer).CommitBackfillResult(ctx, req.(*CommitBackfillResultRequest))
|
||||
}
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func _DataCoord_RefreshExternalCollection_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(RefreshExternalCollectionRequest)
|
||||
if err := dec(in); err != nil {
|
||||
@@ -2302,6 +2339,10 @@ var DataCoord_ServiceDesc = grpc.ServiceDesc{
|
||||
MethodName: "BatchUpdateManifest",
|
||||
Handler: _DataCoord_BatchUpdateManifest_Handler,
|
||||
},
|
||||
{
|
||||
MethodName: "CommitBackfillResult",
|
||||
Handler: _DataCoord_CommitBackfillResult_Handler,
|
||||
},
|
||||
{
|
||||
MethodName: "RefreshExternalCollection",
|
||||
Handler: _DataCoord_RefreshExternalCollection_Handler,
|
||||
|
||||
@@ -788,8 +788,19 @@ message BatchUpdateManifestMessageBody {
|
||||
}
|
||||
|
||||
// BatchUpdateManifestItem is an item in batch update manifest message.
|
||||
// Either manifest_version (V3 segments) or v2_column_groups (V2 segments) is
|
||||
// populated. Items with both fields set are rejected by the callback.
|
||||
message BatchUpdateManifestItem {
|
||||
int64 segment_id = 1;
|
||||
// V3 path: advance the segment's manifest pointer to this version.
|
||||
int64 manifest_version = 2;
|
||||
// V2 path: upsert one or more column groups on the segment's FieldBinlogs.
|
||||
BatchUpdateManifestV2ColumnGroups v2_column_groups = 3;
|
||||
}
|
||||
|
||||
// BatchUpdateManifestV2ColumnGroups carries a V2 segment column-group upsert.
|
||||
// Keyed by the top-level fieldID of the group; value.FieldID must equal the key.
|
||||
message BatchUpdateManifestV2ColumnGroups {
|
||||
map<int64, data.FieldBinlog> column_groups = 1;
|
||||
}
|
||||
|
||||
|
||||
+257
-156
@@ -6316,13 +6316,18 @@ func (x *BatchUpdateManifestMessageBody) GetItems() []*BatchUpdateManifestItem {
|
||||
}
|
||||
|
||||
// BatchUpdateManifestItem is an item in batch update manifest message.
|
||||
// Either manifest_version (V3 segments) or v2_column_groups (V2 segments) is
|
||||
// populated. Items with both fields set are rejected by the callback.
|
||||
type BatchUpdateManifestItem struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
unknownFields protoimpl.UnknownFields
|
||||
|
||||
SegmentId int64 `protobuf:"varint,1,opt,name=segment_id,json=segmentId,proto3" json:"segment_id,omitempty"`
|
||||
SegmentId int64 `protobuf:"varint,1,opt,name=segment_id,json=segmentId,proto3" json:"segment_id,omitempty"`
|
||||
// V3 path: advance the segment's manifest pointer to this version.
|
||||
ManifestVersion int64 `protobuf:"varint,2,opt,name=manifest_version,json=manifestVersion,proto3" json:"manifest_version,omitempty"`
|
||||
// V2 path: upsert one or more column groups on the segment's FieldBinlogs.
|
||||
V2ColumnGroups *BatchUpdateManifestV2ColumnGroups `protobuf:"bytes,3,opt,name=v2_column_groups,json=v2ColumnGroups,proto3" json:"v2_column_groups,omitempty"`
|
||||
}
|
||||
|
||||
func (x *BatchUpdateManifestItem) Reset() {
|
||||
@@ -6371,6 +6376,62 @@ func (x *BatchUpdateManifestItem) GetManifestVersion() int64 {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *BatchUpdateManifestItem) GetV2ColumnGroups() *BatchUpdateManifestV2ColumnGroups {
|
||||
if x != nil {
|
||||
return x.V2ColumnGroups
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// BatchUpdateManifestV2ColumnGroups carries a V2 segment column-group upsert.
|
||||
// Keyed by the top-level fieldID of the group; value.FieldID must equal the key.
|
||||
type BatchUpdateManifestV2ColumnGroups struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
unknownFields protoimpl.UnknownFields
|
||||
|
||||
ColumnGroups map[int64]*datapb.FieldBinlog `protobuf:"bytes,1,rep,name=column_groups,json=columnGroups,proto3" json:"column_groups,omitempty" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
|
||||
}
|
||||
|
||||
func (x *BatchUpdateManifestV2ColumnGroups) Reset() {
|
||||
*x = BatchUpdateManifestV2ColumnGroups{}
|
||||
if protoimpl.UnsafeEnabled {
|
||||
mi := &file_messages_proto_msgTypes[114]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
}
|
||||
|
||||
func (x *BatchUpdateManifestV2ColumnGroups) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*BatchUpdateManifestV2ColumnGroups) ProtoMessage() {}
|
||||
|
||||
func (x *BatchUpdateManifestV2ColumnGroups) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_messages_proto_msgTypes[114]
|
||||
if protoimpl.UnsafeEnabled && x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use BatchUpdateManifestV2ColumnGroups.ProtoReflect.Descriptor instead.
|
||||
func (*BatchUpdateManifestV2ColumnGroups) Descriptor() ([]byte, []int) {
|
||||
return file_messages_proto_rawDescGZIP(), []int{114}
|
||||
}
|
||||
|
||||
func (x *BatchUpdateManifestV2ColumnGroups) GetColumnGroups() map[int64]*datapb.FieldBinlog {
|
||||
if x != nil {
|
||||
return x.ColumnGroups
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var File_messages_proto protoreflect.FileDescriptor
|
||||
|
||||
var file_messages_proto_rawDesc = []byte{
|
||||
@@ -7093,99 +7154,121 @@ var file_messages_proto_rawDesc = []byte{
|
||||
0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x6d, 0x65,
|
||||
0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x42, 0x61, 0x74, 0x63, 0x68, 0x55, 0x70, 0x64, 0x61,
|
||||
0x74, 0x65, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x52, 0x05,
|
||||
0x69, 0x74, 0x65, 0x6d, 0x73, 0x22, 0x63, 0x0a, 0x17, 0x42, 0x61, 0x74, 0x63, 0x68, 0x55, 0x70,
|
||||
0x64, 0x61, 0x74, 0x65, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x49, 0x74, 0x65, 0x6d,
|
||||
0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01,
|
||||
0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x73, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12,
|
||||
0x29, 0x0a, 0x10, 0x6d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x5f, 0x76, 0x65, 0x72, 0x73,
|
||||
0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0f, 0x6d, 0x61, 0x6e, 0x69, 0x66,
|
||||
0x65, 0x73, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x2a, 0xbd, 0x07, 0x0a, 0x0b, 0x4d,
|
||||
0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x6e,
|
||||
0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x54, 0x69, 0x6d, 0x65, 0x54,
|
||||
0x69, 0x63, 0x6b, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x49, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x10,
|
||||
0x02, 0x12, 0x0a, 0x0a, 0x06, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x10, 0x03, 0x12, 0x09, 0x0a,
|
||||
0x05, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x10, 0x04, 0x12, 0x14, 0x0a, 0x10, 0x43, 0x72, 0x65, 0x61,
|
||||
0x74, 0x65, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x10, 0x05, 0x12, 0x12,
|
||||
0x0a, 0x0e, 0x44, 0x72, 0x6f, 0x70, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e,
|
||||
0x10, 0x06, 0x12, 0x13, 0x0a, 0x0f, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, 0x61, 0x72, 0x74,
|
||||
0x69, 0x74, 0x69, 0x6f, 0x6e, 0x10, 0x07, 0x12, 0x11, 0x0a, 0x0d, 0x44, 0x72, 0x6f, 0x70, 0x50,
|
||||
0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x10, 0x08, 0x12, 0x0f, 0x0a, 0x0b, 0x4d, 0x61,
|
||||
0x6e, 0x75, 0x61, 0x6c, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x10, 0x09, 0x12, 0x11, 0x0a, 0x0d, 0x43,
|
||||
0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x10, 0x0a, 0x12, 0x0a,
|
||||
0x0a, 0x06, 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x10, 0x0b, 0x12, 0x14, 0x0a, 0x0c, 0x53, 0x63,
|
||||
0x68, 0x65, 0x6d, 0x61, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x10, 0x0c, 0x1a, 0x02, 0x08, 0x01,
|
||||
0x12, 0x13, 0x0a, 0x0f, 0x41, 0x6c, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74,
|
||||
0x69, 0x6f, 0x6e, 0x10, 0x0d, 0x12, 0x13, 0x0a, 0x0f, 0x41, 0x6c, 0x74, 0x65, 0x72, 0x4c, 0x6f,
|
||||
0x61, 0x64, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x10, 0x0e, 0x12, 0x12, 0x0a, 0x0e, 0x44, 0x72,
|
||||
0x6f, 0x70, 0x4c, 0x6f, 0x61, 0x64, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x10, 0x0f, 0x12, 0x12,
|
||||
0x0a, 0x0e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x44, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65,
|
||||
0x10, 0x10, 0x12, 0x11, 0x0a, 0x0d, 0x41, 0x6c, 0x74, 0x65, 0x72, 0x44, 0x61, 0x74, 0x61, 0x62,
|
||||
0x61, 0x73, 0x65, 0x10, 0x11, 0x12, 0x10, 0x0a, 0x0c, 0x44, 0x72, 0x6f, 0x70, 0x44, 0x61, 0x74,
|
||||
0x61, 0x62, 0x61, 0x73, 0x65, 0x10, 0x12, 0x12, 0x0e, 0x0a, 0x0a, 0x41, 0x6c, 0x74, 0x65, 0x72,
|
||||
0x41, 0x6c, 0x69, 0x61, 0x73, 0x10, 0x13, 0x12, 0x0d, 0x0a, 0x09, 0x44, 0x72, 0x6f, 0x70, 0x41,
|
||||
0x6c, 0x69, 0x61, 0x73, 0x10, 0x14, 0x12, 0x0f, 0x0a, 0x0b, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72,
|
||||
0x65, 0x52, 0x42, 0x41, 0x43, 0x10, 0x15, 0x12, 0x0d, 0x0a, 0x09, 0x41, 0x6c, 0x74, 0x65, 0x72,
|
||||
0x55, 0x73, 0x65, 0x72, 0x10, 0x16, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x72, 0x6f, 0x70, 0x55, 0x73,
|
||||
0x65, 0x72, 0x10, 0x17, 0x12, 0x0d, 0x0a, 0x09, 0x41, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x6f, 0x6c,
|
||||
0x65, 0x10, 0x18, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x72, 0x6f, 0x70, 0x52, 0x6f, 0x6c, 0x65, 0x10,
|
||||
0x19, 0x12, 0x11, 0x0a, 0x0d, 0x41, 0x6c, 0x74, 0x65, 0x72, 0x55, 0x73, 0x65, 0x72, 0x52, 0x6f,
|
||||
0x6c, 0x65, 0x10, 0x1a, 0x12, 0x10, 0x0a, 0x0c, 0x44, 0x72, 0x6f, 0x70, 0x55, 0x73, 0x65, 0x72,
|
||||
0x52, 0x6f, 0x6c, 0x65, 0x10, 0x1b, 0x12, 0x12, 0x0a, 0x0e, 0x41, 0x6c, 0x74, 0x65, 0x72, 0x50,
|
||||
0x72, 0x69, 0x76, 0x69, 0x6c, 0x65, 0x67, 0x65, 0x10, 0x1c, 0x12, 0x11, 0x0a, 0x0d, 0x44, 0x72,
|
||||
0x6f, 0x70, 0x50, 0x72, 0x69, 0x76, 0x69, 0x6c, 0x65, 0x67, 0x65, 0x10, 0x1d, 0x12, 0x17, 0x0a,
|
||||
0x13, 0x41, 0x6c, 0x74, 0x65, 0x72, 0x50, 0x72, 0x69, 0x76, 0x69, 0x6c, 0x65, 0x67, 0x65, 0x47,
|
||||
0x72, 0x6f, 0x75, 0x70, 0x10, 0x1e, 0x12, 0x16, 0x0a, 0x12, 0x44, 0x72, 0x6f, 0x70, 0x50, 0x72,
|
||||
0x69, 0x76, 0x69, 0x6c, 0x65, 0x67, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x10, 0x1f, 0x12, 0x16,
|
||||
0x0a, 0x12, 0x41, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x47,
|
||||
0x72, 0x6f, 0x75, 0x70, 0x10, 0x20, 0x12, 0x15, 0x0a, 0x11, 0x44, 0x72, 0x6f, 0x70, 0x52, 0x65,
|
||||
0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x10, 0x21, 0x12, 0x0f, 0x0a,
|
||||
0x0b, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x10, 0x22, 0x12, 0x0e,
|
||||
0x0a, 0x0a, 0x41, 0x6c, 0x74, 0x65, 0x72, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x10, 0x23, 0x12, 0x0d,
|
||||
0x0a, 0x09, 0x44, 0x72, 0x6f, 0x70, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x10, 0x24, 0x12, 0x0c, 0x0a,
|
||||
0x08, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x41, 0x6c, 0x6c, 0x10, 0x25, 0x12, 0x16, 0x0a, 0x12, 0x54,
|
||||
0x72, 0x75, 0x6e, 0x63, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f,
|
||||
0x6e, 0x10, 0x26, 0x12, 0x13, 0x0a, 0x0f, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x53, 0x6e,
|
||||
0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x10, 0x27, 0x12, 0x12, 0x0a, 0x0e, 0x43, 0x72, 0x65, 0x61,
|
||||
0x74, 0x65, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x10, 0x28, 0x12, 0x10, 0x0a, 0x0c,
|
||||
0x44, 0x72, 0x6f, 0x70, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x10, 0x29, 0x12, 0x17,
|
||||
0x0a, 0x13, 0x42, 0x61, 0x74, 0x63, 0x68, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4d, 0x61, 0x6e,
|
||||
0x69, 0x66, 0x65, 0x73, 0x74, 0x10, 0x2a, 0x12, 0x1d, 0x0a, 0x19, 0x52, 0x65, 0x66, 0x72, 0x65,
|
||||
0x73, 0x68, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63,
|
||||
0x74, 0x69, 0x6f, 0x6e, 0x10, 0x2b, 0x12, 0x1d, 0x0a, 0x19, 0x44, 0x72, 0x6f, 0x70, 0x53, 0x6e,
|
||||
0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x73, 0x42, 0x79, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74,
|
||||
0x69, 0x6f, 0x6e, 0x10, 0x2c, 0x12, 0x0d, 0x0a, 0x08, 0x41, 0x6c, 0x74, 0x65, 0x72, 0x57, 0x41,
|
||||
0x4c, 0x10, 0xbc, 0x05, 0x12, 0x19, 0x0a, 0x14, 0x41, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x65, 0x70,
|
||||
0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x10, 0xa0, 0x06, 0x12,
|
||||
0x0d, 0x0a, 0x08, 0x42, 0x65, 0x67, 0x69, 0x6e, 0x54, 0x78, 0x6e, 0x10, 0x84, 0x07, 0x12, 0x0e,
|
||||
0x0a, 0x09, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x54, 0x78, 0x6e, 0x10, 0x85, 0x07, 0x12, 0x10,
|
||||
0x0a, 0x0b, 0x52, 0x6f, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x54, 0x78, 0x6e, 0x10, 0x86, 0x07,
|
||||
0x12, 0x08, 0x0a, 0x03, 0x54, 0x78, 0x6e, 0x10, 0xe7, 0x07, 0x2a, 0x74, 0x0a, 0x08, 0x54, 0x78,
|
||||
0x6e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0e, 0x0a, 0x0a, 0x54, 0x78, 0x6e, 0x55, 0x6e, 0x6b,
|
||||
0x6e, 0x6f, 0x77, 0x6e, 0x10, 0x00, 0x12, 0x0f, 0x0a, 0x0b, 0x54, 0x78, 0x6e, 0x49, 0x6e, 0x46,
|
||||
0x6c, 0x69, 0x67, 0x68, 0x74, 0x10, 0x01, 0x12, 0x0f, 0x0a, 0x0b, 0x54, 0x78, 0x6e, 0x4f, 0x6e,
|
||||
0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x10, 0x02, 0x12, 0x10, 0x0a, 0x0c, 0x54, 0x78, 0x6e, 0x43,
|
||||
0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x74, 0x65, 0x64, 0x10, 0x03, 0x12, 0x11, 0x0a, 0x0d, 0x54, 0x78,
|
||||
0x6e, 0x4f, 0x6e, 0x52, 0x6f, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x10, 0x04, 0x12, 0x11, 0x0a,
|
||||
0x0d, 0x54, 0x78, 0x6e, 0x52, 0x6f, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x64, 0x10, 0x05,
|
||||
0x2a, 0xe2, 0x01, 0x0a, 0x0e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x44, 0x6f, 0x6d,
|
||||
0x61, 0x69, 0x6e, 0x12, 0x19, 0x0a, 0x15, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x44,
|
||||
0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x55, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x10, 0x00, 0x12, 0x21,
|
||||
0x0a, 0x19, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e,
|
||||
0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x4a, 0x6f, 0x62, 0x49, 0x44, 0x10, 0x01, 0x1a, 0x02, 0x08,
|
||||
0x01, 0x12, 0x20, 0x0a, 0x1c, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x44, 0x6f, 0x6d,
|
||||
0x61, 0x69, 0x6e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4e, 0x61, 0x6d,
|
||||
0x65, 0x10, 0x02, 0x12, 0x18, 0x0a, 0x14, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x44,
|
||||
0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x44, 0x42, 0x4e, 0x61, 0x6d, 0x65, 0x10, 0x03, 0x12, 0x1b, 0x0a,
|
||||
0x17, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x50,
|
||||
0x72, 0x69, 0x76, 0x69, 0x6c, 0x65, 0x67, 0x65, 0x10, 0x04, 0x12, 0x1e, 0x0a, 0x1a, 0x52, 0x65,
|
||||
0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x53, 0x6e, 0x61, 0x70,
|
||||
0x73, 0x68, 0x6f, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x10, 0x05, 0x12, 0x19, 0x0a, 0x15, 0x52, 0x65,
|
||||
0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x43, 0x6c, 0x75, 0x73,
|
||||
0x74, 0x65, 0x72, 0x10, 0x7f, 0x42, 0x35, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e,
|
||||
0x63, 0x6f, 0x6d, 0x2f, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2d, 0x69, 0x6f, 0x2f, 0x6d, 0x69,
|
||||
0x6c, 0x76, 0x75, 0x73, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x76, 0x32, 0x2f, 0x70, 0x72, 0x6f, 0x74,
|
||||
0x6f, 0x2f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72,
|
||||
0x6f, 0x74, 0x6f, 0x33,
|
||||
0x69, 0x74, 0x65, 0x6d, 0x73, 0x22, 0xc7, 0x01, 0x0a, 0x17, 0x42, 0x61, 0x74, 0x63, 0x68, 0x55,
|
||||
0x70, 0x64, 0x61, 0x74, 0x65, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x49, 0x74, 0x65,
|
||||
0x6d, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18,
|
||||
0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x73, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x49, 0x64,
|
||||
0x12, 0x29, 0x0a, 0x10, 0x6d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x5f, 0x76, 0x65, 0x72,
|
||||
0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0f, 0x6d, 0x61, 0x6e, 0x69,
|
||||
0x66, 0x65, 0x73, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x62, 0x0a, 0x10, 0x76,
|
||||
0x32, 0x5f, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x18,
|
||||
0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x38, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70,
|
||||
0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x42, 0x61,
|
||||
0x74, 0x63, 0x68, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73,
|
||||
0x74, 0x56, 0x32, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x52,
|
||||
0x0e, 0x76, 0x32, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x22,
|
||||
0xf5, 0x01, 0x0a, 0x21, 0x42, 0x61, 0x74, 0x63, 0x68, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4d,
|
||||
0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x56, 0x32, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x47,
|
||||
0x72, 0x6f, 0x75, 0x70, 0x73, 0x12, 0x6f, 0x0a, 0x0d, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x5f,
|
||||
0x67, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x4a, 0x2e, 0x6d,
|
||||
0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x6d, 0x65, 0x73, 0x73,
|
||||
0x61, 0x67, 0x65, 0x73, 0x2e, 0x42, 0x61, 0x74, 0x63, 0x68, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65,
|
||||
0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x56, 0x32, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e,
|
||||
0x47, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x2e, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x47, 0x72, 0x6f,
|
||||
0x75, 0x70, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0c, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e,
|
||||
0x47, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x1a, 0x5f, 0x0a, 0x11, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e,
|
||||
0x47, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b,
|
||||
0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x34, 0x0a,
|
||||
0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6d,
|
||||
0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x64, 0x61, 0x74, 0x61,
|
||||
0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x42, 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x52, 0x05, 0x76, 0x61,
|
||||
0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x2a, 0xbd, 0x07, 0x0a, 0x0b, 0x4d, 0x65, 0x73, 0x73,
|
||||
0x61, 0x67, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x6e, 0x6b, 0x6e, 0x6f,
|
||||
0x77, 0x6e, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x54, 0x69, 0x6d, 0x65, 0x54, 0x69, 0x63, 0x6b,
|
||||
0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x49, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x10, 0x02, 0x12, 0x0a,
|
||||
0x0a, 0x06, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x10, 0x03, 0x12, 0x09, 0x0a, 0x05, 0x46, 0x6c,
|
||||
0x75, 0x73, 0x68, 0x10, 0x04, 0x12, 0x14, 0x0a, 0x10, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x43,
|
||||
0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x10, 0x05, 0x12, 0x12, 0x0a, 0x0e, 0x44,
|
||||
0x72, 0x6f, 0x70, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x10, 0x06, 0x12,
|
||||
0x13, 0x0a, 0x0f, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69,
|
||||
0x6f, 0x6e, 0x10, 0x07, 0x12, 0x11, 0x0a, 0x0d, 0x44, 0x72, 0x6f, 0x70, 0x50, 0x61, 0x72, 0x74,
|
||||
0x69, 0x74, 0x69, 0x6f, 0x6e, 0x10, 0x08, 0x12, 0x0f, 0x0a, 0x0b, 0x4d, 0x61, 0x6e, 0x75, 0x61,
|
||||
0x6c, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x10, 0x09, 0x12, 0x11, 0x0a, 0x0d, 0x43, 0x72, 0x65, 0x61,
|
||||
0x74, 0x65, 0x53, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x10, 0x0a, 0x12, 0x0a, 0x0a, 0x06, 0x49,
|
||||
0x6d, 0x70, 0x6f, 0x72, 0x74, 0x10, 0x0b, 0x12, 0x14, 0x0a, 0x0c, 0x53, 0x63, 0x68, 0x65, 0x6d,
|
||||
0x61, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x10, 0x0c, 0x1a, 0x02, 0x08, 0x01, 0x12, 0x13, 0x0a,
|
||||
0x0f, 0x41, 0x6c, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e,
|
||||
0x10, 0x0d, 0x12, 0x13, 0x0a, 0x0f, 0x41, 0x6c, 0x74, 0x65, 0x72, 0x4c, 0x6f, 0x61, 0x64, 0x43,
|
||||
0x6f, 0x6e, 0x66, 0x69, 0x67, 0x10, 0x0e, 0x12, 0x12, 0x0a, 0x0e, 0x44, 0x72, 0x6f, 0x70, 0x4c,
|
||||
0x6f, 0x61, 0x64, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x10, 0x0f, 0x12, 0x12, 0x0a, 0x0e, 0x43,
|
||||
0x72, 0x65, 0x61, 0x74, 0x65, 0x44, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, 0x10, 0x10, 0x12,
|
||||
0x11, 0x0a, 0x0d, 0x41, 0x6c, 0x74, 0x65, 0x72, 0x44, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65,
|
||||
0x10, 0x11, 0x12, 0x10, 0x0a, 0x0c, 0x44, 0x72, 0x6f, 0x70, 0x44, 0x61, 0x74, 0x61, 0x62, 0x61,
|
||||
0x73, 0x65, 0x10, 0x12, 0x12, 0x0e, 0x0a, 0x0a, 0x41, 0x6c, 0x74, 0x65, 0x72, 0x41, 0x6c, 0x69,
|
||||
0x61, 0x73, 0x10, 0x13, 0x12, 0x0d, 0x0a, 0x09, 0x44, 0x72, 0x6f, 0x70, 0x41, 0x6c, 0x69, 0x61,
|
||||
0x73, 0x10, 0x14, 0x12, 0x0f, 0x0a, 0x0b, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x52, 0x42,
|
||||
0x41, 0x43, 0x10, 0x15, 0x12, 0x0d, 0x0a, 0x09, 0x41, 0x6c, 0x74, 0x65, 0x72, 0x55, 0x73, 0x65,
|
||||
0x72, 0x10, 0x16, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x72, 0x6f, 0x70, 0x55, 0x73, 0x65, 0x72, 0x10,
|
||||
0x17, 0x12, 0x0d, 0x0a, 0x09, 0x41, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x6f, 0x6c, 0x65, 0x10, 0x18,
|
||||
0x12, 0x0c, 0x0a, 0x08, 0x44, 0x72, 0x6f, 0x70, 0x52, 0x6f, 0x6c, 0x65, 0x10, 0x19, 0x12, 0x11,
|
||||
0x0a, 0x0d, 0x41, 0x6c, 0x74, 0x65, 0x72, 0x55, 0x73, 0x65, 0x72, 0x52, 0x6f, 0x6c, 0x65, 0x10,
|
||||
0x1a, 0x12, 0x10, 0x0a, 0x0c, 0x44, 0x72, 0x6f, 0x70, 0x55, 0x73, 0x65, 0x72, 0x52, 0x6f, 0x6c,
|
||||
0x65, 0x10, 0x1b, 0x12, 0x12, 0x0a, 0x0e, 0x41, 0x6c, 0x74, 0x65, 0x72, 0x50, 0x72, 0x69, 0x76,
|
||||
0x69, 0x6c, 0x65, 0x67, 0x65, 0x10, 0x1c, 0x12, 0x11, 0x0a, 0x0d, 0x44, 0x72, 0x6f, 0x70, 0x50,
|
||||
0x72, 0x69, 0x76, 0x69, 0x6c, 0x65, 0x67, 0x65, 0x10, 0x1d, 0x12, 0x17, 0x0a, 0x13, 0x41, 0x6c,
|
||||
0x74, 0x65, 0x72, 0x50, 0x72, 0x69, 0x76, 0x69, 0x6c, 0x65, 0x67, 0x65, 0x47, 0x72, 0x6f, 0x75,
|
||||
0x70, 0x10, 0x1e, 0x12, 0x16, 0x0a, 0x12, 0x44, 0x72, 0x6f, 0x70, 0x50, 0x72, 0x69, 0x76, 0x69,
|
||||
0x6c, 0x65, 0x67, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x10, 0x1f, 0x12, 0x16, 0x0a, 0x12, 0x41,
|
||||
0x6c, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x47, 0x72, 0x6f, 0x75,
|
||||
0x70, 0x10, 0x20, 0x12, 0x15, 0x0a, 0x11, 0x44, 0x72, 0x6f, 0x70, 0x52, 0x65, 0x73, 0x6f, 0x75,
|
||||
0x72, 0x63, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x10, 0x21, 0x12, 0x0f, 0x0a, 0x0b, 0x43, 0x72,
|
||||
0x65, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x10, 0x22, 0x12, 0x0e, 0x0a, 0x0a, 0x41,
|
||||
0x6c, 0x74, 0x65, 0x72, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x10, 0x23, 0x12, 0x0d, 0x0a, 0x09, 0x44,
|
||||
0x72, 0x6f, 0x70, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x10, 0x24, 0x12, 0x0c, 0x0a, 0x08, 0x46, 0x6c,
|
||||
0x75, 0x73, 0x68, 0x41, 0x6c, 0x6c, 0x10, 0x25, 0x12, 0x16, 0x0a, 0x12, 0x54, 0x72, 0x75, 0x6e,
|
||||
0x63, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x10, 0x26,
|
||||
0x12, 0x13, 0x0a, 0x0f, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x53, 0x6e, 0x61, 0x70, 0x73,
|
||||
0x68, 0x6f, 0x74, 0x10, 0x27, 0x12, 0x12, 0x0a, 0x0e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53,
|
||||
0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x10, 0x28, 0x12, 0x10, 0x0a, 0x0c, 0x44, 0x72, 0x6f,
|
||||
0x70, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x10, 0x29, 0x12, 0x17, 0x0a, 0x13, 0x42,
|
||||
0x61, 0x74, 0x63, 0x68, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65,
|
||||
0x73, 0x74, 0x10, 0x2a, 0x12, 0x1d, 0x0a, 0x19, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x45,
|
||||
0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f,
|
||||
0x6e, 0x10, 0x2b, 0x12, 0x1d, 0x0a, 0x19, 0x44, 0x72, 0x6f, 0x70, 0x53, 0x6e, 0x61, 0x70, 0x73,
|
||||
0x68, 0x6f, 0x74, 0x73, 0x42, 0x79, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e,
|
||||
0x10, 0x2c, 0x12, 0x0d, 0x0a, 0x08, 0x41, 0x6c, 0x74, 0x65, 0x72, 0x57, 0x41, 0x4c, 0x10, 0xbc,
|
||||
0x05, 0x12, 0x19, 0x0a, 0x14, 0x41, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63,
|
||||
0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x10, 0xa0, 0x06, 0x12, 0x0d, 0x0a, 0x08,
|
||||
0x42, 0x65, 0x67, 0x69, 0x6e, 0x54, 0x78, 0x6e, 0x10, 0x84, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x43,
|
||||
0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x54, 0x78, 0x6e, 0x10, 0x85, 0x07, 0x12, 0x10, 0x0a, 0x0b, 0x52,
|
||||
0x6f, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x54, 0x78, 0x6e, 0x10, 0x86, 0x07, 0x12, 0x08, 0x0a,
|
||||
0x03, 0x54, 0x78, 0x6e, 0x10, 0xe7, 0x07, 0x2a, 0x74, 0x0a, 0x08, 0x54, 0x78, 0x6e, 0x53, 0x74,
|
||||
0x61, 0x74, 0x65, 0x12, 0x0e, 0x0a, 0x0a, 0x54, 0x78, 0x6e, 0x55, 0x6e, 0x6b, 0x6e, 0x6f, 0x77,
|
||||
0x6e, 0x10, 0x00, 0x12, 0x0f, 0x0a, 0x0b, 0x54, 0x78, 0x6e, 0x49, 0x6e, 0x46, 0x6c, 0x69, 0x67,
|
||||
0x68, 0x74, 0x10, 0x01, 0x12, 0x0f, 0x0a, 0x0b, 0x54, 0x78, 0x6e, 0x4f, 0x6e, 0x43, 0x6f, 0x6d,
|
||||
0x6d, 0x69, 0x74, 0x10, 0x02, 0x12, 0x10, 0x0a, 0x0c, 0x54, 0x78, 0x6e, 0x43, 0x6f, 0x6d, 0x6d,
|
||||
0x69, 0x74, 0x74, 0x65, 0x64, 0x10, 0x03, 0x12, 0x11, 0x0a, 0x0d, 0x54, 0x78, 0x6e, 0x4f, 0x6e,
|
||||
0x52, 0x6f, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x10, 0x04, 0x12, 0x11, 0x0a, 0x0d, 0x54, 0x78,
|
||||
0x6e, 0x52, 0x6f, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x64, 0x10, 0x05, 0x2a, 0xe2, 0x01,
|
||||
0x0a, 0x0e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e,
|
||||
0x12, 0x19, 0x0a, 0x15, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x44, 0x6f, 0x6d, 0x61,
|
||||
0x69, 0x6e, 0x55, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x10, 0x00, 0x12, 0x21, 0x0a, 0x19, 0x52,
|
||||
0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x49, 0x6d, 0x70,
|
||||
0x6f, 0x72, 0x74, 0x4a, 0x6f, 0x62, 0x49, 0x44, 0x10, 0x01, 0x1a, 0x02, 0x08, 0x01, 0x12, 0x20,
|
||||
0x0a, 0x1c, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e,
|
||||
0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x10, 0x02,
|
||||
0x12, 0x18, 0x0a, 0x14, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x44, 0x6f, 0x6d, 0x61,
|
||||
0x69, 0x6e, 0x44, 0x42, 0x4e, 0x61, 0x6d, 0x65, 0x10, 0x03, 0x12, 0x1b, 0x0a, 0x17, 0x52, 0x65,
|
||||
0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x50, 0x72, 0x69, 0x76,
|
||||
0x69, 0x6c, 0x65, 0x67, 0x65, 0x10, 0x04, 0x12, 0x1e, 0x0a, 0x1a, 0x52, 0x65, 0x73, 0x6f, 0x75,
|
||||
0x72, 0x63, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f,
|
||||
0x74, 0x4e, 0x61, 0x6d, 0x65, 0x10, 0x05, 0x12, 0x19, 0x0a, 0x15, 0x52, 0x65, 0x73, 0x6f, 0x75,
|
||||
0x72, 0x63, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72,
|
||||
0x10, 0x7f, 0x42, 0x35, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d,
|
||||
0x2f, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2d, 0x69, 0x6f, 0x2f, 0x6d, 0x69, 0x6c, 0x76, 0x75,
|
||||
0x73, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x76, 0x32, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x6d,
|
||||
0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f,
|
||||
0x33,
|
||||
}
|
||||
|
||||
var (
|
||||
@@ -7201,7 +7284,7 @@ func file_messages_proto_rawDescGZIP() []byte {
|
||||
}
|
||||
|
||||
var file_messages_proto_enumTypes = make([]protoimpl.EnumInfo, 3)
|
||||
var file_messages_proto_msgTypes = make([]protoimpl.MessageInfo, 118)
|
||||
var file_messages_proto_msgTypes = make([]protoimpl.MessageInfo, 120)
|
||||
var file_messages_proto_goTypes = []interface{}{
|
||||
(MessageType)(0), // 0: milvus.proto.messages.MessageType
|
||||
(TxnState)(0), // 1: milvus.proto.messages.TxnState
|
||||
@@ -7320,84 +7403,90 @@ var file_messages_proto_goTypes = []interface{}{
|
||||
(*BatchUpdateManifestMessageHeader)(nil), // 114: milvus.proto.messages.BatchUpdateManifestMessageHeader
|
||||
(*BatchUpdateManifestMessageBody)(nil), // 115: milvus.proto.messages.BatchUpdateManifestMessageBody
|
||||
(*BatchUpdateManifestItem)(nil), // 116: milvus.proto.messages.BatchUpdateManifestItem
|
||||
nil, // 117: milvus.proto.messages.Message.PropertiesEntry
|
||||
nil, // 118: milvus.proto.messages.AlterResourceGroupMessageHeader.ResourceGroupConfigsEntry
|
||||
nil, // 119: milvus.proto.messages.AlterWALMessageHeader.ConfigEntry
|
||||
nil, // 120: milvus.proto.messages.RMQMessageLayout.PropertiesEntry
|
||||
(datapb.SegmentLevel)(0), // 121: milvus.proto.data.SegmentLevel
|
||||
(*commonpb.ReplicateConfiguration)(nil), // 122: milvus.proto.common.ReplicateConfiguration
|
||||
(*schemapb.CollectionSchema)(nil), // 123: milvus.proto.schema.CollectionSchema
|
||||
(*fieldmaskpb.FieldMask)(nil), // 124: google.protobuf.FieldMask
|
||||
(commonpb.ConsistencyLevel)(0), // 125: milvus.proto.common.ConsistencyLevel
|
||||
(*commonpb.KeyValuePair)(nil), // 126: milvus.proto.common.KeyValuePair
|
||||
(commonpb.LoadPriority)(0), // 127: milvus.proto.common.LoadPriority
|
||||
(*milvuspb.UserEntity)(nil), // 128: milvus.proto.milvus.UserEntity
|
||||
(*internalpb.CredentialInfo)(nil), // 129: milvus.proto.internal.CredentialInfo
|
||||
(*milvuspb.RoleEntity)(nil), // 130: milvus.proto.milvus.RoleEntity
|
||||
(*milvuspb.RBACMeta)(nil), // 131: milvus.proto.milvus.RBACMeta
|
||||
(*milvuspb.GrantEntity)(nil), // 132: milvus.proto.milvus.GrantEntity
|
||||
(*milvuspb.PrivilegeGroupInfo)(nil), // 133: milvus.proto.milvus.PrivilegeGroupInfo
|
||||
(*indexpb.FieldIndex)(nil), // 134: milvus.proto.index.FieldIndex
|
||||
(commonpb.WALName)(0), // 135: milvus.proto.common.WALName
|
||||
(commonpb.MsgType)(0), // 136: milvus.proto.common.MsgType
|
||||
(*commonpb.MessageID)(nil), // 137: milvus.proto.common.MessageID
|
||||
(*rgpb.ResourceGroupConfig)(nil), // 138: milvus.proto.rg.ResourceGroupConfig
|
||||
(*BatchUpdateManifestV2ColumnGroups)(nil), // 117: milvus.proto.messages.BatchUpdateManifestV2ColumnGroups
|
||||
nil, // 118: milvus.proto.messages.Message.PropertiesEntry
|
||||
nil, // 119: milvus.proto.messages.AlterResourceGroupMessageHeader.ResourceGroupConfigsEntry
|
||||
nil, // 120: milvus.proto.messages.AlterWALMessageHeader.ConfigEntry
|
||||
nil, // 121: milvus.proto.messages.RMQMessageLayout.PropertiesEntry
|
||||
nil, // 122: milvus.proto.messages.BatchUpdateManifestV2ColumnGroups.ColumnGroupsEntry
|
||||
(datapb.SegmentLevel)(0), // 123: milvus.proto.data.SegmentLevel
|
||||
(*commonpb.ReplicateConfiguration)(nil), // 124: milvus.proto.common.ReplicateConfiguration
|
||||
(*schemapb.CollectionSchema)(nil), // 125: milvus.proto.schema.CollectionSchema
|
||||
(*fieldmaskpb.FieldMask)(nil), // 126: google.protobuf.FieldMask
|
||||
(commonpb.ConsistencyLevel)(0), // 127: milvus.proto.common.ConsistencyLevel
|
||||
(*commonpb.KeyValuePair)(nil), // 128: milvus.proto.common.KeyValuePair
|
||||
(commonpb.LoadPriority)(0), // 129: milvus.proto.common.LoadPriority
|
||||
(*milvuspb.UserEntity)(nil), // 130: milvus.proto.milvus.UserEntity
|
||||
(*internalpb.CredentialInfo)(nil), // 131: milvus.proto.internal.CredentialInfo
|
||||
(*milvuspb.RoleEntity)(nil), // 132: milvus.proto.milvus.RoleEntity
|
||||
(*milvuspb.RBACMeta)(nil), // 133: milvus.proto.milvus.RBACMeta
|
||||
(*milvuspb.GrantEntity)(nil), // 134: milvus.proto.milvus.GrantEntity
|
||||
(*milvuspb.PrivilegeGroupInfo)(nil), // 135: milvus.proto.milvus.PrivilegeGroupInfo
|
||||
(*indexpb.FieldIndex)(nil), // 136: milvus.proto.index.FieldIndex
|
||||
(commonpb.WALName)(0), // 137: milvus.proto.common.WALName
|
||||
(commonpb.MsgType)(0), // 138: milvus.proto.common.MsgType
|
||||
(*commonpb.MessageID)(nil), // 139: milvus.proto.common.MessageID
|
||||
(*rgpb.ResourceGroupConfig)(nil), // 140: milvus.proto.rg.ResourceGroupConfig
|
||||
(*datapb.FieldBinlog)(nil), // 141: milvus.proto.data.FieldBinlog
|
||||
}
|
||||
var file_messages_proto_depIdxs = []int32{
|
||||
117, // 0: milvus.proto.messages.Message.properties:type_name -> milvus.proto.messages.Message.PropertiesEntry
|
||||
118, // 0: milvus.proto.messages.Message.properties:type_name -> milvus.proto.messages.Message.PropertiesEntry
|
||||
3, // 1: milvus.proto.messages.TxnMessageBody.messages:type_name -> milvus.proto.messages.Message
|
||||
13, // 2: milvus.proto.messages.InsertMessageHeader.partitions:type_name -> milvus.proto.messages.PartitionSegmentAssignment
|
||||
14, // 3: milvus.proto.messages.PartitionSegmentAssignment.segment_assignment:type_name -> milvus.proto.messages.SegmentAssignment
|
||||
121, // 4: milvus.proto.messages.CreateSegmentMessageHeader.level:type_name -> milvus.proto.data.SegmentLevel
|
||||
122, // 5: milvus.proto.messages.AlterReplicateConfigMessageHeader.replicate_configuration:type_name -> milvus.proto.common.ReplicateConfiguration
|
||||
123, // 6: milvus.proto.messages.SchemaChangeMessageBody.schema:type_name -> milvus.proto.schema.CollectionSchema
|
||||
124, // 7: milvus.proto.messages.AlterCollectionMessageHeader.update_mask:type_name -> google.protobuf.FieldMask
|
||||
123, // 4: milvus.proto.messages.CreateSegmentMessageHeader.level:type_name -> milvus.proto.data.SegmentLevel
|
||||
124, // 5: milvus.proto.messages.AlterReplicateConfigMessageHeader.replicate_configuration:type_name -> milvus.proto.common.ReplicateConfiguration
|
||||
125, // 6: milvus.proto.messages.SchemaChangeMessageBody.schema:type_name -> milvus.proto.schema.CollectionSchema
|
||||
126, // 7: milvus.proto.messages.AlterCollectionMessageHeader.update_mask:type_name -> google.protobuf.FieldMask
|
||||
100, // 8: milvus.proto.messages.AlterCollectionMessageHeader.cache_expirations:type_name -> milvus.proto.messages.CacheExpirations
|
||||
34, // 9: milvus.proto.messages.AlterCollectionMessageBody.updates:type_name -> milvus.proto.messages.AlterCollectionMessageUpdates
|
||||
123, // 10: milvus.proto.messages.AlterCollectionMessageUpdates.schema:type_name -> milvus.proto.schema.CollectionSchema
|
||||
125, // 11: milvus.proto.messages.AlterCollectionMessageUpdates.consistency_level:type_name -> milvus.proto.common.ConsistencyLevel
|
||||
126, // 12: milvus.proto.messages.AlterCollectionMessageUpdates.properties:type_name -> milvus.proto.common.KeyValuePair
|
||||
125, // 10: milvus.proto.messages.AlterCollectionMessageUpdates.schema:type_name -> milvus.proto.schema.CollectionSchema
|
||||
127, // 11: milvus.proto.messages.AlterCollectionMessageUpdates.consistency_level:type_name -> milvus.proto.common.ConsistencyLevel
|
||||
128, // 12: milvus.proto.messages.AlterCollectionMessageUpdates.properties:type_name -> milvus.proto.common.KeyValuePair
|
||||
35, // 13: milvus.proto.messages.AlterCollectionMessageUpdates.alter_load_config:type_name -> milvus.proto.messages.AlterLoadConfigOfAlterCollection
|
||||
38, // 14: milvus.proto.messages.AlterLoadConfigMessageHeader.load_fields:type_name -> milvus.proto.messages.LoadFieldConfig
|
||||
39, // 15: milvus.proto.messages.AlterLoadConfigMessageHeader.replicas:type_name -> milvus.proto.messages.LoadReplicaConfig
|
||||
127, // 16: milvus.proto.messages.LoadReplicaConfig.priority:type_name -> milvus.proto.common.LoadPriority
|
||||
126, // 17: milvus.proto.messages.CreateDatabaseMessageBody.properties:type_name -> milvus.proto.common.KeyValuePair
|
||||
126, // 18: milvus.proto.messages.AlterDatabaseMessageBody.properties:type_name -> milvus.proto.common.KeyValuePair
|
||||
129, // 16: milvus.proto.messages.LoadReplicaConfig.priority:type_name -> milvus.proto.common.LoadPriority
|
||||
128, // 17: milvus.proto.messages.CreateDatabaseMessageBody.properties:type_name -> milvus.proto.common.KeyValuePair
|
||||
128, // 18: milvus.proto.messages.AlterDatabaseMessageBody.properties:type_name -> milvus.proto.common.KeyValuePair
|
||||
46, // 19: milvus.proto.messages.AlterDatabaseMessageBody.alter_load_config:type_name -> milvus.proto.messages.AlterLoadConfigOfAlterDatabase
|
||||
128, // 20: milvus.proto.messages.CreateUserMessageHeader.user_entity:type_name -> milvus.proto.milvus.UserEntity
|
||||
129, // 21: milvus.proto.messages.CreateUserMessageBody.credential_info:type_name -> milvus.proto.internal.CredentialInfo
|
||||
128, // 22: milvus.proto.messages.AlterUserMessageHeader.user_entity:type_name -> milvus.proto.milvus.UserEntity
|
||||
129, // 23: milvus.proto.messages.AlterUserMessageBody.credential_info:type_name -> milvus.proto.internal.CredentialInfo
|
||||
130, // 24: milvus.proto.messages.AlterRoleMessageHeader.role_entity:type_name -> milvus.proto.milvus.RoleEntity
|
||||
128, // 25: milvus.proto.messages.RoleBinding.user_entity:type_name -> milvus.proto.milvus.UserEntity
|
||||
130, // 26: milvus.proto.messages.RoleBinding.role_entity:type_name -> milvus.proto.milvus.RoleEntity
|
||||
130, // 20: milvus.proto.messages.CreateUserMessageHeader.user_entity:type_name -> milvus.proto.milvus.UserEntity
|
||||
131, // 21: milvus.proto.messages.CreateUserMessageBody.credential_info:type_name -> milvus.proto.internal.CredentialInfo
|
||||
130, // 22: milvus.proto.messages.AlterUserMessageHeader.user_entity:type_name -> milvus.proto.milvus.UserEntity
|
||||
131, // 23: milvus.proto.messages.AlterUserMessageBody.credential_info:type_name -> milvus.proto.internal.CredentialInfo
|
||||
132, // 24: milvus.proto.messages.AlterRoleMessageHeader.role_entity:type_name -> milvus.proto.milvus.RoleEntity
|
||||
130, // 25: milvus.proto.messages.RoleBinding.user_entity:type_name -> milvus.proto.milvus.UserEntity
|
||||
132, // 26: milvus.proto.messages.RoleBinding.role_entity:type_name -> milvus.proto.milvus.RoleEntity
|
||||
63, // 27: milvus.proto.messages.AlterUserRoleMessageHeader.role_binding:type_name -> milvus.proto.messages.RoleBinding
|
||||
63, // 28: milvus.proto.messages.DropUserRoleMessageHeader.role_binding:type_name -> milvus.proto.messages.RoleBinding
|
||||
131, // 29: milvus.proto.messages.RestoreRBACMessageBody.rbac_meta:type_name -> milvus.proto.milvus.RBACMeta
|
||||
132, // 30: milvus.proto.messages.AlterPrivilegeMessageHeader.entity:type_name -> milvus.proto.milvus.GrantEntity
|
||||
132, // 31: milvus.proto.messages.DropPrivilegeMessageHeader.entity:type_name -> milvus.proto.milvus.GrantEntity
|
||||
133, // 32: milvus.proto.messages.AlterPrivilegeGroupMessageHeader.privilege_group_info:type_name -> milvus.proto.milvus.PrivilegeGroupInfo
|
||||
133, // 33: milvus.proto.messages.DropPrivilegeGroupMessageHeader.privilege_group_info:type_name -> milvus.proto.milvus.PrivilegeGroupInfo
|
||||
118, // 34: milvus.proto.messages.AlterResourceGroupMessageHeader.resource_group_configs:type_name -> milvus.proto.messages.AlterResourceGroupMessageHeader.ResourceGroupConfigsEntry
|
||||
134, // 35: milvus.proto.messages.CreateIndexMessageBody.field_index:type_name -> milvus.proto.index.FieldIndex
|
||||
134, // 36: milvus.proto.messages.AlterIndexMessageBody.field_indexes:type_name -> milvus.proto.index.FieldIndex
|
||||
135, // 37: milvus.proto.messages.AlterWALMessageHeader.target_wal_name:type_name -> milvus.proto.common.WALName
|
||||
119, // 38: milvus.proto.messages.AlterWALMessageHeader.config:type_name -> milvus.proto.messages.AlterWALMessageHeader.ConfigEntry
|
||||
133, // 29: milvus.proto.messages.RestoreRBACMessageBody.rbac_meta:type_name -> milvus.proto.milvus.RBACMeta
|
||||
134, // 30: milvus.proto.messages.AlterPrivilegeMessageHeader.entity:type_name -> milvus.proto.milvus.GrantEntity
|
||||
134, // 31: milvus.proto.messages.DropPrivilegeMessageHeader.entity:type_name -> milvus.proto.milvus.GrantEntity
|
||||
135, // 32: milvus.proto.messages.AlterPrivilegeGroupMessageHeader.privilege_group_info:type_name -> milvus.proto.milvus.PrivilegeGroupInfo
|
||||
135, // 33: milvus.proto.messages.DropPrivilegeGroupMessageHeader.privilege_group_info:type_name -> milvus.proto.milvus.PrivilegeGroupInfo
|
||||
119, // 34: milvus.proto.messages.AlterResourceGroupMessageHeader.resource_group_configs:type_name -> milvus.proto.messages.AlterResourceGroupMessageHeader.ResourceGroupConfigsEntry
|
||||
136, // 35: milvus.proto.messages.CreateIndexMessageBody.field_index:type_name -> milvus.proto.index.FieldIndex
|
||||
136, // 36: milvus.proto.messages.AlterIndexMessageBody.field_indexes:type_name -> milvus.proto.index.FieldIndex
|
||||
137, // 37: milvus.proto.messages.AlterWALMessageHeader.target_wal_name:type_name -> milvus.proto.common.WALName
|
||||
120, // 38: milvus.proto.messages.AlterWALMessageHeader.config:type_name -> milvus.proto.messages.AlterWALMessageHeader.ConfigEntry
|
||||
101, // 39: milvus.proto.messages.CacheExpirations.cache_expirations:type_name -> milvus.proto.messages.CacheExpiration
|
||||
102, // 40: milvus.proto.messages.CacheExpiration.legacy_proxy_collection_meta_cache:type_name -> milvus.proto.messages.LegacyProxyCollectionMetaCache
|
||||
136, // 41: milvus.proto.messages.LegacyProxyCollectionMetaCache.msg_type:type_name -> milvus.proto.common.MsgType
|
||||
120, // 42: milvus.proto.messages.RMQMessageLayout.properties:type_name -> milvus.proto.messages.RMQMessageLayout.PropertiesEntry
|
||||
138, // 41: milvus.proto.messages.LegacyProxyCollectionMetaCache.msg_type:type_name -> milvus.proto.common.MsgType
|
||||
121, // 42: milvus.proto.messages.RMQMessageLayout.properties:type_name -> milvus.proto.messages.RMQMessageLayout.PropertiesEntry
|
||||
110, // 43: milvus.proto.messages.BroadcastHeader.Resource_keys:type_name -> milvus.proto.messages.ResourceKey
|
||||
137, // 44: milvus.proto.messages.ReplicateHeader.message_id:type_name -> milvus.proto.common.MessageID
|
||||
137, // 45: milvus.proto.messages.ReplicateHeader.last_confirmed_message_id:type_name -> milvus.proto.common.MessageID
|
||||
139, // 44: milvus.proto.messages.ReplicateHeader.message_id:type_name -> milvus.proto.common.MessageID
|
||||
139, // 45: milvus.proto.messages.ReplicateHeader.last_confirmed_message_id:type_name -> milvus.proto.common.MessageID
|
||||
2, // 46: milvus.proto.messages.ResourceKey.domain:type_name -> milvus.proto.messages.ResourceDomain
|
||||
116, // 47: milvus.proto.messages.BatchUpdateManifestMessageBody.items:type_name -> milvus.proto.messages.BatchUpdateManifestItem
|
||||
138, // 48: milvus.proto.messages.AlterResourceGroupMessageHeader.ResourceGroupConfigsEntry.value:type_name -> milvus.proto.rg.ResourceGroupConfig
|
||||
49, // [49:49] is the sub-list for method output_type
|
||||
49, // [49:49] is the sub-list for method input_type
|
||||
49, // [49:49] is the sub-list for extension type_name
|
||||
49, // [49:49] is the sub-list for extension extendee
|
||||
0, // [0:49] is the sub-list for field type_name
|
||||
117, // 48: milvus.proto.messages.BatchUpdateManifestItem.v2_column_groups:type_name -> milvus.proto.messages.BatchUpdateManifestV2ColumnGroups
|
||||
122, // 49: milvus.proto.messages.BatchUpdateManifestV2ColumnGroups.column_groups:type_name -> milvus.proto.messages.BatchUpdateManifestV2ColumnGroups.ColumnGroupsEntry
|
||||
140, // 50: milvus.proto.messages.AlterResourceGroupMessageHeader.ResourceGroupConfigsEntry.value:type_name -> milvus.proto.rg.ResourceGroupConfig
|
||||
141, // 51: milvus.proto.messages.BatchUpdateManifestV2ColumnGroups.ColumnGroupsEntry.value:type_name -> milvus.proto.data.FieldBinlog
|
||||
52, // [52:52] is the sub-list for method output_type
|
||||
52, // [52:52] is the sub-list for method input_type
|
||||
52, // [52:52] is the sub-list for extension type_name
|
||||
52, // [52:52] is the sub-list for extension extendee
|
||||
0, // [0:52] is the sub-list for field type_name
|
||||
}
|
||||
|
||||
func init() { file_messages_proto_init() }
|
||||
@@ -8774,6 +8863,18 @@ func file_messages_proto_init() {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
file_messages_proto_msgTypes[114].Exporter = func(v interface{}, i int) interface{} {
|
||||
switch v := v.(*BatchUpdateManifestV2ColumnGroups); i {
|
||||
case 0:
|
||||
return &v.state
|
||||
case 1:
|
||||
return &v.sizeCache
|
||||
case 2:
|
||||
return &v.unknownFields
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
file_messages_proto_msgTypes[98].OneofWrappers = []interface{}{
|
||||
(*CacheExpiration_LegacyProxyCollectionMetaCache)(nil),
|
||||
@@ -8784,7 +8885,7 @@ func file_messages_proto_init() {
|
||||
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
|
||||
RawDescriptor: file_messages_proto_rawDesc,
|
||||
NumEnums: 3,
|
||||
NumMessages: 118,
|
||||
NumMessages: 120,
|
||||
NumExtensions: 0,
|
||||
NumServices: 0,
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user