milvus/internal/proxy/service_provider.go

package proxy

import (
	"context"
	"fmt"
	"strconv"

	"github.com/cockroachdb/errors"
	"github.com/samber/lo"
	"go.opentelemetry.io/otel"

	"github.com/milvus-io/milvus-proto/go-api/v3/commonpb"
	"github.com/milvus-io/milvus-proto/go-api/v3/milvuspb"
	"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
	"github.com/milvus-io/milvus/pkg/v3/common"
	"github.com/milvus-io/milvus/pkg/v3/metrics"
	"github.com/milvus-io/milvus/pkg/v3/mlog"
	"github.com/milvus-io/milvus/pkg/v3/util/merr"
	"github.com/milvus-io/milvus/pkg/v3/util/paramtable"
	"github.com/milvus-io/milvus/pkg/v3/util/timerecord"
	"github.com/milvus-io/milvus/pkg/v3/util/timestamptz"
	"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
)

type Request interface {
	GetDbName() string
	GetCollectionName() string
}

type Response interface {
	GetStatus() *commonpb.Status
}

type ServiceInterceptor[Req Request, Resp Response] interface {
	Call(ctx context.Context, request Req) (Resp, error)
}

func NewInterceptor[Req Request, Resp Response](proxy *Proxy, method string) (*InterceptorImpl[Req, Resp], error) {
	var provider milvuspb.MilvusServiceServer
	cached := paramtable.Get().ProxyCfg.EnableCachedServiceProvider.GetAsBool()
	if cached {
		provider = &CachedProxyServiceProvider{Proxy: proxy}
	} else {
		provider = &RemoteProxyServiceProvider{Proxy: proxy}
	}
	switch method {
	case "DescribeCollection":
		interceptor := &InterceptorImpl[*milvuspb.DescribeCollectionRequest, *milvuspb.DescribeCollectionResponse]{
			proxy:  proxy,
			method: method,
			onCall: provider.DescribeCollection,
			onError: func(err error) (*milvuspb.DescribeCollectionResponse, error) {
				return &milvuspb.DescribeCollectionResponse{
					Status: merr.Status(err),
				}, nil
			},
		}
		return interface{}(interceptor).(*InterceptorImpl[Req, Resp]), nil
	default:
		return nil, merr.WrapErrParameterInvalidMsg("method %s not supported", method)
	}
}

type InterceptorImpl[Req Request, Resp Response] struct {
	proxy   *Proxy
	method  string
	onCall  func(ctx context.Context, request Req) (Resp, error)
	onError func(err error) (Resp, error)
}

func (i *InterceptorImpl[Req, Resp]) Call(ctx context.Context, request Req,
) (Resp, error) {
	if err := merr.CheckHealthy(i.proxy.GetStateCode()); err != nil {
		return i.onError(err)
	}

	ctx, sp := otel.Tracer(typeutil.ProxyRole).Start(ctx, fmt.Sprintf("Proxy-%s", i.method))
	defer sp.End()
	tr := timerecord.NewTimeRecorder(i.method)
	metrics.ProxyFunctionCall.WithLabelValues(strconv.FormatInt(paramtable.GetNodeID(), 10), i.method,
		metrics.TotalLabel, metrics.CauseNA, request.GetDbName(), request.GetCollectionName()).Inc()

	resp, err := i.onCall(ctx, request)
	if err != nil {
		return i.onError(err)
	}

	metrics.ProxyFunctionCall.WithLabelValues(strconv.FormatInt(paramtable.GetNodeID(), 10), i.method,
		metrics.SuccessLabel, metrics.CauseNA, request.GetDbName(), request.GetCollectionName()).Inc()
	metrics.ProxyReqLatency.WithLabelValues(strconv.FormatInt(paramtable.GetNodeID(), 10), i.method).Observe(float64(tr.ElapseSpan().Milliseconds()))

	return resp, err
}

type CachedProxyServiceProvider struct {
	*Proxy
}

// cloneStructArrayFields creates a deep copy of struct array fields to avoid modifying cached data
func cloneStructArrayFields(fields []*schemapb.StructArrayFieldSchema) []*schemapb.StructArrayFieldSchema {
	if fields == nil {
		return nil
	}

	cloned := make([]*schemapb.StructArrayFieldSchema, len(fields))
	for i, field := range fields {
		cloned[i] = &schemapb.StructArrayFieldSchema{
			FieldID:     field.FieldID,
			Name:        field.Name,
			Description: field.Description,
			Fields:      make([]*schemapb.FieldSchema, len(field.Fields)),
			Nullable:    field.Nullable,
		}

		// Deep copy sub-fields
		for j, subField := range field.Fields {
			cloned[i].Fields[j] = &schemapb.FieldSchema{
				FieldID:          subField.FieldID,
				Name:             subField.Name,
				IsPrimaryKey:     subField.IsPrimaryKey,
				Description:      subField.Description,
				DataType:         subField.DataType,
				TypeParams:       subField.TypeParams,
				IndexParams:      subField.IndexParams,
				AutoID:           subField.AutoID,
				State:            subField.State,
				ElementType:      subField.ElementType,
				DefaultValue:     subField.DefaultValue,
				IsDynamic:        subField.IsDynamic,
				IsPartitionKey:   subField.IsPartitionKey,
				IsClusteringKey:  subField.IsClusteringKey,
				Nullable:         subField.Nullable,
				IsFunctionOutput: subField.IsFunctionOutput,
			}
		}
	}

	return cloned
}

func (node *CachedProxyServiceProvider) DescribeCollection(ctx context.Context,
	request *milvuspb.DescribeCollectionRequest,
) (resp *milvuspb.DescribeCollectionResponse, err error) {
	log := mlog.With(
		mlog.String("role", typeutil.ProxyRole),
		mlog.String("db", request.GetDbName()),
		mlog.String("collection", request.GetCollectionName()),
		mlog.FieldCollectionID(request.GetCollectionID()),
		mlog.Uint64("timestamp", request.GetTimeStamp()),
	)

	log.Debug(ctx, "DescribeCollection received")

	resp = &milvuspb.DescribeCollectionResponse{
		Status:         merr.Success(),
		CollectionName: request.CollectionName,
		DbName:         request.DbName,
	}

	wrapErrorStatus := func(err error) *commonpb.Status {
		status := &commonpb.Status{}
		if errors.Is(err, merr.ErrCollectionNotFound) {
			// nolint
			status.ErrorCode = commonpb.ErrorCode_CollectionNotExists
			// nolint
			status.Reason = fmt.Sprintf("can't find collection[database=%s][collection=%s]", request.DbName, request.CollectionName)
			status.ExtraInfo = map[string]string{merr.InputErrorFlagKey: "true"}
		} else {
			status = merr.Status(err)
		}
		return status
	}

	if request.CollectionName == "" && request.CollectionID > 0 {
		collName, err := globalMetaCache.GetCollectionName(ctx, request.DbName, request.CollectionID)
		if err != nil {
			resp.Status = wrapErrorStatus(err)
			return resp, nil
		}
		request.CollectionName = collName
	}

	// validate collection name, ref describeCollectionTask.PreExecute
	if err = validateCollectionName(request.CollectionName); err != nil {
		resp.Status = wrapErrorStatus(err)
		return resp, nil
	}

	request.CollectionID, err = globalMetaCache.GetCollectionID(ctx, request.DbName, request.CollectionName)
	if err != nil {
		resp.Status = wrapErrorStatus(err)
		return resp, nil
	}

	c, err := globalMetaCache.GetCollectionInfo(ctx, request.DbName, request.CollectionName, request.CollectionID)
	if err != nil {
		resp.Status = wrapErrorStatus(err)
		return resp, nil
	}

	// skip dynamic fields, see describeCollectionTask.Execute
	resp.Schema = &schemapb.CollectionSchema{
		Name:        c.schema.Name,
		Description: c.schema.Description,
		AutoID:      c.schema.AutoID,
		Fields: lo.Filter(c.schema.Fields, func(field *schemapb.FieldSchema, _ int) bool {
			return !field.IsDynamic && field.Name != common.NamespaceFieldName
		}),
		StructArrayFields:  cloneStructArrayFields(c.schema.StructArrayFields),
		EnableDynamicField: c.schema.EnableDynamicField,
		EnableNamespace:    c.schema.EnableNamespace,
		Properties:         c.schema.Properties,
		Functions:          c.schema.Functions,
		DbName:             c.schema.DbName,
		ExternalSource:     c.schema.ExternalSource,
		ExternalSpec:       c.schema.ExternalSpec,
		Version:            c.schema.Version,
	}

	// Restore struct field names from internal format (structName[fieldName]) to original format
	if err := restoreStructFieldNames(resp.Schema); err != nil {
		log.Error(ctx, "failed to restore struct field names", mlog.Err(err))
		return nil, err
	}

	err = timestamptz.RewriteTimestampTzDefaultValueToString(resp.Schema)
	if err != nil {
		log.Info(ctx, "failed to rewrite timestamp value", mlog.Err(err))
		return nil, err
	}

	// prefer the actual database resolved by the coordinator and carried in the
	// cache, the request db name may be empty/default when querying by collection id
	if c.dbName != "" {
		resp.DbName = c.dbName
	}
	resp.DbId = c.dbID
	resp.CollectionID = c.collID
	resp.UpdateTimestamp = c.updateTimestamp
	resp.UpdateTimestampStr = fmt.Sprintf("%d", c.updateTimestamp)
	resp.CreatedTimestamp = c.createdTimestamp
	resp.CreatedUtcTimestamp = c.createdUtcTimestamp
	resp.ConsistencyLevel = c.consistencyLevel
	resp.VirtualChannelNames = c.vChannels
	resp.PhysicalChannelNames = c.pChannels
	resp.NumPartitions = c.numPartitions
	resp.ShardsNum = c.shardsNum
	resp.Aliases = c.aliases
	resp.Properties = c.properties
	log.Debug(ctx, "DescribeCollection done",
		mlog.FieldCollectionID(resp.GetCollectionID()),
		mlog.Any("schema", resp.GetSchema()),
	)
	return resp, nil
}

type RemoteProxyServiceProvider struct {
	*Proxy
}

func (node *RemoteProxyServiceProvider) DescribeCollection(ctx context.Context,
	request *milvuspb.DescribeCollectionRequest,
) (*milvuspb.DescribeCollectionResponse, error) {
	dct := &describeCollectionTask{
		ctx:                       ctx,
		Condition:                 NewTaskCondition(ctx),
		DescribeCollectionRequest: request,
		mixCoord:                  node.mixCoord,
	}

	log := mlog.With(
		mlog.String("role", typeutil.ProxyRole),
		mlog.String("db", request.DbName),
		mlog.String("collection", request.CollectionName))

	method := "DescribeCollection"
	log.Debug(ctx, "DescribeCollection received")

	if err := node.sched.ddQueue.Enqueue(dct); err != nil {
		log.Warn(ctx, "DescribeCollection failed to enqueue",
			mlog.Err(err))

		metrics.ProxyFunctionCall.WithLabelValues(strconv.FormatInt(paramtable.GetNodeID(), 10), method,
			metrics.AbandonLabel, metrics.CauseNA, request.GetDbName(), request.GetCollectionName()).Inc()
		return nil, err
	}

	log.Debug(ctx, "DescribeCollection enqueued",
		mlog.Uint64("BeginTS", dct.BeginTs()),
		mlog.Uint64("EndTS", dct.EndTs()))

	if err := dct.WaitToFinish(); err != nil {
		log.Warn(ctx, "DescribeCollection failed to WaitToFinish",
			mlog.Err(err),
			mlog.Uint64("BeginTS", dct.BeginTs()),
			mlog.Uint64("EndTS", dct.EndTs()))

		failStatus, failCause := failMetricLabel(err)
		metrics.ProxyFunctionCall.WithLabelValues(strconv.FormatInt(paramtable.GetNodeID(), 10), method,
			failStatus, failCause, request.GetDbName(), request.GetCollectionName()).Inc()

		return nil, err
	}

	log.Debug(ctx, "DescribeCollection done",
		mlog.Uint64("BeginTS", dct.BeginTs()),
		mlog.Uint64("EndTS", dct.EndTs()),
	)

	return dct.result, nil
}