furyhawk/milvus
1
0
Fork 0
mirror of https://github.com/milvus-io/milvus.git synced 2026-07-21 10:15:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

enhance: return external spec in refresh job info (#49531)

Browse Source 
issue: milvus-io/milvus#45881

This change returns the external spec used by refresh external
collection jobs through the public refresh job info paths.

Changes:
- Adds external spec to client refresh job info conversion.
- Returns redacted external spec from proxy refresh job responses.
- Exposes externalSpec in REST v2 refresh job response maps.
- Bumps milvus-proto go-api/v3 to the version containing
RefreshExternalCollectionJobInfo.external_spec.

Validation:
- go test -tags dynamic,test
github.com/milvus-io/milvus/client/v2/milvusclient -run
'Test.*External|Test.*Refresh' -count=1 -v

Note:
- internal/proxy and httpserver targeted tests were blocked locally by
missing C++ pkg-config artifacts: milvus_core, rocksdb, and
milvus-storage.

---------

Signed-off-by: Wei Liu <wei.liu@zilliz.com>
This commit is contained in:
wei liu
2026-05-07 14:12:09 +08:00
committed by GitHub
parent 3650408fc1
commit 39cd786ebb
13 changed files with 26 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client/entity/external_table.go
+1
View File
@@ -41,6 +41,7 @@ type RefreshExternalCollectionJobInfo struct {
Progress int64
Reason string
ExternalSource string
ExternalSpec string
StartTime int64
EndTime int64
}
client/go.mod
+1 -1
View File
@@ -7,7 +7,7 @@ require (
github.com/cockroachdb/errors v1.9.1
github.com/google/uuid v1.6.0
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260428092345-c9f705f3cc7f
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260506064405-f5b77584c710
github.com/milvus-io/milvus/pkg/v3 v3.0.0-beta
github.com/quasilyte/go-ruleguard/dsl v0.3.23
github.com/samber/lo v1.52.0
client/go.sum
+2
View File
@@ -224,6 +224,8 @@ github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i
github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260428092345-c9f705f3cc7f h1:Tmo+3s4VHQ8EUGmuLoo5PKo7k3eF90w6hCk/acXG0oQ=
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260428092345-c9f705f3cc7f/go.mod h1:rbKpv5JToISTKTTLl0duL5r6wbYnjJ9SsD0QgXMzKy0=
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260506064405-f5b77584c710 h1:vEaS+KqeIJZbByJFLjVZVA+630oqD40u4jMtdj5FjyA=
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260506064405-f5b77584c710/go.mod h1:rbKpv5JToISTKTTLl0duL5r6wbYnjJ9SsD0QgXMzKy0=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
client/milvusclient/external_table.go
+1
View File
@@ -104,6 +104,7 @@ func convertToEntityJobInfo(info *milvuspb.RefreshExternalCollectionJobInfo) *en
Progress: info.GetProgress(),
Reason: info.GetReason(),
ExternalSource: info.GetExternalSource(),
ExternalSpec: info.GetExternalSpec(),
StartTime: info.GetStartTime(),
EndTime: info.GetEndTime(),
}
client/milvusclient/external_table_test.go
+2
View File
@@ -97,6 +97,7 @@ func TestConvertToEntityJobInfo(t *testing.T) {
Progress: 50,
Reason: "",
ExternalSource: "s3://bucket/path",
ExternalSpec: `{"format":"parquet"}`,
StartTime: 1000,
EndTime: 0,
}
@@ -109,6 +110,7 @@ func TestConvertToEntityJobInfo(t *testing.T) {
assert.Equal(t, int64(50), result.Progress)
assert.Equal(t, "", result.Reason)
assert.Equal(t, "s3://bucket/path", result.ExternalSource)
assert.Equal(t, `{"format":"parquet"}`, result.ExternalSpec)
assert.Equal(t, int64(1000), result.StartTime)
assert.Equal(t, int64(0), result.EndTime)
})
go.mod
+1 -1
View File
@@ -69,7 +69,7 @@ require (
github.com/hamba/avro/v2 v2.29.0
github.com/hashicorp/golang-lru/v2 v2.0.7
github.com/magiconair/properties v1.8.7
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260428092345-c9f705f3cc7f
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260506064405-f5b77584c710
github.com/milvus-io/milvus/client/v2 v2.6.2
github.com/milvus-io/milvus/pkg/v3 v3.0.0-beta
github.com/shirou/gopsutil/v4 v4.25.10
go.sum
+2
View File
@@ -816,6 +816,8 @@ github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZz
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4=
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260428092345-c9f705f3cc7f h1:Tmo+3s4VHQ8EUGmuLoo5PKo7k3eF90w6hCk/acXG0oQ=
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260428092345-c9f705f3cc7f/go.mod h1:rbKpv5JToISTKTTLl0duL5r6wbYnjJ9SsD0QgXMzKy0=
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260506064405-f5b77584c710 h1:vEaS+KqeIJZbByJFLjVZVA+630oqD40u4jMtdj5FjyA=
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260506064405-f5b77584c710/go.mod h1:rbKpv5JToISTKTTLl0duL5r6wbYnjJ9SsD0QgXMzKy0=
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs=
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY=
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI=
internal/distributed/proxy/httpserver/handler_v2.go
+1
View File
@@ -3220,6 +3220,7 @@ func refreshExternalCollectionJobInfoToMap(jobInfo *milvuspb.RefreshExternalColl
detail["state"] = jobInfo.GetState().String()
detail["progress"] = jobInfo.GetProgress()
detail["externalSource"] = jobInfo.GetExternalSource()
detail["externalSpec"] = jobInfo.GetExternalSpec()
detail["startTime"] = jobInfo.GetStartTime()
detail["endTime"] = jobInfo.GetEndTime()
if jobInfo.GetReason() != "" {
internal/distributed/proxy/httpserver/handler_v2_test.go
+4
View File
@@ -1666,6 +1666,7 @@ func (m *externalCollectionRESTProxy) GetRefreshExternalCollectionProgress(ctx c
State: milvuspb.RefreshExternalCollectionState_RefreshInProgress,
Progress: 42,
ExternalSource: "s3://bucket/books",
ExternalSpec: `{"format":"parquet"}`,
StartTime: 10,
},
}, nil
@@ -1682,6 +1683,7 @@ func (m *externalCollectionRESTProxy) ListRefreshExternalCollectionJobs(ctx cont
State: milvuspb.RefreshExternalCollectionState_RefreshCompleted,
Progress: 100,
ExternalSource: "s3://bucket/books",
ExternalSpec: `{"format":"parquet"}`,
StartTime: 10,
EndTime: 20,
},
@@ -1871,6 +1873,7 @@ func TestExternalCollectionJobRoutesRESTV2(t *testing.T) {
assert.Equal(t, http.StatusOK, w.Code)
assert.Equal(t, int64(1001), proxy.progressReq.GetJobId())
assert.Contains(t, w.Body.String(), `"progress":42`)
assert.Contains(t, w.Body.String(), `"externalSpec":"{\"format\":\"parquet\"}"`)
req = httptest.NewRequest(http.MethodPost, versionalV2(ExternalCollectionJobCategory, ListAction), bytes.NewReader([]byte(`{"collectionName": "external_books"}`)))
w = httptest.NewRecorder()
@@ -1878,6 +1881,7 @@ func TestExternalCollectionJobRoutesRESTV2(t *testing.T) {
assert.Equal(t, http.StatusOK, w.Code)
assert.Equal(t, "external_books", proxy.listReq.GetCollectionName())
assert.Contains(t, w.Body.String(), `"state":"RefreshCompleted"`)
assert.Contains(t, w.Body.String(), `"externalSpec":"{\"format\":\"parquet\"}"`)
}
func TestExternalCollectionJobDescribeRequiresJobIDRESTV2(t *testing.T) {
internal/proxy/impl.go
+1
View File
@@ -7954,6 +7954,7 @@ func convertToExternalCollectionJobInfo(internal *datapb.ExternalCollectionRefre
Progress: internal.GetProgress(),
Reason: internal.GetFailReason(),
ExternalSource: internal.GetExternalSource(),
ExternalSpec: externalspec.RedactExternalSpec(internal.GetExternalSpec()),
StartTime: internal.GetStartTime(),
EndTime: internal.GetEndTime(),
}
internal/proxy/impl_test.go
+7
View File
@@ -3223,6 +3223,7 @@ func TestProxy_ListRefreshExternalCollectionJobs_ByCollection(t *testing.T) {
CollectionName: "external_collection",
State: indexpb.JobState_JobStateFinished,
Progress: 100,
ExternalSpec: `{"format":"parquet","extfs":{"access_key_id":"AKIAEXAMPLE","access_key_value":"SUPERSECRET","region":"us-east-1"}}`,
},
},
}, nil
@@ -3238,4 +3239,10 @@ func TestProxy_ListRefreshExternalCollectionJobs_ByCollection(t *testing.T) {
require.True(t, merr.Ok(resp.GetStatus()))
assert.Equal(t, int64(101), capturedCollectionID)
require.Len(t, resp.GetJobs(), 1)
redactedSpec := resp.GetJobs()[0].GetExternalSpec()
assert.Contains(t, redactedSpec, `"access_key_id":"***"`)
assert.Contains(t, redactedSpec, `"access_key_value":"***"`)
assert.Contains(t, redactedSpec, `"region":"us-east-1"`)
assert.NotContains(t, redactedSpec, "AKIAEXAMPLE")
assert.NotContains(t, redactedSpec, "SUPERSECRET")
}
tests/go_client/go.mod
+1 -1
View File
@@ -4,7 +4,7 @@ go 1.25.9
require (
github.com/apache/arrow/go/v17 v17.0.0
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260428092345-c9f705f3cc7f
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260506064405-f5b77584c710
github.com/milvus-io/milvus/client/v2 v2.0.0-20241125024034-0b9edb62a92d
github.com/milvus-io/milvus/pkg/v3 v3.0.0-beta
github.com/minio/minio-go/v7 v7.0.73
tests/go_client/go.sum
+2 -2
View File
@@ -245,8 +245,8 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k
github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8=
github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260428092345-c9f705f3cc7f h1:Tmo+3s4VHQ8EUGmuLoo5PKo7k3eF90w6hCk/acXG0oQ=
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260428092345-c9f705f3cc7f/go.mod h1:rbKpv5JToISTKTTLl0duL5r6wbYnjJ9SsD0QgXMzKy0=
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260506064405-f5b77584c710 h1:vEaS+KqeIJZbByJFLjVZVA+630oqD40u4jMtdj5FjyA=
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260506064405-f5b77584c710/go.mod h1:rbKpv5JToISTKTTLl0duL5r6wbYnjJ9SsD0QgXMzKy0=
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs=
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY=
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI=