mirror of
https://github.com/milvus-io/milvus.git
synced 2026-07-21 10:15:43 +00:00
feat: impl StructArray -- support element-level query (#47906)
issue: https://github.com/milvus-io/milvus/issues/42148 design doc: https://github.com/milvus-io/milvus-design-docs/blob/main/design_docs/20260306-struct.md --------- Signed-off-by: SpadeA <tangchenjie1210@gmail.com>
This commit is contained in:
Spade A
+3
-9
@@ -7,16 +7,13 @@ require (
|
||||
github.com/cockroachdb/errors v1.9.1
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
|
||||
github.com/milvus-io/milvus-proto/go-api/v2 v2.6.6-0.20260202035249-b2c299650f7e
|
||||
github.com/milvus-io/milvus-proto/go-api/v2 v2.6.6-0.20260304062516-e7e54d966ef2
|
||||
github.com/milvus-io/milvus/pkg/v2 v2.6.4-0.20251104142533-a2ce70d25256
|
||||
github.com/minio/minio-go/v7 v7.0.73
|
||||
github.com/quasilyte/go-ruleguard/dsl v0.3.23
|
||||
github.com/samber/lo v1.27.0
|
||||
github.com/stretchr/testify v1.11.1
|
||||
github.com/tidwall/gjson v1.17.1
|
||||
go.opentelemetry.io/otel v1.40.0
|
||||
go.uber.org/atomic v1.11.0
|
||||
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842
|
||||
google.golang.org/grpc v1.71.0
|
||||
google.golang.org/protobuf v1.36.5
|
||||
)
|
||||
@@ -37,11 +34,9 @@ require (
|
||||
github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect
|
||||
github.com/fxamacker/cbor/v2 v2.7.0 // indirect
|
||||
github.com/getsentry/sentry-go v0.12.0 // indirect
|
||||
github.com/go-ini/ini v1.67.0 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-ole/go-ole v1.2.6 // indirect
|
||||
github.com/goccy/go-json v0.10.3 // indirect
|
||||
github.com/godbus/dbus/v5 v5.0.4 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang/protobuf v1.5.4 // indirect
|
||||
@@ -53,11 +48,9 @@ require (
|
||||
github.com/jonboulle/clockwork v0.2.2 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/klauspost/compress v1.17.9 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.8 // indirect
|
||||
github.com/kr/pretty v0.3.1 // indirect
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
@@ -71,7 +64,6 @@ require (
|
||||
github.com/prometheus/common v0.55.0 // indirect
|
||||
github.com/prometheus/procfs v0.15.1 // indirect
|
||||
github.com/rogpeppe/go-internal v1.14.1 // indirect
|
||||
github.com/rs/xid v1.5.0 // indirect
|
||||
github.com/shirou/gopsutil/v3 v3.23.12 // indirect
|
||||
github.com/shoenig/go-m1cpu v0.1.6 // indirect
|
||||
github.com/sirupsen/logrus v1.9.3 // indirect
|
||||
@@ -100,6 +92,7 @@ require (
|
||||
go.etcd.io/etcd/server/v3 v3.5.5 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0 // indirect
|
||||
go.opentelemetry.io/otel v1.40.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.40.0 // indirect
|
||||
@@ -110,6 +103,7 @@ require (
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.uber.org/zap v1.27.0 // indirect
|
||||
golang.org/x/crypto v0.45.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect
|
||||
golang.org/x/net v0.47.0 // indirect
|
||||
golang.org/x/sync v0.18.0 // indirect
|
||||
golang.org/x/sys v0.40.0 // indirect
|
||||
|
||||
+2
-16
@@ -135,8 +135,6 @@ github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclK
|
||||
github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=
|
||||
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
|
||||
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
|
||||
@@ -155,8 +153,6 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me
|
||||
github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
|
||||
github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
|
||||
github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
|
||||
github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
|
||||
github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
|
||||
github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA=
|
||||
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
|
||||
github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
|
||||
@@ -301,9 +297,6 @@ github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0
|
||||
github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
|
||||
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
|
||||
github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM=
|
||||
github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
|
||||
@@ -338,14 +331,10 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5
|
||||
github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8=
|
||||
github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
|
||||
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
|
||||
github.com/milvus-io/milvus-proto/go-api/v2 v2.6.6-0.20260202035249-b2c299650f7e h1:PAyiC+wYbWnuP1sRhpIeaG3EOEZoeRA1lE6x6jWcmX8=
|
||||
github.com/milvus-io/milvus-proto/go-api/v2 v2.6.6-0.20260202035249-b2c299650f7e/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
|
||||
github.com/milvus-io/milvus-proto/go-api/v2 v2.6.6-0.20260304062516-e7e54d966ef2 h1:OEqYnraL/Tf2nGMjRJji6xaaDe40zSQVBvruzEBZ1hU=
|
||||
github.com/milvus-io/milvus-proto/go-api/v2 v2.6.6-0.20260304062516-e7e54d966ef2/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
|
||||
github.com/milvus-io/milvus/pkg/v2 v2.6.4-0.20251104142533-a2ce70d25256 h1:M2waty0w2k4YT2HHzJk3fx6EFPD4DKxNJatitIV+gGU=
|
||||
github.com/milvus-io/milvus/pkg/v2 v2.6.4-0.20251104142533-a2ce70d25256/go.mod h1:HT6Wxahwj/l8+i+D/C3iwDzCjDa36U9gyVw6CjjK4pE=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v7 v7.0.73 h1:qr2vi96Qm7kZ4v7LLebjte+MQh621fFWnv93p12htEo=
|
||||
github.com/minio/minio-go/v7 v7.0.73/go.mod h1:qydcVzV8Hqtj1VtEocfxbmVFa2siu6HGa+LDEPogjD8=
|
||||
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
|
||||
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
@@ -437,8 +426,6 @@ github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4
|
||||
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
|
||||
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
|
||||
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
|
||||
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
|
||||
@@ -736,7 +723,6 @@ golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
|
||||
@@ -19,7 +19,7 @@ require (
|
||||
github.com/google/btree v1.1.2
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
|
||||
github.com/klauspost/compress v1.18.0
|
||||
github.com/milvus-io/milvus-proto/go-api/v2 v2.6.6-0.20260202035249-b2c299650f7e
|
||||
github.com/milvus-io/milvus-proto/go-api/v2 v2.6.6-0.20260304062516-e7e54d966ef2
|
||||
github.com/minio/minio-go/v7 v7.0.73
|
||||
github.com/panjf2000/ants/v2 v2.11.3 // indirect
|
||||
github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81 // indirect
|
||||
|
||||
@@ -797,8 +797,8 @@ github.com/milvus-io/cgosymbolizer v0.0.0-20250318084424-114f4050c3a6 h1:YHMFI6L
|
||||
github.com/milvus-io/cgosymbolizer v0.0.0-20250318084424-114f4050c3a6/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg=
|
||||
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZzUfIfYe5qYDBzt4ZYRqzUjTR6CvUzjat8=
|
||||
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4=
|
||||
github.com/milvus-io/milvus-proto/go-api/v2 v2.6.6-0.20260202035249-b2c299650f7e h1:PAyiC+wYbWnuP1sRhpIeaG3EOEZoeRA1lE6x6jWcmX8=
|
||||
github.com/milvus-io/milvus-proto/go-api/v2 v2.6.6-0.20260202035249-b2c299650f7e/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
|
||||
github.com/milvus-io/milvus-proto/go-api/v2 v2.6.6-0.20260304062516-e7e54d966ef2 h1:OEqYnraL/Tf2nGMjRJji6xaaDe40zSQVBvruzEBZ1hU=
|
||||
github.com/milvus-io/milvus-proto/go-api/v2 v2.6.6-0.20260304062516-e7e54d966ef2/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
|
||||
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs=
|
||||
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY=
|
||||
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI=
|
||||
|
||||
@@ -368,6 +368,13 @@ struct RetrieveResult {
|
||||
bool has_more_result = true;
|
||||
// record the storage usage in retrieve
|
||||
StorageCost retrieve_storage_cost_;
|
||||
|
||||
// Element-level query support
|
||||
// When element_level_ is true:
|
||||
// - result_offsets_ contains unique doc_ids (no duplicates)
|
||||
// - element_indices_[i] contains all matching element indices for result_offsets_[i]
|
||||
bool element_level_{false};
|
||||
std::vector<std::vector<int32_t>> element_indices_;
|
||||
};
|
||||
|
||||
using RetrieveResultPtr = std::shared_ptr<RetrieveResult>;
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
#include "ElementFilterNode.h"
|
||||
#include "exec/operator/Utils.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <chrono>
|
||||
@@ -51,7 +52,8 @@ PhyElementFilterNode::PhyElementFilterNode(
|
||||
operator_id,
|
||||
element_filter_node->id(),
|
||||
"PhyElementFilterNode"),
|
||||
struct_name_(element_filter_node->struct_name()) {
|
||||
struct_name_(element_filter_node->struct_name()),
|
||||
has_doc_predicate_(element_filter_node->has_doc_predicate()) {
|
||||
ExecContext* exec_context = operator_context_->get_exec_context();
|
||||
query_context_ = exec_context->get_query_context();
|
||||
std::vector<expr::TypedExprPtr> exprs;
|
||||
@@ -125,9 +127,19 @@ PhyElementFilterNode::GetOutput() {
|
||||
|
||||
// Step 3: Update search result with wrapped iterators
|
||||
search_result.vector_iterators_ = std::move(wrapped_iterators);
|
||||
size_t num_iterators = search_result.vector_iterators_.has_value()
|
||||
? search_result.vector_iterators_->size()
|
||||
: 0;
|
||||
|
||||
// Step 4: If no doc-level predicate, collect results directly
|
||||
// (otherwise, downstream FilterNode will do this)
|
||||
if (!has_doc_predicate_) {
|
||||
CollectResults(search_result, array_offsets.get());
|
||||
}
|
||||
|
||||
query_context_->set_search_result(std::move(search_result));
|
||||
|
||||
// Step 4: Record metrics
|
||||
// Step 5: Record metrics
|
||||
std::chrono::high_resolution_clock::time_point end_time =
|
||||
std::chrono::high_resolution_clock::now();
|
||||
double cost =
|
||||
@@ -136,14 +148,91 @@ PhyElementFilterNode::GetOutput() {
|
||||
|
||||
tracer::AddEvent(
|
||||
fmt::format("PhyElementFilterNode: wrapped {} iterators, struct_name: "
|
||||
"{}, cost_us: {}",
|
||||
wrapped_iterators.size(),
|
||||
"{}, has_doc_predicate: {}, cost_us: {}",
|
||||
num_iterators,
|
||||
struct_name_,
|
||||
has_doc_predicate_,
|
||||
cost));
|
||||
|
||||
// Pass through input to downstream
|
||||
return input_;
|
||||
}
|
||||
|
||||
void
|
||||
PhyElementFilterNode::CollectResults(SearchResult& search_result,
|
||||
const IArrayOffsets* array_offsets) {
|
||||
// When there's no doc-level predicate, we need to consume the iterators
|
||||
// and collect the top-K results ourselves.
|
||||
//
|
||||
// Note: knowhere iterator doesn't guarantee strictly ordered output,
|
||||
// so we must use binary insertion to maintain sorted order.
|
||||
|
||||
auto& iterators = search_result.vector_iterators_.value();
|
||||
int64_t nq = search_result.total_nq_;
|
||||
int64_t topk = search_result.unity_topK_;
|
||||
|
||||
knowhere::MetricType metric_type = query_context_->get_metric_type();
|
||||
bool large_is_better = PositivelyRelated(metric_type);
|
||||
|
||||
// Initialize result arrays
|
||||
search_result.seg_offsets_.resize(nq * topk, INVALID_SEG_OFFSET);
|
||||
search_result.distances_.resize(nq * topk, 0.0f);
|
||||
search_result.element_indices_.resize(nq * topk, -1);
|
||||
|
||||
for (int64_t q = 0; q < nq; ++q) {
|
||||
auto& iterator = iterators[q];
|
||||
int64_t count = 0;
|
||||
int64_t base_idx = q * topk;
|
||||
|
||||
while (iterator->HasNext() && count < topk) {
|
||||
auto result = iterator->Next();
|
||||
if (!result.has_value()) {
|
||||
break;
|
||||
}
|
||||
|
||||
auto [element_id, distance] = result.value();
|
||||
auto [doc_id, elem_idx] =
|
||||
array_offsets->ElementIDToRowID(element_id);
|
||||
|
||||
// Find insert position using binary search
|
||||
size_t pos =
|
||||
large_is_better
|
||||
? find_binsert_position<true>(search_result.distances_,
|
||||
base_idx,
|
||||
base_idx + count,
|
||||
distance)
|
||||
: find_binsert_position<false>(search_result.distances_,
|
||||
base_idx,
|
||||
base_idx + count,
|
||||
distance);
|
||||
|
||||
// Shift elements to make room for insertion
|
||||
if (count > 0 && pos < base_idx + count) {
|
||||
std::memmove(&search_result.distances_[pos + 1],
|
||||
&search_result.distances_[pos],
|
||||
(base_idx + count - pos) * sizeof(float));
|
||||
std::memmove(&search_result.seg_offsets_[pos + 1],
|
||||
&search_result.seg_offsets_[pos],
|
||||
(base_idx + count - pos) * sizeof(int64_t));
|
||||
std::memmove(&search_result.element_indices_[pos + 1],
|
||||
&search_result.element_indices_[pos],
|
||||
(base_idx + count - pos) * sizeof(int32_t));
|
||||
}
|
||||
|
||||
// Insert the new result
|
||||
search_result.seg_offsets_[pos] = doc_id;
|
||||
search_result.element_indices_[pos] = elem_idx;
|
||||
search_result.distances_[pos] = distance;
|
||||
++count;
|
||||
}
|
||||
}
|
||||
|
||||
// Clear iterators to indicate results have been collected
|
||||
search_result.vector_iterators_.reset();
|
||||
|
||||
tracer::AddEvent(fmt::format(
|
||||
"PhyElementFilterNode::CollectResults: nq={}, topk={}", nq, topk));
|
||||
}
|
||||
|
||||
} // namespace exec
|
||||
} // namespace milvus
|
||||
|
||||
@@ -79,9 +79,14 @@ class PhyElementFilterNode : public Operator {
|
||||
}
|
||||
|
||||
private:
|
||||
void
|
||||
CollectResults(SearchResult& search_result,
|
||||
const IArrayOffsets* array_offsets);
|
||||
|
||||
std::unique_ptr<ExprSet> element_exprs_;
|
||||
QueryContext* query_context_;
|
||||
std::string struct_name_;
|
||||
bool has_doc_predicate_{true};
|
||||
bool is_finished_{false};
|
||||
};
|
||||
|
||||
|
||||
@@ -47,6 +47,7 @@
|
||||
#include "monitor/Monitor.h"
|
||||
#include "plan/PlanNode.h"
|
||||
#include "prometheus/histogram.h"
|
||||
#include "exec/operator/Utils.h"
|
||||
|
||||
namespace milvus {
|
||||
namespace exec {
|
||||
@@ -83,31 +84,6 @@ PhyIterativeFilterNode::IsFinished() {
|
||||
return is_finished_;
|
||||
}
|
||||
|
||||
template <bool large_is_better>
|
||||
inline size_t
|
||||
find_binsert_position(const std::vector<float>& distances,
|
||||
size_t lo,
|
||||
size_t hi,
|
||||
float dist) {
|
||||
while (lo < hi) {
|
||||
size_t mid = lo + ((hi - lo) >> 1);
|
||||
if constexpr (large_is_better) {
|
||||
if (distances[mid] < dist) {
|
||||
hi = mid;
|
||||
} else {
|
||||
lo = mid + 1;
|
||||
}
|
||||
} else {
|
||||
if (distances[mid] > dist) {
|
||||
hi = mid;
|
||||
} else {
|
||||
lo = mid + 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
return lo;
|
||||
}
|
||||
|
||||
inline void
|
||||
insert_helper(milvus::SearchResult& search_result,
|
||||
int& topk,
|
||||
|
||||
@@ -76,7 +76,7 @@ PhyProjectNode::GetOutput() {
|
||||
return row_vector;
|
||||
}
|
||||
|
||||
auto result_pair = segment_->find_first(-1, raw_data_view);
|
||||
auto result_pair = segment_->find_first_n(-1, raw_data_view);
|
||||
auto& selected_offsets = result_pair.first;
|
||||
auto selected_count = selected_offsets.size();
|
||||
// When all rows are filtered out, return nullptr.
|
||||
|
||||
@@ -30,6 +30,21 @@
|
||||
|
||||
namespace milvus {
|
||||
namespace exec {
|
||||
// Binary search to find insert position for sorted order
|
||||
// Used by iterative filter and element filter nodes
|
||||
template <bool large_is_better>
|
||||
inline size_t
|
||||
find_binsert_position(const std::vector<float>& distances,
|
||||
size_t lo,
|
||||
size_t hi,
|
||||
float dist) {
|
||||
auto first = distances.begin() + lo;
|
||||
auto last = distances.begin() + hi;
|
||||
auto it = large_is_better
|
||||
? std::upper_bound(first, last, dist, std::greater<float>{})
|
||||
: std::upper_bound(first, last, dist);
|
||||
return static_cast<size_t>(it - distances.begin());
|
||||
}
|
||||
|
||||
[[maybe_unused]] static bool
|
||||
UseVectorIterator(const SearchInfo& search_info) {
|
||||
@@ -118,6 +133,12 @@ sort_search_result(milvus::SearchResult& result, bool large_is_better) {
|
||||
new_distances.reserve(size);
|
||||
new_seg_offsets.reserve(size);
|
||||
|
||||
bool has_element_indices = !result.element_indices_.empty();
|
||||
std::vector<int32_t> new_element_indices;
|
||||
if (has_element_indices) {
|
||||
new_element_indices.reserve(size);
|
||||
}
|
||||
|
||||
std::vector<size_t> idx(topk);
|
||||
|
||||
for (size_t start = 0; start < size; start += topk) {
|
||||
@@ -141,11 +162,17 @@ sort_search_result(milvus::SearchResult& result, bool large_is_better) {
|
||||
for (auto i : idx) {
|
||||
new_distances.push_back(result.distances_[i]);
|
||||
new_seg_offsets.push_back(result.seg_offsets_[i]);
|
||||
if (has_element_indices) {
|
||||
new_element_indices.push_back(result.element_indices_[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result.distances_ = new_distances;
|
||||
result.seg_offsets_ = new_seg_offsets;
|
||||
result.distances_ = std::move(new_distances);
|
||||
result.seg_offsets_ = std::move(new_seg_offsets);
|
||||
if (has_element_indices) {
|
||||
result.element_indices_ = std::move(new_element_indices);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace exec
|
||||
|
||||
@@ -191,11 +191,13 @@ class ElementFilterNode : public PlanNode {
|
||||
ElementFilterNode(const PlanNodeId& id,
|
||||
expr::TypedExprPtr element_filter,
|
||||
std::string struct_name,
|
||||
std::vector<PlanNodePtr> sources)
|
||||
std::vector<PlanNodePtr> sources,
|
||||
bool has_doc_predicate = true)
|
||||
: PlanNode(id),
|
||||
sources_{std::move(sources)},
|
||||
element_filter_(std::move(element_filter)),
|
||||
struct_name_(std::move(struct_name)) {
|
||||
struct_name_(std::move(struct_name)),
|
||||
has_doc_predicate_(has_doc_predicate) {
|
||||
AssertInfo(
|
||||
element_filter_->type() == DataType::BOOL,
|
||||
fmt::format(
|
||||
@@ -223,6 +225,11 @@ class ElementFilterNode : public PlanNode {
|
||||
return struct_name_;
|
||||
}
|
||||
|
||||
bool
|
||||
has_doc_predicate() const {
|
||||
return has_doc_predicate_;
|
||||
}
|
||||
|
||||
std::string_view
|
||||
name() const override {
|
||||
return "ElementFilter";
|
||||
@@ -231,15 +238,18 @@ class ElementFilterNode : public PlanNode {
|
||||
std::string
|
||||
ToString() const override {
|
||||
return fmt::format(
|
||||
"ElementFilterNode:[struct_name:{}, element_filter:{}]",
|
||||
"ElementFilterNode:[struct_name:{}, element_filter:{}, "
|
||||
"has_doc_predicate:{}]",
|
||||
struct_name_,
|
||||
element_filter_->ToString());
|
||||
element_filter_->ToString(),
|
||||
has_doc_predicate_);
|
||||
}
|
||||
|
||||
private:
|
||||
const std::vector<PlanNodePtr> sources_;
|
||||
const expr::TypedExprPtr element_filter_;
|
||||
const std::string struct_name_;
|
||||
const bool has_doc_predicate_;
|
||||
};
|
||||
|
||||
class ElementFilterBitsNode : public PlanNode {
|
||||
|
||||
@@ -299,7 +299,8 @@ ExecPlanNodeVisitor::visit(RetrievePlanNode& node) {
|
||||
|
||||
// Do task execution
|
||||
auto result = ExecuteTask(plan, query_context);
|
||||
setupRetrieveResult(result, op_context, node, retrieve_result, segment);
|
||||
setupRetrieveResult(
|
||||
result, op_context, node, retrieve_result, segment, query_context);
|
||||
}
|
||||
|
||||
void
|
||||
@@ -308,7 +309,8 @@ ExecPlanNodeVisitor::setupRetrieveResult(
|
||||
const OpContext& op_context,
|
||||
const RetrievePlanNode& node,
|
||||
RetrieveResult& tmp_retrieve_result,
|
||||
const segcore::SegmentInternalInterface* segment) {
|
||||
const segcore::SegmentInternalInterface* segment,
|
||||
std::shared_ptr<milvus::exec::QueryContext> query_context) {
|
||||
if (result == nullptr) {
|
||||
// Return empty field_data arrays with correct schema (0 rows, N columns)
|
||||
// to ensure result structure matches the expected output type.
|
||||
@@ -335,11 +337,25 @@ ExecPlanNodeVisitor::setupRetrieveResult(
|
||||
"children inside row vector must be of column vector for now");
|
||||
tmp_retrieve_result.total_data_cnt_ = first_column->size();
|
||||
if (first_column->IsBitmap()) {
|
||||
tracer::AutoSpan _("Find Limit Pk", tracer::GetRootSpan());
|
||||
BitsetTypeView view(first_column->GetRawData(), first_column->size());
|
||||
auto results_pair = segment->find_first(node.limit_, view);
|
||||
tmp_retrieve_result.result_offsets_ = std::move(results_pair.first);
|
||||
tmp_retrieve_result.has_more_result = results_pair.second;
|
||||
if (query_context->element_level_query()) {
|
||||
// Element-level query: bitset is element-level, need to convert to (doc_id, element_index)
|
||||
tmp_retrieve_result.element_level_ = true;
|
||||
tracer::AutoSpan _(
|
||||
"Element Level Find", tracer::GetRootSpan(), true);
|
||||
auto array_offsets = query_context->get_array_offsets();
|
||||
auto [doc_offsets, element_indices, has_more] =
|
||||
segment->find_first_n_element(
|
||||
node.limit_, view, array_offsets.get());
|
||||
tmp_retrieve_result.result_offsets_ = std::move(doc_offsets);
|
||||
tmp_retrieve_result.element_indices_ = std::move(element_indices);
|
||||
tmp_retrieve_result.has_more_result = has_more;
|
||||
} else {
|
||||
tracer::AutoSpan _("Find Limit Pk", tracer::GetRootSpan());
|
||||
auto results_pair = segment->find_first_n(node.limit_, view);
|
||||
tmp_retrieve_result.result_offsets_ = std::move(results_pair.first);
|
||||
tmp_retrieve_result.has_more_result = results_pair.second;
|
||||
}
|
||||
retrieve_result_opt_ = std::move(tmp_retrieve_result);
|
||||
} else {
|
||||
// load data in the result vector into retrieve_result
|
||||
|
||||
@@ -129,11 +129,13 @@ class ExecPlanNodeVisitor : public PlanNodeVisitor {
|
||||
std::shared_ptr<milvus::exec::QueryContext> query_context);
|
||||
|
||||
void
|
||||
setupRetrieveResult(const RowVectorPtr& result,
|
||||
const OpContext& op_context,
|
||||
const RetrievePlanNode& node,
|
||||
RetrieveResult& tmp_retrieve_result,
|
||||
const segcore::SegmentInternalInterface* segment);
|
||||
setupRetrieveResult(
|
||||
const RowVectorPtr& result,
|
||||
const OpContext& op_context,
|
||||
const RetrievePlanNode& node,
|
||||
RetrieveResult& tmp_retrieve_result,
|
||||
const segcore::SegmentInternalInterface* segment,
|
||||
std::shared_ptr<milvus::exec::QueryContext> query_context);
|
||||
|
||||
private:
|
||||
const segcore::SegmentInterface& segment_;
|
||||
|
||||
@@ -175,54 +175,6 @@ getAggregateOpName(planpb::AggregateOp op) {
|
||||
}
|
||||
|
||||
namespace {
|
||||
// Helper function to build FilterBitsNode and RandomSampleNode
|
||||
plan::PlanNodePtr
|
||||
BuildFilterAndSampleNodes(const proto::plan::QueryPlanNode& query,
|
||||
const planpb::PlanNode& plan_node_proto,
|
||||
const SchemaPtr& schema,
|
||||
const std::vector<plan::PlanNodePtr>& sources,
|
||||
ProtoParser* parser) {
|
||||
if (!query.has_predicates()) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
auto parse_expr_to_filter_node =
|
||||
[&](const proto::plan::Expr& predicate_proto) -> plan::PlanNodePtr {
|
||||
auto expr = parser->ParseExprs(predicate_proto);
|
||||
if (plan_node_proto.has_namespace_()) {
|
||||
expr = MergeExprWithNamespace(
|
||||
schema, expr, plan_node_proto.namespace_());
|
||||
}
|
||||
return std::make_shared<plan::FilterBitsNode>(
|
||||
milvus::plan::GetNextPlanNodeId(), expr, sources);
|
||||
};
|
||||
|
||||
auto* predicate_proto = &query.predicates();
|
||||
if (predicate_proto->expr_case() == proto::plan::Expr::kRandomSampleExpr) {
|
||||
// Predicate exists in random_sample_expr means we encounter expression
|
||||
// like "`predicate expression` && random_sample(...)". Extract it to construct
|
||||
// FilterBitsNode and make it be executed before RandomSampleNode.
|
||||
auto& sample_expr = predicate_proto->random_sample_expr();
|
||||
plan::PlanNodePtr filter_node = nullptr;
|
||||
if (sample_expr.has_predicate()) {
|
||||
filter_node = parse_expr_to_filter_node(sample_expr.predicate());
|
||||
}
|
||||
|
||||
std::vector<plan::PlanNodePtr> sample_sources;
|
||||
if (filter_node) {
|
||||
sample_sources = {filter_node};
|
||||
} else {
|
||||
sample_sources = sources;
|
||||
}
|
||||
|
||||
return std::make_shared<plan::RandomSampleNode>(
|
||||
milvus::plan::GetNextPlanNodeId(),
|
||||
sample_expr.sample_factor(),
|
||||
sample_sources);
|
||||
} else {
|
||||
return parse_expr_to_filter_node(query.predicates());
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function to process group_by fields
|
||||
void
|
||||
@@ -423,11 +375,13 @@ ProtoParser::PlanNodeFromProto(const planpb::PlanNode& plan_node_proto) {
|
||||
|
||||
// Add element-level filter if needed
|
||||
if (is_element_level) {
|
||||
bool has_doc_pred = (doc_expr != nullptr);
|
||||
plannode = std::make_shared<plan::ElementFilterNode>(
|
||||
milvus::plan::GetNextPlanNodeId(),
|
||||
element_expr,
|
||||
struct_name,
|
||||
sources);
|
||||
sources,
|
||||
has_doc_pred);
|
||||
sources = std::vector<milvus::plan::PlanNodePtr>{plannode};
|
||||
}
|
||||
|
||||
@@ -543,86 +497,126 @@ ProtoParser::RetrievePlanNodeFromProto(
|
||||
milvus::plan::PlanNodePtr plannode;
|
||||
std::vector<milvus::plan::PlanNodePtr> sources;
|
||||
|
||||
auto plan_node = [&]() -> std::unique_ptr<RetrievePlanNode> {
|
||||
auto node = std::make_unique<RetrievePlanNode>();
|
||||
if (plan_node_proto.has_predicates()) { // version before 2023.03.30.
|
||||
auto& predicate_proto = plan_node_proto.predicates();
|
||||
auto expr_parser = [&]() -> plan::PlanNodePtr {
|
||||
auto plan_node = std::make_unique<RetrievePlanNode>();
|
||||
if (plan_node_proto.has_predicates()) { // version before 2023.03.30.
|
||||
auto& predicate_proto = plan_node_proto.predicates();
|
||||
auto expr = ParseExprs(predicate_proto);
|
||||
if (plan_node_proto.has_namespace_()) {
|
||||
expr = MergeExprWithNamespace(
|
||||
schema, expr, plan_node_proto.namespace_());
|
||||
}
|
||||
plannode = std::make_shared<plan::FilterBitsNode>(
|
||||
milvus::plan::GetNextPlanNodeId(), expr);
|
||||
sources = std::vector<milvus::plan::PlanNodePtr>{plannode};
|
||||
plannode = std::make_shared<milvus::plan::MvccNode>(
|
||||
milvus::plan::GetNextPlanNodeId(), sources);
|
||||
plan_node->plannodes_ = std::move(plannode);
|
||||
} else {
|
||||
// mvccNode--->FilterBitsNode or
|
||||
// aggNode---> projectNode --->mvccNode--->FilterBitsNode
|
||||
auto& query = plan_node_proto.query();
|
||||
|
||||
// 1. Parse predicates and build FilterBitsNode / RandomSampleNode
|
||||
expr::TypedExprPtr element_expr = nullptr;
|
||||
std::string struct_name;
|
||||
bool is_element_level = false;
|
||||
|
||||
auto parse_expr_to_filter_node =
|
||||
[&](const proto::plan::Expr& predicate_proto) {
|
||||
auto expr = ParseExprs(predicate_proto);
|
||||
if (plan_node_proto.has_namespace_()) {
|
||||
expr = MergeExprWithNamespace(
|
||||
schema, expr, plan_node_proto.namespace_());
|
||||
}
|
||||
return std::make_shared<plan::FilterBitsNode>(
|
||||
milvus::plan::GetNextPlanNodeId(), expr);
|
||||
}();
|
||||
plannode = std::move(expr_parser);
|
||||
sources = std::vector<milvus::plan::PlanNodePtr>{plannode};
|
||||
plannode = std::make_shared<milvus::plan::MvccNode>(
|
||||
milvus::plan::GetNextPlanNodeId(), sources);
|
||||
node->plannodes_ = std::move(plannode);
|
||||
} else {
|
||||
// mvccNode--->FilterBitsNode or
|
||||
// aggNode---> projectNode --->mvccNode--->FilterBitsNode
|
||||
auto& query = plan_node_proto.query();
|
||||
|
||||
// 1. Build FilterBitsNode and RandomSampleNode if needed
|
||||
auto filter_node = BuildFilterAndSampleNodes(
|
||||
query, plan_node_proto, schema, sources, this);
|
||||
if (filter_node) {
|
||||
plannode = filter_node;
|
||||
plannode = std::make_shared<plan::FilterBitsNode>(
|
||||
milvus::plan::GetNextPlanNodeId(), expr, sources);
|
||||
sources = std::vector<milvus::plan::PlanNodePtr>{plannode};
|
||||
};
|
||||
|
||||
if (query.has_predicates()) {
|
||||
auto* predicate_proto = &query.predicates();
|
||||
if (predicate_proto->expr_case() ==
|
||||
proto::plan::Expr::kElementFilterExpr) {
|
||||
is_element_level = true;
|
||||
auto& ef = predicate_proto->element_filter_expr();
|
||||
element_expr = ParseExprs(ef.element_expr());
|
||||
struct_name = ef.struct_name();
|
||||
if (ef.has_predicate()) {
|
||||
parse_expr_to_filter_node(ef.predicate());
|
||||
}
|
||||
} else if (predicate_proto->expr_case() ==
|
||||
proto::plan::Expr::kRandomSampleExpr) {
|
||||
auto& sample_expr = predicate_proto->random_sample_expr();
|
||||
if (sample_expr.has_predicate()) {
|
||||
parse_expr_to_filter_node(sample_expr.predicate());
|
||||
}
|
||||
plannode = std::make_shared<plan::RandomSampleNode>(
|
||||
milvus::plan::GetNextPlanNodeId(),
|
||||
sample_expr.sample_factor(),
|
||||
sources);
|
||||
sources = std::vector<milvus::plan::PlanNodePtr>{plannode};
|
||||
} else {
|
||||
parse_expr_to_filter_node(query.predicates());
|
||||
}
|
||||
|
||||
// 2. Build MvccNode
|
||||
plannode = std::make_shared<milvus::plan::MvccNode>(
|
||||
milvus::plan::GetNextPlanNodeId(), sources);
|
||||
sources = std::vector<milvus::plan::PlanNodePtr>{plannode};
|
||||
|
||||
// 3. Build ProjectNode and AggregationNode if needed
|
||||
auto group_by_field_count = query.group_by_field_ids_size();
|
||||
auto agg_functions_count = query.aggregates_size();
|
||||
if (group_by_field_count > 0 || agg_functions_count > 0) {
|
||||
std::vector<FieldId> project_id_list;
|
||||
std::vector<std::string> project_name_list;
|
||||
std::vector<milvus::DataType> project_type_list;
|
||||
std::vector<expr::FieldAccessTypeExprPtr> groupingKeys;
|
||||
std::vector<plan::AggregationNode::Aggregate> aggregates;
|
||||
std::vector<std::string> agg_names;
|
||||
|
||||
// Process group_by fields
|
||||
ProcessGroupByFields(query,
|
||||
schema,
|
||||
groupingKeys,
|
||||
project_id_list,
|
||||
project_name_list,
|
||||
project_type_list);
|
||||
|
||||
// Process aggregates
|
||||
ProcessAggregates(query,
|
||||
schema,
|
||||
aggregates,
|
||||
agg_names,
|
||||
project_id_list,
|
||||
project_name_list,
|
||||
project_type_list);
|
||||
|
||||
// Build ProjectNode and AggregationNode
|
||||
plannode = BuildProjectAndAggregationNodes(
|
||||
query,
|
||||
sources,
|
||||
std::move(groupingKeys),
|
||||
std::move(agg_names),
|
||||
std::move(aggregates),
|
||||
std::move(project_id_list),
|
||||
std::move(project_name_list),
|
||||
std::move(project_type_list));
|
||||
}
|
||||
node->plannodes_ = plannode;
|
||||
node->limit_ = query.limit();
|
||||
}
|
||||
return node;
|
||||
}();
|
||||
|
||||
// 2. Build MvccNode
|
||||
plannode = std::make_shared<milvus::plan::MvccNode>(
|
||||
milvus::plan::GetNextPlanNodeId(), sources);
|
||||
sources = std::vector<milvus::plan::PlanNodePtr>{plannode};
|
||||
|
||||
// 3. Build ElementFilterBitsNode if element-level
|
||||
if (is_element_level) {
|
||||
plannode = std::make_shared<plan::ElementFilterBitsNode>(
|
||||
milvus::plan::GetNextPlanNodeId(),
|
||||
element_expr,
|
||||
struct_name,
|
||||
sources);
|
||||
sources = std::vector<milvus::plan::PlanNodePtr>{plannode};
|
||||
}
|
||||
|
||||
// 4. Build ProjectNode and AggregationNode if needed
|
||||
auto group_by_field_count = query.group_by_field_ids_size();
|
||||
auto agg_functions_count = query.aggregates_size();
|
||||
if (group_by_field_count > 0 || agg_functions_count > 0) {
|
||||
std::vector<FieldId> project_id_list;
|
||||
std::vector<std::string> project_name_list;
|
||||
std::vector<milvus::DataType> project_type_list;
|
||||
std::vector<expr::FieldAccessTypeExprPtr> groupingKeys;
|
||||
std::vector<plan::AggregationNode::Aggregate> aggregates;
|
||||
std::vector<std::string> agg_names;
|
||||
|
||||
// Process group_by fields
|
||||
ProcessGroupByFields(query,
|
||||
schema,
|
||||
groupingKeys,
|
||||
project_id_list,
|
||||
project_name_list,
|
||||
project_type_list);
|
||||
|
||||
// Process aggregates
|
||||
ProcessAggregates(query,
|
||||
schema,
|
||||
aggregates,
|
||||
agg_names,
|
||||
project_id_list,
|
||||
project_name_list,
|
||||
project_type_list);
|
||||
|
||||
// Build ProjectNode and AggregationNode
|
||||
plannode =
|
||||
BuildProjectAndAggregationNodes(query,
|
||||
sources,
|
||||
std::move(groupingKeys),
|
||||
std::move(agg_names),
|
||||
std::move(aggregates),
|
||||
std::move(project_id_list),
|
||||
std::move(project_name_list),
|
||||
std::move(project_type_list));
|
||||
}
|
||||
plan_node->plannodes_ = plannode;
|
||||
plan_node->limit_ = query.limit();
|
||||
}
|
||||
|
||||
PlanOptionsFromProto(plan_node_proto.plan_options(),
|
||||
plan_node->plan_options_);
|
||||
|
||||
@@ -1441,10 +1441,10 @@ ChunkedSegmentSealedImpl::pk_binary_range(milvus::OpContext* op_ctx,
|
||||
}
|
||||
|
||||
std::pair<std::vector<OffsetMap::OffsetType>, bool>
|
||||
ChunkedSegmentSealedImpl::find_first(int64_t limit,
|
||||
const BitsetTypeView& bitset) const {
|
||||
ChunkedSegmentSealedImpl::find_first_n(int64_t limit,
|
||||
const BitsetTypeView& bitset) const {
|
||||
if (!is_sorted_by_pk_) {
|
||||
return insert_record_.pk2offset_->find_first(limit, bitset);
|
||||
return insert_record_.pk2offset_->find_first_n(limit, bitset);
|
||||
}
|
||||
if (limit == Unlimited || limit == NoLimit) {
|
||||
limit = num_rows_.value();
|
||||
@@ -1474,6 +1474,56 @@ ChunkedSegmentSealedImpl::find_first(int64_t limit,
|
||||
return {seg_offsets, more_hit_than_limit && result.has_value()};
|
||||
}
|
||||
|
||||
std::tuple<std::vector<int64_t>, std::vector<std::vector<int32_t>>, bool>
|
||||
ChunkedSegmentSealedImpl::find_first_n_element(
|
||||
int64_t limit,
|
||||
const BitsetTypeView& element_bitset,
|
||||
const IArrayOffsets* array_offsets) const {
|
||||
if (!is_sorted_by_pk_) {
|
||||
// Not sorted by PK, use pk2offset_ to iterate in PK order
|
||||
return insert_record_.pk2offset_->find_first_n_element(
|
||||
limit, element_bitset, array_offsets);
|
||||
}
|
||||
|
||||
// Sorted by PK, element_id order = (PK, element_index) order
|
||||
// Directly iterate element_bitset in order
|
||||
if (limit == Unlimited || limit == NoLimit) {
|
||||
limit = static_cast<int64_t>(element_bitset.size());
|
||||
}
|
||||
|
||||
int64_t hit_num = 0;
|
||||
auto element_size = static_cast<int64_t>(element_bitset.size());
|
||||
int64_t cnt = element_size - element_bitset.count();
|
||||
auto more_hit_than_limit = cnt > limit;
|
||||
limit = std::min(limit, cnt);
|
||||
|
||||
std::vector<int64_t> doc_offsets;
|
||||
std::vector<std::vector<int32_t>> element_indices;
|
||||
|
||||
int64_t current_doc_id = -1;
|
||||
std::optional<size_t> elem_opt = element_bitset.find_first(false);
|
||||
while (elem_opt.has_value() && hit_num < limit) {
|
||||
int64_t elem_id = static_cast<int64_t>(elem_opt.value());
|
||||
auto [doc_id, elem_idx] = array_offsets->ElementIDToRowID(elem_id);
|
||||
|
||||
if (doc_id != current_doc_id) {
|
||||
// New document - start a new entry
|
||||
doc_offsets.push_back(doc_id);
|
||||
element_indices.push_back({static_cast<int32_t>(elem_idx)});
|
||||
current_doc_id = doc_id;
|
||||
} else {
|
||||
// Same document - append to existing entry
|
||||
element_indices.back().push_back(static_cast<int32_t>(elem_idx));
|
||||
}
|
||||
hit_num++;
|
||||
elem_opt = element_bitset.find_next(elem_id, false);
|
||||
}
|
||||
|
||||
return {std::move(doc_offsets),
|
||||
std::move(element_indices),
|
||||
more_hit_than_limit && elem_opt.has_value()};
|
||||
}
|
||||
|
||||
ChunkedSegmentSealedImpl::ChunkedSegmentSealedImpl(
|
||||
SchemaPtr schema,
|
||||
IndexMetaPtr index_meta,
|
||||
|
||||
@@ -338,7 +338,12 @@ class ChunkedSegmentSealedImpl : public SegmentSealed {
|
||||
const Timestamp* timestamps) override;
|
||||
|
||||
std::pair<std::vector<OffsetMap::OffsetType>, bool>
|
||||
find_first(int64_t limit, const BitsetTypeView& bitset) const override;
|
||||
find_first_n(int64_t limit, const BitsetTypeView& bitset) const override;
|
||||
|
||||
std::tuple<std::vector<int64_t>, std::vector<std::vector<int32_t>>, bool>
|
||||
find_first_n_element(int64_t limit,
|
||||
const BitsetTypeView& element_bitset,
|
||||
const IArrayOffsets* array_offsets) const override;
|
||||
|
||||
// Calculate: output[i] = Vec[seg_offset[i]]
|
||||
// where Vec is determined from field_offset
|
||||
|
||||
@@ -23,6 +23,7 @@
|
||||
|
||||
#include "TimestampData.h"
|
||||
#include "TimestampIndex.h"
|
||||
#include "common/ArrayOffsets.h"
|
||||
#include "common/EasyAssert.h"
|
||||
#include "common/Schema.h"
|
||||
#include "common/TrackingStdAllocator.h"
|
||||
@@ -30,6 +31,7 @@
|
||||
#include "mmap/ChunkedColumn.h"
|
||||
#include "segcore/AckResponder.h"
|
||||
#include "segcore/ConcurrentVector.h"
|
||||
#include <tuple>
|
||||
#include <type_traits>
|
||||
|
||||
namespace milvus::segcore {
|
||||
@@ -70,7 +72,19 @@ class OffsetMap {
|
||||
using OffsetType = int64_t;
|
||||
// TODO: in fact, we can retrieve the pk here. Not sure which way is more efficient.
|
||||
virtual std::pair<std::vector<OffsetMap::OffsetType>, bool>
|
||||
find_first(int64_t limit, const BitsetTypeView& bitset) const = 0;
|
||||
find_first_n(int64_t limit, const BitsetTypeView& bitset) const = 0;
|
||||
|
||||
// Element-level version of find_first_n.
|
||||
// Find first N elements that pass filter, ordered by PK then element_index.
|
||||
// Returns:
|
||||
// - vector of unique doc_offsets (no duplicates)
|
||||
// - vector of element_indices per doc (element_indices[i] for doc_offsets[i])
|
||||
// - has_more flag indicating if there are more results
|
||||
virtual std::
|
||||
tuple<std::vector<int64_t>, std::vector<std::vector<int32_t>>, bool>
|
||||
find_first_n_element(int64_t limit,
|
||||
const BitsetTypeView& element_bitset,
|
||||
const IArrayOffsets* array_offsets) const = 0;
|
||||
|
||||
virtual void
|
||||
clear() = 0;
|
||||
@@ -185,7 +199,7 @@ class OffsetOrderedMap : public OffsetMap {
|
||||
}
|
||||
|
||||
std::pair<std::vector<OffsetMap::OffsetType>, bool>
|
||||
find_first(int64_t limit, const BitsetTypeView& bitset) const override {
|
||||
find_first_n(int64_t limit, const BitsetTypeView& bitset) const override {
|
||||
std::shared_lock<std::shared_mutex> lck(mtx_);
|
||||
|
||||
if (limit == Unlimited || limit == NoLimit) {
|
||||
@@ -194,7 +208,21 @@ class OffsetOrderedMap : public OffsetMap {
|
||||
|
||||
// TODO: we can't retrieve pk by offset very conveniently.
|
||||
// Selectivity should be done outside.
|
||||
return find_first_by_index(limit, bitset);
|
||||
return find_first_n_by_index(limit, bitset);
|
||||
}
|
||||
|
||||
std::tuple<std::vector<int64_t>, std::vector<std::vector<int32_t>>, bool>
|
||||
find_first_n_element(int64_t limit,
|
||||
const BitsetTypeView& element_bitset,
|
||||
const IArrayOffsets* array_offsets) const override {
|
||||
std::shared_lock<std::shared_mutex> lck(mtx_);
|
||||
|
||||
if (limit == Unlimited || limit == NoLimit) {
|
||||
limit = static_cast<int64_t>(element_bitset.size());
|
||||
}
|
||||
|
||||
return find_first_n_element_by_index(
|
||||
limit, element_bitset, array_offsets);
|
||||
}
|
||||
|
||||
void
|
||||
@@ -211,10 +239,11 @@ class OffsetOrderedMap : public OffsetMap {
|
||||
|
||||
private:
|
||||
std::pair<std::vector<OffsetMap::OffsetType>, bool>
|
||||
find_first_by_index(int64_t limit, const BitsetTypeView& bitset) const {
|
||||
find_first_n_by_index(int64_t limit, const BitsetTypeView& bitset) const {
|
||||
int64_t hit_num = 0; // avoid counting the number everytime.
|
||||
auto size = bitset.size();
|
||||
int64_t cnt = size - bitset.count();
|
||||
auto more_hit_than_limit = cnt > limit;
|
||||
limit = std::min(limit, cnt);
|
||||
std::vector<int64_t> seg_offsets;
|
||||
seg_offsets.reserve(limit);
|
||||
@@ -237,7 +266,66 @@ class OffsetOrderedMap : public OffsetMap {
|
||||
}
|
||||
}
|
||||
}
|
||||
return {seg_offsets, it != map_.end()};
|
||||
return {seg_offsets, more_hit_than_limit && it != map_.end()};
|
||||
}
|
||||
|
||||
std::tuple<std::vector<int64_t>, std::vector<std::vector<int32_t>>, bool>
|
||||
find_first_n_element_by_index(int64_t limit,
|
||||
const BitsetTypeView& element_bitset,
|
||||
const IArrayOffsets* array_offsets) const {
|
||||
std::vector<int64_t> doc_offsets;
|
||||
std::vector<std::vector<int32_t>> element_indices;
|
||||
|
||||
int64_t hit_num = 0;
|
||||
auto element_size = static_cast<int64_t>(element_bitset.size());
|
||||
// Clamp limit to the actual number of matching elements,
|
||||
// same as find_first_n_by_index does for doc-level queries.
|
||||
int64_t cnt = element_size - element_bitset.count();
|
||||
auto more_hit_than_limit = cnt > limit;
|
||||
limit = std::min(limit, cnt);
|
||||
|
||||
// Traverse map_ in PK order
|
||||
std::vector<int32_t> matching_indices;
|
||||
auto it = map_.begin();
|
||||
for (; hit_num < limit && it != map_.end(); ++it) {
|
||||
// For each PK, traverse from back to front to obtain the latest offset.
|
||||
// Same as find_first_n_by_index: only use the first (newest) offset
|
||||
// that has matching elements, then break to avoid returning stale versions.
|
||||
for (int i = it->second.size() - 1; i >= 0 && hit_num < limit;
|
||||
--i) {
|
||||
auto doc_offset = it->second[i];
|
||||
|
||||
// Get element range for this doc
|
||||
auto [first_elem, last_elem] =
|
||||
array_offsets->ElementIDRangeOfRow(doc_offset);
|
||||
|
||||
// Collect all matching element indices for this doc
|
||||
matching_indices.clear();
|
||||
for (int64_t elem_id = first_elem;
|
||||
elem_id < last_elem && hit_num < limit;
|
||||
++elem_id) {
|
||||
if (elem_id >= element_size) {
|
||||
continue;
|
||||
}
|
||||
if (!element_bitset[elem_id]) { // 0 means pass filter
|
||||
matching_indices.push_back(
|
||||
static_cast<int32_t>(elem_id - first_elem));
|
||||
hit_num++;
|
||||
}
|
||||
}
|
||||
|
||||
// Only add doc if it has matching elements
|
||||
if (!matching_indices.empty()) {
|
||||
doc_offsets.push_back(doc_offset);
|
||||
element_indices.push_back(std::move(matching_indices));
|
||||
// PK hit, no need to continue traversing older offsets with the same PK.
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool has_more = more_hit_than_limit && (it != map_.end());
|
||||
return {std::move(doc_offsets), std::move(element_indices), has_more};
|
||||
}
|
||||
|
||||
private:
|
||||
@@ -365,7 +453,7 @@ class OffsetOrderedArray : public OffsetMap {
|
||||
}
|
||||
|
||||
std::pair<std::vector<OffsetMap::OffsetType>, bool>
|
||||
find_first(int64_t limit, const BitsetTypeView& bitset) const override {
|
||||
find_first_n(int64_t limit, const BitsetTypeView& bitset) const override {
|
||||
check_search();
|
||||
|
||||
if (limit == Unlimited || limit == NoLimit) {
|
||||
@@ -374,7 +462,21 @@ class OffsetOrderedArray : public OffsetMap {
|
||||
|
||||
// TODO: we can't retrieve pk by offset very conveniently.
|
||||
// Selectivity should be done outside.
|
||||
return find_first_by_index(limit, bitset);
|
||||
return find_first_n_by_index(limit, bitset);
|
||||
}
|
||||
|
||||
std::tuple<std::vector<int64_t>, std::vector<std::vector<int32_t>>, bool>
|
||||
find_first_n_element(int64_t limit,
|
||||
const BitsetTypeView& element_bitset,
|
||||
const IArrayOffsets* array_offsets) const override {
|
||||
check_search();
|
||||
|
||||
if (limit == Unlimited || limit == NoLimit) {
|
||||
limit = static_cast<int64_t>(element_bitset.size());
|
||||
}
|
||||
|
||||
return find_first_n_element_by_index(
|
||||
limit, element_bitset, array_offsets);
|
||||
}
|
||||
|
||||
void
|
||||
@@ -390,7 +492,7 @@ class OffsetOrderedArray : public OffsetMap {
|
||||
|
||||
private:
|
||||
std::pair<std::vector<OffsetMap::OffsetType>, bool>
|
||||
find_first_by_index(int64_t limit, const BitsetTypeView& bitset) const {
|
||||
find_first_n_by_index(int64_t limit, const BitsetTypeView& bitset) const {
|
||||
int64_t hit_num = 0; // avoid counting the number everytime.
|
||||
auto size = bitset.size();
|
||||
int64_t cnt = size - bitset.count();
|
||||
@@ -414,6 +516,57 @@ class OffsetOrderedArray : public OffsetMap {
|
||||
return {seg_offsets, more_hit_than_limit && it != array_.end()};
|
||||
}
|
||||
|
||||
std::tuple<std::vector<int64_t>, std::vector<std::vector<int32_t>>, bool>
|
||||
find_first_n_element_by_index(int64_t limit,
|
||||
const BitsetTypeView& element_bitset,
|
||||
const IArrayOffsets* array_offsets) const {
|
||||
std::vector<int64_t> doc_offsets;
|
||||
std::vector<std::vector<int32_t>> element_indices;
|
||||
|
||||
int64_t hit_num = 0;
|
||||
auto element_size = static_cast<int64_t>(element_bitset.size());
|
||||
// Clamp limit to the actual number of matching elements,
|
||||
// same as find_first_n_by_index does for doc-level queries.
|
||||
int64_t cnt = element_size - element_bitset.count();
|
||||
auto more_hit_than_limit = cnt > limit;
|
||||
limit = std::min(limit, cnt);
|
||||
|
||||
// Traverse array_ in PK order (already sorted)
|
||||
std::vector<int32_t> matching_indices;
|
||||
auto it = array_.begin();
|
||||
for (; hit_num < limit && it != array_.end(); ++it) {
|
||||
auto doc_offset = it->second;
|
||||
|
||||
// Get element range for this doc
|
||||
auto [first_elem, last_elem] =
|
||||
array_offsets->ElementIDRangeOfRow(doc_offset);
|
||||
|
||||
// Collect all matching element indices for this doc
|
||||
matching_indices.clear();
|
||||
for (int64_t elem_id = first_elem;
|
||||
elem_id < last_elem && hit_num < limit;
|
||||
++elem_id) {
|
||||
if (elem_id >= element_size) {
|
||||
continue;
|
||||
}
|
||||
if (!element_bitset[elem_id]) { // 0 means pass filter
|
||||
matching_indices.push_back(
|
||||
static_cast<int32_t>(elem_id - first_elem));
|
||||
hit_num++;
|
||||
}
|
||||
}
|
||||
|
||||
// Only add doc if it has matching elements
|
||||
if (!matching_indices.empty()) {
|
||||
doc_offsets.push_back(doc_offset);
|
||||
element_indices.push_back(std::move(matching_indices));
|
||||
}
|
||||
}
|
||||
|
||||
bool has_more = more_hit_than_limit && (it != array_.end());
|
||||
return {std::move(doc_offsets), std::move(element_indices), has_more};
|
||||
}
|
||||
|
||||
void
|
||||
check_search() const {
|
||||
AssertInfo(is_sealed,
|
||||
|
||||
@@ -72,9 +72,9 @@ class TypedOffsetOrderedArrayTest : public testing::Test {
|
||||
using TypeOfPks = testing::Types<int64_t, std::string>;
|
||||
TYPED_TEST_SUITE_P(TypedOffsetOrderedArrayTest);
|
||||
|
||||
TYPED_TEST_P(TypedOffsetOrderedArrayTest, find_first) {
|
||||
TYPED_TEST_P(TypedOffsetOrderedArrayTest, find_first_n) {
|
||||
// not sealed.
|
||||
ASSERT_ANY_THROW(this->map_.find_first(Unlimited, {}));
|
||||
ASSERT_ANY_THROW(this->map_.find_first_n(Unlimited, {}));
|
||||
|
||||
// insert 10 entities.
|
||||
int num = 10;
|
||||
@@ -92,7 +92,7 @@ TYPED_TEST_P(TypedOffsetOrderedArrayTest, find_first) {
|
||||
BitsetTypeView all_view(all.data(), num);
|
||||
{
|
||||
auto [offsets, has_more_res] =
|
||||
this->map_.find_first(num / 2, all_view);
|
||||
this->map_.find_first_n(num / 2, all_view);
|
||||
ASSERT_EQ(num / 2, offsets.size());
|
||||
ASSERT_TRUE(has_more_res);
|
||||
for (int i = 1; i < offsets.size(); i++) {
|
||||
@@ -101,7 +101,7 @@ TYPED_TEST_P(TypedOffsetOrderedArrayTest, find_first) {
|
||||
}
|
||||
{
|
||||
auto [offsets, has_more_res] =
|
||||
this->map_.find_first(Unlimited, all_view);
|
||||
this->map_.find_first_n(Unlimited, all_view);
|
||||
ASSERT_EQ(num, offsets.size());
|
||||
ASSERT_FALSE(has_more_res);
|
||||
for (int i = 1; i < offsets.size(); i++) {
|
||||
@@ -115,7 +115,7 @@ TYPED_TEST_P(TypedOffsetOrderedArrayTest, find_first) {
|
||||
BitsetTypeView all_minus_1_view(all_minus_1.data(), num - 1);
|
||||
{
|
||||
auto [offsets, has_more_res] =
|
||||
this->map_.find_first(num / 2, all_minus_1_view);
|
||||
this->map_.find_first_n(num / 2, all_minus_1_view);
|
||||
ASSERT_EQ(num / 2, offsets.size());
|
||||
ASSERT_TRUE(has_more_res);
|
||||
for (int i = 1; i < offsets.size(); i++) {
|
||||
@@ -124,7 +124,7 @@ TYPED_TEST_P(TypedOffsetOrderedArrayTest, find_first) {
|
||||
}
|
||||
{
|
||||
auto [offsets, has_more_res] =
|
||||
this->map_.find_first(Unlimited, all_minus_1_view);
|
||||
this->map_.find_first_n(Unlimited, all_minus_1_view);
|
||||
ASSERT_EQ(all_minus_1.size(), offsets.size());
|
||||
ASSERT_FALSE(has_more_res);
|
||||
for (int i = 1; i < offsets.size(); i++) {
|
||||
@@ -137,14 +137,216 @@ TYPED_TEST_P(TypedOffsetOrderedArrayTest, find_first) {
|
||||
BitsetType none(num);
|
||||
none.set();
|
||||
BitsetTypeView none_view(none.data(), num);
|
||||
auto result_pair = this->map_.find_first(num / 2, none_view);
|
||||
auto result_pair = this->map_.find_first_n(num / 2, none_view);
|
||||
ASSERT_EQ(0, result_pair.first.size());
|
||||
ASSERT_FALSE(result_pair.second);
|
||||
result_pair = this->map_.find_first(NoLimit, none_view);
|
||||
result_pair = this->map_.find_first_n(NoLimit, none_view);
|
||||
ASSERT_EQ(0, result_pair.first.size());
|
||||
ASSERT_FALSE(result_pair.second);
|
||||
}
|
||||
}
|
||||
|
||||
REGISTER_TYPED_TEST_SUITE_P(TypedOffsetOrderedArrayTest, find_first);
|
||||
TYPED_TEST_P(TypedOffsetOrderedArrayTest, find_first_n_element) {
|
||||
// Setup: insert 5 docs with sequential PKs
|
||||
int num = 5;
|
||||
int array_len = 3;
|
||||
auto data = this->random_generate(num);
|
||||
for (const auto& x : data) {
|
||||
this->insert(x);
|
||||
}
|
||||
this->seal();
|
||||
|
||||
// Build ArrayOffsets: each doc has array_len elements
|
||||
std::vector<int32_t> element_row_ids;
|
||||
std::vector<int32_t> row_to_element_start = {0};
|
||||
for (int doc = 0; doc < num; doc++) {
|
||||
for (int e = 0; e < array_len; e++) {
|
||||
element_row_ids.push_back(doc);
|
||||
}
|
||||
row_to_element_start.push_back(
|
||||
static_cast<int32_t>((doc + 1) * array_len));
|
||||
}
|
||||
auto array_offsets = std::make_shared<ArrayOffsetsSealed>(
|
||||
std::move(element_row_ids), std::move(row_to_element_start));
|
||||
|
||||
int total_elements = num * array_len; // 15
|
||||
|
||||
// Case 1: all elements pass filter
|
||||
{
|
||||
BitsetType all(total_elements);
|
||||
all.reset();
|
||||
BitsetTypeView view(all.data(), total_elements);
|
||||
auto [doc_offsets, elem_indices, has_more] =
|
||||
this->map_.find_first_n_element(
|
||||
total_elements, view, array_offsets.get());
|
||||
ASSERT_EQ(doc_offsets.size(), num);
|
||||
for (size_t i = 0; i < doc_offsets.size(); i++) {
|
||||
ASSERT_EQ(elem_indices[i].size(), array_len);
|
||||
}
|
||||
ASSERT_FALSE(has_more);
|
||||
}
|
||||
|
||||
// Case 2: limit counts elements, not docs
|
||||
{
|
||||
BitsetType all(total_elements);
|
||||
all.reset();
|
||||
BitsetTypeView view(all.data(), total_elements);
|
||||
auto [doc_offsets, elem_indices, has_more] =
|
||||
this->map_.find_first_n_element(4, view, array_offsets.get());
|
||||
int total = 0;
|
||||
for (auto& indices : elem_indices) {
|
||||
total += indices.size();
|
||||
}
|
||||
ASSERT_EQ(total, 4);
|
||||
ASSERT_EQ(doc_offsets.size(), 2);
|
||||
ASSERT_EQ(elem_indices[0].size(), 3);
|
||||
ASSERT_EQ(elem_indices[1].size(), 1);
|
||||
ASSERT_TRUE(has_more);
|
||||
}
|
||||
|
||||
// Case 3: partial elements pass (only elem_idx=1 per doc)
|
||||
{
|
||||
BitsetType partial(total_elements);
|
||||
partial.set();
|
||||
for (int doc = 0; doc < num; doc++) {
|
||||
partial.reset(doc * array_len + 1);
|
||||
}
|
||||
BitsetTypeView view(partial.data(), total_elements);
|
||||
auto [doc_offsets, elem_indices, has_more] =
|
||||
this->map_.find_first_n_element(
|
||||
total_elements, view, array_offsets.get());
|
||||
ASSERT_EQ(doc_offsets.size(), num);
|
||||
for (size_t i = 0; i < doc_offsets.size(); i++) {
|
||||
ASSERT_EQ(elem_indices[i].size(), 1);
|
||||
ASSERT_EQ(elem_indices[i][0], 1);
|
||||
}
|
||||
}
|
||||
|
||||
// Case 4: no elements pass
|
||||
{
|
||||
BitsetType none(total_elements);
|
||||
none.set();
|
||||
BitsetTypeView view(none.data(), total_elements);
|
||||
auto [doc_offsets, elem_indices, has_more] =
|
||||
this->map_.find_first_n_element(
|
||||
total_elements, view, array_offsets.get());
|
||||
ASSERT_EQ(doc_offsets.size(), 0);
|
||||
ASSERT_EQ(elem_indices.size(), 0);
|
||||
ASSERT_FALSE(has_more);
|
||||
}
|
||||
|
||||
// Case 5: element bitset smaller than array_offsets
|
||||
{
|
||||
int smaller_size = total_elements - array_len;
|
||||
BitsetType small(smaller_size);
|
||||
small.reset();
|
||||
BitsetTypeView view(small.data(), smaller_size);
|
||||
auto [doc_offsets, elem_indices, has_more] =
|
||||
this->map_.find_first_n_element(
|
||||
total_elements, view, array_offsets.get());
|
||||
int total = 0;
|
||||
for (auto& indices : elem_indices) {
|
||||
total += indices.size();
|
||||
}
|
||||
ASSERT_EQ(total, smaller_size);
|
||||
}
|
||||
}
|
||||
|
||||
TYPED_TEST_P(TypedOffsetOrderedArrayTest, find_first_n_element_has_more) {
|
||||
// Test has_more correctness when limit exactly equals total matching
|
||||
// elements. Previously, has_more used (it != end) || (hit_num >= limit)
|
||||
// which incorrectly returned true when all data was exhausted.
|
||||
|
||||
int num = 3;
|
||||
int array_len = 2;
|
||||
// Insert docs with deterministic PKs for reproducibility
|
||||
std::vector<TypeParam> data;
|
||||
for (int i = 0; i < num; i++) {
|
||||
TypeParam pk;
|
||||
if constexpr (std::is_same_v<std::string, TypeParam>) {
|
||||
pk = std::to_string(i);
|
||||
} else {
|
||||
pk = static_cast<TypeParam>(i);
|
||||
}
|
||||
this->insert(pk);
|
||||
data.push_back(pk);
|
||||
}
|
||||
this->seal();
|
||||
|
||||
// Build ArrayOffsets: each doc has array_len elements
|
||||
// doc0: [elem0, elem1], doc1: [elem2, elem3], doc2: [elem4, elem5]
|
||||
std::vector<int32_t> element_row_ids;
|
||||
std::vector<int32_t> row_to_element_start = {0};
|
||||
for (int doc = 0; doc < num; doc++) {
|
||||
for (int e = 0; e < array_len; e++) {
|
||||
element_row_ids.push_back(doc);
|
||||
}
|
||||
row_to_element_start.push_back(
|
||||
static_cast<int32_t>((doc + 1) * array_len));
|
||||
}
|
||||
auto array_offsets = std::make_shared<ArrayOffsetsSealed>(
|
||||
std::move(element_row_ids), std::move(row_to_element_start));
|
||||
|
||||
int total_elements = num * array_len; // 6
|
||||
|
||||
// Case 1: limit == total matching elements (aligned on doc boundary)
|
||||
// All 6 elements pass, limit=6 → should return all, has_more=false
|
||||
{
|
||||
BitsetType all(total_elements);
|
||||
all.reset();
|
||||
BitsetTypeView view(all.data(), total_elements);
|
||||
auto [doc_offsets, elem_indices, has_more] =
|
||||
this->map_.find_first_n_element(
|
||||
total_elements, view, array_offsets.get());
|
||||
int collected = 0;
|
||||
for (auto& indices : elem_indices) {
|
||||
collected += indices.size();
|
||||
}
|
||||
ASSERT_EQ(collected, total_elements);
|
||||
ASSERT_FALSE(has_more) << "has_more should be false when limit equals "
|
||||
"total matching elements";
|
||||
}
|
||||
|
||||
// Case 2: limit == total matching elements (NOT aligned on doc boundary)
|
||||
// Only elem_idx=0 per doc passes → 3 matching elements, limit=3
|
||||
{
|
||||
BitsetType partial(total_elements);
|
||||
partial.set();
|
||||
for (int doc = 0; doc < num; doc++) {
|
||||
partial.reset(doc *
|
||||
array_len); // only first element per doc passes
|
||||
}
|
||||
BitsetTypeView view(partial.data(), total_elements);
|
||||
auto [doc_offsets, elem_indices, has_more] =
|
||||
this->map_.find_first_n_element(num, view, array_offsets.get());
|
||||
int collected = 0;
|
||||
for (auto& indices : elem_indices) {
|
||||
collected += indices.size();
|
||||
}
|
||||
ASSERT_EQ(collected, num);
|
||||
ASSERT_FALSE(has_more) << "has_more should be false when limit equals "
|
||||
"total matching elements (non-aligned)";
|
||||
}
|
||||
|
||||
// Case 3: limit < total matching elements → has_more=true
|
||||
{
|
||||
BitsetType all(total_elements);
|
||||
all.reset();
|
||||
BitsetTypeView view(all.data(), total_elements);
|
||||
auto [doc_offsets, elem_indices, has_more] =
|
||||
this->map_.find_first_n_element(3, view, array_offsets.get());
|
||||
int collected = 0;
|
||||
for (auto& indices : elem_indices) {
|
||||
collected += indices.size();
|
||||
}
|
||||
ASSERT_EQ(collected, 3);
|
||||
ASSERT_TRUE(has_more)
|
||||
<< "has_more should be true when more elements remain";
|
||||
}
|
||||
}
|
||||
|
||||
REGISTER_TYPED_TEST_SUITE_P(TypedOffsetOrderedArrayTest,
|
||||
find_first_n,
|
||||
find_first_n_element,
|
||||
find_first_n_element_has_more);
|
||||
INSTANTIATE_TYPED_TEST_SUITE_P(Prefix, TypedOffsetOrderedArrayTest, TypeOfPks);
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "common/ArrayOffsets.h"
|
||||
#include "common/Types.h"
|
||||
#include "filemanager/InputStream.h"
|
||||
#include "gtest/gtest.h"
|
||||
@@ -67,10 +68,10 @@ class TypedOffsetOrderedMapTest : public testing::Test {
|
||||
using TypeOfPks = testing::Types<int64_t, std::string>;
|
||||
TYPED_TEST_SUITE_P(TypedOffsetOrderedMapTest);
|
||||
|
||||
TYPED_TEST_P(TypedOffsetOrderedMapTest, find_first) {
|
||||
TYPED_TEST_P(TypedOffsetOrderedMapTest, find_first_n) {
|
||||
// no data.
|
||||
{
|
||||
auto [offsets, has_more_res] = this->map_.find_first(Unlimited, {});
|
||||
auto [offsets, has_more_res] = this->map_.find_first_n(Unlimited, {});
|
||||
ASSERT_EQ(0, offsets.size());
|
||||
ASSERT_FALSE(has_more_res);
|
||||
}
|
||||
@@ -86,7 +87,8 @@ TYPED_TEST_P(TypedOffsetOrderedMapTest, find_first) {
|
||||
all.reset();
|
||||
BitsetTypeView all_view(all.data(), num);
|
||||
{
|
||||
auto [offsets, has_more_res] = this->map_.find_first(num / 2, all_view);
|
||||
auto [offsets, has_more_res] =
|
||||
this->map_.find_first_n(num / 2, all_view);
|
||||
ASSERT_EQ(num / 2, offsets.size());
|
||||
ASSERT_TRUE(has_more_res);
|
||||
for (int i = 1; i < offsets.size(); i++) {
|
||||
@@ -95,7 +97,7 @@ TYPED_TEST_P(TypedOffsetOrderedMapTest, find_first) {
|
||||
}
|
||||
{
|
||||
auto [offsets, has_more_res] =
|
||||
this->map_.find_first(Unlimited, all_view);
|
||||
this->map_.find_first_n(Unlimited, all_view);
|
||||
ASSERT_EQ(num, offsets.size());
|
||||
ASSERT_FALSE(has_more_res);
|
||||
for (int i = 1; i < offsets.size(); i++) {
|
||||
@@ -109,7 +111,7 @@ TYPED_TEST_P(TypedOffsetOrderedMapTest, find_first) {
|
||||
BitsetTypeView all_minus_1_view(all_minus_1.data(), num - 1);
|
||||
{
|
||||
auto [offsets, has_more_res] =
|
||||
this->map_.find_first(num / 2, all_minus_1_view);
|
||||
this->map_.find_first_n(num / 2, all_minus_1_view);
|
||||
ASSERT_EQ(num / 2, offsets.size());
|
||||
ASSERT_TRUE(has_more_res);
|
||||
for (int i = 1; i < offsets.size(); i++) {
|
||||
@@ -118,7 +120,7 @@ TYPED_TEST_P(TypedOffsetOrderedMapTest, find_first) {
|
||||
}
|
||||
{
|
||||
auto [offsets, has_more_res] =
|
||||
this->map_.find_first(Unlimited, all_minus_1_view);
|
||||
this->map_.find_first_n(Unlimited, all_minus_1_view);
|
||||
ASSERT_EQ(all_minus_1.size(), offsets.size());
|
||||
ASSERT_FALSE(has_more_res);
|
||||
for (int i = 1; i < offsets.size(); i++) {
|
||||
@@ -132,17 +134,218 @@ TYPED_TEST_P(TypedOffsetOrderedMapTest, find_first) {
|
||||
BitsetTypeView none_view(none.data(), num);
|
||||
{
|
||||
auto [offsets, has_more_res] =
|
||||
this->map_.find_first(num / 2, none_view);
|
||||
ASSERT_TRUE(has_more_res);
|
||||
this->map_.find_first_n(num / 2, none_view);
|
||||
ASSERT_FALSE(has_more_res);
|
||||
ASSERT_EQ(0, offsets.size());
|
||||
}
|
||||
{
|
||||
auto [offsets, has_more_res] =
|
||||
this->map_.find_first(NoLimit, none_view);
|
||||
ASSERT_TRUE(has_more_res);
|
||||
this->map_.find_first_n(NoLimit, none_view);
|
||||
ASSERT_FALSE(has_more_res);
|
||||
ASSERT_EQ(0, offsets.size());
|
||||
}
|
||||
}
|
||||
|
||||
REGISTER_TYPED_TEST_SUITE_P(TypedOffsetOrderedMapTest, find_first);
|
||||
TYPED_TEST_P(TypedOffsetOrderedMapTest, find_first_n_element) {
|
||||
// Setup: insert 5 docs with sequential PKs
|
||||
int num = 5;
|
||||
int array_len = 3;
|
||||
auto data = this->random_generate(num);
|
||||
for (const auto& x : data) {
|
||||
this->insert(x);
|
||||
}
|
||||
|
||||
// Build ArrayOffsets: each doc has array_len elements
|
||||
// element_row_ids: [0,0,0, 1,1,1, 2,2,2, 3,3,3, 4,4,4]
|
||||
// row_to_element_start: [0, 3, 6, 9, 12, 15]
|
||||
std::vector<int32_t> element_row_ids;
|
||||
std::vector<int32_t> row_to_element_start = {0};
|
||||
for (int doc = 0; doc < num; doc++) {
|
||||
for (int e = 0; e < array_len; e++) {
|
||||
element_row_ids.push_back(doc);
|
||||
}
|
||||
row_to_element_start.push_back(
|
||||
static_cast<int32_t>((doc + 1) * array_len));
|
||||
}
|
||||
auto array_offsets = std::make_shared<ArrayOffsetsSealed>(
|
||||
std::move(element_row_ids), std::move(row_to_element_start));
|
||||
|
||||
int total_elements = num * array_len; // 15
|
||||
|
||||
// Case 1: all elements pass filter → get all docs
|
||||
{
|
||||
BitsetType all(total_elements);
|
||||
all.reset(); // 0 = pass
|
||||
BitsetTypeView view(all.data(), total_elements);
|
||||
auto [doc_offsets, elem_indices, has_more] =
|
||||
this->map_.find_first_n_element(
|
||||
total_elements, view, array_offsets.get());
|
||||
ASSERT_EQ(doc_offsets.size(), num);
|
||||
for (size_t i = 0; i < doc_offsets.size(); i++) {
|
||||
ASSERT_EQ(elem_indices[i].size(), array_len)
|
||||
<< "Each doc should have all elements";
|
||||
}
|
||||
ASSERT_FALSE(has_more);
|
||||
}
|
||||
|
||||
// Case 2: limit counts elements, not docs
|
||||
{
|
||||
BitsetType all(total_elements);
|
||||
all.reset();
|
||||
BitsetTypeView view(all.data(), total_elements);
|
||||
// limit=4: first doc contributes 3 elements, second doc contributes 1
|
||||
auto [doc_offsets, elem_indices, has_more] =
|
||||
this->map_.find_first_n_element(4, view, array_offsets.get());
|
||||
int total = 0;
|
||||
for (auto& indices : elem_indices) {
|
||||
total += indices.size();
|
||||
}
|
||||
ASSERT_EQ(total, 4) << "Should collect exactly 4 elements";
|
||||
ASSERT_EQ(doc_offsets.size(), 2) << "4 elements spans 2 docs (3 + 1)";
|
||||
ASSERT_EQ(elem_indices[0].size(), 3);
|
||||
ASSERT_EQ(elem_indices[1].size(), 1);
|
||||
ASSERT_TRUE(has_more);
|
||||
}
|
||||
|
||||
// Case 3: partial elements pass (only elem_idx=1 per doc)
|
||||
{
|
||||
BitsetType partial(total_elements);
|
||||
partial.set(); // 1 = filtered out
|
||||
for (int doc = 0; doc < num; doc++) {
|
||||
partial.reset(doc * array_len + 1); // only elem_idx=1 passes
|
||||
}
|
||||
BitsetTypeView view(partial.data(), total_elements);
|
||||
auto [doc_offsets, elem_indices, has_more] =
|
||||
this->map_.find_first_n_element(
|
||||
total_elements, view, array_offsets.get());
|
||||
ASSERT_EQ(doc_offsets.size(), num);
|
||||
for (size_t i = 0; i < doc_offsets.size(); i++) {
|
||||
ASSERT_EQ(elem_indices[i].size(), 1);
|
||||
ASSERT_EQ(elem_indices[i][0], 1) << "Only elem_idx=1 should pass";
|
||||
}
|
||||
}
|
||||
|
||||
// Case 4: no elements pass
|
||||
{
|
||||
BitsetType none(total_elements);
|
||||
none.set(); // all filtered out
|
||||
BitsetTypeView view(none.data(), total_elements);
|
||||
auto [doc_offsets, elem_indices, has_more] =
|
||||
this->map_.find_first_n_element(
|
||||
total_elements, view, array_offsets.get());
|
||||
ASSERT_EQ(doc_offsets.size(), 0);
|
||||
ASSERT_EQ(elem_indices.size(), 0);
|
||||
}
|
||||
|
||||
// Case 5: element bitset smaller than array_offsets (concurrent insert)
|
||||
{
|
||||
int smaller_size = total_elements - array_len; // 12 (missing last doc)
|
||||
BitsetType small(smaller_size);
|
||||
small.reset();
|
||||
BitsetTypeView view(small.data(), smaller_size);
|
||||
auto [doc_offsets, elem_indices, has_more] =
|
||||
this->map_.find_first_n_element(
|
||||
total_elements, view, array_offsets.get());
|
||||
// Last doc's elements are beyond bitset, should be skipped
|
||||
int total = 0;
|
||||
for (auto& indices : elem_indices) {
|
||||
total += indices.size();
|
||||
}
|
||||
ASSERT_EQ(total, smaller_size)
|
||||
<< "Should only collect elements within bitset range";
|
||||
}
|
||||
}
|
||||
|
||||
TYPED_TEST_P(TypedOffsetOrderedMapTest, find_first_n_element_has_more) {
|
||||
// Test has_more correctness when limit exactly equals total matching
|
||||
// elements. Previously, has_more used (it != end) || (hit_num >= limit)
|
||||
// which incorrectly returned true when all data was exhausted.
|
||||
|
||||
int num = 3;
|
||||
int array_len = 2;
|
||||
std::vector<TypeParam> data;
|
||||
for (int i = 0; i < num; i++) {
|
||||
TypeParam pk;
|
||||
if constexpr (std::is_same_v<std::string, TypeParam>) {
|
||||
pk = std::to_string(i);
|
||||
} else {
|
||||
pk = static_cast<TypeParam>(i);
|
||||
}
|
||||
this->insert(pk);
|
||||
data.push_back(pk);
|
||||
}
|
||||
|
||||
// Build ArrayOffsets: each doc has array_len elements
|
||||
std::vector<int32_t> element_row_ids;
|
||||
std::vector<int32_t> row_to_element_start = {0};
|
||||
for (int doc = 0; doc < num; doc++) {
|
||||
for (int e = 0; e < array_len; e++) {
|
||||
element_row_ids.push_back(doc);
|
||||
}
|
||||
row_to_element_start.push_back(
|
||||
static_cast<int32_t>((doc + 1) * array_len));
|
||||
}
|
||||
auto array_offsets = std::make_shared<ArrayOffsetsSealed>(
|
||||
std::move(element_row_ids), std::move(row_to_element_start));
|
||||
|
||||
int total_elements = num * array_len; // 6
|
||||
|
||||
// Case 1: limit == total matching elements (aligned on doc boundary)
|
||||
{
|
||||
BitsetType all(total_elements);
|
||||
all.reset();
|
||||
BitsetTypeView view(all.data(), total_elements);
|
||||
auto [doc_offsets, elem_indices, has_more] =
|
||||
this->map_.find_first_n_element(
|
||||
total_elements, view, array_offsets.get());
|
||||
int collected = 0;
|
||||
for (auto& indices : elem_indices) {
|
||||
collected += indices.size();
|
||||
}
|
||||
ASSERT_EQ(collected, total_elements);
|
||||
ASSERT_FALSE(has_more) << "has_more should be false when limit equals "
|
||||
"total matching elements";
|
||||
}
|
||||
|
||||
// Case 2: limit == total matching elements (NOT aligned on doc boundary)
|
||||
// Only elem_idx=0 per doc passes → 3 matching elements, limit=3
|
||||
{
|
||||
BitsetType partial(total_elements);
|
||||
partial.set();
|
||||
for (int doc = 0; doc < num; doc++) {
|
||||
partial.reset(doc * array_len);
|
||||
}
|
||||
BitsetTypeView view(partial.data(), total_elements);
|
||||
auto [doc_offsets, elem_indices, has_more] =
|
||||
this->map_.find_first_n_element(num, view, array_offsets.get());
|
||||
int collected = 0;
|
||||
for (auto& indices : elem_indices) {
|
||||
collected += indices.size();
|
||||
}
|
||||
ASSERT_EQ(collected, num);
|
||||
ASSERT_FALSE(has_more) << "has_more should be false when limit equals "
|
||||
"total matching elements (non-aligned)";
|
||||
}
|
||||
|
||||
// Case 3: limit < total matching elements → has_more=true
|
||||
{
|
||||
BitsetType all(total_elements);
|
||||
all.reset();
|
||||
BitsetTypeView view(all.data(), total_elements);
|
||||
auto [doc_offsets, elem_indices, has_more] =
|
||||
this->map_.find_first_n_element(3, view, array_offsets.get());
|
||||
int collected = 0;
|
||||
for (auto& indices : elem_indices) {
|
||||
collected += indices.size();
|
||||
}
|
||||
ASSERT_EQ(collected, 3);
|
||||
ASSERT_TRUE(has_more)
|
||||
<< "has_more should be true when more elements remain";
|
||||
}
|
||||
}
|
||||
|
||||
REGISTER_TYPED_TEST_SUITE_P(TypedOffsetOrderedMapTest,
|
||||
find_first_n,
|
||||
find_first_n_element,
|
||||
find_first_n_element_has_more);
|
||||
INSTANTIATE_TYPED_TEST_SUITE_P(Prefix, TypedOffsetOrderedMapTest, TypeOfPks);
|
||||
|
||||
@@ -521,8 +521,16 @@ class SegmentGrowingImpl : public SegmentGrowing {
|
||||
}
|
||||
|
||||
std::pair<std::vector<OffsetMap::OffsetType>, bool>
|
||||
find_first(int64_t limit, const BitsetTypeView& bitset) const override {
|
||||
return insert_record_.pk2offset_->find_first(limit, bitset);
|
||||
find_first_n(int64_t limit, const BitsetTypeView& bitset) const override {
|
||||
return insert_record_.pk2offset_->find_first_n(limit, bitset);
|
||||
}
|
||||
|
||||
std::tuple<std::vector<int64_t>, std::vector<std::vector<int32_t>>, bool>
|
||||
find_first_n_element(int64_t limit,
|
||||
const BitsetTypeView& element_bitset,
|
||||
const IArrayOffsets* array_offsets) const override {
|
||||
return insert_record_.pk2offset_->find_first_n_element(
|
||||
limit, element_bitset, array_offsets);
|
||||
}
|
||||
|
||||
bool
|
||||
|
||||
@@ -181,6 +181,17 @@ SegmentInternalInterface::Retrieve(tracer::TraceContext* trace_ctx,
|
||||
results->mutable_offset()->Add(retrieve_results.result_offsets_.begin(),
|
||||
retrieve_results.result_offsets_.end());
|
||||
|
||||
// Element-level query support: serialize element_level flag and element_indices
|
||||
if (retrieve_results.element_level_) {
|
||||
results->set_element_level(true);
|
||||
// element_indices_ is vector<vector<int32_t>>, serialize each doc's indices
|
||||
for (const auto& indices : retrieve_results.element_indices_) {
|
||||
auto* elem_indices = results->add_element_indices();
|
||||
elem_indices->mutable_indices()->Add(indices.begin(),
|
||||
indices.end());
|
||||
}
|
||||
}
|
||||
|
||||
std::chrono::high_resolution_clock::time_point get_target_entry_start =
|
||||
std::chrono::high_resolution_clock::now();
|
||||
if (retrieve_results.field_data_.empty()) {
|
||||
|
||||
@@ -545,7 +545,25 @@ class SegmentInternalInterface : public SegmentInterface {
|
||||
* @return All candidates offsets.
|
||||
*/
|
||||
virtual std::pair<std::vector<OffsetMap::OffsetType>, bool>
|
||||
find_first(int64_t limit, const BitsetTypeView& bitset) const = 0;
|
||||
find_first_n(int64_t limit, const BitsetTypeView& bitset) const = 0;
|
||||
|
||||
/**
|
||||
* Element-level version of find_first_n.
|
||||
* Find the first N elements that pass the filter from an element-level bitset.
|
||||
*
|
||||
* @param limit Maximum number of elements to return
|
||||
* @param element_bitset Element-level bitset (size = total_element_count)
|
||||
* @param array_offsets Mapping between element IDs and (doc_id, element_index)
|
||||
* @return tuple of:
|
||||
* - vector of unique doc_offsets (no duplicates)
|
||||
* - vector of element_indices per doc (element_indices[i] for doc_offsets[i])
|
||||
* - has_more_result flag
|
||||
*/
|
||||
virtual std::
|
||||
tuple<std::vector<int64_t>, std::vector<std::vector<int32_t>>, bool>
|
||||
find_first_n_element(int64_t limit,
|
||||
const BitsetTypeView& element_bitset,
|
||||
const IArrayOffsets* array_offsets) const = 0;
|
||||
|
||||
void
|
||||
FillTargetEntryDirectly(
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -873,12 +873,23 @@ func reduceRetrieveResults(ctx context.Context, retrieveResults []*internalpb.Re
|
||||
loopEnd int
|
||||
)
|
||||
|
||||
// Detect if this is an element-level query
|
||||
isElementLevel := len(retrieveResults) > 0 && retrieveResults[0].GetElementLevel()
|
||||
|
||||
validRetrieveResults := []*internalpb.RetrieveResults{}
|
||||
for _, r := range retrieveResults {
|
||||
size := typeutil.GetSizeOfIDs(r.GetIds())
|
||||
if r == nil || len(r.GetFieldsData()) == 0 || size == 0 {
|
||||
continue
|
||||
}
|
||||
// Validate element-level consistency: if any result is element-level, all must be
|
||||
if isElementLevel && !r.GetElementLevel() {
|
||||
return nil, fmt.Errorf("inconsistent element-level flag: expected all results to be element-level")
|
||||
}
|
||||
// Validate element_indices length matches ids length for element-level
|
||||
if isElementLevel && len(r.GetElementIndices()) != size {
|
||||
return nil, fmt.Errorf("element_indices length (%d) does not match ids length (%d)", len(r.GetElementIndices()), size)
|
||||
}
|
||||
validRetrieveResults = append(validRetrieveResults, r)
|
||||
loopEnd += size
|
||||
}
|
||||
@@ -893,36 +904,65 @@ func reduceRetrieveResults(ctx context.Context, retrieveResults []*internalpb.Re
|
||||
idxComputers[i] = typeutil.NewFieldDataIdxComputer(vr.GetFieldsData())
|
||||
}
|
||||
|
||||
// Used in element-level query to limit the number of elements returned
|
||||
var elementLimit int = -1
|
||||
if queryParams != nil && queryParams.limit != typeutil.Unlimited {
|
||||
// IReduceInOrderForBest will try to get as many results as possible
|
||||
// so loopEnd in this case will be set to the sum of all results' size
|
||||
// to get as many qualified results as possible
|
||||
if reduce.ShouldUseInputLimit(queryParams.reduceType) {
|
||||
loopEnd = int(queryParams.limit)
|
||||
if !isElementLevel {
|
||||
loopEnd = int(queryParams.limit)
|
||||
}
|
||||
elementLimit = int(queryParams.limit)
|
||||
}
|
||||
}
|
||||
|
||||
// handle offset
|
||||
if queryParams != nil && queryParams.offset > 0 {
|
||||
for i := int64(0); i < queryParams.offset; i++ {
|
||||
var skipped int64
|
||||
for skipped < queryParams.offset {
|
||||
sel, drainOneResult := typeutil.SelectMinPK(validRetrieveResults, cursors)
|
||||
if sel == -1 || (reduce.ShouldStopWhenDrained(queryParams.reduceType) && drainOneResult) {
|
||||
return ret, nil
|
||||
}
|
||||
if isElementLevel {
|
||||
elemIndices := validRetrieveResults[sel].GetElementIndices()[cursors[sel]]
|
||||
indicesCount := int64(len(elemIndices.GetIndices()))
|
||||
if skipped+indicesCount > queryParams.offset {
|
||||
elemIndices.Indices = elemIndices.Indices[queryParams.offset-skipped:]
|
||||
break
|
||||
} else {
|
||||
skipped += indicesCount
|
||||
}
|
||||
} else {
|
||||
skipped++
|
||||
}
|
||||
cursors[sel]++
|
||||
}
|
||||
}
|
||||
|
||||
ret.FieldsData = typeutil.PrepareResultFieldData(validRetrieveResults[0].GetFieldsData(), int64(loopEnd))
|
||||
var retSize int64
|
||||
var availableCount int // for element-level: element count; for doc-level: doc count
|
||||
maxOutputSize := paramtable.Get().QuotaConfig.MaxOutputSize.GetAsInt64()
|
||||
for j := 0; j < loopEnd; j++ {
|
||||
for j := 0; j < loopEnd && (elementLimit == -1 || availableCount < elementLimit); j++ {
|
||||
sel, drainOneResult := typeutil.SelectMinPK(validRetrieveResults, cursors)
|
||||
if sel == -1 || (reduce.ShouldStopWhenDrained(queryParams.reduceType) && drainOneResult) {
|
||||
break
|
||||
}
|
||||
|
||||
// Get element indices for element-level query
|
||||
var elemCount int = 1 // default for doc-level
|
||||
if isElementLevel {
|
||||
elemIndices := validRetrieveResults[sel].GetElementIndices()[cursors[sel]]
|
||||
elemCount = len(elemIndices.GetIndices())
|
||||
ret.ElementIndices = append(ret.ElementIndices, convertInternalElementIndicesToMilvus(elemIndices))
|
||||
}
|
||||
|
||||
fieldIdxs := idxComputers[sel].Compute(cursors[sel])
|
||||
retSize += typeutil.AppendFieldData(ret.FieldsData, validRetrieveResults[sel].GetFieldsData(), cursors[sel], fieldIdxs...)
|
||||
availableCount += elemCount
|
||||
|
||||
// limit retrieve result to avoid oom
|
||||
if retSize > maxOutputSize {
|
||||
@@ -935,6 +975,21 @@ func reduceRetrieveResults(ctx context.Context, retrieveResults []*internalpb.Re
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
// convertInternalElementIndicesToMilvus converts internalpb.ElementIndices (int32) to milvuspb.ElementIndices (int64)
|
||||
func convertInternalElementIndicesToMilvus(src *internalpb.ElementIndices) *milvuspb.ElementIndices {
|
||||
if src == nil {
|
||||
return nil
|
||||
}
|
||||
indices := src.GetIndices()
|
||||
data := make([]int64, len(indices))
|
||||
for i, v := range indices {
|
||||
data[i] = int64(v)
|
||||
}
|
||||
return &milvuspb.ElementIndices{
|
||||
Indices: &schemapb.LongArray{Data: data},
|
||||
}
|
||||
}
|
||||
|
||||
func reduceRetrieveResultsAndFillIfEmpty(ctx context.Context, retrieveResults []*internalpb.RetrieveResults, queryParams *queryParams, outputFieldsID []int64, schema *schemapb.CollectionSchema) (*milvuspb.QueryResults, error) {
|
||||
result, err := reduceRetrieveResults(ctx, retrieveResults, queryParams)
|
||||
if err != nil {
|
||||
|
||||
@@ -1013,6 +1013,266 @@ func TestTaskQuery_functions(t *testing.T) {
|
||||
// Should include the maxInt64PK result
|
||||
assert.Equal(t, []int64{maxInt64PK}, result.GetFieldsData()[0].GetScalars().GetLongData().Data)
|
||||
})
|
||||
|
||||
t.Run("test element-level reduce", func(t *testing.T) {
|
||||
// Helper to create element-level retrieve result
|
||||
makeElementLevelResult := func(pks []int64, elemIndices [][]int32) *internalpb.RetrieveResults {
|
||||
elemIdxList := make([]*internalpb.ElementIndices, len(elemIndices))
|
||||
for i, indices := range elemIndices {
|
||||
elemIdxList[i] = &internalpb.ElementIndices{Indices: indices}
|
||||
}
|
||||
return &internalpb.RetrieveResults{
|
||||
Ids: &schemapb.IDs{
|
||||
IdField: &schemapb.IDs_IntId{
|
||||
IntId: &schemapb.LongArray{Data: pks},
|
||||
},
|
||||
},
|
||||
FieldsData: []*schemapb.FieldData{
|
||||
getFieldData(Int64FieldName, Int64FieldID, schemapb.DataType_Int64, pks, 1),
|
||||
},
|
||||
ElementLevel: true,
|
||||
ElementIndices: elemIdxList,
|
||||
}
|
||||
}
|
||||
|
||||
t.Run("basic element-level merge", func(t *testing.T) {
|
||||
r1 := makeElementLevelResult([]int64{1, 3}, [][]int32{{0, 1}, {2}})
|
||||
r2 := makeElementLevelResult([]int64{2, 4}, [][]int32{{0}, {1, 2}})
|
||||
result, err := reduceRetrieveResults(context.Background(),
|
||||
[]*internalpb.RetrieveResults{r1, r2},
|
||||
&queryParams{limit: typeutil.Unlimited})
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 4, len(result.GetElementIndices()))
|
||||
// Verify element_indices are converted to int64
|
||||
assert.Equal(t, []int64{0, 1}, result.GetElementIndices()[0].GetIndices().GetData())
|
||||
assert.Equal(t, []int64{0}, result.GetElementIndices()[1].GetIndices().GetData())
|
||||
assert.Equal(t, []int64{2}, result.GetElementIndices()[2].GetIndices().GetData())
|
||||
assert.Equal(t, []int64{1, 2}, result.GetElementIndices()[3].GetIndices().GetData())
|
||||
})
|
||||
|
||||
t.Run("element-level limit counts elements not docs", func(t *testing.T) {
|
||||
// doc1 has 3 elements, doc2 has 2 elements
|
||||
r := makeElementLevelResult([]int64{1, 2}, [][]int32{{0, 1, 2}, {0, 1}})
|
||||
result, err := reduceRetrieveResults(context.Background(),
|
||||
[]*internalpb.RetrieveResults{r},
|
||||
&queryParams{limit: 3}) // limit=3 elements
|
||||
assert.NoError(t, err)
|
||||
// Should get doc1 (3 elements) only
|
||||
assert.Equal(t, 1, len(result.GetFieldsData()[0].GetScalars().GetLongData().GetData()))
|
||||
assert.Equal(t, int64(1), result.GetFieldsData()[0].GetScalars().GetLongData().GetData()[0])
|
||||
assert.Equal(t, 1, len(result.GetElementIndices()))
|
||||
})
|
||||
|
||||
t.Run("element-level limit allows partial fill", func(t *testing.T) {
|
||||
// doc1 has 2 elements, doc2 has 2 elements
|
||||
r := makeElementLevelResult([]int64{1, 2}, [][]int32{{0, 1}, {0, 1}})
|
||||
result, err := reduceRetrieveResults(context.Background(),
|
||||
[]*internalpb.RetrieveResults{r},
|
||||
&queryParams{limit: 3}) // limit=3 elements
|
||||
assert.NoError(t, err)
|
||||
// Behavior: process docs while availableCount < limit
|
||||
// doc1 (2 elements): 0 < 3, process, count=2
|
||||
// doc2 (2 elements): 2 < 3, process, count=4
|
||||
// Both docs are included because the check is done before processing each doc
|
||||
assert.Equal(t, 2, len(result.GetFieldsData()[0].GetScalars().GetLongData().GetData()))
|
||||
})
|
||||
|
||||
t.Run("element-level with offset", func(t *testing.T) {
|
||||
r := makeElementLevelResult([]int64{1, 2, 3}, [][]int32{{0}, {1}, {2}})
|
||||
result, err := reduceRetrieveResults(context.Background(),
|
||||
[]*internalpb.RetrieveResults{r},
|
||||
&queryParams{limit: 2, offset: 1}) // skip 1 doc, get 2 docs
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 2, len(result.GetFieldsData()[0].GetScalars().GetLongData().GetData()))
|
||||
assert.Equal(t, []int64{2, 3}, result.GetFieldsData()[0].GetScalars().GetLongData().GetData())
|
||||
})
|
||||
|
||||
t.Run("element-level offset should skip by element count not doc count", func(t *testing.T) {
|
||||
// Doc layout:
|
||||
// pk=1: 3 elements [0,1,2]
|
||||
// pk=2: 2 elements [0,1]
|
||||
// pk=3: 1 element [0]
|
||||
// Total: 6 elements
|
||||
//
|
||||
// Query: offset=3, limit=3
|
||||
// Correct (element-level offset):
|
||||
// Skip 3 elements (all from pk=1) → return pk=2 (2 elem) + pk=3 (1 elem) = 3 elements
|
||||
// Bug (doc-level offset):
|
||||
// Skip 3 docs (pk=1, pk=2, pk=3) → nothing left → empty result
|
||||
r := makeElementLevelResult([]int64{1, 2, 3}, [][]int32{{0, 1, 2}, {0, 1}, {0}})
|
||||
result, err := reduceRetrieveResults(context.Background(),
|
||||
[]*internalpb.RetrieveResults{r},
|
||||
&queryParams{limit: 3, offset: 3})
|
||||
assert.NoError(t, err)
|
||||
pks := result.GetFieldsData()[0].GetScalars().GetLongData().GetData()
|
||||
assert.Equal(t, []int64{2, 3}, pks, "offset=3 should skip 3 elements (all of pk=1), returning pk=2 and pk=3")
|
||||
assert.Equal(t, 2, len(result.GetElementIndices()))
|
||||
assert.Equal(t, []int64{0, 1}, result.GetElementIndices()[0].GetIndices().GetData())
|
||||
assert.Equal(t, []int64{0}, result.GetElementIndices()[1].GetIndices().GetData())
|
||||
})
|
||||
|
||||
t.Run("element-level offset falls mid-document trims elements", func(t *testing.T) {
|
||||
// Doc layout:
|
||||
// pk=1: 3 elements [0,1,2]
|
||||
// pk=2: 2 elements [3,4]
|
||||
// Total: 5 elements
|
||||
//
|
||||
// Query: offset=2, limit=10
|
||||
// offset=2 falls in the middle of pk=1 (which has 3 elements).
|
||||
// Should skip elements [0,1] of pk=1, keep element [2].
|
||||
// Result: pk=1 with trimmed indices [2], pk=2 with [3,4] → 3 elements total
|
||||
r := makeElementLevelResult([]int64{1, 2}, [][]int32{{0, 1, 2}, {3, 4}})
|
||||
result, err := reduceRetrieveResults(context.Background(),
|
||||
[]*internalpb.RetrieveResults{r},
|
||||
&queryParams{limit: 10, offset: 2})
|
||||
assert.NoError(t, err)
|
||||
pks := result.GetFieldsData()[0].GetScalars().GetLongData().GetData()
|
||||
assert.Equal(t, []int64{1, 2}, pks, "both docs should be present since offset lands mid-doc on pk=1")
|
||||
assert.Equal(t, 2, len(result.GetElementIndices()))
|
||||
// pk=1: original [0,1,2], trimmed first 2 → [2]
|
||||
assert.Equal(t, []int64{2}, result.GetElementIndices()[0].GetIndices().GetData())
|
||||
// pk=2: unchanged [3,4]
|
||||
assert.Equal(t, []int64{3, 4}, result.GetElementIndices()[1].GetIndices().GetData())
|
||||
})
|
||||
|
||||
t.Run("element-level offset+limit pagination consistency", func(t *testing.T) {
|
||||
// Verify that two consecutive pages cover all elements without gap or overlap.
|
||||
// Doc layout (across 2 shards):
|
||||
// shard0: pk=1 (2 elements [0,1]), pk=3 (1 element [0])
|
||||
// shard1: pk=2 (3 elements [0,1,2]), pk=4 (2 elements [0,1])
|
||||
// Merged by PK: pk=1(2), pk=2(3), pk=3(1), pk=4(2) → total 8 elements
|
||||
r1 := makeElementLevelResult([]int64{1, 3}, [][]int32{{0, 1}, {0}})
|
||||
r2 := makeElementLevelResult([]int64{2, 4}, [][]int32{{0, 1, 2}, {0, 1}})
|
||||
|
||||
// Page 1: offset=0, limit=5 → first 5 elements
|
||||
page1, err := reduceRetrieveResults(context.Background(),
|
||||
[]*internalpb.RetrieveResults{r1, r2},
|
||||
&queryParams{limit: 5, offset: 0})
|
||||
assert.NoError(t, err)
|
||||
page1PKs := page1.GetFieldsData()[0].GetScalars().GetLongData().GetData()
|
||||
|
||||
// Page 2: offset=5, limit=5 → next 3 elements (only 8 total)
|
||||
// Recreate results since cursors are consumed
|
||||
r1 = makeElementLevelResult([]int64{1, 3}, [][]int32{{0, 1}, {0}})
|
||||
r2 = makeElementLevelResult([]int64{2, 4}, [][]int32{{0, 1, 2}, {0, 1}})
|
||||
page2, err := reduceRetrieveResults(context.Background(),
|
||||
[]*internalpb.RetrieveResults{r1, r2},
|
||||
&queryParams{limit: 5, offset: 5})
|
||||
assert.NoError(t, err)
|
||||
page2PKs := page2.GetFieldsData()[0].GetScalars().GetLongData().GetData()
|
||||
|
||||
// Count total elements across both pages
|
||||
page1ElemCount := 0
|
||||
for _, ei := range page1.GetElementIndices() {
|
||||
page1ElemCount += len(ei.GetIndices().GetData())
|
||||
}
|
||||
page2ElemCount := 0
|
||||
for _, ei := range page2.GetElementIndices() {
|
||||
page2ElemCount += len(ei.GetIndices().GetData())
|
||||
}
|
||||
|
||||
// The two pages together should cover exactly 8 elements total
|
||||
assert.Equal(t, 8, page1ElemCount+page2ElemCount,
|
||||
"page1 (%v, %d elems) + page2 (%v, %d elems) should cover all 8 elements",
|
||||
page1PKs, page1ElemCount, page2PKs, page2ElemCount)
|
||||
})
|
||||
|
||||
t.Run("element-level validation: inconsistent flag", func(t *testing.T) {
|
||||
r1 := makeElementLevelResult([]int64{1}, [][]int32{{0, 1}})
|
||||
r2 := &internalpb.RetrieveResults{
|
||||
Ids: &schemapb.IDs{
|
||||
IdField: &schemapb.IDs_IntId{
|
||||
IntId: &schemapb.LongArray{Data: []int64{2}},
|
||||
},
|
||||
},
|
||||
FieldsData: []*schemapb.FieldData{
|
||||
getFieldData(Int64FieldName, Int64FieldID, schemapb.DataType_Int64, []int64{2}, 1),
|
||||
},
|
||||
ElementLevel: false, // inconsistent
|
||||
}
|
||||
_, err := reduceRetrieveResults(context.Background(),
|
||||
[]*internalpb.RetrieveResults{r1, r2},
|
||||
&queryParams{limit: typeutil.Unlimited})
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "inconsistent element-level flag")
|
||||
})
|
||||
|
||||
t.Run("element-level validation: mismatched element_indices length", func(t *testing.T) {
|
||||
r := &internalpb.RetrieveResults{
|
||||
Ids: &schemapb.IDs{
|
||||
IdField: &schemapb.IDs_IntId{
|
||||
IntId: &schemapb.LongArray{Data: []int64{1, 2}}, // 2 ids
|
||||
},
|
||||
},
|
||||
FieldsData: []*schemapb.FieldData{
|
||||
getFieldData(Int64FieldName, Int64FieldID, schemapb.DataType_Int64, []int64{1, 2}, 1),
|
||||
},
|
||||
ElementLevel: true,
|
||||
ElementIndices: []*internalpb.ElementIndices{
|
||||
{Indices: []int32{0}}, // only 1 element_indices, should be 2
|
||||
},
|
||||
}
|
||||
_, err := reduceRetrieveResults(context.Background(),
|
||||
[]*internalpb.RetrieveResults{r},
|
||||
&queryParams{limit: typeutil.Unlimited})
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "element_indices length")
|
||||
})
|
||||
|
||||
t.Run("element-level type conversion", func(t *testing.T) {
|
||||
r := makeElementLevelResult([]int64{1}, [][]int32{{0, 100, 200}})
|
||||
result, err := reduceRetrieveResults(context.Background(),
|
||||
[]*internalpb.RetrieveResults{r},
|
||||
&queryParams{limit: typeutil.Unlimited})
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 1, len(result.GetElementIndices()))
|
||||
// Verify int32 -> int64 conversion
|
||||
indices := result.GetElementIndices()[0].GetIndices().GetData()
|
||||
assert.Equal(t, []int64{0, 100, 200}, indices)
|
||||
})
|
||||
|
||||
t.Run("element-level with IReduceInOrderForBest", func(t *testing.T) {
|
||||
// 2 results from different segments, both have HasMoreResult=true
|
||||
// r1: PK=1 (2 elems), PK=3 (1 elem)
|
||||
// r2: PK=2 (3 elems), PK=4 (1 elem)
|
||||
// Merged order by PK: 1,2,3,4
|
||||
// IReduceInOrderForBest does not use input limit for elements
|
||||
// Stops when r1 drains after PK3 (ShouldStopWhenDrained=true)
|
||||
r1 := makeElementLevelResult([]int64{1, 3}, [][]int32{{0, 1}, {2}})
|
||||
r1.HasMoreResult = true
|
||||
r2 := makeElementLevelResult([]int64{2, 4}, [][]int32{{0, 1, 2}, {1}})
|
||||
r2.HasMoreResult = true
|
||||
result, err := reduceRetrieveResults(context.Background(),
|
||||
[]*internalpb.RetrieveResults{r1, r2},
|
||||
&queryParams{limit: 4, reduceType: reduce.IReduceInOrderForBest})
|
||||
assert.NoError(t, err)
|
||||
// IReduceInOrderForBest does not enforce element limit (ShouldUseInputLimit=false)
|
||||
// It stops when one result is drained (ShouldStopWhenDrained=true)
|
||||
// r1 drains after PK3, so result is {1, 2, 3}
|
||||
pks := result.GetFieldsData()[0].GetScalars().GetLongData().GetData()
|
||||
assert.Equal(t, []int64{1, 2, 3}, pks)
|
||||
assert.Equal(t, 3, len(result.GetElementIndices()))
|
||||
})
|
||||
|
||||
t.Run("element-level with IReduceInOrderForBest drains one result", func(t *testing.T) {
|
||||
// r1 has no more (HasMoreResult=false), r2 has more (HasMoreResult=true)
|
||||
// drainResult only triggers when HasMoreResult=true, so r1 draining
|
||||
// does not stop the loop — it's safe to continue with r2's remaining PKs
|
||||
r1 := makeElementLevelResult([]int64{1}, [][]int32{{0, 1}})
|
||||
r1.HasMoreResult = false
|
||||
r2 := makeElementLevelResult([]int64{2, 4}, [][]int32{{0}, {1, 2}})
|
||||
r2.HasMoreResult = true
|
||||
result, err := reduceRetrieveResults(context.Background(),
|
||||
[]*internalpb.RetrieveResults{r1, r2},
|
||||
&queryParams{limit: 10, reduceType: reduce.IReduceInOrderForBest})
|
||||
assert.NoError(t, err)
|
||||
// r1 HasMoreResult=false, so drain does not trigger stop
|
||||
// All PKs are returned: 1, 2, 4
|
||||
pks := result.GetFieldsData()[0].GetScalars().GetLongData().GetData()
|
||||
assert.Equal(t, []int64{1, 2, 4}, pks)
|
||||
assert.Equal(t, 3, len(result.GetElementIndices()))
|
||||
})
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -276,7 +276,9 @@ func MergeInternalRetrieveResult(ctx context.Context, retrieveResults []*interna
|
||||
zap.Int64("limit", param.limit),
|
||||
zap.Int("resultNum", len(retrieveResults)),
|
||||
)
|
||||
if len(retrieveResults) == 1 {
|
||||
// For doc-level with single result, return directly (no merge/limit needed)
|
||||
// For element-level, need to continue to apply element-count limit
|
||||
if len(retrieveResults) == 1 && !retrieveResults[0].GetElementLevel() {
|
||||
return retrieveResults[0], nil
|
||||
}
|
||||
var (
|
||||
@@ -291,6 +293,10 @@ func MergeInternalRetrieveResult(ctx context.Context, retrieveResults []*interna
|
||||
_, sp := otel.Tracer(typeutil.QueryNodeRole).Start(ctx, "MergeInternalRetrieveResult")
|
||||
defer sp.End()
|
||||
|
||||
// Detect if this is an element-level query
|
||||
isElementLevel := len(retrieveResults) > 0 && retrieveResults[0].GetElementLevel()
|
||||
ret.ElementLevel = isElementLevel
|
||||
|
||||
validRetrieveResults := []*TimestampedRetrieveResult[*internalpb.RetrieveResults]{}
|
||||
relatedDataSize := int64(0)
|
||||
hasMoreResult := false
|
||||
@@ -303,6 +309,14 @@ func MergeInternalRetrieveResult(ctx context.Context, retrieveResults []*interna
|
||||
if r == nil || len(r.GetFieldsData()) == 0 || size == 0 {
|
||||
continue
|
||||
}
|
||||
// Validate element-level consistency: if any result is element-level, all must be
|
||||
if isElementLevel && !r.GetElementLevel() {
|
||||
return nil, fmt.Errorf("inconsistent element-level flag: expected all results to be element-level")
|
||||
}
|
||||
// Validate element_indices length matches ids length for element-level
|
||||
if isElementLevel && len(r.GetElementIndices()) != size {
|
||||
return nil, fmt.Errorf("element_indices length (%d) does not match ids length (%d)", len(r.GetElementIndices()), size)
|
||||
}
|
||||
tr, err := NewTimestampedRetrieveResult(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -317,8 +331,12 @@ func MergeInternalRetrieveResult(ctx context.Context, retrieveResults []*interna
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
var limit int = -1
|
||||
if param.limit != typeutil.Unlimited && reduce.ShouldUseInputLimit(param.reduceType) {
|
||||
loopEnd = int(param.limit)
|
||||
limit = int(param.limit)
|
||||
if !isElementLevel {
|
||||
loopEnd = int(param.limit)
|
||||
}
|
||||
}
|
||||
|
||||
ret.FieldsData = typeutil.PrepareResultFieldData(validRetrieveResults[0].Result.GetFieldsData(), int64(loopEnd))
|
||||
@@ -330,9 +348,19 @@ func MergeInternalRetrieveResult(ctx context.Context, retrieveResults []*interna
|
||||
idxComputers[i] = typeutil.NewFieldDataIdxComputer(vr.Result.GetFieldsData())
|
||||
}
|
||||
|
||||
// Track element indices for element-level query
|
||||
type docSelection struct {
|
||||
batchIndex int
|
||||
resultIndex int64
|
||||
elementIndices *internalpb.ElementIndices
|
||||
}
|
||||
docSelections := make([]docSelection, 0, loopEnd)
|
||||
idxInSelections := make(map[interface{}]int) // pk -> index in docSelections
|
||||
|
||||
var availableCount int // for doc-level: doc count; for element-level: element count
|
||||
var retSize int64
|
||||
maxOutputSize := paramtable.Get().QuotaConfig.MaxOutputSize.GetAsInt64()
|
||||
for j := 0; j < loopEnd; {
|
||||
for j := 0; j < loopEnd && (limit == -1 || availableCount < limit); {
|
||||
sel, drainOneResult := typeutil.SelectMinPKWithTimestamp(validRetrieveResults, cursors)
|
||||
if sel == -1 || (reduce.ShouldStopWhenDrained(param.reduceType) && drainOneResult) {
|
||||
break
|
||||
@@ -340,12 +368,33 @@ func MergeInternalRetrieveResult(ctx context.Context, retrieveResults []*interna
|
||||
|
||||
pk := typeutil.GetPK(validRetrieveResults[sel].GetIds(), cursors[sel])
|
||||
ts := validRetrieveResults[sel].Timestamps[cursors[sel]]
|
||||
|
||||
// Get element indices for element-level query
|
||||
var elemIndices *internalpb.ElementIndices
|
||||
var elemCount int = 1 // default for doc-level
|
||||
if isElementLevel {
|
||||
elemIndicesList := validRetrieveResults[sel].Result.GetElementIndices()
|
||||
if int(cursors[sel]) < len(elemIndicesList) {
|
||||
elemIndices = elemIndicesList[cursors[sel]]
|
||||
elemCount = len(elemIndices.GetIndices())
|
||||
}
|
||||
}
|
||||
|
||||
fieldsData := validRetrieveResults[sel].Result.GetFieldsData()
|
||||
fieldIdxs := idxComputers[sel].Compute(cursors[sel])
|
||||
if _, ok := idTsMap[pk]; !ok {
|
||||
typeutil.AppendPKs(ret.Ids, pk)
|
||||
retSize += typeutil.AppendFieldData(ret.FieldsData, fieldsData, cursors[sel], fieldIdxs...)
|
||||
idTsMap[pk] = ts
|
||||
if isElementLevel {
|
||||
idxInSelections[pk] = len(docSelections)
|
||||
docSelections = append(docSelections, docSelection{
|
||||
batchIndex: sel,
|
||||
resultIndex: cursors[sel],
|
||||
elementIndices: elemIndices,
|
||||
})
|
||||
}
|
||||
availableCount += elemCount
|
||||
j++
|
||||
} else {
|
||||
// primary keys duplicate
|
||||
@@ -354,6 +403,16 @@ func MergeInternalRetrieveResult(ctx context.Context, retrieveResults []*interna
|
||||
idTsMap[pk] = ts
|
||||
typeutil.DeleteFieldData(ret.FieldsData)
|
||||
retSize += typeutil.AppendFieldData(ret.FieldsData, fieldsData, cursors[sel], fieldIdxs...)
|
||||
if isElementLevel {
|
||||
idx := idxInSelections[pk]
|
||||
oldElemCount := len(docSelections[idx].elementIndices.GetIndices())
|
||||
availableCount = availableCount - oldElemCount + elemCount
|
||||
docSelections[idx] = docSelection{
|
||||
batchIndex: sel,
|
||||
resultIndex: cursors[sel],
|
||||
elementIndices: elemIndices,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -369,6 +428,13 @@ func MergeInternalRetrieveResult(ctx context.Context, retrieveResults []*interna
|
||||
log.Debug("skip duplicated query result while reducing internal.RetrieveResults", zap.Int64("dupCount", skipDupCnt))
|
||||
}
|
||||
|
||||
// Fill ElementIndices for element-level query
|
||||
if isElementLevel {
|
||||
for _, sel := range docSelections {
|
||||
ret.ElementIndices = append(ret.ElementIndices, sel.elementIndices)
|
||||
}
|
||||
}
|
||||
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
@@ -404,6 +470,10 @@ func MergeSegcoreRetrieveResults(ctx context.Context, retrieveResults []*segcore
|
||||
loopEnd int
|
||||
)
|
||||
|
||||
// Detect if this is an element-level query
|
||||
isElementLevel := len(retrieveResults) > 0 && retrieveResults[0].GetElementLevel()
|
||||
ret.ElementLevel = isElementLevel
|
||||
|
||||
validRetrieveResults := []*TimestampedRetrieveResult[*segcorepb.RetrieveResults]{}
|
||||
validSegments := make([]Segment, 0, len(segments))
|
||||
hasMoreResult := false
|
||||
@@ -416,6 +486,14 @@ func MergeSegcoreRetrieveResults(ctx context.Context, retrieveResults []*segcore
|
||||
log.Debug("filter out invalid retrieve result")
|
||||
continue
|
||||
}
|
||||
// Validate element-level consistency: if any result is element-level, all must be
|
||||
if isElementLevel && !r.GetElementLevel() {
|
||||
return nil, fmt.Errorf("inconsistent element-level flag: expected all results to be element-level")
|
||||
}
|
||||
// Validate element_indices length matches ids length for element-level
|
||||
if isElementLevel && len(r.GetElementIndices()) != size {
|
||||
return nil, fmt.Errorf("element_indices length (%d) does not match ids length (%d)", len(r.GetElementIndices()), size)
|
||||
}
|
||||
tr, err := NewTimestampedRetrieveResult(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -440,16 +518,21 @@ func MergeSegcoreRetrieveResults(ctx context.Context, retrieveResults []*segcore
|
||||
|
||||
ret.FieldsData = typeutil.PrepareResultFieldData(validRetrieveResults[0].Result.GetFieldsData(), int64(loopEnd))
|
||||
cursors := make([]int64, len(validRetrieveResults))
|
||||
idTsMap := make(map[any]int64, limit*len(validRetrieveResults))
|
||||
mapCap := 0
|
||||
if limit > 0 {
|
||||
mapCap = limit * len(validRetrieveResults)
|
||||
}
|
||||
idTsMap := make(map[any]int64, mapCap)
|
||||
|
||||
var availableCount int
|
||||
var availableCount int // for doc-level: doc count; for element-level: element count
|
||||
var retSize int64
|
||||
maxOutputSize := paramtable.Get().QuotaConfig.MaxOutputSize.GetAsInt64()
|
||||
|
||||
type selection struct {
|
||||
batchIndex int // index of validate retrieve results
|
||||
resultIndex int64 // index of selection in selected result item
|
||||
offset int64 // offset of the result
|
||||
batchIndex int // index of validate retrieve results
|
||||
resultIndex int64 // index of selection in selected result item
|
||||
offset int64 // offset of the result
|
||||
elementIndices *segcorepb.ElementIndices // element indices for element-level query
|
||||
}
|
||||
|
||||
var selections []selection
|
||||
@@ -462,15 +545,30 @@ func MergeSegcoreRetrieveResults(ctx context.Context, retrieveResults []*segcore
|
||||
|
||||
pk := typeutil.GetPK(validRetrieveResults[sel].GetIds(), cursors[sel])
|
||||
ts := validRetrieveResults[sel].Timestamps[cursors[sel]]
|
||||
|
||||
// Get element indices for element-level query
|
||||
var elemIndices *segcorepb.ElementIndices
|
||||
// For doc-level filter, we always count 1 for each document
|
||||
// For element-level filter, we count the number of elements for each document
|
||||
var elemCount int = 1
|
||||
if isElementLevel {
|
||||
elemIndicesList := validRetrieveResults[sel].Result.GetElementIndices()
|
||||
if int(cursors[sel]) < len(elemIndicesList) {
|
||||
elemIndices = elemIndicesList[cursors[sel]]
|
||||
elemCount = len(elemIndices.GetIndices())
|
||||
}
|
||||
}
|
||||
|
||||
if _, ok := idTsMap[pk]; !ok {
|
||||
typeutil.AppendPKs(ret.Ids, pk)
|
||||
selections = append(selections, selection{
|
||||
batchIndex: sel,
|
||||
resultIndex: cursors[sel],
|
||||
offset: validRetrieveResults[sel].Result.GetOffset()[cursors[sel]],
|
||||
batchIndex: sel,
|
||||
resultIndex: cursors[sel],
|
||||
offset: validRetrieveResults[sel].Result.GetOffset()[cursors[sel]],
|
||||
elementIndices: elemIndices,
|
||||
})
|
||||
idTsMap[pk] = ts
|
||||
availableCount++
|
||||
availableCount += elemCount
|
||||
} else {
|
||||
// primary keys duplicate
|
||||
skipDupCnt++
|
||||
@@ -485,10 +583,16 @@ func MergeSegcoreRetrieveResults(ctx context.Context, retrieveResults []*segcore
|
||||
}
|
||||
}
|
||||
if idx >= 0 {
|
||||
// For element-level: adjust availableCount
|
||||
if isElementLevel {
|
||||
oldElemCount := len(selections[idx].elementIndices.GetIndices())
|
||||
availableCount = availableCount - oldElemCount + elemCount
|
||||
}
|
||||
selections[idx] = selection{
|
||||
batchIndex: sel,
|
||||
resultIndex: cursors[sel],
|
||||
offset: validRetrieveResults[sel].Result.GetOffset()[cursors[sel]],
|
||||
batchIndex: sel,
|
||||
resultIndex: cursors[sel],
|
||||
offset: validRetrieveResults[sel].Result.GetOffset()[cursors[sel]],
|
||||
elementIndices: elemIndices,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -587,6 +691,13 @@ func MergeSegcoreRetrieveResults(ctx context.Context, retrieveResults []*segcore
|
||||
}
|
||||
}
|
||||
|
||||
// Fill ElementIndices for element-level query
|
||||
if isElementLevel {
|
||||
for _, sel := range selections {
|
||||
ret.ElementIndices = append(ret.ElementIndices, sel.elementIndices)
|
||||
}
|
||||
}
|
||||
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -1054,6 +1054,94 @@ func TestResult_MergeRequestCost(t *testing.T) {
|
||||
assert.Equal(t, int64(43), channelCost.TotalNQ)
|
||||
}
|
||||
|
||||
func (suite *ResultSuite) TestResult_MergeSegcoreRetrieveResults_ElementLevel() {
|
||||
// Helper to create element-level retrieve result
|
||||
makeResult := func(pks []int64, ts []int64, elemIndices [][]int32) *segcorepb.RetrieveResults {
|
||||
offsets := make([]int64, len(pks))
|
||||
for i := range pks {
|
||||
offsets[i] = int64(i)
|
||||
}
|
||||
elemIdxList := make([]*segcorepb.ElementIndices, len(elemIndices))
|
||||
for i, indices := range elemIndices {
|
||||
elemIdxList[i] = &segcorepb.ElementIndices{Indices: indices}
|
||||
}
|
||||
return &segcorepb.RetrieveResults{
|
||||
Ids: &schemapb.IDs{IdField: &schemapb.IDs_IntId{IntId: &schemapb.LongArray{Data: pks}}},
|
||||
Offset: offsets,
|
||||
FieldsData: []*schemapb.FieldData{mock_segcore.GenFieldData(common.TimeStampFieldName, common.TimeStampField, schemapb.DataType_Int64, ts, 1)},
|
||||
ElementLevel: true,
|
||||
ElementIndices: elemIdxList,
|
||||
}
|
||||
}
|
||||
|
||||
suite.Run("merge different pks", func() {
|
||||
r1 := makeResult([]int64{1, 3}, []int64{100, 100}, [][]int32{{0, 1}, {2}})
|
||||
r2 := makeResult([]int64{2, 4}, []int64{100, 100}, [][]int32{{0}, {1, 2}})
|
||||
result, err := MergeSegcoreRetrieveResultsV1(context.Background(), []*segcorepb.RetrieveResults{r1, r2},
|
||||
NewMergeParam(typeutil.Unlimited, nil, nil, reduce.IReduceNoOrder))
|
||||
suite.NoError(err)
|
||||
suite.True(result.GetElementLevel())
|
||||
suite.Equal([]int64{1, 2, 3, 4}, result.GetIds().GetIntId().GetData())
|
||||
suite.Equal(4, len(result.GetElementIndices()))
|
||||
suite.Equal([]int32{0, 1}, result.GetElementIndices()[0].GetIndices())
|
||||
})
|
||||
|
||||
suite.Run("dedup by pk with newer ts", func() {
|
||||
r1 := makeResult([]int64{1}, []int64{100}, [][]int32{{0, 1}}) // older
|
||||
r2 := makeResult([]int64{1}, []int64{200}, [][]int32{{0, 1, 2}}) // newer, should win
|
||||
result, err := MergeSegcoreRetrieveResultsV1(context.Background(), []*segcorepb.RetrieveResults{r1, r2},
|
||||
NewMergeParam(typeutil.Unlimited, nil, nil, reduce.IReduceNoOrder))
|
||||
suite.NoError(err)
|
||||
suite.Equal([]int64{1}, result.GetIds().GetIntId().GetData())
|
||||
suite.Equal([]int32{0, 1, 2}, result.GetElementIndices()[0].GetIndices()) // newer wins
|
||||
})
|
||||
|
||||
suite.Run("limit counts elements", func() {
|
||||
// doc1 has 3 elements, doc2 has 2 elements
|
||||
r1 := makeResult([]int64{1, 2}, []int64{100, 100}, [][]int32{{0, 1, 2}, {0, 1}})
|
||||
result, err := MergeSegcoreRetrieveResultsV1(context.Background(), []*segcorepb.RetrieveResults{r1},
|
||||
NewMergeParam(3, nil, nil, reduce.IReduceNoOrder)) // limit=3 elements
|
||||
suite.NoError(err)
|
||||
// Should get doc1 (3 elements) only, not doc2
|
||||
suite.Equal([]int64{1}, result.GetIds().GetIntId().GetData())
|
||||
suite.Equal(1, len(result.GetElementIndices()))
|
||||
})
|
||||
}
|
||||
|
||||
func (suite *ResultSuite) TestResult_MergeInternalRetrieveResult_ElementLevel() {
|
||||
makeResult := func(pks []int64, ts []int64, elemIndices [][]int32) *internalpb.RetrieveResults {
|
||||
elemIdxList := make([]*internalpb.ElementIndices, len(elemIndices))
|
||||
for i, indices := range elemIndices {
|
||||
elemIdxList[i] = &internalpb.ElementIndices{Indices: indices}
|
||||
}
|
||||
return &internalpb.RetrieveResults{
|
||||
Ids: &schemapb.IDs{IdField: &schemapb.IDs_IntId{IntId: &schemapb.LongArray{Data: pks}}},
|
||||
FieldsData: []*schemapb.FieldData{mock_segcore.GenFieldData(common.TimeStampFieldName, common.TimeStampField, schemapb.DataType_Int64, ts, 1)},
|
||||
ElementLevel: true,
|
||||
ElementIndices: elemIdxList,
|
||||
}
|
||||
}
|
||||
|
||||
suite.Run("merge and dedup", func() {
|
||||
r1 := makeResult([]int64{1, 2}, []int64{100, 100}, [][]int32{{0}, {1, 2}})
|
||||
r2 := makeResult([]int64{1, 3}, []int64{200, 100}, [][]int32{{0, 1}, {0}}) // pk=1 newer
|
||||
result, err := MergeInternalRetrieveResult(context.Background(), []*internalpb.RetrieveResults{r1, r2},
|
||||
NewMergeParam(typeutil.Unlimited, nil, nil, reduce.IReduceNoOrder))
|
||||
suite.NoError(err)
|
||||
suite.True(result.GetElementLevel())
|
||||
suite.Equal([]int64{1, 2, 3}, result.GetIds().GetIntId().GetData())
|
||||
suite.Equal([]int32{0, 1}, result.GetElementIndices()[0].GetIndices()) // pk=1 uses newer
|
||||
})
|
||||
|
||||
suite.Run("limit counts elements", func() {
|
||||
r1 := makeResult([]int64{1, 2}, []int64{100, 100}, [][]int32{{0, 1}, {0, 1, 2}})
|
||||
result, err := MergeInternalRetrieveResult(context.Background(), []*internalpb.RetrieveResults{r1},
|
||||
NewMergeParam(2, nil, nil, reduce.IReduceNoOrder)) // limit=2 elements
|
||||
suite.NoError(err)
|
||||
suite.Equal([]int64{1}, result.GetIds().GetIntId().GetData())
|
||||
})
|
||||
}
|
||||
|
||||
func TestResult(t *testing.T) {
|
||||
paramtable.Init()
|
||||
suite.Run(t, new(ResultSuite))
|
||||
|
||||
@@ -172,6 +172,8 @@ func (t *QueryTask) Execute() error {
|
||||
HasMoreResult: reducedResult.HasMoreResult,
|
||||
ScannedRemoteBytes: reducedResult.GetScannedRemoteBytes(),
|
||||
ScannedTotalBytes: reducedResult.GetScannedTotalBytes(),
|
||||
ElementLevel: reducedResult.GetElementLevel(),
|
||||
ElementIndices: convertSegcoreElementIndicesToInternal(reducedResult.GetElementIndices()),
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -195,3 +197,17 @@ func (t *QueryTask) Result() *internalpb.RetrieveResults {
|
||||
func (t *QueryTask) NQ() int64 {
|
||||
return 1
|
||||
}
|
||||
|
||||
// convertSegcoreElementIndicesToInternal converts segcorepb.ElementIndices to internalpb.ElementIndices
|
||||
func convertSegcoreElementIndicesToInternal(src []*segcorepb.ElementIndices) []*internalpb.ElementIndices {
|
||||
if src == nil {
|
||||
return nil
|
||||
}
|
||||
dst := make([]*internalpb.ElementIndices, len(src))
|
||||
for i, s := range src {
|
||||
if s != nil {
|
||||
dst[i] = &internalpb.ElementIndices{Indices: s.GetIndices()}
|
||||
}
|
||||
}
|
||||
return dst
|
||||
}
|
||||
|
||||
+1
-1
@@ -22,7 +22,7 @@ require (
|
||||
github.com/jolestar/go-commons-pool/v2 v2.1.2
|
||||
github.com/json-iterator/go v1.1.13-0.20220915233716-71ac16282d12
|
||||
github.com/klauspost/compress v1.18.0
|
||||
github.com/milvus-io/milvus-proto/go-api/v2 v2.6.6-0.20260202035249-b2c299650f7e
|
||||
github.com/milvus-io/milvus-proto/go-api/v2 v2.6.6-0.20260304062516-e7e54d966ef2
|
||||
github.com/minio/minio-go/v7 v7.0.73
|
||||
github.com/panjf2000/ants/v2 v2.11.3
|
||||
github.com/prometheus/client_golang v1.20.5
|
||||
|
||||
+2
-2
@@ -482,8 +482,8 @@ github.com/milvus-io/cgosymbolizer v0.0.0-20250318084424-114f4050c3a6 h1:YHMFI6L
|
||||
github.com/milvus-io/cgosymbolizer v0.0.0-20250318084424-114f4050c3a6/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg=
|
||||
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZzUfIfYe5qYDBzt4ZYRqzUjTR6CvUzjat8=
|
||||
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4=
|
||||
github.com/milvus-io/milvus-proto/go-api/v2 v2.6.6-0.20260202035249-b2c299650f7e h1:PAyiC+wYbWnuP1sRhpIeaG3EOEZoeRA1lE6x6jWcmX8=
|
||||
github.com/milvus-io/milvus-proto/go-api/v2 v2.6.6-0.20260202035249-b2c299650f7e/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
|
||||
github.com/milvus-io/milvus-proto/go-api/v2 v2.6.6-0.20260304062516-e7e54d966ef2 h1:OEqYnraL/Tf2nGMjRJji6xaaDe40zSQVBvruzEBZ1hU=
|
||||
github.com/milvus-io/milvus-proto/go-api/v2 v2.6.6-0.20260304062516-e7e54d966ef2/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v7 v7.0.73 h1:qr2vi96Qm7kZ4v7LLebjte+MQh621fFWnv93p12htEo=
|
||||
|
||||
@@ -211,6 +211,10 @@ message RetrieveRequest {
|
||||
uint64 entity_ttl_physical_time = 23;
|
||||
}
|
||||
|
||||
// Element indices for element-level query results
|
||||
message ElementIndices {
|
||||
repeated int32 indices = 1;
|
||||
}
|
||||
|
||||
message RetrieveResults {
|
||||
common.MsgBase base = 1;
|
||||
@@ -228,6 +232,10 @@ message RetrieveResults {
|
||||
bool has_more_result = 15;
|
||||
int64 scanned_remote_bytes = 16;
|
||||
int64 scanned_total_bytes = 17;
|
||||
|
||||
// Element-level query support
|
||||
bool element_level = 18;
|
||||
repeated ElementIndices element_indices = 19;
|
||||
}
|
||||
|
||||
message LoadIndex {
|
||||
|
||||
+721
-632
File diff suppressed because it is too large
Load Diff
@@ -41,6 +41,11 @@ message JsonKeyStats {
|
||||
int64 json_key_stats_data_format = 7;
|
||||
}
|
||||
|
||||
// Element indices for a single document in element-level query
|
||||
message ElementIndices {
|
||||
repeated int32 indices = 1;
|
||||
}
|
||||
|
||||
message RetrieveResults {
|
||||
schema.IDs ids = 1;
|
||||
repeated int64 offset = 2;
|
||||
@@ -49,6 +54,11 @@ message RetrieveResults {
|
||||
bool has_more_result = 5;
|
||||
int64 scanned_remote_bytes = 6;
|
||||
int64 scanned_total_bytes = 7;
|
||||
// Element-level query support
|
||||
bool element_level = 8;
|
||||
// Element indices per document (aligned with offset array)
|
||||
// element_indices[i] contains all matching element indices for offset[i]
|
||||
repeated ElementIndices element_indices = 9;
|
||||
}
|
||||
|
||||
message LoadFieldMeta {
|
||||
|
||||
@@ -370,6 +370,54 @@ func (x *JsonKeyStats) GetJsonKeyStatsDataFormat() int64 {
|
||||
return 0
|
||||
}
|
||||
|
||||
// Element indices for a single document in element-level query
|
||||
type ElementIndices struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
unknownFields protoimpl.UnknownFields
|
||||
|
||||
Indices []int32 `protobuf:"varint,1,rep,packed,name=indices,proto3" json:"indices,omitempty"`
|
||||
}
|
||||
|
||||
func (x *ElementIndices) Reset() {
|
||||
*x = ElementIndices{}
|
||||
if protoimpl.UnsafeEnabled {
|
||||
mi := &file_segcore_proto_msgTypes[4]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
}
|
||||
|
||||
func (x *ElementIndices) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*ElementIndices) ProtoMessage() {}
|
||||
|
||||
func (x *ElementIndices) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_segcore_proto_msgTypes[4]
|
||||
if protoimpl.UnsafeEnabled && x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use ElementIndices.ProtoReflect.Descriptor instead.
|
||||
func (*ElementIndices) Descriptor() ([]byte, []int) {
|
||||
return file_segcore_proto_rawDescGZIP(), []int{4}
|
||||
}
|
||||
|
||||
func (x *ElementIndices) GetIndices() []int32 {
|
||||
if x != nil {
|
||||
return x.Indices
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type RetrieveResults struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
@@ -382,12 +430,17 @@ type RetrieveResults struct {
|
||||
HasMoreResult bool `protobuf:"varint,5,opt,name=has_more_result,json=hasMoreResult,proto3" json:"has_more_result,omitempty"`
|
||||
ScannedRemoteBytes int64 `protobuf:"varint,6,opt,name=scanned_remote_bytes,json=scannedRemoteBytes,proto3" json:"scanned_remote_bytes,omitempty"`
|
||||
ScannedTotalBytes int64 `protobuf:"varint,7,opt,name=scanned_total_bytes,json=scannedTotalBytes,proto3" json:"scanned_total_bytes,omitempty"`
|
||||
// Element-level query support
|
||||
ElementLevel bool `protobuf:"varint,8,opt,name=element_level,json=elementLevel,proto3" json:"element_level,omitempty"`
|
||||
// Element indices per document (aligned with offset array)
|
||||
// element_indices[i] contains all matching element indices for offset[i]
|
||||
ElementIndices []*ElementIndices `protobuf:"bytes,9,rep,name=element_indices,json=elementIndices,proto3" json:"element_indices,omitempty"`
|
||||
}
|
||||
|
||||
func (x *RetrieveResults) Reset() {
|
||||
*x = RetrieveResults{}
|
||||
if protoimpl.UnsafeEnabled {
|
||||
mi := &file_segcore_proto_msgTypes[4]
|
||||
mi := &file_segcore_proto_msgTypes[5]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -400,7 +453,7 @@ func (x *RetrieveResults) String() string {
|
||||
func (*RetrieveResults) ProtoMessage() {}
|
||||
|
||||
func (x *RetrieveResults) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_segcore_proto_msgTypes[4]
|
||||
mi := &file_segcore_proto_msgTypes[5]
|
||||
if protoimpl.UnsafeEnabled && x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -413,7 +466,7 @@ func (x *RetrieveResults) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use RetrieveResults.ProtoReflect.Descriptor instead.
|
||||
func (*RetrieveResults) Descriptor() ([]byte, []int) {
|
||||
return file_segcore_proto_rawDescGZIP(), []int{4}
|
||||
return file_segcore_proto_rawDescGZIP(), []int{5}
|
||||
}
|
||||
|
||||
func (x *RetrieveResults) GetIds() *schemapb.IDs {
|
||||
@@ -465,6 +518,20 @@ func (x *RetrieveResults) GetScannedTotalBytes() int64 {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *RetrieveResults) GetElementLevel() bool {
|
||||
if x != nil {
|
||||
return x.ElementLevel
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (x *RetrieveResults) GetElementIndices() []*ElementIndices {
|
||||
if x != nil {
|
||||
return x.ElementIndices
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type LoadFieldMeta struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
@@ -478,7 +545,7 @@ type LoadFieldMeta struct {
|
||||
func (x *LoadFieldMeta) Reset() {
|
||||
*x = LoadFieldMeta{}
|
||||
if protoimpl.UnsafeEnabled {
|
||||
mi := &file_segcore_proto_msgTypes[5]
|
||||
mi := &file_segcore_proto_msgTypes[6]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -491,7 +558,7 @@ func (x *LoadFieldMeta) String() string {
|
||||
func (*LoadFieldMeta) ProtoMessage() {}
|
||||
|
||||
func (x *LoadFieldMeta) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_segcore_proto_msgTypes[5]
|
||||
mi := &file_segcore_proto_msgTypes[6]
|
||||
if protoimpl.UnsafeEnabled && x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -504,7 +571,7 @@ func (x *LoadFieldMeta) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use LoadFieldMeta.ProtoReflect.Descriptor instead.
|
||||
func (*LoadFieldMeta) Descriptor() ([]byte, []int) {
|
||||
return file_segcore_proto_rawDescGZIP(), []int{5}
|
||||
return file_segcore_proto_rawDescGZIP(), []int{6}
|
||||
}
|
||||
|
||||
func (x *LoadFieldMeta) GetMinTimestamp() int64 {
|
||||
@@ -541,7 +608,7 @@ type LoadSegmentMeta struct {
|
||||
func (x *LoadSegmentMeta) Reset() {
|
||||
*x = LoadSegmentMeta{}
|
||||
if protoimpl.UnsafeEnabled {
|
||||
mi := &file_segcore_proto_msgTypes[6]
|
||||
mi := &file_segcore_proto_msgTypes[7]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -554,7 +621,7 @@ func (x *LoadSegmentMeta) String() string {
|
||||
func (*LoadSegmentMeta) ProtoMessage() {}
|
||||
|
||||
func (x *LoadSegmentMeta) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_segcore_proto_msgTypes[6]
|
||||
mi := &file_segcore_proto_msgTypes[7]
|
||||
if protoimpl.UnsafeEnabled && x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -567,7 +634,7 @@ func (x *LoadSegmentMeta) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use LoadSegmentMeta.ProtoReflect.Descriptor instead.
|
||||
func (*LoadSegmentMeta) Descriptor() ([]byte, []int) {
|
||||
return file_segcore_proto_rawDescGZIP(), []int{6}
|
||||
return file_segcore_proto_rawDescGZIP(), []int{7}
|
||||
}
|
||||
|
||||
func (x *LoadSegmentMeta) GetMetas() []*LoadFieldMeta {
|
||||
@@ -596,7 +663,7 @@ type InsertRecord struct {
|
||||
func (x *InsertRecord) Reset() {
|
||||
*x = InsertRecord{}
|
||||
if protoimpl.UnsafeEnabled {
|
||||
mi := &file_segcore_proto_msgTypes[7]
|
||||
mi := &file_segcore_proto_msgTypes[8]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -609,7 +676,7 @@ func (x *InsertRecord) String() string {
|
||||
func (*InsertRecord) ProtoMessage() {}
|
||||
|
||||
func (x *InsertRecord) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_segcore_proto_msgTypes[7]
|
||||
mi := &file_segcore_proto_msgTypes[8]
|
||||
if protoimpl.UnsafeEnabled && x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -622,7 +689,7 @@ func (x *InsertRecord) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use InsertRecord.ProtoReflect.Descriptor instead.
|
||||
func (*InsertRecord) Descriptor() ([]byte, []int) {
|
||||
return file_segcore_proto_rawDescGZIP(), []int{7}
|
||||
return file_segcore_proto_rawDescGZIP(), []int{8}
|
||||
}
|
||||
|
||||
func (x *InsertRecord) GetFieldsData() []*schemapb.FieldData {
|
||||
@@ -657,7 +724,7 @@ type FieldIndexMeta struct {
|
||||
func (x *FieldIndexMeta) Reset() {
|
||||
*x = FieldIndexMeta{}
|
||||
if protoimpl.UnsafeEnabled {
|
||||
mi := &file_segcore_proto_msgTypes[8]
|
||||
mi := &file_segcore_proto_msgTypes[9]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -670,7 +737,7 @@ func (x *FieldIndexMeta) String() string {
|
||||
func (*FieldIndexMeta) ProtoMessage() {}
|
||||
|
||||
func (x *FieldIndexMeta) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_segcore_proto_msgTypes[8]
|
||||
mi := &file_segcore_proto_msgTypes[9]
|
||||
if protoimpl.UnsafeEnabled && x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -683,7 +750,7 @@ func (x *FieldIndexMeta) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use FieldIndexMeta.ProtoReflect.Descriptor instead.
|
||||
func (*FieldIndexMeta) Descriptor() ([]byte, []int) {
|
||||
return file_segcore_proto_rawDescGZIP(), []int{8}
|
||||
return file_segcore_proto_rawDescGZIP(), []int{9}
|
||||
}
|
||||
|
||||
func (x *FieldIndexMeta) GetFieldID() int64 {
|
||||
@@ -754,7 +821,7 @@ type CollectionIndexMeta struct {
|
||||
func (x *CollectionIndexMeta) Reset() {
|
||||
*x = CollectionIndexMeta{}
|
||||
if protoimpl.UnsafeEnabled {
|
||||
mi := &file_segcore_proto_msgTypes[9]
|
||||
mi := &file_segcore_proto_msgTypes[10]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -767,7 +834,7 @@ func (x *CollectionIndexMeta) String() string {
|
||||
func (*CollectionIndexMeta) ProtoMessage() {}
|
||||
|
||||
func (x *CollectionIndexMeta) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_segcore_proto_msgTypes[9]
|
||||
mi := &file_segcore_proto_msgTypes[10]
|
||||
if protoimpl.UnsafeEnabled && x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -780,7 +847,7 @@ func (x *CollectionIndexMeta) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use CollectionIndexMeta.ProtoReflect.Descriptor instead.
|
||||
func (*CollectionIndexMeta) Descriptor() ([]byte, []int) {
|
||||
return file_segcore_proto_rawDescGZIP(), []int{9}
|
||||
return file_segcore_proto_rawDescGZIP(), []int{10}
|
||||
}
|
||||
|
||||
func (x *CollectionIndexMeta) GetMaxIndexRowCount() int64 {
|
||||
@@ -822,7 +889,7 @@ type FieldIndexInfo struct {
|
||||
func (x *FieldIndexInfo) Reset() {
|
||||
*x = FieldIndexInfo{}
|
||||
if protoimpl.UnsafeEnabled {
|
||||
mi := &file_segcore_proto_msgTypes[10]
|
||||
mi := &file_segcore_proto_msgTypes[11]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -835,7 +902,7 @@ func (x *FieldIndexInfo) String() string {
|
||||
func (*FieldIndexInfo) ProtoMessage() {}
|
||||
|
||||
func (x *FieldIndexInfo) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_segcore_proto_msgTypes[10]
|
||||
mi := &file_segcore_proto_msgTypes[11]
|
||||
if protoimpl.UnsafeEnabled && x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -848,7 +915,7 @@ func (x *FieldIndexInfo) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use FieldIndexInfo.ProtoReflect.Descriptor instead.
|
||||
func (*FieldIndexInfo) Descriptor() ([]byte, []int) {
|
||||
return file_segcore_proto_rawDescGZIP(), []int{10}
|
||||
return file_segcore_proto_rawDescGZIP(), []int{11}
|
||||
}
|
||||
|
||||
func (x *FieldIndexInfo) GetFieldID() int64 {
|
||||
@@ -976,7 +1043,7 @@ type SegmentLoadInfo struct {
|
||||
func (x *SegmentLoadInfo) Reset() {
|
||||
*x = SegmentLoadInfo{}
|
||||
if protoimpl.UnsafeEnabled {
|
||||
mi := &file_segcore_proto_msgTypes[11]
|
||||
mi := &file_segcore_proto_msgTypes[12]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -989,7 +1056,7 @@ func (x *SegmentLoadInfo) String() string {
|
||||
func (*SegmentLoadInfo) ProtoMessage() {}
|
||||
|
||||
func (x *SegmentLoadInfo) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_segcore_proto_msgTypes[11]
|
||||
mi := &file_segcore_proto_msgTypes[12]
|
||||
if protoimpl.UnsafeEnabled && x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -1002,7 +1069,7 @@ func (x *SegmentLoadInfo) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use SegmentLoadInfo.ProtoReflect.Descriptor instead.
|
||||
func (*SegmentLoadInfo) Descriptor() ([]byte, []int) {
|
||||
return file_segcore_proto_rawDescGZIP(), []int{11}
|
||||
return file_segcore_proto_rawDescGZIP(), []int{12}
|
||||
}
|
||||
|
||||
func (x *SegmentLoadInfo) GetSegmentID() int64 {
|
||||
@@ -1212,28 +1279,38 @@ var file_segcore_proto_rawDesc = []byte{
|
||||
0x0a, 0x1a, 0x6a, 0x73, 0x6f, 0x6e, 0x5f, 0x6b, 0x65, 0x79, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x73,
|
||||
0x5f, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x07, 0x20, 0x01,
|
||||
0x28, 0x03, 0x52, 0x16, 0x6a, 0x73, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x53, 0x74, 0x61, 0x74, 0x73,
|
||||
0x44, 0x61, 0x74, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x22, 0xce, 0x02, 0x0a, 0x0f, 0x52,
|
||||
0x65, 0x74, 0x72, 0x69, 0x65, 0x76, 0x65, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x12, 0x2a,
|
||||
0x0a, 0x03, 0x69, 0x64, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6d, 0x69,
|
||||
0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d,
|
||||
0x61, 0x2e, 0x49, 0x44, 0x73, 0x52, 0x03, 0x69, 0x64, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x66,
|
||||
0x66, 0x73, 0x65, 0x74, 0x18, 0x02, 0x20, 0x03, 0x28, 0x03, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73,
|
||||
0x65, 0x74, 0x12, 0x3f, 0x0a, 0x0b, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x5f, 0x64, 0x61, 0x74,
|
||||
0x61, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73,
|
||||
0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x46, 0x69,
|
||||
0x65, 0x6c, 0x64, 0x44, 0x61, 0x74, 0x61, 0x52, 0x0a, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x44,
|
||||
0x61, 0x74, 0x61, 0x12, 0x2c, 0x0a, 0x12, 0x61, 0x6c, 0x6c, 0x5f, 0x72, 0x65, 0x74, 0x72, 0x69,
|
||||
0x65, 0x76, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52,
|
||||
0x10, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, 0x76, 0x65, 0x43, 0x6f, 0x75, 0x6e,
|
||||
0x74, 0x12, 0x26, 0x0a, 0x0f, 0x68, 0x61, 0x73, 0x5f, 0x6d, 0x6f, 0x72, 0x65, 0x5f, 0x72, 0x65,
|
||||
0x73, 0x75, 0x6c, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x68, 0x61, 0x73, 0x4d,
|
||||
0x6f, 0x72, 0x65, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x30, 0x0a, 0x14, 0x73, 0x63, 0x61,
|
||||
0x6e, 0x6e, 0x65, 0x64, 0x5f, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x62, 0x79, 0x74, 0x65,
|
||||
0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x12, 0x73, 0x63, 0x61, 0x6e, 0x6e, 0x65, 0x64,
|
||||
0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x2e, 0x0a, 0x13, 0x73,
|
||||
0x63, 0x61, 0x6e, 0x6e, 0x65, 0x64, 0x5f, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x62, 0x79, 0x74,
|
||||
0x65, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x11, 0x73, 0x63, 0x61, 0x6e, 0x6e, 0x65,
|
||||
0x64, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x42, 0x79, 0x74, 0x65, 0x73, 0x22, 0x76, 0x0a, 0x0d, 0x4c,
|
||||
0x44, 0x61, 0x74, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x22, 0x2a, 0x0a, 0x0e, 0x45, 0x6c,
|
||||
0x65, 0x6d, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x64, 0x69, 0x63, 0x65, 0x73, 0x12, 0x18, 0x0a, 0x07,
|
||||
0x69, 0x6e, 0x64, 0x69, 0x63, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x05, 0x52, 0x07, 0x69,
|
||||
0x6e, 0x64, 0x69, 0x63, 0x65, 0x73, 0x22, 0xc2, 0x03, 0x0a, 0x0f, 0x52, 0x65, 0x74, 0x72, 0x69,
|
||||
0x65, 0x76, 0x65, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x12, 0x2a, 0x0a, 0x03, 0x69, 0x64,
|
||||
0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73,
|
||||
0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x49, 0x44,
|
||||
0x73, 0x52, 0x03, 0x69, 0x64, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74,
|
||||
0x18, 0x02, 0x20, 0x03, 0x28, 0x03, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x3f,
|
||||
0x0a, 0x0b, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x18, 0x03, 0x20,
|
||||
0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f,
|
||||
0x74, 0x6f, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x44,
|
||||
0x61, 0x74, 0x61, 0x52, 0x0a, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x44, 0x61, 0x74, 0x61, 0x12,
|
||||
0x2c, 0x0a, 0x12, 0x61, 0x6c, 0x6c, 0x5f, 0x72, 0x65, 0x74, 0x72, 0x69, 0x65, 0x76, 0x65, 0x5f,
|
||||
0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x10, 0x61, 0x6c, 0x6c,
|
||||
0x52, 0x65, 0x74, 0x72, 0x69, 0x65, 0x76, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x26, 0x0a,
|
||||
0x0f, 0x68, 0x61, 0x73, 0x5f, 0x6d, 0x6f, 0x72, 0x65, 0x5f, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74,
|
||||
0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x68, 0x61, 0x73, 0x4d, 0x6f, 0x72, 0x65, 0x52,
|
||||
0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x30, 0x0a, 0x14, 0x73, 0x63, 0x61, 0x6e, 0x6e, 0x65, 0x64,
|
||||
0x5f, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x06, 0x20,
|
||||
0x01, 0x28, 0x03, 0x52, 0x12, 0x73, 0x63, 0x61, 0x6e, 0x6e, 0x65, 0x64, 0x52, 0x65, 0x6d, 0x6f,
|
||||
0x74, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x2e, 0x0a, 0x13, 0x73, 0x63, 0x61, 0x6e, 0x6e,
|
||||
0x65, 0x64, 0x5f, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x07,
|
||||
0x20, 0x01, 0x28, 0x03, 0x52, 0x11, 0x73, 0x63, 0x61, 0x6e, 0x6e, 0x65, 0x64, 0x54, 0x6f, 0x74,
|
||||
0x61, 0x6c, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x65, 0x6c, 0x65, 0x6d, 0x65,
|
||||
0x6e, 0x74, 0x5f, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c,
|
||||
0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x4d, 0x0a, 0x0f,
|
||||
0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x6e, 0x64, 0x69, 0x63, 0x65, 0x73, 0x18,
|
||||
0x09, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70,
|
||||
0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x73, 0x65, 0x67, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x45, 0x6c, 0x65,
|
||||
0x6d, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x64, 0x69, 0x63, 0x65, 0x73, 0x52, 0x0e, 0x65, 0x6c, 0x65,
|
||||
0x6d, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x64, 0x69, 0x63, 0x65, 0x73, 0x22, 0x76, 0x0a, 0x0d, 0x4c,
|
||||
0x6f, 0x61, 0x64, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x23, 0x0a, 0x0d,
|
||||
0x6d, 0x69, 0x6e, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x01, 0x20,
|
||||
0x01, 0x28, 0x03, 0x52, 0x0c, 0x6d, 0x69, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d,
|
||||
@@ -1420,53 +1497,55 @@ func file_segcore_proto_rawDescGZIP() []byte {
|
||||
return file_segcore_proto_rawDescData
|
||||
}
|
||||
|
||||
var file_segcore_proto_msgTypes = make([]protoimpl.MessageInfo, 14)
|
||||
var file_segcore_proto_msgTypes = make([]protoimpl.MessageInfo, 15)
|
||||
var file_segcore_proto_goTypes = []interface{}{
|
||||
(*Binlog)(nil), // 0: milvus.proto.segcore.Binlog
|
||||
(*FieldBinlog)(nil), // 1: milvus.proto.segcore.FieldBinlog
|
||||
(*TextIndexStats)(nil), // 2: milvus.proto.segcore.TextIndexStats
|
||||
(*JsonKeyStats)(nil), // 3: milvus.proto.segcore.JsonKeyStats
|
||||
(*RetrieveResults)(nil), // 4: milvus.proto.segcore.RetrieveResults
|
||||
(*LoadFieldMeta)(nil), // 5: milvus.proto.segcore.LoadFieldMeta
|
||||
(*LoadSegmentMeta)(nil), // 6: milvus.proto.segcore.LoadSegmentMeta
|
||||
(*InsertRecord)(nil), // 7: milvus.proto.segcore.InsertRecord
|
||||
(*FieldIndexMeta)(nil), // 8: milvus.proto.segcore.FieldIndexMeta
|
||||
(*CollectionIndexMeta)(nil), // 9: milvus.proto.segcore.CollectionIndexMeta
|
||||
(*FieldIndexInfo)(nil), // 10: milvus.proto.segcore.FieldIndexInfo
|
||||
(*SegmentLoadInfo)(nil), // 11: milvus.proto.segcore.SegmentLoadInfo
|
||||
nil, // 12: milvus.proto.segcore.SegmentLoadInfo.TextStatsLogsEntry
|
||||
nil, // 13: milvus.proto.segcore.SegmentLoadInfo.JsonKeyStatsLogsEntry
|
||||
(*schemapb.IDs)(nil), // 14: milvus.proto.schema.IDs
|
||||
(*schemapb.FieldData)(nil), // 15: milvus.proto.schema.FieldData
|
||||
(*commonpb.KeyValuePair)(nil), // 16: milvus.proto.common.KeyValuePair
|
||||
(commonpb.LoadPriority)(0), // 17: milvus.proto.common.LoadPriority
|
||||
(*ElementIndices)(nil), // 4: milvus.proto.segcore.ElementIndices
|
||||
(*RetrieveResults)(nil), // 5: milvus.proto.segcore.RetrieveResults
|
||||
(*LoadFieldMeta)(nil), // 6: milvus.proto.segcore.LoadFieldMeta
|
||||
(*LoadSegmentMeta)(nil), // 7: milvus.proto.segcore.LoadSegmentMeta
|
||||
(*InsertRecord)(nil), // 8: milvus.proto.segcore.InsertRecord
|
||||
(*FieldIndexMeta)(nil), // 9: milvus.proto.segcore.FieldIndexMeta
|
||||
(*CollectionIndexMeta)(nil), // 10: milvus.proto.segcore.CollectionIndexMeta
|
||||
(*FieldIndexInfo)(nil), // 11: milvus.proto.segcore.FieldIndexInfo
|
||||
(*SegmentLoadInfo)(nil), // 12: milvus.proto.segcore.SegmentLoadInfo
|
||||
nil, // 13: milvus.proto.segcore.SegmentLoadInfo.TextStatsLogsEntry
|
||||
nil, // 14: milvus.proto.segcore.SegmentLoadInfo.JsonKeyStatsLogsEntry
|
||||
(*schemapb.IDs)(nil), // 15: milvus.proto.schema.IDs
|
||||
(*schemapb.FieldData)(nil), // 16: milvus.proto.schema.FieldData
|
||||
(*commonpb.KeyValuePair)(nil), // 17: milvus.proto.common.KeyValuePair
|
||||
(commonpb.LoadPriority)(0), // 18: milvus.proto.common.LoadPriority
|
||||
}
|
||||
var file_segcore_proto_depIdxs = []int32{
|
||||
0, // 0: milvus.proto.segcore.FieldBinlog.binlogs:type_name -> milvus.proto.segcore.Binlog
|
||||
14, // 1: milvus.proto.segcore.RetrieveResults.ids:type_name -> milvus.proto.schema.IDs
|
||||
15, // 2: milvus.proto.segcore.RetrieveResults.fields_data:type_name -> milvus.proto.schema.FieldData
|
||||
5, // 3: milvus.proto.segcore.LoadSegmentMeta.metas:type_name -> milvus.proto.segcore.LoadFieldMeta
|
||||
15, // 4: milvus.proto.segcore.InsertRecord.fields_data:type_name -> milvus.proto.schema.FieldData
|
||||
16, // 5: milvus.proto.segcore.FieldIndexMeta.type_params:type_name -> milvus.proto.common.KeyValuePair
|
||||
16, // 6: milvus.proto.segcore.FieldIndexMeta.index_params:type_name -> milvus.proto.common.KeyValuePair
|
||||
16, // 7: milvus.proto.segcore.FieldIndexMeta.user_index_params:type_name -> milvus.proto.common.KeyValuePair
|
||||
8, // 8: milvus.proto.segcore.CollectionIndexMeta.index_metas:type_name -> milvus.proto.segcore.FieldIndexMeta
|
||||
16, // 9: milvus.proto.segcore.FieldIndexInfo.index_params:type_name -> milvus.proto.common.KeyValuePair
|
||||
1, // 10: milvus.proto.segcore.SegmentLoadInfo.binlog_paths:type_name -> milvus.proto.segcore.FieldBinlog
|
||||
1, // 11: milvus.proto.segcore.SegmentLoadInfo.statslogs:type_name -> milvus.proto.segcore.FieldBinlog
|
||||
1, // 12: milvus.proto.segcore.SegmentLoadInfo.deltalogs:type_name -> milvus.proto.segcore.FieldBinlog
|
||||
10, // 13: milvus.proto.segcore.SegmentLoadInfo.index_infos:type_name -> milvus.proto.segcore.FieldIndexInfo
|
||||
12, // 14: milvus.proto.segcore.SegmentLoadInfo.textStatsLogs:type_name -> milvus.proto.segcore.SegmentLoadInfo.TextStatsLogsEntry
|
||||
1, // 15: milvus.proto.segcore.SegmentLoadInfo.bm25logs:type_name -> milvus.proto.segcore.FieldBinlog
|
||||
13, // 16: milvus.proto.segcore.SegmentLoadInfo.jsonKeyStatsLogs:type_name -> milvus.proto.segcore.SegmentLoadInfo.JsonKeyStatsLogsEntry
|
||||
17, // 17: milvus.proto.segcore.SegmentLoadInfo.priority:type_name -> milvus.proto.common.LoadPriority
|
||||
2, // 18: milvus.proto.segcore.SegmentLoadInfo.TextStatsLogsEntry.value:type_name -> milvus.proto.segcore.TextIndexStats
|
||||
3, // 19: milvus.proto.segcore.SegmentLoadInfo.JsonKeyStatsLogsEntry.value:type_name -> milvus.proto.segcore.JsonKeyStats
|
||||
20, // [20:20] is the sub-list for method output_type
|
||||
20, // [20:20] is the sub-list for method input_type
|
||||
20, // [20:20] is the sub-list for extension type_name
|
||||
20, // [20:20] is the sub-list for extension extendee
|
||||
0, // [0:20] is the sub-list for field type_name
|
||||
15, // 1: milvus.proto.segcore.RetrieveResults.ids:type_name -> milvus.proto.schema.IDs
|
||||
16, // 2: milvus.proto.segcore.RetrieveResults.fields_data:type_name -> milvus.proto.schema.FieldData
|
||||
4, // 3: milvus.proto.segcore.RetrieveResults.element_indices:type_name -> milvus.proto.segcore.ElementIndices
|
||||
6, // 4: milvus.proto.segcore.LoadSegmentMeta.metas:type_name -> milvus.proto.segcore.LoadFieldMeta
|
||||
16, // 5: milvus.proto.segcore.InsertRecord.fields_data:type_name -> milvus.proto.schema.FieldData
|
||||
17, // 6: milvus.proto.segcore.FieldIndexMeta.type_params:type_name -> milvus.proto.common.KeyValuePair
|
||||
17, // 7: milvus.proto.segcore.FieldIndexMeta.index_params:type_name -> milvus.proto.common.KeyValuePair
|
||||
17, // 8: milvus.proto.segcore.FieldIndexMeta.user_index_params:type_name -> milvus.proto.common.KeyValuePair
|
||||
9, // 9: milvus.proto.segcore.CollectionIndexMeta.index_metas:type_name -> milvus.proto.segcore.FieldIndexMeta
|
||||
17, // 10: milvus.proto.segcore.FieldIndexInfo.index_params:type_name -> milvus.proto.common.KeyValuePair
|
||||
1, // 11: milvus.proto.segcore.SegmentLoadInfo.binlog_paths:type_name -> milvus.proto.segcore.FieldBinlog
|
||||
1, // 12: milvus.proto.segcore.SegmentLoadInfo.statslogs:type_name -> milvus.proto.segcore.FieldBinlog
|
||||
1, // 13: milvus.proto.segcore.SegmentLoadInfo.deltalogs:type_name -> milvus.proto.segcore.FieldBinlog
|
||||
11, // 14: milvus.proto.segcore.SegmentLoadInfo.index_infos:type_name -> milvus.proto.segcore.FieldIndexInfo
|
||||
13, // 15: milvus.proto.segcore.SegmentLoadInfo.textStatsLogs:type_name -> milvus.proto.segcore.SegmentLoadInfo.TextStatsLogsEntry
|
||||
1, // 16: milvus.proto.segcore.SegmentLoadInfo.bm25logs:type_name -> milvus.proto.segcore.FieldBinlog
|
||||
14, // 17: milvus.proto.segcore.SegmentLoadInfo.jsonKeyStatsLogs:type_name -> milvus.proto.segcore.SegmentLoadInfo.JsonKeyStatsLogsEntry
|
||||
18, // 18: milvus.proto.segcore.SegmentLoadInfo.priority:type_name -> milvus.proto.common.LoadPriority
|
||||
2, // 19: milvus.proto.segcore.SegmentLoadInfo.TextStatsLogsEntry.value:type_name -> milvus.proto.segcore.TextIndexStats
|
||||
3, // 20: milvus.proto.segcore.SegmentLoadInfo.JsonKeyStatsLogsEntry.value:type_name -> milvus.proto.segcore.JsonKeyStats
|
||||
21, // [21:21] is the sub-list for method output_type
|
||||
21, // [21:21] is the sub-list for method input_type
|
||||
21, // [21:21] is the sub-list for extension type_name
|
||||
21, // [21:21] is the sub-list for extension extendee
|
||||
0, // [0:21] is the sub-list for field type_name
|
||||
}
|
||||
|
||||
func init() { file_segcore_proto_init() }
|
||||
@@ -1524,7 +1603,7 @@ func file_segcore_proto_init() {
|
||||
}
|
||||
}
|
||||
file_segcore_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
|
||||
switch v := v.(*RetrieveResults); i {
|
||||
switch v := v.(*ElementIndices); i {
|
||||
case 0:
|
||||
return &v.state
|
||||
case 1:
|
||||
@@ -1536,7 +1615,7 @@ func file_segcore_proto_init() {
|
||||
}
|
||||
}
|
||||
file_segcore_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} {
|
||||
switch v := v.(*LoadFieldMeta); i {
|
||||
switch v := v.(*RetrieveResults); i {
|
||||
case 0:
|
||||
return &v.state
|
||||
case 1:
|
||||
@@ -1548,7 +1627,7 @@ func file_segcore_proto_init() {
|
||||
}
|
||||
}
|
||||
file_segcore_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} {
|
||||
switch v := v.(*LoadSegmentMeta); i {
|
||||
switch v := v.(*LoadFieldMeta); i {
|
||||
case 0:
|
||||
return &v.state
|
||||
case 1:
|
||||
@@ -1560,7 +1639,7 @@ func file_segcore_proto_init() {
|
||||
}
|
||||
}
|
||||
file_segcore_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} {
|
||||
switch v := v.(*InsertRecord); i {
|
||||
switch v := v.(*LoadSegmentMeta); i {
|
||||
case 0:
|
||||
return &v.state
|
||||
case 1:
|
||||
@@ -1572,7 +1651,7 @@ func file_segcore_proto_init() {
|
||||
}
|
||||
}
|
||||
file_segcore_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} {
|
||||
switch v := v.(*FieldIndexMeta); i {
|
||||
switch v := v.(*InsertRecord); i {
|
||||
case 0:
|
||||
return &v.state
|
||||
case 1:
|
||||
@@ -1584,7 +1663,7 @@ func file_segcore_proto_init() {
|
||||
}
|
||||
}
|
||||
file_segcore_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} {
|
||||
switch v := v.(*CollectionIndexMeta); i {
|
||||
switch v := v.(*FieldIndexMeta); i {
|
||||
case 0:
|
||||
return &v.state
|
||||
case 1:
|
||||
@@ -1596,7 +1675,7 @@ func file_segcore_proto_init() {
|
||||
}
|
||||
}
|
||||
file_segcore_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} {
|
||||
switch v := v.(*FieldIndexInfo); i {
|
||||
switch v := v.(*CollectionIndexMeta); i {
|
||||
case 0:
|
||||
return &v.state
|
||||
case 1:
|
||||
@@ -1608,6 +1687,18 @@ func file_segcore_proto_init() {
|
||||
}
|
||||
}
|
||||
file_segcore_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} {
|
||||
switch v := v.(*FieldIndexInfo); i {
|
||||
case 0:
|
||||
return &v.state
|
||||
case 1:
|
||||
return &v.sizeCache
|
||||
case 2:
|
||||
return &v.unknownFields
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
file_segcore_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} {
|
||||
switch v := v.(*SegmentLoadInfo); i {
|
||||
case 0:
|
||||
return &v.state
|
||||
@@ -1626,7 +1717,7 @@ func file_segcore_proto_init() {
|
||||
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
|
||||
RawDescriptor: file_segcore_proto_rawDesc,
|
||||
NumEnums: 0,
|
||||
NumMessages: 14,
|
||||
NumMessages: 15,
|
||||
NumExtensions: 0,
|
||||
NumServices: 0,
|
||||
},
|
||||
|
||||
@@ -4,7 +4,7 @@ go 1.24.12
|
||||
|
||||
require (
|
||||
github.com/apache/arrow/go/v17 v17.0.0
|
||||
github.com/milvus-io/milvus-proto/go-api/v2 v2.6.6-0.20260202035249-b2c299650f7e
|
||||
github.com/milvus-io/milvus-proto/go-api/v2 v2.6.6-0.20260304062516-e7e54d966ef2
|
||||
github.com/milvus-io/milvus/client/v2 v2.0.0-20241125024034-0b9edb62a92d
|
||||
github.com/milvus-io/milvus/pkg/v2 v2.6.4-0.20251104142533-a2ce70d25256
|
||||
github.com/minio/minio-go/v7 v7.0.73
|
||||
|
||||
@@ -354,8 +354,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5
|
||||
github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8=
|
||||
github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
|
||||
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
|
||||
github.com/milvus-io/milvus-proto/go-api/v2 v2.6.6-0.20260202035249-b2c299650f7e h1:PAyiC+wYbWnuP1sRhpIeaG3EOEZoeRA1lE6x6jWcmX8=
|
||||
github.com/milvus-io/milvus-proto/go-api/v2 v2.6.6-0.20260202035249-b2c299650f7e/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
|
||||
github.com/milvus-io/milvus-proto/go-api/v2 v2.6.6-0.20260304062516-e7e54d966ef2 h1:OEqYnraL/Tf2nGMjRJji6xaaDe40zSQVBvruzEBZ1hU=
|
||||
github.com/milvus-io/milvus-proto/go-api/v2 v2.6.6-0.20260304062516-e7e54d966ef2/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
|
||||
github.com/milvus-io/milvus/pkg/v2 v2.6.4-0.20251104142533-a2ce70d25256 h1:M2waty0w2k4YT2HHzJk3fx6EFPD4DKxNJatitIV+gGU=
|
||||
github.com/milvus-io/milvus/pkg/v2 v2.6.4-0.20251104142533-a2ce70d25256/go.mod h1:HT6Wxahwj/l8+i+D/C3iwDzCjDa36U9gyVw6CjjK4pE=
|
||||
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs=
|
||||
|
||||
Reference in New Issue
Block a user