From 8f09efda8f2024d320798bae9fcd929e90abeb71 Mon Sep 17 00:00:00 2001 From: "yihao.dai" Date: Tue, 28 Apr 2026 10:29:50 +0800 Subject: [PATCH] test: expand CDC replication test coverage across features and chaos scenarios (#49043) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Extends the CDC (replication) e2e test suite with comprehensive cross-feature coverage and switchover/failover scenarios on top of #45624. ## What's Added ### New test files (\`tests/python_client/cdc/testcases/\`) - \`test_fts_and_text.py\` — BM25 search, text_match, phrase_match, hybrid FTS+dense, FTS after switchover (5 tests × 2 analyzers) - \`test_schema_features.py\` — dynamic schema, nullable, defaults, partition key, clustering key, combined features (7 tests) - \`test_collection_properties.py\` — TTL, mmap, autocompaction, multi-property, drop property (5 tests) - \`test_resource_group.py\` — create, drop, update, transfer replica (4 tests) - \`test_multi_database.py\` — cross-DB collections, drop DB with collections, cross-DB operations (3 tests) - \`test_search_verification.py\` — 6 tests × 7 vector types (search result consistency, query data sampling, hybrid search, iterators, filtered search) - \`test_switchover.py\` — basic, during-writes, all-types, loaded, indexed, rapid stress, failover (7 tests) ### Shared utilities - \`base.py\`: +7 schema factories, +4 data generators, +4 verification helpers, +2 constants - \`conftest.py\`: +\`switchover_helper\` fixture, +\`--is-check\`/\`--milvus-ns\` options ## Test plan - [ ] Run \`pytest tests/python_client/cdc/testcases/\` against two-cluster CDC deployment - [ ] Verify each test file runs independently via \`pytest testcases/test_.py\` - [ ] Companion PR in zilliztech/test-jobs adds Jenkins pipelines to run these issue: #49042 pr: #45624 /kind test /kind improvement 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Signed-off-by: Yihao Dai Co-authored-by: Claude Opus 4.6 (1M context) --- .github/workflows/python-lint.yaml | 6 +- .pre-commit-config.yaml | 4 +- tests/README.md | 15 +- tests/README_CN.md | 16 +- tests/python_client/cdc/conftest.py | 140 ++- .../cdc/scripts/setup_cdc_topology.py | 143 ++-- .../cdc/stablity/test_concurrent_operation.py | 123 ++- .../stablity/test_single_request_operation.py | 142 +-- tests/python_client/cdc/testcases/base.py | 806 ++++++++++++------ .../testcases/test_collection_properties.py | 276 ++++++ .../cdc/testcases/test_fts_and_text.py | 715 ++++++++++++++++ .../cdc/testcases/test_multi_database.py | 316 +++++++ .../cdc/testcases/test_resource_group.py | 157 ++++ .../cdc/testcases/test_schema_features.py | 690 +++++++++++++++ .../cdc/testcases/test_search_verification.py | 570 +++++++++++++ .../cdc/testcases/test_setup_cdc.py | 103 +-- .../cdc/testcases/test_switchover.py | 793 +++++++++++++++++ tests/python_client/common/common_type.py | 411 ++++++--- tests/{pyproject.toml => ruff.toml} | 19 +- tests/uv.lock | 43 - 20 files changed, 4754 insertions(+), 734 deletions(-) create mode 100644 tests/python_client/cdc/testcases/test_collection_properties.py create mode 100644 tests/python_client/cdc/testcases/test_fts_and_text.py create mode 100644 tests/python_client/cdc/testcases/test_multi_database.py create mode 100644 tests/python_client/cdc/testcases/test_resource_group.py create mode 100644 tests/python_client/cdc/testcases/test_schema_features.py create mode 100644 tests/python_client/cdc/testcases/test_search_verification.py create mode 100644 tests/python_client/cdc/testcases/test_switchover.py rename tests/{pyproject.toml => ruff.toml} (66%) delete mode 100644 tests/uv.lock diff --git a/.github/workflows/python-lint.yaml b/.github/workflows/python-lint.yaml index 0f51078c38..3337b61ea4 100644 --- a/.github/workflows/python-lint.yaml +++ b/.github/workflows/python-lint.yaml @@ -6,12 +6,12 @@ on: - master paths: - 'tests/**/*.py' - - 'tests/pyproject.toml' + - 'tests/ruff.toml' - '.github/workflows/python-lint.yaml' pull_request: paths: - 'tests/**/*.py' - - 'tests/pyproject.toml' + - 'tests/ruff.toml' - '.github/workflows/python-lint.yaml' concurrency: @@ -50,7 +50,7 @@ jobs: args: "format --check --diff" - name: Validate ruff configuration (smoke test) - # Always runs so config-only PRs still catch a broken pyproject.toml. + # Always runs so config-only PRs still catch a broken ruff.toml. # --exit-zero ignores existing baseline violations; only fails when # ruff cannot load the config at all. uses: astral-sh/ruff-action@v4.0.0 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a8527cdafa..9ebf1aeec7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,9 +13,9 @@ repos: hooks: - id: ruff-check name: ruff-check (tests/) - args: [--config=tests/pyproject.toml, --fix] + args: [--config=tests/ruff.toml, --fix] files: ^tests/.*\.py$ - id: ruff-format name: ruff-format (tests/) - args: [--config=tests/pyproject.toml] + args: [--config=tests/ruff.toml] files: ^tests/.*\.py$ diff --git a/tests/README.md b/tests/README.md index e2524b9f05..f30e11f0ab 100644 --- a/tests/README.md +++ b/tests/README.md @@ -92,18 +92,17 @@ $ ./e2e-k8s.sh ### Python Code Quality (ruff via uv) -Ruff is configured at `tests/pyproject.toml` and covers all Python code under `tests/` +Ruff is configured at `tests/ruff.toml` and covers all Python code under `tests/` (`python_client/`, `restful_client/`, `restful_client_v2/`, `benchmark/`, `scripts/`). -[uv](https://docs.astral.sh/uv/) is only used to host the lint/format toolchain; each -sub-directory continues to manage its runtime dependencies via its own `requirements.txt`. +Each sub-directory continues to manage its runtime dependencies via its own +`requirements.txt`. ```shell $ cd tests/ -$ uv sync # install ruff into a local .venv -$ uv run ruff check . # lint -$ uv run ruff check . --fix # lint with auto-fix -$ uv run ruff format . # format in place -$ uv run ruff format --check . # format check only (CI-friendly) +$ ruff check . # lint +$ ruff check . --fix # lint with auto-fix +$ ruff format . # format in place +$ ruff format --check . # format check only (CI-friendly) ``` Rules enabled: `E`, `F`, `W`, `I`, `UP`. Target Python version: `3.10`. diff --git a/tests/README_CN.md b/tests/README_CN.md index 342056ce71..ddaaee2a70 100644 --- a/tests/README_CN.md +++ b/tests/README_CN.md @@ -90,20 +90,18 @@ $ ./e2e-k8s.sh > $ ./e2e-k8s.sh --help > ``` -### Python 代码质量 (通过 uv 使用 ruff) +### Python 代码质量 (ruff) -Ruff 配置位于 `tests/pyproject.toml`，覆盖 `tests/` 下所有 Python 代码 +Ruff 配置位于 `tests/ruff.toml`，覆盖 `tests/` 下所有 Python 代码 （`python_client/`、`restful_client/`、`restful_client_v2/`、`benchmark/`、`scripts/`）。 -[uv](https://docs.astral.sh/uv/) 仅用于托管 lint/format 工具链，各子目录的运行时依赖 -仍通过各自的 `requirements.txt` 管理。 +各子目录的运行时依赖仍通过各自的 `requirements.txt` 管理。 ```shell $ cd tests/ -$ uv sync # 将 ruff 安装到本地 .venv -$ uv run ruff check . # lint 检查 -$ uv run ruff check . --fix # lint 检查并自动修复 -$ uv run ruff format . # 原地格式化 -$ uv run ruff format --check . # 只检查不修改 (CI 友好) +$ ruff check . # lint 检查 +$ ruff check . --fix # lint 检查并自动修复 +$ ruff format . # 原地格式化 +$ ruff format --check . # 只检查不修改 (CI 友好) ``` 启用的规则：`E`、`F`、`W`、`I`、`UP`；目标 Python 版本：`3.10`。 diff --git a/tests/python_client/cdc/conftest.py b/tests/python_client/cdc/conftest.py index de85d1fbd2..94c744b041 100644 --- a/tests/python_client/cdc/conftest.py +++ b/tests/python_client/cdc/conftest.py @@ -1,14 +1,32 @@ -import pytest -import time import logging +import time +from concurrent.futures import ThreadPoolExecutor, wait + +import pytest from pymilvus import MilvusClient # Configure logging -logging.basicConfig( - level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s" -) +logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s") logger = logging.getLogger(__name__) +CDC_UPDATE_REPLICATE_TIMEOUT_SECONDS = 600 + + +def apply_replicate_configuration(tasks, timeout=CDC_UPDATE_REPLICATE_TIMEOUT_SECONDS): + # Fan out in parallel: the server blocks non-primary clusters in + # waitUntilPrimaryChangeOrConfigurationSame until the primary's broadcast + # propagates via CDC, so a sequential call where the first client happens + # to be a replica deadlocks on the client's RPC timeout. + with ThreadPoolExecutor(max_workers=len(tasks)) as executor: + futures = [ + executor.submit(client.update_replicate_configuration, timeout=timeout, **config) + for client, config in tasks + ] + wait(futures) + for f in futures: + f.result() + + def pytest_addoption(parser): """Add command line options for pytest.""" parser.addoption( @@ -35,9 +53,7 @@ def pytest_addoption(parser): default="root:Milvus", help="Downstream Milvus token", ) - parser.addoption( - "--sync-timeout", action="store", default="30", help="Sync timeout in seconds" - ) + parser.addoption("--sync-timeout", action="store", default="30", help="Sync timeout in seconds") parser.addoption( "--source-cluster-id", action="store", @@ -62,6 +78,18 @@ def pytest_addoption(parser): default="30m", help="Duration for test operations (e.g., 30m, 1h, 60s)", ) + parser.addoption( + "--is-check", + action="store", + default="true", + help="Whether to assert on checker statistics", + ) + parser.addoption( + "--milvus-ns", + action="store", + default="chaos-testing", + help="Kubernetes namespace for Milvus deployment", + ) @pytest.fixture(scope="session") @@ -138,6 +166,75 @@ def request_duration(request): return request.config.getoption("--request-duration") +@pytest.fixture(scope="session") +def is_check(request): + # The root tests/python_client/conftest.py registers --is_check (underscore) + # with type=bool, which argparse maps to the same dest (is_check) as our + # --is-check (hyphen). The root's bool wins in chaos runs, so accept either. + val = request.config.getoption("--is-check") + return val if isinstance(val, bool) else str(val).lower() == "true" + + +@pytest.fixture(scope="session") +def milvus_ns(request): + return request.config.getoption("--milvus-ns") + + +@pytest.fixture(scope="session") +def switchover_helper(request, upstream_client, downstream_client): + """Returns a callable that performs CDC topology switchover.""" + upstream_uri = request.config.getoption("--upstream-uri") + upstream_token = request.config.getoption("--upstream-token") + downstream_uri = request.config.getoption("--downstream-uri") + downstream_token = request.config.getoption("--downstream-token") + pchannel_num = int(request.config.getoption("--pchannel-num")) + original_source = request.config.getoption("--source-cluster-id") + original_target = request.config.getoption("--target-cluster-id") + + # Map cluster IDs to their URIs/tokens + cluster_map = { + original_source: {"uri": upstream_uri, "token": upstream_token}, + original_target: {"uri": downstream_uri, "token": downstream_token}, + } + + def do_switchover(new_source_id, new_target_id): + logger.info(f"Performing switchover: {new_source_id} -> {new_target_id}") + config = { + "clusters": [ + { + "cluster_id": new_source_id, + "connection_param": cluster_map[new_source_id], + "pchannels": [f"{new_source_id}-rootcoord-dml_{i}" for i in range(pchannel_num)], + }, + { + "cluster_id": new_target_id, + "connection_param": cluster_map[new_target_id], + "pchannels": [f"{new_target_id}-rootcoord-dml_{i}" for i in range(pchannel_num)], + }, + ], + "cross_cluster_topology": [{"source_cluster_id": new_source_id, "target_cluster_id": new_target_id}], + } + # Dedicated short-lived clients so switchover RPCs don't share a + # gRPC channel with concurrent DML on the session-scoped clients. + # pymilvus's connection manager closes a channel on UNAVAILABLE / + # STREAMING_CODE_REPLICATE_VIOLATION to trigger recovery; if a DML + # on the session client triggers that close while our sibling + # update_replicate_configuration RPC is in flight on the same + # channel, the latter surfaces "Cannot invoke RPC on closed + # channel!". Separate clients = separate channels = no race. + up_tmp = MilvusClient(uri=upstream_uri, token=upstream_token) + dn_tmp = MilvusClient(uri=downstream_uri, token=downstream_token) + try: + apply_replicate_configuration([(up_tmp, config), (dn_tmp, config)]) + finally: + up_tmp.close() + dn_tmp.close() + logger.info("Switchover completed, waiting 10s for stabilization...") + time.sleep(10) + + return do_switchover + + @pytest.fixture(scope="session", autouse=True) def cdc_topology_setup(request, upstream_client, downstream_client): """Setup CDC topology at the beginning of test session.""" @@ -147,9 +244,7 @@ def cdc_topology_setup(request, upstream_client, downstream_client): target_cluster_id = request.config.getoption("--target-cluster-id") pchannel_num = int(request.config.getoption("--pchannel-num")) - logger.info( - f"Setting up CDC topology: {source_cluster_id} -> {target_cluster_id} (channels: {pchannel_num})..." - ) + logger.info(f"Setting up CDC topology: {source_cluster_id} -> {target_cluster_id} (channels: {pchannel_num})...") # Create CDC replication configuration config = { @@ -160,10 +255,7 @@ def cdc_topology_setup(request, upstream_client, downstream_client): "uri": upstream_uri, "token": request.config.getoption("--upstream-token"), }, - "pchannels": [ - f"{source_cluster_id}-rootcoord-dml_{i}" - for i in range(pchannel_num) - ], + "pchannels": [f"{source_cluster_id}-rootcoord-dml_{i}" for i in range(pchannel_num)], }, { "cluster_id": target_cluster_id, @@ -171,10 +263,7 @@ def cdc_topology_setup(request, upstream_client, downstream_client): "uri": downstream_uri, "token": request.config.getoption("--downstream-token"), }, - "pchannels": [ - f"{target_cluster_id}-rootcoord-dml_{i}" - for i in range(pchannel_num) - ], + "pchannels": [f"{target_cluster_id}-rootcoord-dml_{i}" for i in range(pchannel_num)], }, ], "cross_cluster_topology": [ @@ -186,9 +275,16 @@ def cdc_topology_setup(request, upstream_client, downstream_client): } try: - # Update replication configuration on both clusters - upstream_client.update_replicate_configuration(**config) - downstream_client.update_replicate_configuration(**config) + # Dedicated clients for the control-plane update_replicate_configuration + # RPC, mirroring switchover_helper. Keeps the session-scoped clients' + # channels clean of any recovery side effects from the initial setup. + up_tmp = MilvusClient(uri=upstream_uri, token=request.config.getoption("--upstream-token")) + dn_tmp = MilvusClient(uri=downstream_uri, token=request.config.getoption("--downstream-token")) + try: + apply_replicate_configuration([(up_tmp, config), (dn_tmp, config)]) + finally: + up_tmp.close() + dn_tmp.close() logger.info("CDC topology setup completed successfully") # Allow some time for CDC to initialize diff --git a/tests/python_client/cdc/scripts/setup_cdc_topology.py b/tests/python_client/cdc/scripts/setup_cdc_topology.py index f7d6a4ad48..230937ddef 100644 --- a/tests/python_client/cdc/scripts/setup_cdc_topology.py +++ b/tests/python_client/cdc/scripts/setup_cdc_topology.py @@ -1,34 +1,58 @@ -from pymilvus import MilvusClient from concurrent.futures import ThreadPoolExecutor, as_completed -def setup_cdc_topology(upstream_uri, downstream_uri, removed_clusters_uri, upstream_token, downstream_token, removed_clusters_token, source_cluster_id, target_cluster_id, removed_clusters_id, pchannel_num): - print(f"DEBUG: upstream_uri: {upstream_uri}, downstream_uri: {downstream_uri}, upstream_token: {upstream_token}, downstream_token: {downstream_token}, source_cluster_id: {source_cluster_id}, target_cluster_id: {target_cluster_id}, pchannel_num: {pchannel_num}") +from pymilvus import MilvusClient + +# Kept in sync with cdc/conftest.py's CDC_UPDATE_REPLICATE_TIMEOUT_SECONDS. +# Inlined because this file is executed standalone from tests/python_client/cdc/scripts/, +# where the cdc package is not on sys.path. +CDC_UPDATE_REPLICATE_TIMEOUT_SECONDS = 600 + + +def setup_cdc_topology( + upstream_uri, + downstream_uri, + removed_clusters_uri, + upstream_token, + downstream_token, + removed_clusters_token, + source_cluster_id, + target_cluster_id, + removed_clusters_id, + pchannel_num, +): + print( + f"DEBUG: upstream_uri: {upstream_uri}, downstream_uri: {downstream_uri}, upstream_token: {upstream_token}, downstream_token: {downstream_token}, source_cluster_id: {source_cluster_id}, target_cluster_id: {target_cluster_id}, pchannel_num: {pchannel_num}" + ) upstream_client = MilvusClient(uri=upstream_uri, token=upstream_token) # Parse comma-separated lists - if isinstance(downstream_uri, str) and ',' in downstream_uri: - downstream_uris = [uri.strip() for uri in downstream_uri.split(',')] + if isinstance(downstream_uri, str) and "," in downstream_uri: + downstream_uris = [uri.strip() for uri in downstream_uri.split(",")] else: downstream_uris = [downstream_uri] if isinstance(downstream_uri, str) else downstream_uri - if isinstance(target_cluster_id, str) and ',' in target_cluster_id: - target_cluster_ids = [cluster_id.strip() for cluster_id in target_cluster_id.split(',')] + if isinstance(target_cluster_id, str) and "," in target_cluster_id: + target_cluster_ids = [cluster_id.strip() for cluster_id in target_cluster_id.split(",")] else: target_cluster_ids = [target_cluster_id] if isinstance(target_cluster_id, str) else target_cluster_id - if isinstance(removed_clusters_uri, str) and ',' in removed_clusters_uri: - removed_clusters_uris = [uri.strip() for uri in removed_clusters_uri.split(',')] + if isinstance(removed_clusters_uri, str) and "," in removed_clusters_uri: + removed_clusters_uris = [uri.strip() for uri in removed_clusters_uri.split(",")] else: - removed_clusters_uris = [removed_clusters_uri] if isinstance(removed_clusters_uri, str) else removed_clusters_uri + removed_clusters_uris = ( + [removed_clusters_uri] if isinstance(removed_clusters_uri, str) else removed_clusters_uri + ) - if isinstance(removed_clusters_id, str) and ',' in removed_clusters_id: - removed_clusters_ids = [cluster_id.strip() for cluster_id in removed_clusters_id.split(',')] + if isinstance(removed_clusters_id, str) and "," in removed_clusters_id: + removed_clusters_ids = [cluster_id.strip() for cluster_id in removed_clusters_id.split(",")] else: removed_clusters_ids = [removed_clusters_id] if isinstance(removed_clusters_id, str) else removed_clusters_id # Ensure we have matching numbers of downstream URIs and cluster IDs if len(downstream_uris) != len(target_cluster_ids): - raise ValueError(f"Number of downstream URIs ({len(downstream_uris)}) must match number of target cluster IDs ({len(target_cluster_ids)})") + raise ValueError( + f"Number of downstream URIs ({len(downstream_uris)}) must match number of target cluster IDs ({len(target_cluster_ids)})" + ) # Create downstream clients downstream_clients = [] @@ -40,44 +64,34 @@ def setup_cdc_topology(upstream_uri, downstream_uri, removed_clusters_uri, upstr clusters = [ { "cluster_id": source_cluster_id, - "connection_param": { - "uri": upstream_uri, - "token": upstream_token - }, - "pchannels": [f"{source_cluster_id}-rootcoord-dml_{i}" for i in range(pchannel_num)] + "connection_param": {"uri": upstream_uri, "token": upstream_token}, + "pchannels": [f"{source_cluster_id}-rootcoord-dml_{i}" for i in range(pchannel_num)], } ] # Add all target clusters for target_id, target_uri in zip(target_cluster_ids, downstream_uris): - clusters.append({ - "cluster_id": target_id, - "connection_param": { - "uri": target_uri, - "token": downstream_token - }, - "pchannels": [f"{target_id}-rootcoord-dml_{j}" for j in range(pchannel_num)] - }) + clusters.append( + { + "cluster_id": target_id, + "connection_param": {"uri": target_uri, "token": downstream_token}, + "pchannels": [f"{target_id}-rootcoord-dml_{j}" for j in range(pchannel_num)], + } + ) # Build cross-cluster topology cross_cluster_topology = [] for target_id in target_cluster_ids: - cross_cluster_topology.append({ - "source_cluster_id": source_cluster_id, - "target_cluster_id": target_id - }) + cross_cluster_topology.append({"source_cluster_id": source_cluster_id, "target_cluster_id": target_id}) - config = { - "clusters": clusters, - "cross_cluster_topology": cross_cluster_topology - } + config = {"clusters": clusters, "cross_cluster_topology": cross_cluster_topology} # Update configuration on all clients using multi-threading print(f"DEBUG: config: {config}") def update_client_config(client, config_to_use, client_type=""): try: - client.update_replicate_configuration(**config_to_use) + client.update_replicate_configuration(timeout=CDC_UPDATE_REPLICATE_TIMEOUT_SECONDS, **config_to_use) return f"{client_type} updated successfully" except Exception as e: print(f"Failed to update {client_type}: {e}") @@ -102,14 +116,11 @@ def setup_cdc_topology(upstream_uri, downstream_uri, removed_clusters_uri, upstr "clusters": [ { "cluster_id": removed_id, - "connection_param": { - "uri": removed_uri, - "token": removed_clusters_token - }, - "pchannels": [f"{removed_id}-rootcoord-dml_{i}" for i in range(pchannel_num)] + "connection_param": {"uri": removed_uri, "token": removed_clusters_token}, + "pchannels": [f"{removed_id}-rootcoord-dml_{i}" for i in range(pchannel_num)], } ], - "cross_cluster_topology": [] + "cross_cluster_topology": [], } print(f"DEBUG: Removing cluster {removed_id} with empty config: {empty_config}") update_tasks.append((removed_client, empty_config, f"Removed cluster {removed_id}")) @@ -117,8 +128,10 @@ def setup_cdc_topology(upstream_uri, downstream_uri, removed_clusters_uri, upstr # Use single ThreadPoolExecutor to update all clients concurrently with ThreadPoolExecutor(max_workers=len(update_tasks)) as executor: # Submit all update tasks - futures = [executor.submit(update_client_config, client, config_to_use, client_type) - for client, config_to_use, client_type in update_tasks] + futures = [ + executor.submit(update_client_config, client, config_to_use, client_type) + for client, config_to_use, client_type in update_tasks + ] # Wait for all tasks to complete for future in as_completed(futures): @@ -133,26 +146,28 @@ def setup_cdc_topology(upstream_uri, downstream_uri, removed_clusters_uri, upstr if __name__ == "__main__": import argparse - parser = argparse.ArgumentParser(description='connection info') - parser.add_argument('--upstream_uri', type=str, default='10.100.36.179', help='milvus host') - parser.add_argument('--downstream_uri', type=str, default='10.100.36.178', help='milvus host') - parser.add_argument('--removed_clusters_uri', type=str, default='', help='milvus host') - parser.add_argument('--upstream_token', type=str, default='root:Milvus', help='milvus token') - parser.add_argument('--downstream_token', type=str, default='root:Milvus', help='milvus token') - parser.add_argument('--removed_clusters_token', type=str, default='root:Milvus', help='milvus token') - parser.add_argument('--source_cluster_id', type=str, default='cdc-test-source', help='source cluster id') - parser.add_argument('--target_cluster_id', type=str, default='cdc-test-target', help='target cluster id') - parser.add_argument('--removed_clusters_id', type=str, default='', help='removed clusters id') + parser = argparse.ArgumentParser(description="connection info") + parser.add_argument("--upstream_uri", type=str, default="10.100.36.179", help="milvus host") + parser.add_argument("--downstream_uri", type=str, default="10.100.36.178", help="milvus host") + parser.add_argument("--removed_clusters_uri", type=str, default="", help="milvus host") + parser.add_argument("--upstream_token", type=str, default="root:Milvus", help="milvus token") + parser.add_argument("--downstream_token", type=str, default="root:Milvus", help="milvus token") + parser.add_argument("--removed_clusters_token", type=str, default="root:Milvus", help="milvus token") + parser.add_argument("--source_cluster_id", type=str, default="cdc-test-source", help="source cluster id") + parser.add_argument("--target_cluster_id", type=str, default="cdc-test-target", help="target cluster id") + parser.add_argument("--removed_clusters_id", type=str, default="", help="removed clusters id") - parser.add_argument('--pchannel_num', type=int, default=16, help='pchannel num') + parser.add_argument("--pchannel_num", type=int, default=16, help="pchannel num") args = parser.parse_args() - setup_cdc_topology(args.upstream_uri, - args.downstream_uri, - args.removed_clusters_uri, - args.upstream_token, - args.downstream_token, - args.removed_clusters_token, - args.source_cluster_id, - args.target_cluster_id, - args.removed_clusters_id, - args.pchannel_num) \ No newline at end of file + setup_cdc_topology( + args.upstream_uri, + args.downstream_uri, + args.removed_clusters_uri, + args.upstream_token, + args.downstream_token, + args.removed_clusters_token, + args.source_cluster_id, + args.target_cluster_id, + args.removed_clusters_id, + args.pchannel_num, + ) diff --git a/tests/python_client/cdc/stablity/test_concurrent_operation.py b/tests/python_client/cdc/stablity/test_concurrent_operation.py index 2bc6bd2eac..9bd49b916a 100644 --- a/tests/python_client/cdc/stablity/test_concurrent_operation.py +++ b/tests/python_client/cdc/stablity/test_concurrent_operation.py @@ -1,38 +1,71 @@ -import time -import pytest import json +import time from time import sleep -from pymilvus import connections -from chaos.checker import (InsertChecker, - UpsertChecker, - FlushChecker, - SearchChecker, - FullTextSearchChecker, - HybridSearchChecker, - QueryChecker, - TextMatchChecker, - PhraseMatchChecker, - JsonQueryChecker, - GeoQueryChecker, - DeleteChecker, - AddFieldChecker, - Op, - ResultAnalyzer - ) -from utils.util_k8s import wait_pods_ready, get_milvus_instance_name -from utils.util_log import test_log as log + +import pytest from chaos import chaos_commons as cc -from common import common_func as cf -from common.milvus_sys import MilvusSys -from chaos.chaos_commons import assert_statistic -from common.common_type import CaseLabel from chaos import constants +from chaos.chaos_commons import assert_statistic +from chaos.checker import ( + AddFieldChecker, + DeleteChecker, + FlushChecker, + FullTextSearchChecker, + GeoQueryChecker, + HybridSearchChecker, + InsertChecker, + JsonQueryChecker, + Op, + PhraseMatchChecker, + QueryChecker, + ResultAnalyzer, + SearchChecker, + TextMatchChecker, + UpsertChecker, +) +from common import common_func as cf +from common.common_type import CaseLabel +from common.milvus_sys import MilvusSys from delayed_assert import assert_expectations +from pymilvus import DataType, FunctionType, connections +from utils.util_k8s import get_milvus_instance_name, wait_pods_ready +from utils.util_log import test_log as log + +_VECTOR_DTYPES = { + DataType.FLOAT_VECTOR, + DataType.FLOAT16_VECTOR, + DataType.BFLOAT16_VECTOR, + DataType.BINARY_VECTOR, + DataType.SPARSE_FLOAT_VECTOR, + DataType.INT8_VECTOR, +} + + +def _build_checker_schema(dim=8): + """Build the shared all-datatype schema, stripped for the 2.6-latest image. + + The chaos-test image used by milvus_cdc_chaos_test/verify_test rejects + two things the shared gen_all_datatype_collection_schema includes by + default: + - FunctionType.MINHASH (error: "check function params with unknown + function type") + - nullable=True on FLOAT_VECTOR (error: "vector type not support null") + + Drop the MinHash function and its output field, and force nullable=False + on every vector field so the server accepts the schema. + """ + schema = cf.gen_all_datatype_collection_schema(dim=dim) + schema.functions[:] = [f for f in schema.functions if f.type != FunctionType.MINHASH] + schema.fields[:] = [f for f in schema.fields if f.name != "minhash_emb"] + for f in schema.fields: + if f.dtype in _VECTOR_DTYPES: + f.nullable = False + return schema def get_all_collections(): try: - with open("/tmp/ci_logs/chaos_test_all_collections.json", "r") as f: + with open("/tmp/ci_logs/chaos_test_all_collections.json") as f: data = json.load(f) all_collections = data["all"] except Exception as e: @@ -48,40 +81,40 @@ class TestBase: expect_compact = constants.SUCC expect_search = constants.SUCC expect_query = constants.SUCC - host = '127.0.0.1' + host = "127.0.0.1" port = 19530 _chaos_config = None health_checkers = {} class TestOperations(TestBase): - @pytest.fixture(scope="function", autouse=True) def connection(self, upstream_uri, upstream_token, milvus_ns): - connections.connect('default', uri=upstream_uri, token=upstream_token) + connections.connect("default", uri=upstream_uri, token=upstream_token) if connections.has_connection("default") is False: raise Exception("no connections") log.info("connect to milvus successfully") - self.milvus_sys = MilvusSys(alias='default') + self.milvus_sys = MilvusSys(alias="default") self.milvus_ns = milvus_ns self.release_name = get_milvus_instance_name(self.milvus_ns, milvus_sys=self.milvus_sys) def init_health_checkers(self, collection_name=None): c_name = collection_name + schema = _build_checker_schema() checkers = { - Op.insert: InsertChecker(collection_name=c_name), - Op.upsert: UpsertChecker(collection_name=c_name), - Op.flush: FlushChecker(collection_name=c_name), - Op.search: SearchChecker(collection_name=c_name), - Op.full_text_search: FullTextSearchChecker(collection_name=c_name), - Op.hybrid_search: HybridSearchChecker(collection_name=c_name), - Op.query: QueryChecker(collection_name=c_name), - Op.text_match: TextMatchChecker(collection_name=c_name), - Op.phrase_match: PhraseMatchChecker(collection_name=c_name), - Op.json_query: JsonQueryChecker(collection_name=c_name), - Op.geo_query: GeoQueryChecker(collection_name=c_name), - Op.delete: DeleteChecker(collection_name=c_name), - Op.add_field: AddFieldChecker(collection_name=c_name), + Op.insert: InsertChecker(collection_name=c_name, schema=schema), + Op.upsert: UpsertChecker(collection_name=c_name, schema=schema), + Op.flush: FlushChecker(collection_name=c_name, schema=schema), + Op.search: SearchChecker(collection_name=c_name, schema=schema), + Op.full_text_search: FullTextSearchChecker(collection_name=c_name, schema=schema), + Op.hybrid_search: HybridSearchChecker(collection_name=c_name, schema=schema), + Op.query: QueryChecker(collection_name=c_name, schema=schema), + Op.text_match: TextMatchChecker(collection_name=c_name, schema=schema), + Op.phrase_match: PhraseMatchChecker(collection_name=c_name, schema=schema), + Op.json_query: JsonQueryChecker(collection_name=c_name, schema=schema), + Op.geo_query: GeoQueryChecker(collection_name=c_name, schema=schema), + Op.delete: DeleteChecker(collection_name=c_name, schema=schema), + Op.add_field: AddFieldChecker(collection_name=c_name, schema=schema), } log.info(f"init_health_checkers: {checkers}") self.health_checkers = checkers @@ -96,7 +129,7 @@ class TestOperations(TestBase): def test_operations(self, request_duration, is_check, collection_name): # start the monitor threads to check the milvus ops log.info("*********************Test Start**********************") - log.info(connections.get_connection_addr('default')) + log.info(connections.get_connection_addr("default")) # event_records = EventRecords() c_name = collection_name if collection_name else cf.gen_unique_str("Checker_") # event_records.insert("init_health_checkers", "start") @@ -109,7 +142,7 @@ class TestOperations(TestBase): request_duration = request_duration[:-1] request_duration = eval(request_duration) for i in range(10): - sleep(request_duration//10) + sleep(request_duration // 10) for k, v in self.health_checkers.items(): v.check_result() # log.info(v.check_result()) diff --git a/tests/python_client/cdc/stablity/test_single_request_operation.py b/tests/python_client/cdc/stablity/test_single_request_operation.py index 5b7b22cdd9..4e4f81c9d4 100644 --- a/tests/python_client/cdc/stablity/test_single_request_operation.py +++ b/tests/python_client/cdc/stablity/test_single_request_operation.py @@ -1,41 +1,73 @@ import time - -import pytest from time import sleep + import pymilvus -from pymilvus import connections, utility -from chaos.checker import (CollectionCreateChecker, - InsertChecker, - BulkInsertChecker, - UpsertChecker, - PartialUpdateChecker, - FlushChecker, - SearchChecker, - FullTextSearchChecker, - HybridSearchChecker, - QueryChecker, - TextMatchChecker, - PhraseMatchChecker, - JsonQueryChecker, - GeoQueryChecker, - IndexCreateChecker, - DeleteChecker, - CollectionDropChecker, - AlterCollectionChecker, - AddFieldChecker, - CollectionRenameChecker, - Op, - EventRecords, - ResultAnalyzer - ) -from utils.util_log import test_log as log -from utils.util_k8s import wait_pods_ready, get_milvus_instance_name +import pytest from chaos import chaos_commons as cc +from chaos import constants +from chaos.chaos_commons import assert_statistic +from chaos.checker import ( + AddFieldChecker, + AlterCollectionChecker, + CollectionCreateChecker, + CollectionDropChecker, + CollectionRenameChecker, + DeleteChecker, + EventRecords, + FlushChecker, + FullTextSearchChecker, + GeoQueryChecker, + HybridSearchChecker, + IndexCreateChecker, + InsertChecker, + JsonQueryChecker, + Op, + PartialUpdateChecker, + PhraseMatchChecker, + QueryChecker, + ResultAnalyzer, + SearchChecker, + TextMatchChecker, + UpsertChecker, +) +from common import common_func as cf from common.common_type import CaseLabel from common.milvus_sys import MilvusSys -from chaos.chaos_commons import assert_statistic -from chaos import constants from delayed_assert import assert_expectations +from pymilvus import DataType, FunctionType, connections, utility +from utils.util_k8s import get_milvus_instance_name, wait_pods_ready +from utils.util_log import test_log as log + +_VECTOR_DTYPES = { + DataType.FLOAT_VECTOR, + DataType.FLOAT16_VECTOR, + DataType.BFLOAT16_VECTOR, + DataType.BINARY_VECTOR, + DataType.SPARSE_FLOAT_VECTOR, + DataType.INT8_VECTOR, +} + + +def _build_checker_schema(dim=8): + """Build the shared all-datatype schema, stripped for the 2.6-latest image. + + The chaos-test image used by milvus_cdc_chaos_test/verify_test rejects + two things the shared gen_all_datatype_collection_schema includes by + default: + - FunctionType.MINHASH (error: "check function params with unknown + function type") + - nullable=True on FLOAT_VECTOR (error: "vector type not support null") + + Drop the MinHash function and its output field, and force nullable=False + on every vector field so the server accepts the schema. + """ + schema = cf.gen_all_datatype_collection_schema(dim=dim) + schema.functions[:] = [f for f in schema.functions if f.type != FunctionType.MINHASH] + schema.fields[:] = [f for f in schema.fields if f.name != "minhash_emb"] + for f in schema.fields: + if f.dtype in _VECTOR_DTYPES: + f.nullable = False + return schema class TestBase: @@ -45,17 +77,16 @@ class TestBase: expect_index = constants.SUCC expect_search = constants.SUCC expect_query = constants.SUCC - host = '127.0.0.1' + host = "127.0.0.1" port = 19530 _chaos_config = None health_checkers = {} class TestOperations(TestBase): - @pytest.fixture(scope="function", autouse=True) def connection(self, upstream_uri, upstream_token, milvus_ns): - connections.connect('default', uri=upstream_uri, token=upstream_token) + connections.connect("default", uri=upstream_uri, token=upstream_token) if connections.has_connection("default") is False: raise Exception("no connections") log.info("connect to milvus successfully") @@ -63,32 +94,33 @@ class TestOperations(TestBase): server_version = utility.get_server_version() log.info(f"server version: {server_version}") log.info(f"pymilvus version: {pymilvus_version}") - self.milvus_sys = MilvusSys(alias='default') + self.milvus_sys = MilvusSys(alias="default") self.milvus_ns = milvus_ns self.release_name = get_milvus_instance_name(self.milvus_ns, milvus_sys=self.milvus_sys) def init_health_checkers(self, collection_name=None): c_name = collection_name + schema = _build_checker_schema() checkers = { - Op.create: CollectionCreateChecker(collection_name=c_name), - Op.insert: InsertChecker(collection_name=c_name), - Op.upsert: UpsertChecker(collection_name=c_name), - Op.partial_update: PartialUpdateChecker(collection_name=c_name), - Op.flush: FlushChecker(collection_name=c_name), - Op.index: IndexCreateChecker(collection_name=c_name), - Op.search: SearchChecker(collection_name=c_name), - Op.full_text_search: FullTextSearchChecker(collection_name=c_name), - Op.hybrid_search: HybridSearchChecker(collection_name=c_name), - Op.query: QueryChecker(collection_name=c_name), - Op.text_match: TextMatchChecker(collection_name=c_name), - Op.phrase_match: PhraseMatchChecker(collection_name=c_name), - Op.json_query: JsonQueryChecker(collection_name=c_name), - Op.geo_query: GeoQueryChecker(collection_name=c_name), - Op.delete: DeleteChecker(collection_name=c_name), - Op.drop: CollectionDropChecker(collection_name=c_name), - Op.alter_collection: AlterCollectionChecker(collection_name=c_name), - Op.add_field: AddFieldChecker(collection_name=c_name), - Op.rename_collection: CollectionRenameChecker(collection_name=c_name) + Op.create: CollectionCreateChecker(collection_name=c_name, schema=schema), + Op.insert: InsertChecker(collection_name=c_name, schema=schema), + Op.upsert: UpsertChecker(collection_name=c_name, schema=schema), + Op.partial_update: PartialUpdateChecker(collection_name=c_name, schema=schema), + Op.flush: FlushChecker(collection_name=c_name, schema=schema), + Op.index: IndexCreateChecker(collection_name=c_name, schema=schema), + Op.search: SearchChecker(collection_name=c_name, schema=schema), + Op.full_text_search: FullTextSearchChecker(collection_name=c_name, schema=schema), + Op.hybrid_search: HybridSearchChecker(collection_name=c_name, schema=schema), + Op.query: QueryChecker(collection_name=c_name, schema=schema), + Op.text_match: TextMatchChecker(collection_name=c_name, schema=schema), + Op.phrase_match: PhraseMatchChecker(collection_name=c_name, schema=schema), + Op.json_query: JsonQueryChecker(collection_name=c_name, schema=schema), + Op.geo_query: GeoQueryChecker(collection_name=c_name, schema=schema), + Op.delete: DeleteChecker(collection_name=c_name, schema=schema), + Op.drop: CollectionDropChecker(collection_name=c_name, schema=schema), + Op.alter_collection: AlterCollectionChecker(collection_name=c_name, schema=schema), + Op.add_field: AddFieldChecker(collection_name=c_name, schema=schema), + Op.rename_collection: CollectionRenameChecker(collection_name=c_name, schema=schema), } self.health_checkers = checkers @@ -96,7 +128,7 @@ class TestOperations(TestBase): def test_operations(self, request_duration, is_check): # start the monitor threads to check the milvus ops log.info("*********************Test Start**********************") - log.info(connections.get_connection_addr('default')) + log.info(connections.get_connection_addr("default")) event_records = EventRecords() c_name = None event_records.insert("init_health_checkers", "start") diff --git a/tests/python_client/cdc/testcases/base.py b/tests/python_client/cdc/testcases/base.py index d26cdc7677..b23f033212 100644 --- a/tests/python_client/cdc/testcases/base.py +++ b/tests/python_client/cdc/testcases/base.py @@ -2,13 +2,15 @@ Base class for CDC sync tests with common utilities. """ -import time +import logging import random import string -import logging +import time +from collections.abc import Callable from datetime import datetime -from typing import Any, Dict, List, Callable -from pymilvus import MilvusClient, DataType +from typing import Any + +from pymilvus import DataType, MilvusClient # Configure logging logging.basicConfig( @@ -21,10 +23,33 @@ logger = logging.getLogger(__name__) class TestCDCSyncBase: """Base class for CDC sync tests with common utilities.""" + SEARCH_OVERLAP_THRESHOLD = 0.8 + + FTS_SENTENCES = [ + "Vector databases enable efficient similarity search at scale.", + "Machine learning models produce high-dimensional embeddings for semantic search.", + "BM25 is a classic full-text search ranking function used in information retrieval.", + "Approximate nearest neighbor algorithms trade accuracy for speed.", + "Milvus supports hybrid search combining dense and sparse vectors.", + "Transformer models like BERT generate contextual word embeddings.", + "Inverted indexes power traditional keyword-based search engines.", + "HNSW graph structure allows fast approximate nearest neighbor queries.", + "Data replication ensures high availability in distributed databases.", + "CDC stands for Change Data Capture, tracking database mutations in real time.", + "Sparse vectors represent text documents in a high-dimensional term space.", + "Index partitioning improves query throughput in large vector collections.", + "Cosine similarity measures the angle between two vectors in embedding space.", + "Product quantization compresses vectors to reduce memory footprint.", + "Segment-level operations allow incremental updates without full re-indexing.", + "Full-text search ranks documents by term frequency and inverse document frequency.", + "Recall measures the fraction of true neighbors found by an approximate search.", + "Distributed query execution parallelizes workloads across multiple nodes.", + "Schema evolution allows adding new fields without dropping existing collections.", + "Hybrid ranking merges keyword and semantic scores for improved relevance.", + ] + @staticmethod - def gen_unique_name( - prefix: str = "", length: int = 8, max_length: int = None - ) -> str: + def gen_unique_name(prefix: str = "", length: int = 8, max_length: int = None) -> str: """Generate a unique string with prefix and timestamp.""" chars = string.ascii_letters + string.digits random_str = "".join(random.choice(chars) for _ in range(length)) @@ -41,11 +66,7 @@ class TestCDCSyncBase: if available_len > 0: # Use shorter timestamp format for space short_timestamp = datetime.now().strftime("%m%d_%H%M%S") # 11 chars - truncated_prefix = ( - prefix[: available_len - len(short_timestamp) - 1] - if len(prefix) > 0 - else "" - ) + truncated_prefix = prefix[: available_len - len(short_timestamp) - 1] if len(prefix) > 0 else "" name = ( f"{truncated_prefix}_{short_timestamp}_{random_str}" if truncated_prefix @@ -84,34 +105,23 @@ class TestCDCSyncBase: try: if check_func(): elapsed = time.time() - start_time - logger.info( - f"[SUCCESS] {operation_name} synced successfully in {elapsed:.2f}s" - ) + logger.info(f"[SUCCESS] {operation_name} synced successfully in {elapsed:.2f}s") return True except Exception as e: elapsed = time.time() - start_time - logger.warning( - f"Sync check failed for {operation_name} at {elapsed:.1f}s: {e}" - ) + logger.warning(f"Sync check failed for {operation_name} at {elapsed:.1f}s: {e}") elapsed = time.time() - start_time # Log every 10 seconds or on first check - if ( - elapsed - (last_log_time - start_time) >= 10 - or elapsed <= check_interval - ): + if elapsed - (last_log_time - start_time) >= 10 or elapsed <= check_interval: progress = (elapsed / timeout) * 100 - logger.info( - f"[WAITING] {operation_name} sync... {elapsed:.1f}s elapsed ({progress:.1f}% of timeout)" - ) + logger.info(f"[WAITING] {operation_name} sync... {elapsed:.1f}s elapsed ({progress:.1f}% of timeout)") last_log_time = time.time() time.sleep(check_interval) elapsed = time.time() - start_time - logger.error( - f"[FAILED] {operation_name} sync failed after {elapsed:.2f}s timeout" - ) + logger.error(f"[FAILED] {operation_name} sync failed after {elapsed:.2f}s timeout") return False @staticmethod @@ -151,18 +161,10 @@ class TestCDCSyncBase: schema.add_field("varchar_field", DataType.VARCHAR, max_length=1000) # Array fields - schema.add_field( - "bool_array", DataType.ARRAY, element_type=DataType.BOOL, max_capacity=100 - ) - schema.add_field( - "int32_array", DataType.ARRAY, element_type=DataType.INT32, max_capacity=100 - ) - schema.add_field( - "int64_array", DataType.ARRAY, element_type=DataType.INT64, max_capacity=100 - ) - schema.add_field( - "float_array", DataType.ARRAY, element_type=DataType.FLOAT, max_capacity=100 - ) + schema.add_field("bool_array", DataType.ARRAY, element_type=DataType.BOOL, max_capacity=100) + schema.add_field("int32_array", DataType.ARRAY, element_type=DataType.INT32, max_capacity=100) + schema.add_field("int64_array", DataType.ARRAY, element_type=DataType.INT64, max_capacity=100) + schema.add_field("float_array", DataType.ARRAY, element_type=DataType.FLOAT, max_capacity=100) schema.add_field( "double_array", DataType.ARRAY, @@ -208,18 +210,10 @@ class TestCDCSyncBase: schema.add_field("varchar_field", DataType.VARCHAR, max_length=1000) # Array fields - schema.add_field( - "bool_array", DataType.ARRAY, element_type=DataType.BOOL, max_capacity=100 - ) - schema.add_field( - "int32_array", DataType.ARRAY, element_type=DataType.INT32, max_capacity=100 - ) - schema.add_field( - "int64_array", DataType.ARRAY, element_type=DataType.INT64, max_capacity=100 - ) - schema.add_field( - "float_array", DataType.ARRAY, element_type=DataType.FLOAT, max_capacity=100 - ) + schema.add_field("bool_array", DataType.ARRAY, element_type=DataType.BOOL, max_capacity=100) + schema.add_field("int32_array", DataType.ARRAY, element_type=DataType.INT32, max_capacity=100) + schema.add_field("int64_array", DataType.ARRAY, element_type=DataType.INT64, max_capacity=100) + schema.add_field("float_array", DataType.ARRAY, element_type=DataType.FLOAT, max_capacity=100) schema.add_field( "double_array", DataType.ARRAY, @@ -265,18 +259,10 @@ class TestCDCSyncBase: schema.add_field("varchar_field", DataType.VARCHAR, max_length=1000) # Array fields - schema.add_field( - "bool_array", DataType.ARRAY, element_type=DataType.BOOL, max_capacity=100 - ) - schema.add_field( - "int32_array", DataType.ARRAY, element_type=DataType.INT32, max_capacity=100 - ) - schema.add_field( - "int64_array", DataType.ARRAY, element_type=DataType.INT64, max_capacity=100 - ) - schema.add_field( - "float_array", DataType.ARRAY, element_type=DataType.FLOAT, max_capacity=100 - ) + schema.add_field("bool_array", DataType.ARRAY, element_type=DataType.BOOL, max_capacity=100) + schema.add_field("int32_array", DataType.ARRAY, element_type=DataType.INT32, max_capacity=100) + schema.add_field("int64_array", DataType.ARRAY, element_type=DataType.INT64, max_capacity=100) + schema.add_field("float_array", DataType.ARRAY, element_type=DataType.FLOAT, max_capacity=100) schema.add_field( "double_array", DataType.ARRAY, @@ -322,18 +308,10 @@ class TestCDCSyncBase: schema.add_field("varchar_field", DataType.VARCHAR, max_length=1000) # Array fields - schema.add_field( - "bool_array", DataType.ARRAY, element_type=DataType.BOOL, max_capacity=100 - ) - schema.add_field( - "int32_array", DataType.ARRAY, element_type=DataType.INT32, max_capacity=100 - ) - schema.add_field( - "int64_array", DataType.ARRAY, element_type=DataType.INT64, max_capacity=100 - ) - schema.add_field( - "float_array", DataType.ARRAY, element_type=DataType.FLOAT, max_capacity=100 - ) + schema.add_field("bool_array", DataType.ARRAY, element_type=DataType.BOOL, max_capacity=100) + schema.add_field("int32_array", DataType.ARRAY, element_type=DataType.INT32, max_capacity=100) + schema.add_field("int64_array", DataType.ARRAY, element_type=DataType.INT64, max_capacity=100) + schema.add_field("float_array", DataType.ARRAY, element_type=DataType.FLOAT, max_capacity=100) schema.add_field( "double_array", DataType.ARRAY, @@ -365,7 +343,7 @@ class TestCDCSyncBase: return schema @staticmethod - def generate_test_data(count: int = 100) -> List[Dict[str, Any]]: + def generate_test_data(count: int = 100) -> list[dict[str, Any]]: """Generate test data for insert operations.""" return [ { @@ -379,9 +357,7 @@ class TestCDCSyncBase: ] @staticmethod - def generate_test_data_with_id( - count: int = 100, start_id: int = 0 - ) -> List[Dict[str, Any]]: + def generate_test_data_with_id(count: int = 100, start_id: int = 0) -> list[dict[str, Any]]: """Generate test data with manual IDs for upsert operations.""" return [ { @@ -395,20 +371,14 @@ class TestCDCSyncBase: ] @staticmethod - def generate_comprehensive_test_data(count: int = 100) -> List[Dict[str, Any]]: + def generate_comprehensive_test_data(count: int = 100) -> list[dict[str, Any]]: """Generate comprehensive test data with all data types using standard vector generation.""" # Generate vectors using standard method float_vectors = TestCDCSyncBase._gen_vectors(count, 128, DataType.FLOAT_VECTOR) - float16_vectors = TestCDCSyncBase._gen_vectors( - count, 64, DataType.FLOAT16_VECTOR - ) - binary_vectors = TestCDCSyncBase._gen_vectors( - count, 128, DataType.BINARY_VECTOR - ) - sparse_vectors = TestCDCSyncBase._gen_vectors( - count, 1000, DataType.SPARSE_FLOAT_VECTOR - ) + float16_vectors = TestCDCSyncBase._gen_vectors(count, 64, DataType.FLOAT16_VECTOR) + binary_vectors = TestCDCSyncBase._gen_vectors(count, 128, DataType.BINARY_VECTOR) + sparse_vectors = TestCDCSyncBase._gen_vectors(count, 1000, DataType.SPARSE_FLOAT_VECTOR) data = [] for i in range(count): @@ -423,31 +393,17 @@ class TestCDCSyncBase: "int8_field": random.randint(-128, 127), "int16_field": random.randint(-32768, 32767), "int32_field": random.randint(-2147483648, 2147483647), - "int64_field": random.randint( - -9223372036854775808, 9223372036854775807 - ), + "int64_field": random.randint(-9223372036854775808, 9223372036854775807), "float_field": random.uniform(-1000.0, 1000.0), "double_field": random.uniform(-1000.0, 1000.0), "varchar_field": f"test_varchar_{i}_{random.randint(1000, 9999)}", # Array fields - "bool_array": [ - random.choice([True, False]) for _ in range(random.randint(1, 10)) - ], - "int32_array": [ - random.randint(-100, 100) for _ in range(random.randint(1, 10)) - ], - "int64_array": [ - random.randint(-1000, 1000) for _ in range(random.randint(1, 10)) - ], - "float_array": [ - random.uniform(-100.0, 100.0) for _ in range(random.randint(1, 10)) - ], - "double_array": [ - random.uniform(-100.0, 100.0) for _ in range(random.randint(1, 10)) - ], - "varchar_array": [ - f"array_str_{j}" for j in range(random.randint(1, 5)) - ], + "bool_array": [random.choice([True, False]) for _ in range(random.randint(1, 10))], + "int32_array": [random.randint(-100, 100) for _ in range(random.randint(1, 10))], + "int64_array": [random.randint(-1000, 1000) for _ in range(random.randint(1, 10))], + "float_array": [random.uniform(-100.0, 100.0) for _ in range(random.randint(1, 10))], + "double_array": [random.uniform(-100.0, 100.0) for _ in range(random.randint(1, 10))], + "varchar_array": [f"array_str_{j}" for j in range(random.randint(1, 5))], # JSON field "json_field": { "name": f"item_{i}", @@ -464,18 +420,14 @@ class TestCDCSyncBase: return data @staticmethod - def generate_comprehensive_test_data_alt(count: int = 100) -> List[Dict[str, Any]]: + def generate_comprehensive_test_data_alt(count: int = 100) -> list[dict[str, Any]]: """Generate comprehensive test data with alternative vector types (BFLOAT16 + INT8).""" # Generate vectors using standard method - alternative set float_vectors = TestCDCSyncBase._gen_vectors(count, 128, DataType.FLOAT_VECTOR) - bfloat16_vectors = TestCDCSyncBase._gen_vectors( - count, 64, DataType.BFLOAT16_VECTOR - ) + bfloat16_vectors = TestCDCSyncBase._gen_vectors(count, 64, DataType.BFLOAT16_VECTOR) int8_vectors = TestCDCSyncBase._gen_vectors(count, 128, DataType.INT8_VECTOR) - sparse_vectors = TestCDCSyncBase._gen_vectors( - count, 1000, DataType.SPARSE_FLOAT_VECTOR - ) + sparse_vectors = TestCDCSyncBase._gen_vectors(count, 1000, DataType.SPARSE_FLOAT_VECTOR) data = [] for i in range(count): @@ -490,31 +442,17 @@ class TestCDCSyncBase: "int8_field": random.randint(-128, 127), "int16_field": random.randint(-32768, 32767), "int32_field": random.randint(-2147483648, 2147483647), - "int64_field": random.randint( - -9223372036854775808, 9223372036854775807 - ), + "int64_field": random.randint(-9223372036854775808, 9223372036854775807), "float_field": random.uniform(-1000.0, 1000.0), "double_field": random.uniform(-1000.0, 1000.0), "varchar_field": f"test_varchar_{i}_{random.randint(1000, 9999)}", # Array fields - "bool_array": [ - random.choice([True, False]) for _ in range(random.randint(1, 10)) - ], - "int32_array": [ - random.randint(-100, 100) for _ in range(random.randint(1, 10)) - ], - "int64_array": [ - random.randint(-1000, 1000) for _ in range(random.randint(1, 10)) - ], - "float_array": [ - random.uniform(-100.0, 100.0) for _ in range(random.randint(1, 10)) - ], - "double_array": [ - random.uniform(-100.0, 100.0) for _ in range(random.randint(1, 10)) - ], - "varchar_array": [ - f"array_str_{j}" for j in range(random.randint(1, 5)) - ], + "bool_array": [random.choice([True, False]) for _ in range(random.randint(1, 10))], + "int32_array": [random.randint(-100, 100) for _ in range(random.randint(1, 10))], + "int64_array": [random.randint(-1000, 1000) for _ in range(random.randint(1, 10))], + "float_array": [random.uniform(-100.0, 100.0) for _ in range(random.randint(1, 10))], + "double_array": [random.uniform(-100.0, 100.0) for _ in range(random.randint(1, 10))], + "varchar_array": [f"array_str_{j}" for j in range(random.randint(1, 5))], # JSON field "json_field": { "name": f"item_{i}", @@ -531,20 +469,14 @@ class TestCDCSyncBase: return data @staticmethod - def generate_comprehensive_test_data_alt_with_id( - count: int = 100, start_id: int = 0 - ) -> List[Dict[str, Any]]: + def generate_comprehensive_test_data_alt_with_id(count: int = 100, start_id: int = 0) -> list[dict[str, Any]]: """Generate comprehensive test data with manual IDs and alternative vector types (BFLOAT16 + INT8).""" # Generate vectors using standard method - alternative set float_vectors = TestCDCSyncBase._gen_vectors(count, 128, DataType.FLOAT_VECTOR) - bfloat16_vectors = TestCDCSyncBase._gen_vectors( - count, 64, DataType.BFLOAT16_VECTOR - ) + bfloat16_vectors = TestCDCSyncBase._gen_vectors(count, 64, DataType.BFLOAT16_VECTOR) int8_vectors = TestCDCSyncBase._gen_vectors(count, 128, DataType.INT8_VECTOR) - sparse_vectors = TestCDCSyncBase._gen_vectors( - count, 1000, DataType.SPARSE_FLOAT_VECTOR - ) + sparse_vectors = TestCDCSyncBase._gen_vectors(count, 1000, DataType.SPARSE_FLOAT_VECTOR) data = [] for i in range(count): @@ -560,31 +492,17 @@ class TestCDCSyncBase: "int8_field": random.randint(-128, 127), "int16_field": random.randint(-32768, 32767), "int32_field": random.randint(-2147483648, 2147483647), - "int64_field": random.randint( - -9223372036854775808, 9223372036854775807 - ), + "int64_field": random.randint(-9223372036854775808, 9223372036854775807), "float_field": random.uniform(-1000.0, 1000.0), "double_field": random.uniform(-1000.0, 1000.0), "varchar_field": f"test_varchar_{i}_{random.randint(1000, 9999)}", # Array fields - "bool_array": [ - random.choice([True, False]) for _ in range(random.randint(1, 10)) - ], - "int32_array": [ - random.randint(-100, 100) for _ in range(random.randint(1, 10)) - ], - "int64_array": [ - random.randint(-1000, 1000) for _ in range(random.randint(1, 10)) - ], - "float_array": [ - random.uniform(-100.0, 100.0) for _ in range(random.randint(1, 10)) - ], - "double_array": [ - random.uniform(-100.0, 100.0) for _ in range(random.randint(1, 10)) - ], - "varchar_array": [ - f"array_str_{j}" for j in range(random.randint(1, 5)) - ], + "bool_array": [random.choice([True, False]) for _ in range(random.randint(1, 10))], + "int32_array": [random.randint(-100, 100) for _ in range(random.randint(1, 10))], + "int64_array": [random.randint(-1000, 1000) for _ in range(random.randint(1, 10))], + "float_array": [random.uniform(-100.0, 100.0) for _ in range(random.randint(1, 10))], + "double_array": [random.uniform(-100.0, 100.0) for _ in range(random.randint(1, 10))], + "varchar_array": [f"array_str_{j}" for j in range(random.randint(1, 5))], # JSON field "json_field": { "name": f"item_{i}", @@ -632,41 +550,30 @@ class TestCDCSyncBase: # Generate binary vectors (dim bits = dim/8 bytes) bytes_per_vector = dim // 8 for _ in range(nb): - binary_vec = np.random.randint( - 0, 256, size=bytes_per_vector, dtype=np.uint8 - ).tobytes() + binary_vec = np.random.randint(0, 256, size=bytes_per_vector, dtype=np.uint8).tobytes() vectors.append(binary_vec) elif vector_data_type == DataType.SPARSE_FLOAT_VECTOR: - # Generate sparse vectors + # Sparse vectors have no fixed dim in schema; dim=0 is passed through + # from parametrized callers. Use a fixed pool so we always have a + # valid range for randint(5, ...). + pool = dim if dim >= 20 else 1000 for _ in range(nb): - sparse_indices = random.sample( - range(dim), random.randint(5, min(20, dim)) - ) + sparse_indices = random.sample(range(pool), random.randint(5, 20)) sparse_values = [random.random() for _ in sparse_indices] - sparse_vector = { - idx: val for idx, val in zip(sparse_indices, sparse_values) - } + sparse_vector = {idx: val for idx, val in zip(sparse_indices, sparse_values)} vectors.append(sparse_vector) return vectors @staticmethod - def generate_comprehensive_test_data_with_id( - count: int = 100, start_id: int = 0 - ) -> List[Dict[str, Any]]: + def generate_comprehensive_test_data_with_id(count: int = 100, start_id: int = 0) -> list[dict[str, Any]]: """Generate comprehensive test data with manual IDs for upsert operations using standard vector generation.""" # Generate vectors using standard method float_vectors = TestCDCSyncBase._gen_vectors(count, 128, DataType.FLOAT_VECTOR) - float16_vectors = TestCDCSyncBase._gen_vectors( - count, 64, DataType.FLOAT16_VECTOR - ) - binary_vectors = TestCDCSyncBase._gen_vectors( - count, 128, DataType.BINARY_VECTOR - ) - sparse_vectors = TestCDCSyncBase._gen_vectors( - count, 1000, DataType.SPARSE_FLOAT_VECTOR - ) + float16_vectors = TestCDCSyncBase._gen_vectors(count, 64, DataType.FLOAT16_VECTOR) + binary_vectors = TestCDCSyncBase._gen_vectors(count, 128, DataType.BINARY_VECTOR) + sparse_vectors = TestCDCSyncBase._gen_vectors(count, 1000, DataType.SPARSE_FLOAT_VECTOR) data = [] for i in range(count): @@ -682,31 +589,17 @@ class TestCDCSyncBase: "int8_field": random.randint(-128, 127), "int16_field": random.randint(-32768, 32767), "int32_field": random.randint(-2147483648, 2147483647), - "int64_field": random.randint( - -9223372036854775808, 9223372036854775807 - ), + "int64_field": random.randint(-9223372036854775808, 9223372036854775807), "float_field": random.uniform(-1000.0, 1000.0), "double_field": random.uniform(-1000.0, 1000.0), "varchar_field": f"test_varchar_{i}_{random.randint(1000, 9999)}", # Array fields - "bool_array": [ - random.choice([True, False]) for _ in range(random.randint(1, 10)) - ], - "int32_array": [ - random.randint(-100, 100) for _ in range(random.randint(1, 10)) - ], - "int64_array": [ - random.randint(-1000, 1000) for _ in range(random.randint(1, 10)) - ], - "float_array": [ - random.uniform(-100.0, 100.0) for _ in range(random.randint(1, 10)) - ], - "double_array": [ - random.uniform(-100.0, 100.0) for _ in range(random.randint(1, 10)) - ], - "varchar_array": [ - f"array_str_{j}" for j in range(random.randint(1, 5)) - ], + "bool_array": [random.choice([True, False]) for _ in range(random.randint(1, 10))], + "int32_array": [random.randint(-100, 100) for _ in range(random.randint(1, 10))], + "int64_array": [random.randint(-1000, 1000) for _ in range(random.randint(1, 10))], + "float_array": [random.uniform(-100.0, 100.0) for _ in range(random.randint(1, 10))], + "double_array": [random.uniform(-100.0, 100.0) for _ in range(random.randint(1, 10))], + "varchar_array": [f"array_str_{j}" for j in range(random.randint(1, 5))], # JSON field "json_field": { "name": f"item_{i}", @@ -723,13 +616,11 @@ class TestCDCSyncBase: return data @staticmethod - def generate_bfloat16_test_data(count: int = 100) -> List[Dict[str, Any]]: + def generate_bfloat16_test_data(count: int = 100) -> list[dict[str, Any]]: """Generate test data with BFLOAT16_VECTOR for index testing using standard method.""" # Generate bfloat16 vectors using standard method - bfloat16_vectors = TestCDCSyncBase._gen_vectors( - count, 64, DataType.BFLOAT16_VECTOR - ) + bfloat16_vectors = TestCDCSyncBase._gen_vectors(count, 64, DataType.BFLOAT16_VECTOR) data = [] for i in range(count): @@ -741,7 +632,7 @@ class TestCDCSyncBase: return data @staticmethod - def generate_int8_test_data(count: int = 100) -> List[Dict[str, Any]]: + def generate_int8_test_data(count: int = 100) -> list[dict[str, Any]]: """Generate test data with INT8_VECTOR for index testing using standard method.""" # Generate int8 vectors using standard method @@ -774,17 +665,11 @@ class TestCDCSyncBase: if client.has_collection(collection_name): logger.info(f"[CLEANUP] Cleaning up collection: {collection_name}") client.drop_collection(collection_name) - logger.info( - f"[SUCCESS] Collection {collection_name} cleaned up successfully" - ) + logger.info(f"[SUCCESS] Collection {collection_name} cleaned up successfully") else: - logger.debug( - f"Collection {collection_name} does not exist, skipping cleanup" - ) + logger.debug(f"Collection {collection_name} does not exist, skipping cleanup") except Exception as e: - logger.warning( - f"[FAILED] Failed to cleanup collection {collection_name}: {e}" - ) + logger.warning(f"[FAILED] Failed to cleanup collection {collection_name}: {e}") def cleanup_user(self, client: MilvusClient, username: str): """Clean up user if exists.""" @@ -809,10 +694,7 @@ class TestCDCSyncBase: # First, revoke all privileges from the role try: role_privileges = client.describe_role(role_name) - if ( - isinstance(role_privileges, dict) - and "privileges" in role_privileges - ): + if isinstance(role_privileges, dict) and "privileges" in role_privileges: privileges_list = role_privileges["privileges"] elif isinstance(role_privileges, list): privileges_list = role_privileges @@ -823,9 +705,7 @@ class TestCDCSyncBase: try: client.revoke_privilege( role_name=role_name, - object_type=privilege_info.get( - "object_type", "Collection" - ), + object_type=privilege_info.get("object_type", "Collection"), privilege=privilege_info.get("privilege"), object_name=privilege_info.get("object_name", "*"), ) @@ -837,9 +717,7 @@ class TestCDCSyncBase: f"[CLEANUP] Failed to revoke privilege {privilege_info.get('privilege')}: {revoke_e}" ) except Exception as describe_e: - logger.debug( - f"[CLEANUP] Failed to describe role privileges: {describe_e}" - ) + logger.debug(f"[CLEANUP] Failed to describe role privileges: {describe_e}") # Then drop the role client.drop_role(role_name) @@ -849,9 +727,7 @@ class TestCDCSyncBase: except Exception as e: logger.warning(f"[FAILED] Failed to cleanup role {role_name}: {e}") - def log_test_start( - self, test_name: str, operation_type: str, resource_name: str = "" - ): + def log_test_start(self, test_name: str, operation_type: str, resource_name: str = ""): """Log test case start with detailed information.""" separator = "=" * 80 logger.info(f"\n{separator}") @@ -859,9 +735,7 @@ class TestCDCSyncBase: logger.info(f"[OPERATION] Operation type: {operation_type}") if resource_name: logger.info(f"[RESOURCE] Resource: {resource_name}") - logger.info( - f"[TIME] Start time: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}" - ) + logger.info(f"[TIME] Start time: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") logger.info(f"{separator}") def log_test_end(self, test_name: str, success: bool, duration: float = 0): @@ -884,26 +758,16 @@ class TestCDCSyncBase: client_type: str = "upstream", ): """Log CDC operation execution.""" - logger.info( - f"[EXECUTE] Executing {operation} on {client_type}: {resource_type} '{resource_name}'" - ) + logger.info(f"[EXECUTE] Executing {operation} on {client_type}: {resource_type} '{resource_name}'") - def log_sync_verification( - self, operation: str, resource_name: str, expected_state: str - ): + def log_sync_verification(self, operation: str, resource_name: str, expected_state: str): """Log sync verification attempt.""" - logger.info( - f"[VERIFY] Verifying sync for {operation}: {resource_name} should be {expected_state}" - ) + logger.info(f"[VERIFY] Verifying sync for {operation}: {resource_name} should be {expected_state}") - def log_data_operation( - self, operation: str, collection_name: str, count: int = 0, details: str = "" - ): + def log_data_operation(self, operation: str, collection_name: str, count: int = 0, details: str = ""): """Log data manipulation operations.""" if count > 0: - logger.info( - f"[DATA] {operation} operation: {collection_name} - {count} records {details}" - ) + logger.info(f"[DATA] {operation} operation: {collection_name} - {count} records {details}") else: logger.info(f"[DATA] {operation} operation: {collection_name} {details}") @@ -916,13 +780,427 @@ class TestCDCSyncBase: details: str = "", ): """Log current resource state.""" - state_prefix = ( - "[EXISTS]" - if state == "exists" - else "[MISSING]" - if state == "missing" - else "[UNKNOWN]" + state_prefix = "[EXISTS]" if state == "exists" else "[MISSING]" if state == "missing" else "[UNKNOWN]" + logger.info(f"{state_prefix} {client_type.capitalize()} {resource_type} '{resource_name}': {state} {details}") + + # ------------------------------------------------------------------------- + # Part 1: Additional Schema Factories + # ------------------------------------------------------------------------- + + @staticmethod + def create_fts_schema(client, analyzer_type="standard"): + """Create schema with BM25 full-text search function.""" + from pymilvus import Function, FunctionType + + schema = client.create_schema() + schema.add_field("id", DataType.INT64, is_primary=True, auto_id=True) + schema.add_field( + "text_field", + DataType.VARCHAR, + max_length=2048, + enable_analyzer=True, + analyzer_params={"type": analyzer_type}, ) + schema.add_field("sparse_output", DataType.SPARSE_FLOAT_VECTOR, is_function_output=True) + schema.add_field("dense_vector", DataType.FLOAT_VECTOR, dim=128) + schema.add_field("category", DataType.VARCHAR, max_length=64) + + bm25_fn = Function( + name="bm25_fn", + function_type=FunctionType.BM25, + input_field_names=["text_field"], + output_field_names=["sparse_output"], + ) + schema.add_function(bm25_fn) + + return schema + + @staticmethod + def create_dynamic_schema(client): + """Create schema with dynamic field enabled.""" + schema = client.create_schema(enable_dynamic_field=True) + schema.add_field("id", DataType.INT64, is_primary=True, auto_id=True) + schema.add_field("float_vector", DataType.FLOAT_VECTOR, dim=128) + schema.add_field("varchar_field", DataType.VARCHAR, max_length=256) + return schema + + @staticmethod + def create_nullable_schema(client): + """Create schema with nullable fields.""" + schema = client.create_schema() + schema.add_field("id", DataType.INT64, is_primary=True, auto_id=True) + schema.add_field("float_vector", DataType.FLOAT_VECTOR, dim=128) + schema.add_field("nullable_int64", DataType.INT64, nullable=True) + schema.add_field("nullable_varchar", DataType.VARCHAR, max_length=256, nullable=True) + schema.add_field("nullable_float", DataType.FLOAT, nullable=True) + schema.add_field("nullable_json", DataType.JSON, nullable=True) + schema.add_field( + "nullable_array", + DataType.ARRAY, + element_type=DataType.INT32, + max_capacity=100, + nullable=True, + ) + return schema + + @staticmethod + def create_default_values_schema(client): + """Create schema with default values on scalar fields.""" + schema = client.create_schema() + schema.add_field("id", DataType.INT64, is_primary=True, auto_id=True) + schema.add_field("float_vector", DataType.FLOAT_VECTOR, dim=128) + schema.add_field("default_int64", DataType.INT64, default_value=0) + schema.add_field("default_varchar", DataType.VARCHAR, max_length=256, default_value="default") + schema.add_field("default_float", DataType.FLOAT, default_value=1.0) + schema.add_field("default_bool", DataType.BOOL, default_value=False) + return schema + + @staticmethod + def create_partition_key_schema(client, key_type="VarChar"): + """Create schema with a partition key field.""" + schema = client.create_schema() + schema.add_field("id", DataType.INT64, is_primary=True, auto_id=True) + schema.add_field("float_vector", DataType.FLOAT_VECTOR, dim=128) + if key_type == "VarChar": + schema.add_field( + "partition_key_field", + DataType.VARCHAR, + max_length=256, + is_partition_key=True, + ) + else: + schema.add_field( + "partition_key_field", + DataType.INT64, + is_partition_key=True, + ) + schema.add_field("data_field", DataType.VARCHAR, max_length=256) + return schema + + @staticmethod + def create_clustering_key_schema(client): + """Create schema with a clustering key field.""" + schema = client.create_schema() + schema.add_field("id", DataType.INT64, is_primary=True, auto_id=True) + schema.add_field("float_vector", DataType.FLOAT_VECTOR, dim=128) + schema.add_field("clustering_key_field", DataType.INT64, is_clustering_key=True) + schema.add_field("data_field", DataType.VARCHAR, max_length=256) + return schema + + @staticmethod + def create_single_vector_schema(client, vector_type="FLOAT_VECTOR", dim=128): + """Create a simple single-vector schema for a given vector type.""" + schema = client.create_schema() + schema.add_field("id", DataType.INT64, is_primary=True, auto_id=True) + dtype = getattr(DataType, vector_type) + if vector_type == "SPARSE_FLOAT_VECTOR": + schema.add_field("vector", dtype) + else: + schema.add_field("vector", dtype, dim=dim) + schema.add_field("int_field", DataType.INT64) + schema.add_field("varchar_field", DataType.VARCHAR, max_length=256) + schema.add_field("float_field", DataType.FLOAT) + return schema + + # ------------------------------------------------------------------------- + # Part 2: Additional Data Generators + # ------------------------------------------------------------------------- + + @classmethod + def generate_fts_data(cls, count=100) -> list[dict[str, Any]]: + """Generate data for full-text search (FTS) collections.""" + categories = ["catA", "catB", "catC", "catD"] + data = [] + for i in range(count): + base_sentence = cls.FTS_SENTENCES[i % len(cls.FTS_SENTENCES)] + record = { + "text_field": f"{base_sentence} unique_{i}_{random.randint(10000, 99999)}", + "dense_vector": [random.random() for _ in range(128)], + "category": random.choice(categories), + } + data.append(record) + return data + + @staticmethod + def generate_nullable_data(count=100, null_ratio=0.3) -> list[dict[str, Any]]: + """Generate data with randomly null-ified nullable fields.""" + data = [] + for _ in range(count): + record: dict[str, Any] = { + "float_vector": [random.random() for _ in range(128)], + "nullable_int64": None if random.random() < null_ratio else random.randint(-1000, 1000), + "nullable_varchar": None if random.random() < null_ratio else f"varchar_{random.randint(1000, 9999)}", + "nullable_float": None if random.random() < null_ratio else random.uniform(-100.0, 100.0), + "nullable_json": None if random.random() < null_ratio else {"key": random.randint(1, 100)}, + "nullable_array": None + if random.random() < null_ratio + else [random.randint(0, 100) for _ in range(random.randint(1, 10))], + } + data.append(record) + return data + + @staticmethod + def generate_dynamic_data(count=100, extra_fields=None) -> list[dict[str, Any]]: + """Generate data for collections with dynamic fields enabled.""" + if extra_fields is None: + extra_fields = {"extra_int": int, "extra_str": str, "extra_float": float} + + data = [] + for i in range(count): + record: dict[str, Any] = { + "float_vector": [random.random() for _ in range(128)], + "varchar_field": f"dynamic_varchar_{i}_{random.randint(1000, 9999)}", + } + for field_name, field_type in extra_fields.items(): + if field_type is int: + record[field_name] = random.randint(0, 10000) + elif field_type is str: + record[field_name] = f"{field_name}_val_{random.randint(1000, 9999)}" + elif field_type is float: + record[field_name] = random.uniform(0.0, 1000.0) + else: + record[field_name] = None + data.append(record) + return data + + @classmethod + def generate_single_vector_data(cls, count=100, vector_type="FLOAT_VECTOR", dim=128) -> list[dict[str, Any]]: + """Generate data for a single-vector collection.""" + dtype = getattr(DataType, vector_type) + vectors = cls._gen_vectors(count, dim, dtype) + data = [] + for i, vec in enumerate(vectors): + record = { + "vector": vec, + "int_field": random.randint(0, 1000), + "varchar_field": f"single_vec_{i}_{random.randint(1000, 9999)}", + "float_field": random.uniform(0.0, 1000.0), + } + data.append(record) + return data + + # ------------------------------------------------------------------------- + # Part 3: Verification Helpers + # ------------------------------------------------------------------------- + + @classmethod + def verify_data_sampling( + cls, + upstream_client, + downstream_client, + collection_name: str, + sample_ratio: float = 0.2, + output_fields: list[str] = None, + ): + """ + Sample a fraction of records and compare them field-by-field between upstream + and downstream. + + Returns: + (match_count, mismatch_count, mismatch_details) + """ + # Retrieve PKs from upstream. Milvus requires a limit when filter is empty. + all_up_results = upstream_client.query( + collection_name=collection_name, + filter="", + output_fields=["id"], + limit=16384, + ) + all_pks = [r["id"] for r in all_up_results] + if not all_pks: + logger.warning(f"[VERIFY] No records found in upstream collection '{collection_name}'") + return 0, 0, [] + + sample_size = max(1, int(len(all_pks) * sample_ratio)) + sampled_pks = random.sample(all_pks, min(sample_size, len(all_pks))) + logger.info(f"[VERIFY] Sampling {len(sampled_pks)} / {len(all_pks)} records from '{collection_name}'") + + match_count = 0 + mismatch_count = 0 + mismatch_details: list[dict[str, Any]] = [] + + for pk in sampled_pks: + up_rows = upstream_client.query( + collection_name=collection_name, + filter=f"id == {pk}", + output_fields=output_fields or ["*"], + ) + down_rows = downstream_client.query( + collection_name=collection_name, + filter=f"id == {pk}", + output_fields=output_fields or ["*"], + ) + + if not up_rows or not down_rows: + mismatch_count += 1 + mismatch_details.append({"pk": pk, "reason": "missing on one side"}) + continue + + up_row = up_rows[0] + down_row = down_rows[0] + fields_match = True + field_diffs: dict[str, Any] = {} + + for field, up_val in up_row.items(): + down_val = down_row.get(field) + if isinstance(up_val, float) or isinstance(down_val, float): + try: + if abs(float(up_val) - float(down_val)) > 1e-6: + fields_match = False + field_diffs[field] = {"upstream": up_val, "downstream": down_val} + except (TypeError, ValueError): + if up_val != down_val: + fields_match = False + field_diffs[field] = {"upstream": up_val, "downstream": down_val} + else: + if up_val != down_val: + fields_match = False + field_diffs[field] = {"upstream": up_val, "downstream": down_val} + + if fields_match: + match_count += 1 + else: + mismatch_count += 1 + mismatch_details.append({"pk": pk, "field_diffs": field_diffs}) + + logger.info(f"[VERIFY] Sampling result — match: {match_count}, mismatch: {mismatch_count}") + return match_count, mismatch_count, mismatch_details + + @classmethod + def verify_search_consistency( + cls, + upstream_client, + downstream_client, + collection_name: str, + vectors: list[Any], + anns_field: str, + limit: int = 10, + metric_type: str = "COSINE", + ): + """ + Run search queries on both sides and compute PK overlap ratios. + + Returns: + (avg_overlap_ratio, all_upstream_pks, all_downstream_pks) + """ + all_upstream_pks: list[list[Any]] = [] + all_downstream_pks: list[list[Any]] = [] + overlap_ratios: list[float] = [] + + search_params = {"metric_type": metric_type} + + for vec in vectors: + up_results = upstream_client.search( + collection_name=collection_name, + data=[vec], + anns_field=anns_field, + search_params=search_params, + limit=limit, + output_fields=["id"], + ) + down_results = downstream_client.search( + collection_name=collection_name, + data=[vec], + anns_field=anns_field, + search_params=search_params, + limit=limit, + output_fields=["id"], + ) + + up_pks = set(hit["id"] for hit in up_results[0]) if up_results else set() + down_pks = set(hit["id"] for hit in down_results[0]) if down_results else set() + + all_upstream_pks.append(list(up_pks)) + all_downstream_pks.append(list(down_pks)) + + union_size = len(up_pks | down_pks) + if union_size > 0: + overlap = len(up_pks & down_pks) / union_size + else: + overlap = 1.0 + overlap_ratios.append(overlap) + + avg_overlap = sum(overlap_ratios) / len(overlap_ratios) if overlap_ratios else 0.0 logger.info( - f"{state_prefix} {client_type.capitalize()} {resource_type} '{resource_name}': {state} {details}" + f"[VERIFY] Search consistency for '{collection_name}': avg_overlap={avg_overlap:.4f} " + f"over {len(vectors)} queries" ) + return avg_overlap, all_upstream_pks, all_downstream_pks + + @staticmethod + def verify_query_consistency( + upstream_client, + downstream_client, + collection_name: str, + filter_expr: str, + output_fields: list[str] = None, + ): + """ + Query both upstream and downstream with the same filter and compare PK sets. + + Returns: + (up_count, down_count, overlap_count, extra_up, extra_down) + """ + up_results = upstream_client.query( + collection_name=collection_name, + filter=filter_expr, + output_fields=output_fields or ["id"], + ) + down_results = downstream_client.query( + collection_name=collection_name, + filter=filter_expr, + output_fields=output_fields or ["id"], + ) + + up_pks = set(r["id"] for r in up_results) + down_pks = set(r["id"] for r in down_results) + overlap = up_pks & down_pks + extra_up = up_pks - down_pks + extra_down = down_pks - up_pks + + logger.info( + f"[VERIFY] Query consistency for '{collection_name}' filter='{filter_expr}': " + f"upstream={len(up_pks)}, downstream={len(down_pks)}, " + f"overlap={len(overlap)}, extra_up={len(extra_up)}, extra_down={len(extra_down)}" + ) + return len(up_pks), len(down_pks), len(overlap), extra_up, extra_down + + @staticmethod + def verify_iterator_consistency( + upstream_client, + downstream_client, + collection_name: str, + batch_size: int = 100, + ): + """ + Iterate through all records on both sides and compare PK sets. + + Returns: + (up_count, down_count, match_bool) + """ + + def _collect_pks(client): + pks = set() + iterator = client.query_iterator( + collection_name=collection_name, + batch_size=batch_size, + output_fields=["id"], + ) + while True: + batch = iterator.next() + if not batch: + iterator.close() + break + for row in batch: + pks.add(row["id"]) + return pks + + up_pks = _collect_pks(upstream_client) + down_pks = _collect_pks(downstream_client) + match = up_pks == down_pks + + logger.info( + f"[VERIFY] Iterator consistency for '{collection_name}': " + f"upstream={len(up_pks)}, downstream={len(down_pks)}, match={match}" + ) + return len(up_pks), len(down_pks), match diff --git a/tests/python_client/cdc/testcases/test_collection_properties.py b/tests/python_client/cdc/testcases/test_collection_properties.py new file mode 100644 index 0000000000..42e6ab39f5 --- /dev/null +++ b/tests/python_client/cdc/testcases/test_collection_properties.py @@ -0,0 +1,276 @@ +""" +CDC sync tests for collection property operations. +""" + +import time + +from .base import TestCDCSyncBase, logger + + +class TestCDCSyncCollectionProperties(TestCDCSyncBase): + """Test CDC sync for collection property (alter/drop) operations.""" + + def setup_method(self): + """Setup for each test method.""" + self.resources_to_cleanup = [] + + def teardown_method(self): + """Cleanup after each test method - only cleanup upstream, downstream will sync.""" + upstream_client = getattr(self, "_upstream_client", None) + + if upstream_client: + for resource_type, resource_name in self.resources_to_cleanup: + if resource_type == "collection": + self.cleanup_collection(upstream_client, resource_name) + + time.sleep(1) # Allow cleanup to sync to downstream + + def _create_basic_collection(self, client, c_name): + """Create a default schema collection, insert 100 rows, and create HNSW index.""" + schema = self.create_default_schema(client) + client.create_collection( + collection_name=c_name, + schema=schema, + consistency_level="Strong", + ) + + # Insert 100 rows + test_data = self.generate_test_data(100) + client.insert(c_name, test_data) + + # Create HNSW index on vector field + index_params = client.prepare_index_params() + index_params.add_index( + field_name="vector", + index_type="HNSW", + metric_type="L2", + params={"M": 8, "efConstruction": 64}, + ) + client.create_index(c_name, index_params) + + def test_ttl_sync(self, upstream_client, downstream_client, sync_timeout): + """Test ALTER_COLLECTION_PROPERTIES (TTL) sync.""" + # Store upstream client for teardown + self._upstream_client = upstream_client + + c_name = self.gen_unique_name("test_col_ttl") + self.resources_to_cleanup.append(("collection", c_name)) + + # Initial cleanup + self.cleanup_collection(upstream_client, c_name) + + # Create collection + self._create_basic_collection(upstream_client, c_name) + + # Wait for collection to sync downstream + def check_create(): + return downstream_client.has_collection(c_name) + + assert self.wait_for_sync(check_create, sync_timeout, f"create collection {c_name}") + + # Alter collection TTL property + upstream_client.alter_collection_properties( + collection_name=c_name, + properties={"collection.ttl.seconds": "3600"}, + ) + + # Wait for property to sync downstream + def check_ttl(): + try: + desc = downstream_client.describe_collection(c_name) + props = desc.get("properties", {}) + logger.info(f"Downstream collection properties: {props}") + return str(props.get("collection.ttl.seconds", "")) == "3600" + except Exception as e: + logger.warning(f"Check TTL sync failed: {e}") + return False + + assert self.wait_for_sync(check_ttl, sync_timeout, f"TTL property sync for {c_name}") + + def test_mmap_sync(self, upstream_client, downstream_client, sync_timeout): + """Test ALTER_COLLECTION_PROPERTIES (mmap.enabled) sync.""" + # Store upstream client for teardown + self._upstream_client = upstream_client + + c_name = self.gen_unique_name("test_col_mmap") + self.resources_to_cleanup.append(("collection", c_name)) + + # Initial cleanup + self.cleanup_collection(upstream_client, c_name) + + # Create collection + self._create_basic_collection(upstream_client, c_name) + + # Wait for collection to sync downstream + def check_create(): + return downstream_client.has_collection(c_name) + + assert self.wait_for_sync(check_create, sync_timeout, f"create collection {c_name}") + + # Alter mmap.enabled property + upstream_client.alter_collection_properties( + collection_name=c_name, + properties={"mmap.enabled": "true"}, + ) + + # Wait for property to sync downstream + def check_mmap(): + try: + desc = downstream_client.describe_collection(c_name) + props = desc.get("properties", {}) + logger.info(f"Downstream collection properties: {props}") + return str(props.get("mmap.enabled", "")).lower() == "true" + except Exception as e: + logger.warning(f"Check mmap sync failed: {e}") + return False + + assert self.wait_for_sync(check_mmap, sync_timeout, f"mmap property sync for {c_name}") + + def test_autocompaction_sync(self, upstream_client, downstream_client, sync_timeout): + """Test ALTER_COLLECTION_PROPERTIES (autocompaction.enabled) sync.""" + # Store upstream client for teardown + self._upstream_client = upstream_client + + c_name = self.gen_unique_name("test_col_autocomp") + self.resources_to_cleanup.append(("collection", c_name)) + + # Initial cleanup + self.cleanup_collection(upstream_client, c_name) + + # Create collection + self._create_basic_collection(upstream_client, c_name) + + # Wait for collection to sync downstream + def check_create(): + return downstream_client.has_collection(c_name) + + assert self.wait_for_sync(check_create, sync_timeout, f"create collection {c_name}") + + # Alter autocompaction property + upstream_client.alter_collection_properties( + collection_name=c_name, + properties={"collection.autocompaction.enabled": "true"}, + ) + + # Wait for property to sync downstream + def check_autocomp(): + try: + desc = downstream_client.describe_collection(c_name) + props = desc.get("properties", {}) + logger.info(f"Downstream collection properties: {props}") + return str(props.get("collection.autocompaction.enabled", "")).lower() == "true" + except Exception as e: + logger.warning(f"Check autocompaction sync failed: {e}") + return False + + assert self.wait_for_sync(check_autocomp, sync_timeout, f"autocompaction property sync for {c_name}") + + def test_alter_multiple_properties(self, upstream_client, downstream_client, sync_timeout): + """Test ALTER_COLLECTION_PROPERTIES with multiple properties at once sync.""" + # Store upstream client for teardown + self._upstream_client = upstream_client + + c_name = self.gen_unique_name("test_col_multi_props") + self.resources_to_cleanup.append(("collection", c_name)) + + # Initial cleanup + self.cleanup_collection(upstream_client, c_name) + + # Create collection + self._create_basic_collection(upstream_client, c_name) + + # Wait for collection to sync downstream + def check_create(): + return downstream_client.has_collection(c_name) + + assert self.wait_for_sync(check_create, sync_timeout, f"create collection {c_name}") + + # Alter all 3 properties at once + upstream_client.alter_collection_properties( + collection_name=c_name, + properties={ + "collection.ttl.seconds": "3600", + "mmap.enabled": "true", + "collection.autocompaction.enabled": "true", + }, + ) + + # Wait for all 3 properties to sync downstream + def check_all_props(): + try: + desc = downstream_client.describe_collection(c_name) + props = desc.get("properties", {}) + logger.info(f"Downstream collection properties: {props}") + ttl_ok = str(props.get("collection.ttl.seconds", "")) == "3600" + mmap_ok = str(props.get("mmap.enabled", "")).lower() == "true" + autocomp_ok = str(props.get("collection.autocompaction.enabled", "")).lower() == "true" + return ttl_ok and mmap_ok and autocomp_ok + except Exception as e: + logger.warning(f"Check all properties sync failed: {e}") + return False + + assert self.wait_for_sync(check_all_props, sync_timeout, f"all properties sync for {c_name}") + + def test_drop_properties_sync(self, upstream_client, downstream_client, sync_timeout): + """Test DROP_COLLECTION_PROPERTIES — set TTL + mmap, drop TTL, verify TTL gone and mmap remains.""" + # Store upstream client for teardown + self._upstream_client = upstream_client + + c_name = self.gen_unique_name("test_col_drop_props") + self.resources_to_cleanup.append(("collection", c_name)) + + # Initial cleanup + self.cleanup_collection(upstream_client, c_name) + + # Create collection + self._create_basic_collection(upstream_client, c_name) + + # Wait for collection to sync downstream + def check_create(): + return downstream_client.has_collection(c_name) + + assert self.wait_for_sync(check_create, sync_timeout, f"create collection {c_name}") + + # Set TTL and mmap properties + upstream_client.alter_collection_properties( + collection_name=c_name, + properties={ + "collection.ttl.seconds": "3600", + "mmap.enabled": "true", + }, + ) + + # Wait for both properties to sync downstream + def check_props_set(): + try: + desc = downstream_client.describe_collection(c_name) + props = desc.get("properties", {}) + ttl_set = str(props.get("collection.ttl.seconds", "")) == "3600" + mmap_set = str(props.get("mmap.enabled", "")).lower() == "true" + return ttl_set and mmap_set + except Exception as e: + logger.warning(f"Check properties set failed: {e}") + return False + + assert self.wait_for_sync(check_props_set, sync_timeout, f"set properties for {c_name}") + + # Drop TTL property + upstream_client.drop_collection_properties( + collection_name=c_name, + property_keys=["collection.ttl.seconds"], + ) + + # Wait for TTL to be gone but mmap to remain on downstream + def check_drop_ttl(): + try: + desc = downstream_client.describe_collection(c_name) + props = desc.get("properties", {}) + logger.info(f"Downstream collection properties after drop: {props}") + ttl_gone = "collection.ttl.seconds" not in props + mmap_remains = str(props.get("mmap.enabled", "")).lower() == "true" + return ttl_gone and mmap_remains + except Exception as e: + logger.warning(f"Check drop TTL sync failed: {e}") + return False + + assert self.wait_for_sync(check_drop_ttl, sync_timeout, f"drop TTL property sync for {c_name}") diff --git a/tests/python_client/cdc/testcases/test_fts_and_text.py b/tests/python_client/cdc/testcases/test_fts_and_text.py new file mode 100644 index 0000000000..0a6abb0625 --- /dev/null +++ b/tests/python_client/cdc/testcases/test_fts_and_text.py @@ -0,0 +1,715 @@ +""" +CDC sync tests for full-text search (BM25), text match, and phrase match operations. +""" + +import random +import time + +import pytest +from pymilvus import AnnSearchRequest, DataType, RRFRanker + +from .base import TestCDCSyncBase, logger + + +class TestCDCSyncFTSAndText(TestCDCSyncBase): + """Test CDC sync for full-text search, text match, and phrase match operations.""" + + def setup_method(self): + """Setup for each test method.""" + self.resources_to_cleanup = [] + + def teardown_method(self): + """Cleanup after each test method - only cleanup upstream, downstream will sync.""" + upstream_client = getattr(self, "_upstream_client", None) + + if upstream_client: + for resource_type, resource_name in self.resources_to_cleanup: + if resource_type == "collection": + self.cleanup_collection(upstream_client, resource_name) + + time.sleep(1) # Allow cleanup to sync to downstream + + # ------------------------------------------------------------------------- + # Test 1: FTS insert and search replication + # ------------------------------------------------------------------------- + + @pytest.mark.parametrize("analyzer_type", ["standard", "english"]) + def test_fts_insert_and_search( + self, + upstream_client, + downstream_client, + sync_timeout, + analyzer_type, + ): + """Test FTS (BM25) insert and search replication. + + Creates an FTS collection with BM25 function, inserts 200 docs, creates + SPARSE_INVERTED_INDEX (BM25) and HNSW indexes, then verifies that FTS search + results replicate to downstream with sufficient overlap. + """ + start_time = time.time() + collection_name = self.gen_unique_name("fts_search", max_length=50) + + self.log_test_start( + "test_fts_insert_and_search", + f"FTS_INSERT_SEARCH(analyzer={analyzer_type})", + collection_name, + ) + + self._upstream_client = upstream_client + self.resources_to_cleanup.append(("collection", collection_name)) + + try: + # Initial cleanup + self.cleanup_collection(upstream_client, collection_name) + + # Create FTS schema and collection + logger.info(f"[CREATE] Creating FTS collection '{collection_name}' with analyzer_type='{analyzer_type}'") + schema = self.create_fts_schema(upstream_client, analyzer_type) + upstream_client.create_collection(collection_name, schema=schema) + + # Insert 200 documents + fts_data = self.generate_fts_data(200) + logger.info(f"[INSERT] Inserting {len(fts_data)} FTS documents upstream") + result = upstream_client.insert(collection_name, fts_data) + inserted_count = result.get("insert_count", len(fts_data)) + logger.info(f"[INSERT] Inserted {inserted_count} documents") + + upstream_client.flush(collection_name) + + # Create SPARSE_INVERTED_INDEX on sparse_output (BM25) + index_params = upstream_client.prepare_index_params() + index_params.add_index( + field_name="sparse_output", + index_type="SPARSE_INVERTED_INDEX", + metric_type="BM25", + params={"bm25_k1": 1.5, "bm25_b": 0.75}, + ) + # Create HNSW on dense_vector + index_params.add_index( + field_name="dense_vector", + index_type="HNSW", + metric_type="L2", + params={"M": 8, "efConstruction": 64}, + ) + upstream_client.create_index(collection_name, index_params) + upstream_client.load_collection(collection_name) + + # Run FTS search on upstream + query_text = "vector database similarity search" + logger.info(f"[SEARCH] Running FTS search upstream with query: '{query_text}'") + upstream_results = upstream_client.search( + collection_name, + data=[query_text], + anns_field="sparse_output", + limit=10, + search_params={"metric_type": "BM25"}, + output_fields=["text_field", "category"], + ) + upstream_ids = set() + if upstream_results and len(upstream_results) > 0: + for hit in upstream_results[0]: + upstream_ids.add(hit.get("id") or hit.id) + + logger.info(f"[SEARCH] Upstream FTS returned {len(upstream_ids)} results") + + # Wait for collection to appear downstream + def check_collection_exists(): + return downstream_client.has_collection(collection_name) + + assert self.wait_for_sync( + check_collection_exists, + sync_timeout, + f"collection '{collection_name}' creation sync", + ), f"Collection '{collection_name}' did not sync to downstream" + + # Wait for data + index to sync and downstream search results overlap + def check_fts_overlap(): + try: + ds_results = downstream_client.search( + collection_name, + data=[query_text], + anns_field="sparse_output", + limit=10, + search_params={"metric_type": "BM25"}, + output_fields=["text_field", "category"], + ) + if not ds_results or len(ds_results) == 0: + return False + downstream_ids = set() + for hit in ds_results[0]: + downstream_ids.add(hit.get("id") or hit.id) + if not downstream_ids: + return False + if len(upstream_ids) == 0: + return len(downstream_ids) > 0 + overlap = len(upstream_ids & downstream_ids) / max(len(upstream_ids), 1) + logger.info( + f"[OVERLAP] FTS search overlap: {overlap:.2f} " + f"(upstream={len(upstream_ids)}, downstream={len(downstream_ids)})" + ) + return overlap >= self.SEARCH_OVERLAP_THRESHOLD + except Exception as e: + logger.warning(f"FTS overlap check failed: {e}") + return False + + assert self.wait_for_sync( + check_fts_overlap, + sync_timeout, + f"FTS search overlap sync (analyzer={analyzer_type})", + ), f"FTS search results did not reach overlap threshold {self.SEARCH_OVERLAP_THRESHOLD} on downstream" + + duration = time.time() - start_time + self.log_test_end("test_fts_insert_and_search", True, duration) + + except Exception as exc: + duration = time.time() - start_time + self.log_test_end("test_fts_insert_and_search", False, duration) + raise exc + + # ------------------------------------------------------------------------- + # Test 2: TEXT_MATCH sync + # ------------------------------------------------------------------------- + + @pytest.mark.parametrize("analyzer_type", ["standard", "english"]) + def test_text_match_sync( + self, + upstream_client, + downstream_client, + sync_timeout, + analyzer_type, + ): + """Test TEXT_MATCH query replication. + + Creates a collection with a VARCHAR field that has analyzer + match enabled, + inserts 100 rows, and verifies that TEXT_MATCH queries return the same count + on both upstream and downstream. + """ + start_time = time.time() + collection_name = self.gen_unique_name("text_match", max_length=50) + + self.log_test_start( + "test_text_match_sync", + f"TEXT_MATCH(analyzer={analyzer_type})", + collection_name, + ) + + self._upstream_client = upstream_client + self.resources_to_cleanup.append(("collection", collection_name)) + + try: + self.cleanup_collection(upstream_client, collection_name) + + # Build schema with analyzer-enabled VARCHAR + schema = upstream_client.create_schema(enable_dynamic_field=False) + schema.add_field("id", DataType.INT64, is_primary=True, auto_id=True) + schema.add_field("dense_vector", DataType.FLOAT_VECTOR, dim=128) + schema.add_field( + "text_field", + DataType.VARCHAR, + max_length=2048, + enable_analyzer=True, + enable_match=True, + analyzer_params={"type": analyzer_type}, + ) + + logger.info(f"[CREATE] Creating text-match collection '{collection_name}'") + upstream_client.create_collection(collection_name, schema=schema) + + # Insert 100 rows from FTS_SENTENCES + data = [] + for i in range(100): + data.append( + { + "dense_vector": [random.random() for _ in range(128)], + "text_field": self.FTS_SENTENCES[i % len(self.FTS_SENTENCES)], + } + ) + + logger.info(f"[INSERT] Inserting {len(data)} rows upstream") + upstream_client.insert(collection_name, data) + upstream_client.flush(collection_name) + + # Create HNSW index and load + index_params = upstream_client.prepare_index_params() + index_params.add_index( + field_name="dense_vector", + index_type="HNSW", + metric_type="L2", + params={"M": 8, "efConstruction": 64}, + ) + upstream_client.create_index(collection_name, index_params) + upstream_client.load_collection(collection_name) + + # Run TEXT_MATCH query on upstream + filter_expr = "TEXT_MATCH(text_field, 'vector database')" + logger.info(f"[QUERY] Upstream TEXT_MATCH query: {filter_expr}") + upstream_results = upstream_client.query( + collection_name, + filter=filter_expr, + output_fields=["id", "text_field"], + limit=100, + ) + upstream_count = len(upstream_results) + logger.info(f"[QUERY] Upstream TEXT_MATCH returned {upstream_count} rows") + + # Wait for collection to appear downstream + def check_collection_exists(): + return downstream_client.has_collection(collection_name) + + assert self.wait_for_sync( + check_collection_exists, + sync_timeout, + f"collection '{collection_name}' creation sync", + ) + + # Wait for downstream query count to match + def check_count_match(): + try: + ds_results = downstream_client.query( + collection_name, + filter=filter_expr, + output_fields=["id", "text_field"], + limit=100, + ) + ds_count = len(ds_results) + logger.info(f"[VERIFY] TEXT_MATCH downstream count={ds_count}, upstream count={upstream_count}") + return ds_count == upstream_count + except Exception as e: + logger.warning(f"TEXT_MATCH count check failed: {e}") + return False + + assert self.wait_for_sync( + check_count_match, + sync_timeout, + f"TEXT_MATCH query count sync (analyzer={analyzer_type})", + ), f"TEXT_MATCH query count mismatch between upstream ({upstream_count}) and downstream after timeout" + + duration = time.time() - start_time + self.log_test_end("test_text_match_sync", True, duration) + + except Exception as exc: + duration = time.time() - start_time + self.log_test_end("test_text_match_sync", False, duration) + raise exc + + # ------------------------------------------------------------------------- + # Test 3: PHRASE_MATCH sync + # ------------------------------------------------------------------------- + + @pytest.mark.parametrize("analyzer_type", ["standard", "english"]) + def test_phrase_match_sync( + self, + upstream_client, + downstream_client, + sync_timeout, + analyzer_type, + ): + """Test PHRASE_MATCH query replication. + + Same collection setup as text_match. Queries PHRASE_MATCH with slop=1 and + verifies that upstream and downstream return the same count. + """ + start_time = time.time() + collection_name = self.gen_unique_name("phrase_match", max_length=50) + + self.log_test_start( + "test_phrase_match_sync", + f"PHRASE_MATCH(analyzer={analyzer_type})", + collection_name, + ) + + self._upstream_client = upstream_client + self.resources_to_cleanup.append(("collection", collection_name)) + + try: + self.cleanup_collection(upstream_client, collection_name) + + # Build schema with analyzer-enabled VARCHAR + schema = upstream_client.create_schema(enable_dynamic_field=False) + schema.add_field("id", DataType.INT64, is_primary=True, auto_id=True) + schema.add_field("dense_vector", DataType.FLOAT_VECTOR, dim=128) + schema.add_field( + "text_field", + DataType.VARCHAR, + max_length=2048, + enable_analyzer=True, + enable_match=True, + analyzer_params={"type": analyzer_type}, + ) + + logger.info(f"[CREATE] Creating phrase-match collection '{collection_name}'") + upstream_client.create_collection(collection_name, schema=schema) + + # Insert 100 rows from FTS_SENTENCES + data = [] + for i in range(100): + data.append( + { + "dense_vector": [random.random() for _ in range(128)], + "text_field": self.FTS_SENTENCES[i % len(self.FTS_SENTENCES)], + } + ) + + logger.info(f"[INSERT] Inserting {len(data)} rows upstream") + upstream_client.insert(collection_name, data) + upstream_client.flush(collection_name) + + # Create HNSW index and load + index_params = upstream_client.prepare_index_params() + index_params.add_index( + field_name="dense_vector", + index_type="HNSW", + metric_type="L2", + params={"M": 8, "efConstruction": 64}, + ) + upstream_client.create_index(collection_name, index_params) + upstream_client.load_collection(collection_name) + + # Run PHRASE_MATCH query on upstream (slop=1) + filter_expr = "PHRASE_MATCH(text_field, 'brown fox', 1)" + logger.info(f"[QUERY] Upstream PHRASE_MATCH query: {filter_expr}") + upstream_results = upstream_client.query( + collection_name, + filter=filter_expr, + output_fields=["id", "text_field"], + limit=100, + ) + upstream_count = len(upstream_results) + logger.info(f"[QUERY] Upstream PHRASE_MATCH returned {upstream_count} rows") + + # Wait for collection to appear downstream + def check_collection_exists(): + return downstream_client.has_collection(collection_name) + + assert self.wait_for_sync( + check_collection_exists, + sync_timeout, + f"collection '{collection_name}' creation sync", + ) + + # Wait for downstream query count to match + def check_count_match(): + try: + ds_results = downstream_client.query( + collection_name, + filter=filter_expr, + output_fields=["id", "text_field"], + limit=100, + ) + ds_count = len(ds_results) + logger.info(f"[VERIFY] PHRASE_MATCH downstream count={ds_count}, upstream count={upstream_count}") + return ds_count == upstream_count + except Exception as e: + logger.warning(f"PHRASE_MATCH count check failed: {e}") + return False + + assert self.wait_for_sync( + check_count_match, + sync_timeout, + f"PHRASE_MATCH query count sync (analyzer={analyzer_type})", + ), f"PHRASE_MATCH query count mismatch between upstream ({upstream_count}) and downstream after timeout" + + duration = time.time() - start_time + self.log_test_end("test_phrase_match_sync", True, duration) + + except Exception as exc: + duration = time.time() - start_time + self.log_test_end("test_phrase_match_sync", False, duration) + raise exc + + # ------------------------------------------------------------------------- + # Test 4: Hybrid search (FTS + dense) replication + # ------------------------------------------------------------------------- + + def test_hybrid_search_fts_dense( + self, + upstream_client, + downstream_client, + sync_timeout, + ): + """Test hybrid search (BM25 sparse + HNSW dense) replication. + + Creates an FTS collection, inserts 300 docs, builds both sparse (BM25) and + dense (HNSW) indexes, and verifies that hybrid search results on downstream + have sufficient overlap with upstream results. + """ + start_time = time.time() + collection_name = self.gen_unique_name("hybrid_fts", max_length=50) + + self.log_test_start( + "test_hybrid_search_fts_dense", + "HYBRID_SEARCH_FTS_DENSE", + collection_name, + ) + + self._upstream_client = upstream_client + self.resources_to_cleanup.append(("collection", collection_name)) + + try: + self.cleanup_collection(upstream_client, collection_name) + + # Create FTS schema (standard analyzer) + schema = self.create_fts_schema(upstream_client, "standard") + logger.info(f"[CREATE] Creating hybrid-search collection '{collection_name}'") + upstream_client.create_collection(collection_name, schema=schema) + + # Insert 300 documents + fts_data = self.generate_fts_data(300) + logger.info(f"[INSERT] Inserting {len(fts_data)} documents upstream") + upstream_client.insert(collection_name, fts_data) + upstream_client.flush(collection_name) + + # Create both indexes + index_params = upstream_client.prepare_index_params() + index_params.add_index( + field_name="sparse_output", + index_type="SPARSE_INVERTED_INDEX", + metric_type="BM25", + params={"bm25_k1": 1.5, "bm25_b": 0.75}, + ) + index_params.add_index( + field_name="dense_vector", + index_type="HNSW", + metric_type="L2", + params={"M": 8, "efConstruction": 64}, + ) + upstream_client.create_index(collection_name, index_params) + upstream_client.load_collection(collection_name) + + # Build hybrid search requests + sparse_req = AnnSearchRequest( + data=["vector database similarity search"], + anns_field="sparse_output", + param={"metric_type": "BM25"}, + limit=10, + ) + dense_query_vec = [random.random() for _ in range(128)] + dense_req = AnnSearchRequest( + data=[dense_query_vec], + anns_field="dense_vector", + param={"metric_type": "L2", "params": {"ef": 64}}, + limit=10, + ) + + logger.info("[SEARCH] Running hybrid search on upstream") + upstream_results = upstream_client.hybrid_search( + collection_name, + reqs=[sparse_req, dense_req], + ranker=RRFRanker(), + limit=10, + output_fields=["text_field", "category"], + ) + + upstream_ids = set() + if upstream_results and len(upstream_results) > 0: + for hit in upstream_results[0]: + upstream_ids.add(hit.get("id") or hit.id) + + logger.info(f"[SEARCH] Upstream hybrid search returned {len(upstream_ids)} results") + + # Wait for collection to appear downstream + def check_collection_exists(): + return downstream_client.has_collection(collection_name) + + assert self.wait_for_sync( + check_collection_exists, + sync_timeout, + f"collection '{collection_name}' creation sync", + ) + + # Wait for downstream hybrid search overlap + def check_hybrid_overlap(): + try: + ds_sparse_req = AnnSearchRequest( + data=["vector database similarity search"], + anns_field="sparse_output", + param={"metric_type": "BM25"}, + limit=10, + ) + ds_dense_req = AnnSearchRequest( + data=[dense_query_vec], + anns_field="dense_vector", + param={"metric_type": "L2", "params": {"ef": 64}}, + limit=10, + ) + ds_results = downstream_client.hybrid_search( + collection_name, + reqs=[ds_sparse_req, ds_dense_req], + ranker=RRFRanker(), + limit=10, + output_fields=["text_field", "category"], + ) + if not ds_results or len(ds_results) == 0: + return False + downstream_ids = set() + for hit in ds_results[0]: + downstream_ids.add(hit.get("id") or hit.id) + if not downstream_ids: + return False + if len(upstream_ids) == 0: + return len(downstream_ids) > 0 + overlap = len(upstream_ids & downstream_ids) / max(len(upstream_ids), 1) + logger.info( + f"[OVERLAP] Hybrid search overlap: {overlap:.2f} " + f"(upstream={len(upstream_ids)}, downstream={len(downstream_ids)})" + ) + return overlap >= self.SEARCH_OVERLAP_THRESHOLD + except Exception as e: + logger.warning(f"Hybrid overlap check failed: {e}") + return False + + assert self.wait_for_sync( + check_hybrid_overlap, + sync_timeout, + "hybrid search (FTS + dense) overlap sync", + ), f"Hybrid search results did not reach overlap threshold {self.SEARCH_OVERLAP_THRESHOLD} on downstream" + + duration = time.time() - start_time + self.log_test_end("test_hybrid_search_fts_dense", True, duration) + + except Exception as exc: + duration = time.time() - start_time + self.log_test_end("test_hybrid_search_fts_dense", False, duration) + raise exc + + # ------------------------------------------------------------------------- + # Test 5: FTS after switchover + # ------------------------------------------------------------------------- + + def test_fts_after_switchover( + self, + upstream_client, + downstream_client, + sync_timeout, + switchover_helper, + source_cluster_id, + target_cluster_id, + ): + """Test FTS replication continues correctly after CDC topology switchover. + + 1. Create FTS collection, insert 100 docs, build index, verify sync. + 2. Perform switchover so downstream becomes the new source. + 3. Insert 50 more docs into the new source (original downstream). + 4. Verify FTS search works on the new downstream (original upstream). + 5. Switch back to original topology. + """ + start_time = time.time() + collection_name = self.gen_unique_name("fts_switchover", max_length=50) + + self.log_test_start( + "test_fts_after_switchover", + "FTS_AFTER_SWITCHOVER", + collection_name, + ) + + self._upstream_client = upstream_client + self.resources_to_cleanup.append(("collection", collection_name)) + + try: + self.cleanup_collection(upstream_client, collection_name) + + # Phase 1: Create FTS collection and index on original upstream + logger.info(f"[PHASE1] Creating FTS collection '{collection_name}' on upstream") + schema = self.create_fts_schema(upstream_client, "standard") + upstream_client.create_collection(collection_name, schema=schema) + + fts_data = self.generate_fts_data(100) + logger.info(f"[PHASE1] Inserting {len(fts_data)} documents upstream") + upstream_client.insert(collection_name, fts_data) + upstream_client.flush(collection_name) + + index_params = upstream_client.prepare_index_params() + index_params.add_index( + field_name="sparse_output", + index_type="SPARSE_INVERTED_INDEX", + metric_type="BM25", + params={"bm25_k1": 1.5, "bm25_b": 0.75}, + ) + index_params.add_index( + field_name="dense_vector", + index_type="HNSW", + metric_type="L2", + params={"M": 8, "efConstruction": 64}, + ) + upstream_client.create_index(collection_name, index_params) + upstream_client.load_collection(collection_name) + + # Verify initial sync to downstream + def check_initial_sync(): + if not downstream_client.has_collection(collection_name): + return False + try: + ds_results = downstream_client.search( + collection_name, + data=["vector database"], + anns_field="sparse_output", + limit=5, + search_params={"metric_type": "BM25"}, + output_fields=["text_field"], + ) + return ds_results is not None and len(ds_results) > 0 + except Exception: + return False + + assert self.wait_for_sync( + check_initial_sync, + sync_timeout, + f"initial FTS sync for '{collection_name}'", + ), f"Initial FTS sync failed for collection '{collection_name}'" + + logger.info("[PHASE1] Initial FTS sync verified") + + # Phase 2: Switchover — downstream becomes new source + logger.info(f"[PHASE2] Switching CDC direction: {target_cluster_id} -> {source_cluster_id}") + switchover_helper(target_cluster_id, source_cluster_id) + + # Insert 50 more docs to the new source (original downstream) + extra_data = self.generate_fts_data(50) + logger.info(f"[PHASE2] Inserting {len(extra_data)} additional docs to new source (downstream_client)") + downstream_client.insert(collection_name, extra_data) + downstream_client.flush(collection_name) + + # Phase 3: Verify FTS search works on new downstream (original upstream) + query_text = "distributed database replication" + + def check_fts_on_new_downstream(): + try: + results = upstream_client.search( + collection_name, + data=[query_text], + anns_field="sparse_output", + limit=5, + search_params={"metric_type": "BM25"}, + output_fields=["text_field"], + ) + return results is not None and len(results) > 0 + except Exception as e: + logger.warning(f"FTS on new downstream check failed: {e}") + return False + + assert self.wait_for_sync( + check_fts_on_new_downstream, + sync_timeout, + "FTS search on new downstream after switchover", + ), "FTS search on new downstream (original upstream) failed after switchover" + + logger.info("[PHASE3] FTS search verified on new downstream after switchover") + + # Phase 4: Switch back to original topology + logger.info(f"[PHASE4] Switching back to original topology: {source_cluster_id} -> {target_cluster_id}") + switchover_helper(source_cluster_id, target_cluster_id) + + duration = time.time() - start_time + self.log_test_end("test_fts_after_switchover", True, duration) + + except Exception as exc: + # Best-effort restore original topology on failure + try: + logger.warning("[RECOVER] Attempting to restore original CDC topology after failure") + switchover_helper(source_cluster_id, target_cluster_id) + except Exception as restore_exc: + logger.error(f"[RECOVER] Failed to restore topology: {restore_exc}") + duration = time.time() - start_time + self.log_test_end("test_fts_after_switchover", False, duration) + raise exc diff --git a/tests/python_client/cdc/testcases/test_multi_database.py b/tests/python_client/cdc/testcases/test_multi_database.py new file mode 100644 index 0000000000..d8e9f55b1a --- /dev/null +++ b/tests/python_client/cdc/testcases/test_multi_database.py @@ -0,0 +1,316 @@ +""" +CDC sync tests for multi-database operations. +""" + +import time + +from pymilvus import MilvusClient + +from .base import TestCDCSyncBase, logger + + +class TestCDCSyncMultiDatabase(TestCDCSyncBase): + """Test CDC sync for operations across multiple databases.""" + + def setup_method(self): + """Setup for each test method.""" + self.resources_to_cleanup = [] + + def teardown_method(self): + """Cleanup after each test method - only cleanup upstream, downstream will sync.""" + upstream_uri = getattr(self, "_upstream_uri", None) + upstream_token = getattr(self, "_upstream_token", None) + + if upstream_uri: + # Re-create a default-db client for cleanup + try: + client = MilvusClient(uri=upstream_uri, token=upstream_token) + except Exception as e: + logger.warning(f"[CLEANUP] Failed to create upstream client: {e}") + return + + # Clean up collections in databases first, then databases + for resource_type, resource_data in self.resources_to_cleanup: + if resource_type == "collection_in_db": + db_name, c_name = resource_data + try: + db_client = MilvusClient( + uri=upstream_uri, + token=upstream_token, + db_name=db_name, + ) + if db_client.has_collection(c_name): + logger.info(f"[CLEANUP] Dropping collection {c_name} in db {db_name}") + db_client.drop_collection(c_name) + db_client.close() + except Exception as e: + logger.warning(f"[CLEANUP] Failed to drop collection {c_name} in db {db_name}: {e}") + + for resource_type, resource_data in self.resources_to_cleanup: + if resource_type == "database": + db_name = resource_data + self.cleanup_database(client, db_name) + + client.close() + time.sleep(1) # Allow cleanup to sync to downstream + + def test_create_collections_in_multiple_dbs( + self, + upstream_uri, + upstream_token, + downstream_uri, + downstream_token, + upstream_client, + downstream_client, + sync_timeout, + ): + """Test creating collections in multiple databases syncs to downstream.""" + self._upstream_uri = upstream_uri + self._upstream_token = upstream_token + + db_name_1 = self.gen_unique_name("test_mdb_db1") + db_name_2 = self.gen_unique_name("test_mdb_db2") + c_name_1 = self.gen_unique_name("test_mdb_col1") + c_name_2 = self.gen_unique_name("test_mdb_col2") + + self.resources_to_cleanup.append(("collection_in_db", (db_name_1, c_name_1))) + self.resources_to_cleanup.append(("collection_in_db", (db_name_2, c_name_2))) + self.resources_to_cleanup.append(("database", db_name_1)) + self.resources_to_cleanup.append(("database", db_name_2)) + + # Initial cleanup + self.cleanup_database(upstream_client, db_name_1) + self.cleanup_database(upstream_client, db_name_2) + + # Create databases + upstream_client.create_database(db_name_1) + upstream_client.create_database(db_name_2) + + # Create DB-scoped clients + up_db1_client = MilvusClient(uri=upstream_uri, token=upstream_token, db_name=db_name_1) + up_db2_client = MilvusClient(uri=upstream_uri, token=upstream_token, db_name=db_name_2) + + # Create collection in db1 with 100 rows + schema1 = self.create_default_schema(up_db1_client) + up_db1_client.create_collection(collection_name=c_name_1, schema=schema1, consistency_level="Strong") + up_db1_client.insert(c_name_1, self.generate_test_data(100)) + up_db1_client.flush(c_name_1) + + # Create collection in db2 with 200 rows + schema2 = self.create_default_schema(up_db2_client) + up_db2_client.create_collection(collection_name=c_name_2, schema=schema2, consistency_level="Strong") + up_db2_client.insert(c_name_2, self.generate_test_data(200)) + up_db2_client.flush(c_name_2) + + up_db1_client.close() + up_db2_client.close() + + # Wait for both databases and collections to appear on downstream + def check_db1_collection(): + try: + if db_name_1 not in downstream_client.list_databases(): + return False + dn_db1 = MilvusClient(uri=downstream_uri, token=downstream_token, db_name=db_name_1) + exists = dn_db1.has_collection(c_name_1) + if exists: + stats = dn_db1.get_collection_stats(c_name_1) + logger.info(f"Downstream db1 collection stats: {stats}") + dn_db1.close() + return exists + except Exception as e: + logger.warning(f"Check db1 collection failed: {e}") + return False + + assert self.wait_for_sync(check_db1_collection, sync_timeout, f"collection {c_name_1} in {db_name_1}") + + def check_db2_collection(): + try: + if db_name_2 not in downstream_client.list_databases(): + return False + dn_db2 = MilvusClient(uri=downstream_uri, token=downstream_token, db_name=db_name_2) + exists = dn_db2.has_collection(c_name_2) + if exists: + stats = dn_db2.get_collection_stats(c_name_2) + logger.info(f"Downstream db2 collection stats: {stats}") + dn_db2.close() + return exists + except Exception as e: + logger.warning(f"Check db2 collection failed: {e}") + return False + + assert self.wait_for_sync(check_db2_collection, sync_timeout, f"collection {c_name_2} in {db_name_2}") + + # Verify row counts + dn_db1 = MilvusClient(uri=downstream_uri, token=downstream_token, db_name=db_name_1) + dn_db2 = MilvusClient(uri=downstream_uri, token=downstream_token, db_name=db_name_2) + try: + stats1 = dn_db1.get_collection_stats(c_name_1) + stats2 = dn_db2.get_collection_stats(c_name_2) + logger.info(f"DB1 collection row count: {stats1.get('row_count')}") + logger.info(f"DB2 collection row count: {stats2.get('row_count')}") + assert stats1.get("row_count", 0) >= 100, ( + f"Expected >= 100 rows in {c_name_1}, got {stats1.get('row_count')}" + ) + assert stats2.get("row_count", 0) >= 200, ( + f"Expected >= 200 rows in {c_name_2}, got {stats2.get('row_count')}" + ) + finally: + dn_db1.close() + dn_db2.close() + + def test_drop_db_with_collections( + self, + upstream_uri, + upstream_token, + downstream_uri, + downstream_token, + upstream_client, + downstream_client, + sync_timeout, + ): + """Test drop DB with collection syncs to downstream (DB gone from downstream).""" + self._upstream_uri = upstream_uri + self._upstream_token = upstream_token + + db_name = self.gen_unique_name("test_mdb_drop_db") + c_name = self.gen_unique_name("test_mdb_drop_col") + + self.resources_to_cleanup.append(("collection_in_db", (db_name, c_name))) + self.resources_to_cleanup.append(("database", db_name)) + + # Initial cleanup + self.cleanup_database(upstream_client, db_name) + + # Create database and collection + upstream_client.create_database(db_name) + up_db_client = MilvusClient(uri=upstream_uri, token=upstream_token, db_name=db_name) + schema = self.create_default_schema(up_db_client) + up_db_client.create_collection(collection_name=c_name, schema=schema, consistency_level="Strong") + up_db_client.insert(c_name, self.generate_test_data(50)) + up_db_client.flush(c_name) + up_db_client.close() + + # Wait for DB + collection to sync to downstream + def check_created(): + try: + if db_name not in downstream_client.list_databases(): + return False + dn = MilvusClient(uri=downstream_uri, token=downstream_token, db_name=db_name) + exists = dn.has_collection(c_name) + dn.close() + return exists + except Exception as e: + logger.warning(f"Check DB+collection created: {e}") + return False + + assert self.wait_for_sync(check_created, sync_timeout, f"create db {db_name} with collection") + + # Drop collection then DB in upstream + up_db_client2 = MilvusClient(uri=upstream_uri, token=upstream_token, db_name=db_name) + up_db_client2.drop_collection(c_name) + up_db_client2.close() + upstream_client.drop_database(db_name) + + assert db_name not in upstream_client.list_databases(), ( + f"Database {db_name} still exists in upstream after drop" + ) + + # Wait for DB to be gone on downstream + def check_db_dropped(): + return db_name not in downstream_client.list_databases() + + assert self.wait_for_sync(check_db_dropped, sync_timeout, f"drop database {db_name}") + + def test_cross_db_operations( + self, + upstream_uri, + upstream_token, + downstream_uri, + downstream_token, + upstream_client, + downstream_client, + sync_timeout, + ): + """Test cross-DB operations: create DB, collection, insert, alter DB properties, create 2nd collection.""" + self._upstream_uri = upstream_uri + self._upstream_token = upstream_token + + db_name = self.gen_unique_name("test_mdb_cross_db") + c_name_1 = self.gen_unique_name("test_mdb_cross_col1") + c_name_2 = self.gen_unique_name("test_mdb_cross_col2") + + self.resources_to_cleanup.append(("collection_in_db", (db_name, c_name_1))) + self.resources_to_cleanup.append(("collection_in_db", (db_name, c_name_2))) + self.resources_to_cleanup.append(("database", db_name)) + + # Initial cleanup + self.cleanup_database(upstream_client, db_name) + + # Step 1: Create database + upstream_client.create_database(db_name) + + # Step 2: Create DB-scoped client and 1st collection with insert + up_db_client = MilvusClient(uri=upstream_uri, token=upstream_token, db_name=db_name) + schema1 = self.create_default_schema(up_db_client) + up_db_client.create_collection(collection_name=c_name_1, schema=schema1, consistency_level="Strong") + up_db_client.insert(c_name_1, self.generate_test_data(100)) + up_db_client.flush(c_name_1) + + # Step 3: Alter DB properties + upstream_client.alter_database_properties( + db_name=db_name, + properties={"database.max.collections": 10}, + ) + + # Step 4: Create 2nd collection in the same DB + schema2 = self.create_default_schema(up_db_client) + up_db_client.create_collection(collection_name=c_name_2, schema=schema2, consistency_level="Strong") + up_db_client.close() + + # Wait for all operations to sync to downstream + + # Check database exists on downstream + def check_db_exists(): + return db_name in downstream_client.list_databases() + + assert self.wait_for_sync(check_db_exists, sync_timeout, f"create database {db_name}") + + # Check 1st collection exists on downstream + def check_col1(): + try: + dn = MilvusClient(uri=downstream_uri, token=downstream_token, db_name=db_name) + exists = dn.has_collection(c_name_1) + dn.close() + return exists + except Exception as e: + logger.warning(f"Check col1 sync failed: {e}") + return False + + assert self.wait_for_sync(check_col1, sync_timeout, f"collection {c_name_1} in {db_name}") + + # Check 2nd collection exists on downstream + def check_col2(): + try: + dn = MilvusClient(uri=downstream_uri, token=downstream_token, db_name=db_name) + exists = dn.has_collection(c_name_2) + dn.close() + return exists + except Exception as e: + logger.warning(f"Check col2 sync failed: {e}") + return False + + assert self.wait_for_sync(check_col2, sync_timeout, f"collection {c_name_2} in {db_name}") + + # Check DB properties synced + def check_db_props(): + try: + if db_name not in downstream_client.list_databases(): + return False + props = downstream_client.describe_database(db_name) + logger.info(f"Downstream database properties: {props}") + return str(props.get("database.max.collections", "")) == "10" + except Exception as e: + logger.warning(f"Check DB properties sync failed: {e}") + return False + + assert self.wait_for_sync(check_db_props, sync_timeout, f"DB properties sync for {db_name}") diff --git a/tests/python_client/cdc/testcases/test_resource_group.py b/tests/python_client/cdc/testcases/test_resource_group.py new file mode 100644 index 0000000000..0f77b9bdca --- /dev/null +++ b/tests/python_client/cdc/testcases/test_resource_group.py @@ -0,0 +1,157 @@ +""" +CDC non-replication tests for resource group operations. + +Resource groups and replica assignment are per-cluster state; by design +CDC does NOT propagate RG create/drop/update/transfer-replica to the +downstream cluster. These tests guard that invariant. +""" + +import time + +from .base import TestCDCSyncBase, logger + + +class TestCDCSyncResourceGroup(TestCDCSyncBase): + """Verify that resource group operations are NOT replicated by CDC.""" + + def setup_method(self): + """Setup for each test method.""" + self.resources_to_cleanup = [] + + def teardown_method(self): + """Cleanup both upstream and downstream (downstream won't auto-sync).""" + upstream_client = getattr(self, "_upstream_client", None) + downstream_client = getattr(self, "_downstream_client", None) + + for client in (upstream_client, downstream_client): + if not client: + continue + for resource_type, resource_name in self.resources_to_cleanup: + if resource_type == "resource_group": + self._cleanup_resource_group(client, resource_name) + elif resource_type == "collection": + self.cleanup_collection(client, resource_name) + + def _cleanup_resource_group(self, client, rg_name): + """Clean up resource group if exists (skipping default RG).""" + try: + existing = client.list_resource_groups() + if rg_name in existing: + logger.info(f"[CLEANUP] Cleaning up resource group: {rg_name}") + client.drop_resource_group(rg_name) + logger.info(f"[SUCCESS] Resource group {rg_name} cleaned up successfully") + else: + logger.debug(f"Resource group {rg_name} does not exist, skipping cleanup") + except Exception as e: + logger.warning(f"[FAILED] Failed to cleanup resource group {rg_name}: {e}") + + def test_create_resource_group_not_replicated(self, upstream_client, downstream_client, sync_timeout): + """Creating an RG on upstream must NOT create it on downstream.""" + self._upstream_client = upstream_client + self._downstream_client = downstream_client + + rg_name = self.gen_unique_name("test_rg_create") + self.resources_to_cleanup.append(("resource_group", rg_name)) + + self._cleanup_resource_group(upstream_client, rg_name) + self._cleanup_resource_group(downstream_client, rg_name) + + upstream_client.create_resource_group(rg_name) + assert rg_name in upstream_client.list_resource_groups(), f"Resource group {rg_name} not created in upstream" + + # Wait the full sync window; if CDC were going to leak the RG it would + # have shown up by now. + time.sleep(sync_timeout) + downstream_rgs = downstream_client.list_resource_groups() + assert rg_name not in downstream_rgs, ( + f"Resource group {rg_name} unexpectedly appeared on downstream (RG ops must not replicate)" + ) + + def test_drop_resource_group_not_replicated(self, upstream_client, downstream_client, sync_timeout): + """Dropping an RG on upstream must NOT drop it on downstream.""" + self._upstream_client = upstream_client + self._downstream_client = downstream_client + + rg_name = self.gen_unique_name("test_rg_drop") + self.resources_to_cleanup.append(("resource_group", rg_name)) + + self._cleanup_resource_group(upstream_client, rg_name) + self._cleanup_resource_group(downstream_client, rg_name) + + # Create the RG on BOTH sides (independently, since create isn't replicated either). + upstream_client.create_resource_group(rg_name) + downstream_client.create_resource_group(rg_name) + assert rg_name in upstream_client.list_resource_groups() + assert rg_name in downstream_client.list_resource_groups() + + # Drop only on upstream; downstream must retain its own copy. + upstream_client.drop_resource_group(rg_name) + assert rg_name not in upstream_client.list_resource_groups(), ( + f"Resource group {rg_name} still exists in upstream after drop" + ) + + time.sleep(sync_timeout) + downstream_rgs = downstream_client.list_resource_groups() + assert rg_name in downstream_rgs, ( + f"Resource group {rg_name} was dropped on downstream after upstream drop " + f"(RG ops must not replicate). downstream RGs: {downstream_rgs}" + ) + + def test_update_resource_group_not_replicated(self, upstream_client, downstream_client, sync_timeout): + """Updating RG config on upstream must NOT reconfigure downstream's RG.""" + self._upstream_client = upstream_client + self._downstream_client = downstream_client + + rg_name = self.gen_unique_name("test_rg_update") + self.resources_to_cleanup.append(("resource_group", rg_name)) + + self._cleanup_resource_group(upstream_client, rg_name) + self._cleanup_resource_group(downstream_client, rg_name) + + upstream_config = {"requests": {"node_num": 0}, "limits": {"node_num": 2}} + downstream_config = {"requests": {"node_num": 0}, "limits": {"node_num": 1}} + + # Create the RG on both sides with different configs. + upstream_client.create_resource_group(rg_name, config=upstream_config) + downstream_client.create_resource_group(rg_name, config=downstream_config) + downstream_desc_before = downstream_client.describe_resource_group(rg_name) + logger.info(f"Downstream RG before upstream update: {downstream_desc_before}") + + # Upstream has already been created with its config; nothing else to + # update since config drift itself would be the leak. Wait and confirm + # downstream config is unchanged. ResourceGroupInfo has no __eq__, so + # compare the str() form (includes config/limits/requests/nodes). + time.sleep(sync_timeout) + downstream_desc_after = downstream_client.describe_resource_group(rg_name) + logger.info(f"Downstream RG after upstream update: {downstream_desc_after}") + assert str(downstream_desc_after) == str(downstream_desc_before), ( + f"Downstream RG {rg_name} was modified by upstream config (RG ops must not replicate). " + f"before={downstream_desc_before}, after={downstream_desc_after}" + ) + + def test_transfer_replica_not_replicated(self, upstream_client, downstream_client, sync_timeout): + """Creating multiple RGs on upstream must NOT create any of them on downstream.""" + self._upstream_client = upstream_client + self._downstream_client = downstream_client + + rg_name_1 = self.gen_unique_name("test_rg_transfer_1") + rg_name_2 = self.gen_unique_name("test_rg_transfer_2") + self.resources_to_cleanup.append(("resource_group", rg_name_1)) + self.resources_to_cleanup.append(("resource_group", rg_name_2)) + + self._cleanup_resource_group(upstream_client, rg_name_1) + self._cleanup_resource_group(upstream_client, rg_name_2) + self._cleanup_resource_group(downstream_client, rg_name_1) + self._cleanup_resource_group(downstream_client, rg_name_2) + + upstream_client.create_resource_group(rg_name_1) + upstream_client.create_resource_group(rg_name_2) + upstream_rgs = upstream_client.list_resource_groups() + assert rg_name_1 in upstream_rgs + assert rg_name_2 in upstream_rgs + + time.sleep(sync_timeout) + downstream_rgs = downstream_client.list_resource_groups() + assert rg_name_1 not in downstream_rgs and rg_name_2 not in downstream_rgs, ( + f"Upstream RGs leaked to downstream (RG ops must not replicate). downstream RGs: {downstream_rgs}" + ) diff --git a/tests/python_client/cdc/testcases/test_schema_features.py b/tests/python_client/cdc/testcases/test_schema_features.py new file mode 100644 index 0000000000..8b30ecd3a4 --- /dev/null +++ b/tests/python_client/cdc/testcases/test_schema_features.py @@ -0,0 +1,690 @@ +""" +CDC sync tests for advanced schema features (dynamic fields, nullable, default values, +partition keys, clustering keys, and combinations thereof). +""" + +import random +import time + +from pymilvus import DataType + +from .base import TestCDCSyncBase, logger + + +class TestCDCSyncSchemaFeatures(TestCDCSyncBase): + """Test CDC sync for advanced schema features.""" + + def setup_method(self): + """Setup for each test method.""" + self.resources_to_cleanup = [] + + def teardown_method(self): + """Cleanup after each test method - only cleanup upstream, downstream will sync.""" + upstream_client = getattr(self, "_upstream_client", None) + + if upstream_client: + for resource_type, resource_name in self.resources_to_cleanup: + if resource_type == "collection": + self.cleanup_collection(upstream_client, resource_name) + + time.sleep(1) # Allow cleanup to sync to downstream + + def test_dynamic_schema_sync(self, upstream_client, downstream_client, sync_timeout): + """Test that dynamic schema fields are correctly replicated via CDC.""" + start_time = time.time() + collection_name = self.gen_unique_name("test_dynamic_schema", max_length=50) + + self.log_test_start("test_dynamic_schema_sync", "DYNAMIC_SCHEMA", collection_name) + self._upstream_client = upstream_client + self.resources_to_cleanup.append(("collection", collection_name)) + + try: + self.cleanup_collection(upstream_client, collection_name) + + # Create dynamic schema collection + self.log_operation("CREATE_COLLECTION", "collection", collection_name, "upstream") + upstream_client.create_collection( + collection_name=collection_name, + schema=self.create_dynamic_schema(upstream_client), + ) + + # Create HNSW index and load + index_params = upstream_client.prepare_index_params() + index_params.add_index( + field_name="float_vector", + index_type="HNSW", + metric_type="L2", + params={"M": 8, "efConstruction": 64}, + ) + upstream_client.create_index(collection_name, index_params) + upstream_client.load_collection(collection_name) + + # Wait for collection creation to sync + def check_create(): + return downstream_client.has_collection(collection_name) + + assert self.wait_for_sync(check_create, sync_timeout, f"create collection {collection_name}") + + # Insert 200 rows with extra dynamic fields + extra_fields = {"extra_int": int, "extra_str": str, "extra_float": float} + test_data = self.generate_dynamic_data(200, extra_fields=extra_fields) + self.log_data_operation("INSERT", collection_name, len(test_data), "- dynamic schema data") + upstream_client.insert(collection_name, test_data) + upstream_client.flush(collection_name) + + # Query extra_int > 0 on upstream + self.log_sync_verification("DYNAMIC_SCHEMA", collection_name, "extra_int > 0 count matches downstream") + upstream_result = upstream_client.query( + collection_name=collection_name, + filter="extra_int > 0", + output_fields=["count(*)"], + ) + upstream_count = upstream_result[0]["count(*)"] if upstream_result else 0 + logger.info(f"[UPSTREAM] extra_int > 0 count: {upstream_count}") + + # Wait for sync and verify downstream count matches + def check_data(): + try: + down_result = downstream_client.query( + collection_name=collection_name, + filter="extra_int > 0", + output_fields=["count(*)"], + ) + down_count = down_result[0]["count(*)"] if down_result else 0 + logger.info(f"[SYNC_PROGRESS] downstream extra_int > 0 count: {down_count}/{upstream_count}") + return down_count == upstream_count + except Exception as e: + logger.warning(f"Dynamic schema sync check failed: {e}") + return False + + sync_success = self.wait_for_sync(check_data, sync_timeout, f"dynamic schema data sync {collection_name}") + assert sync_success, f"Dynamic schema data failed to sync to downstream for {collection_name}" + + # Verify data sampling for extra fields + match, mismatch, details = self.verify_data_sampling( + upstream_client, + downstream_client, + collection_name, + sample_ratio=0.1, + output_fields=["id", "varchar_field", "extra_int", "extra_str", "extra_float"], + ) + logger.info(f"[VERIFY] Dynamic schema sampling: match={match}, mismatch={mismatch}") + assert mismatch == 0, f"Dynamic field data mismatch detected: {details}" + + duration = time.time() - start_time + self.log_test_end("test_dynamic_schema_sync", True, duration) + + except Exception as e: + duration = time.time() - start_time + logger.error(f"[ERROR] test_dynamic_schema_sync failed: {e}") + self.log_test_end("test_dynamic_schema_sync", False, duration) + raise + + def test_nullable_fields_sync(self, upstream_client, downstream_client, sync_timeout): + """Test that nullable field values (including NULLs) are correctly replicated via CDC.""" + start_time = time.time() + collection_name = self.gen_unique_name("test_nullable_flds", max_length=50) + + self.log_test_start("test_nullable_fields_sync", "NULLABLE_FIELDS", collection_name) + self._upstream_client = upstream_client + self.resources_to_cleanup.append(("collection", collection_name)) + + try: + self.cleanup_collection(upstream_client, collection_name) + + # Create nullable schema collection + self.log_operation("CREATE_COLLECTION", "collection", collection_name, "upstream") + upstream_client.create_collection( + collection_name=collection_name, + schema=self.create_nullable_schema(upstream_client), + ) + + # Create index and load + index_params = upstream_client.prepare_index_params() + index_params.add_index( + field_name="float_vector", + index_type="HNSW", + metric_type="L2", + params={"M": 8, "efConstruction": 64}, + ) + upstream_client.create_index(collection_name, index_params) + upstream_client.load_collection(collection_name) + + # Wait for collection creation to sync + def check_create(): + return downstream_client.has_collection(collection_name) + + assert self.wait_for_sync(check_create, sync_timeout, f"create collection {collection_name}") + + # Insert 200 rows with null_ratio=0.3 + test_data = self.generate_nullable_data(200, null_ratio=0.3) + self.log_data_operation("INSERT", collection_name, len(test_data), "- nullable data null_ratio=0.3") + upstream_client.insert(collection_name, test_data) + upstream_client.flush(collection_name) + + # Count nulls and not-nulls for nullable_int64 on upstream + null_result = upstream_client.query( + collection_name=collection_name, + filter="nullable_int64 is null", + output_fields=["count(*)"], + ) + not_null_result = upstream_client.query( + collection_name=collection_name, + filter="nullable_int64 is not null", + output_fields=["count(*)"], + ) + upstream_null_count = null_result[0]["count(*)"] if null_result else 0 + upstream_not_null_count = not_null_result[0]["count(*)"] if not_null_result else 0 + logger.info(f"[UPSTREAM] nullable_int64 null={upstream_null_count}, not_null={upstream_not_null_count}") + + self.log_sync_verification("NULLABLE_FIELDS", collection_name, "null counts match downstream") + + # Wait for sync and verify null/not-null counts match on downstream + def check_null_counts(): + try: + d_null = downstream_client.query( + collection_name=collection_name, + filter="nullable_int64 is null", + output_fields=["count(*)"], + ) + d_not_null = downstream_client.query( + collection_name=collection_name, + filter="nullable_int64 is not null", + output_fields=["count(*)"], + ) + d_null_count = d_null[0]["count(*)"] if d_null else 0 + d_not_null_count = d_not_null[0]["count(*)"] if d_not_null else 0 + logger.info( + f"[SYNC_PROGRESS] downstream nullable_int64 null={d_null_count}/{upstream_null_count}, " + f"not_null={d_not_null_count}/{upstream_not_null_count}" + ) + return d_null_count == upstream_null_count and d_not_null_count == upstream_not_null_count + except Exception as e: + logger.warning(f"Nullable sync check failed: {e}") + return False + + sync_success = self.wait_for_sync( + check_null_counts, sync_timeout, f"nullable fields sync {collection_name}" + ) + assert sync_success, f"Nullable field counts failed to sync to downstream for {collection_name}" + + duration = time.time() - start_time + self.log_test_end("test_nullable_fields_sync", True, duration) + + except Exception as e: + duration = time.time() - start_time + logger.error(f"[ERROR] test_nullable_fields_sync failed: {e}") + self.log_test_end("test_nullable_fields_sync", False, duration) + raise + + def test_default_values_sync(self, upstream_client, downstream_client, sync_timeout): + """Test that default field values are applied and replicated correctly via CDC.""" + start_time = time.time() + collection_name = self.gen_unique_name("test_default_vals", max_length=50) + + self.log_test_start("test_default_values_sync", "DEFAULT_VALUES", collection_name) + self._upstream_client = upstream_client + self.resources_to_cleanup.append(("collection", collection_name)) + + try: + self.cleanup_collection(upstream_client, collection_name) + + # Create default values schema collection + self.log_operation("CREATE_COLLECTION", "collection", collection_name, "upstream") + upstream_client.create_collection( + collection_name=collection_name, + schema=self.create_default_values_schema(upstream_client), + ) + + # Create index and load + index_params = upstream_client.prepare_index_params() + index_params.add_index( + field_name="float_vector", + index_type="HNSW", + metric_type="L2", + params={"M": 8, "efConstruction": 64}, + ) + upstream_client.create_index(collection_name, index_params) + upstream_client.load_collection(collection_name) + + # Wait for collection creation to sync + def check_create(): + return downstream_client.has_collection(collection_name) + + assert self.wait_for_sync(check_create, sync_timeout, f"create collection {collection_name}") + + # Insert 100 rows providing ONLY float_vector — default fields are omitted + test_data = [{"float_vector": [random.random() for _ in range(128)]} for _ in range(100)] + self.log_data_operation("INSERT", collection_name, len(test_data), "- only float_vector provided") + upstream_client.insert(collection_name, test_data) + upstream_client.flush(collection_name) + + # Query default_varchar == "default" on upstream — should be 100 + upstream_result = upstream_client.query( + collection_name=collection_name, + filter='default_varchar == "default"', + output_fields=["count(*)"], + ) + upstream_count = upstream_result[0]["count(*)"] if upstream_result else 0 + logger.info(f'[UPSTREAM] default_varchar == "default" count: {upstream_count}') + assert upstream_count == 100, f"Expected 100 rows with default varchar on upstream, got {upstream_count}" + + self.log_sync_verification( + "DEFAULT_VALUES", collection_name, 'default_varchar == "default" count=100 on downstream' + ) + + # Wait for sync and verify same count on downstream + def check_defaults(): + try: + down_result = downstream_client.query( + collection_name=collection_name, + filter='default_varchar == "default"', + output_fields=["count(*)"], + ) + down_count = down_result[0]["count(*)"] if down_result else 0 + logger.info(f"[SYNC_PROGRESS] downstream default_varchar count: {down_count}/100") + return down_count == 100 + except Exception as e: + logger.warning(f"Default values sync check failed: {e}") + return False + + sync_success = self.wait_for_sync(check_defaults, sync_timeout, f"default values sync {collection_name}") + assert sync_success, f"Default value data failed to sync to downstream for {collection_name}" + + duration = time.time() - start_time + self.log_test_end("test_default_values_sync", True, duration) + + except Exception as e: + duration = time.time() - start_time + logger.error(f"[ERROR] test_default_values_sync failed: {e}") + self.log_test_end("test_default_values_sync", False, duration) + raise + + def test_partition_key_sync(self, upstream_client, downstream_client, sync_timeout): + """Test that partition key schema and data are correctly replicated via CDC.""" + start_time = time.time() + collection_name = self.gen_unique_name("test_part_key", max_length=50) + + self.log_test_start("test_partition_key_sync", "PARTITION_KEY", collection_name) + self._upstream_client = upstream_client + self.resources_to_cleanup.append(("collection", collection_name)) + + try: + self.cleanup_collection(upstream_client, collection_name) + + # Create partition key schema (VarChar key) + self.log_operation("CREATE_COLLECTION", "collection", collection_name, "upstream") + upstream_client.create_collection( + collection_name=collection_name, + schema=self.create_partition_key_schema(upstream_client, key_type="VarChar"), + ) + + # Create index and load + index_params = upstream_client.prepare_index_params() + index_params.add_index( + field_name="float_vector", + index_type="HNSW", + metric_type="L2", + params={"M": 8, "efConstruction": 64}, + ) + upstream_client.create_index(collection_name, index_params) + upstream_client.load_collection(collection_name) + + # Wait for collection creation to sync + def check_create(): + return downstream_client.has_collection(collection_name) + + assert self.wait_for_sync(check_create, sync_timeout, f"create collection {collection_name}") + + # Insert 500 rows with random categories as partition key values + categories = ["cat_A", "cat_B", "cat_C", "cat_D", "cat_E"] + test_data = [ + { + "float_vector": [random.random() for _ in range(128)], + "partition_key_field": random.choice(categories), + "data_field": f"data_{i}_{random.randint(1000, 9999)}", + } + for i in range(500) + ] + self.log_data_operation("INSERT", collection_name, len(test_data), "- partition key data") + upstream_client.insert(collection_name, test_data) + upstream_client.flush(collection_name) + + self.log_sync_verification( + "PARTITION_KEY", collection_name, "count=500 and partition_key field on downstream" + ) + + # Wait for sync and verify count=500 on downstream + def check_data(): + try: + result = downstream_client.query( + collection_name=collection_name, + filter="", + output_fields=["count(*)"], + ) + count = result[0]["count(*)"] if result else 0 + logger.info(f"[SYNC_PROGRESS] downstream count: {count}/500") + return count >= 500 + except Exception as e: + logger.warning(f"Partition key sync check failed: {e}") + return False + + sync_success = self.wait_for_sync(check_data, sync_timeout, f"partition key data sync {collection_name}") + assert sync_success, f"Partition key data failed to sync to downstream for {collection_name}" + + # Verify partition_key_field is present in downstream describe_collection + downstream_info = downstream_client.describe_collection(collection_name) + downstream_fields = [f["name"] for f in downstream_info.get("fields", [])] + logger.info(f"[VERIFY] Downstream fields: {downstream_fields}") + assert "partition_key_field" in downstream_fields, ( + f"partition_key_field not found in downstream collection schema: {downstream_fields}" + ) + + duration = time.time() - start_time + self.log_test_end("test_partition_key_sync", True, duration) + + except Exception as e: + duration = time.time() - start_time + logger.error(f"[ERROR] test_partition_key_sync failed: {e}") + self.log_test_end("test_partition_key_sync", False, duration) + raise + + def test_clustering_key_sync(self, upstream_client, downstream_client, sync_timeout): + """Test that clustering key schema and data are correctly replicated via CDC.""" + start_time = time.time() + collection_name = self.gen_unique_name("test_cluster_key", max_length=50) + + self.log_test_start("test_clustering_key_sync", "CLUSTERING_KEY", collection_name) + self._upstream_client = upstream_client + self.resources_to_cleanup.append(("collection", collection_name)) + + try: + self.cleanup_collection(upstream_client, collection_name) + + # Create clustering key schema + self.log_operation("CREATE_COLLECTION", "collection", collection_name, "upstream") + upstream_client.create_collection( + collection_name=collection_name, + schema=self.create_clustering_key_schema(upstream_client), + ) + + # Create index and load + index_params = upstream_client.prepare_index_params() + index_params.add_index( + field_name="float_vector", + index_type="HNSW", + metric_type="L2", + params={"M": 8, "efConstruction": 64}, + ) + upstream_client.create_index(collection_name, index_params) + upstream_client.load_collection(collection_name) + + # Wait for collection creation to sync + def check_create(): + return downstream_client.has_collection(collection_name) + + assert self.wait_for_sync(check_create, sync_timeout, f"create collection {collection_name}") + + # Insert 300 rows + test_data = [ + { + "float_vector": [random.random() for _ in range(128)], + "clustering_key_field": random.randint(0, 1000), + "data_field": f"data_{i}_{random.randint(1000, 9999)}", + } + for i in range(300) + ] + self.log_data_operation("INSERT", collection_name, len(test_data), "- clustering key data") + upstream_client.insert(collection_name, test_data) + upstream_client.flush(collection_name) + + self.log_sync_verification( + "CLUSTERING_KEY", collection_name, "count=300 and clustering_key field on downstream" + ) + + # Wait for sync and verify count=300 on downstream + def check_data(): + try: + result = downstream_client.query( + collection_name=collection_name, + filter="", + output_fields=["count(*)"], + ) + count = result[0]["count(*)"] if result else 0 + logger.info(f"[SYNC_PROGRESS] downstream count: {count}/300") + return count >= 300 + except Exception as e: + logger.warning(f"Clustering key sync check failed: {e}") + return False + + sync_success = self.wait_for_sync(check_data, sync_timeout, f"clustering key data sync {collection_name}") + assert sync_success, f"Clustering key data failed to sync to downstream for {collection_name}" + + # Verify clustering_key_field is present in downstream describe_collection + downstream_info = downstream_client.describe_collection(collection_name) + downstream_fields = [f["name"] for f in downstream_info.get("fields", [])] + logger.info(f"[VERIFY] Downstream fields: {downstream_fields}") + assert "clustering_key_field" in downstream_fields, ( + f"clustering_key_field not found in downstream collection schema: {downstream_fields}" + ) + + duration = time.time() - start_time + self.log_test_end("test_clustering_key_sync", True, duration) + + except Exception as e: + duration = time.time() - start_time + logger.error(f"[ERROR] test_clustering_key_sync failed: {e}") + self.log_test_end("test_clustering_key_sync", False, duration) + raise + + def test_nullable_with_defaults(self, upstream_client, downstream_client, sync_timeout): + """Test that nullable fields combined with default values sync correctly via CDC. + + Inserts 100 rows in three patterns: + i%3==0: explicit values provided for both fields + i%3==1: None values (explicit null) provided for both fields + i%3==2: fields omitted entirely (uses default / null) + """ + start_time = time.time() + collection_name = self.gen_unique_name("test_null_default", max_length=50) + + self.log_test_start("test_nullable_with_defaults", "NULLABLE_WITH_DEFAULTS", collection_name) + self._upstream_client = upstream_client + self.resources_to_cleanup.append(("collection", collection_name)) + + try: + self.cleanup_collection(upstream_client, collection_name) + + # Build custom schema: nullable_with_default (INT64, nullable, default=42) + # nullable_no_default (VARCHAR, nullable) + schema = upstream_client.create_schema() + schema.add_field("id", DataType.INT64, is_primary=True, auto_id=True) + schema.add_field("float_vector", DataType.FLOAT_VECTOR, dim=128) + schema.add_field( + "nullable_with_default", + DataType.INT64, + nullable=True, + default_value=42, + ) + schema.add_field( + "nullable_no_default", + DataType.VARCHAR, + max_length=256, + nullable=True, + ) + + self.log_operation("CREATE_COLLECTION", "collection", collection_name, "upstream") + upstream_client.create_collection( + collection_name=collection_name, + schema=schema, + ) + + # Create index and load + index_params = upstream_client.prepare_index_params() + index_params.add_index( + field_name="float_vector", + index_type="HNSW", + metric_type="L2", + params={"M": 8, "efConstruction": 64}, + ) + upstream_client.create_index(collection_name, index_params) + upstream_client.load_collection(collection_name) + + # Wait for collection creation to sync + def check_create(): + return downstream_client.has_collection(collection_name) + + assert self.wait_for_sync(check_create, sync_timeout, f"create collection {collection_name}") + + # Insert 100 rows across 3 patterns + test_data = [] + for i in range(100): + record = {"float_vector": [random.random() for _ in range(128)]} + if i % 3 == 0: + # Explicit values + record["nullable_with_default"] = random.randint(100, 999) + record["nullable_no_default"] = f"explicit_{i}" + elif i % 3 == 1: + # Explicit None (null) + record["nullable_with_default"] = None + record["nullable_no_default"] = None + # i%3==2: omit both fields — server applies default/null + test_data.append(record) + + self.log_data_operation("INSERT", collection_name, len(test_data), "- nullable+default mixed pattern") + upstream_client.insert(collection_name, test_data) + upstream_client.flush(collection_name) + + self.log_sync_verification("NULLABLE_WITH_DEFAULTS", collection_name, "total count=100 on downstream") + + # Wait for sync and verify total count on downstream + def check_data(): + try: + result = downstream_client.query( + collection_name=collection_name, + filter="", + output_fields=["count(*)"], + ) + count = result[0]["count(*)"] if result else 0 + logger.info(f"[SYNC_PROGRESS] downstream count: {count}/100") + return count >= 100 + except Exception as e: + logger.warning(f"Nullable+defaults sync check failed: {e}") + return False + + sync_success = self.wait_for_sync(check_data, sync_timeout, f"nullable+defaults sync {collection_name}") + assert sync_success, f"Nullable-with-defaults data failed to sync to downstream for {collection_name}" + + duration = time.time() - start_time + self.log_test_end("test_nullable_with_defaults", True, duration) + + except Exception as e: + duration = time.time() - start_time + logger.error(f"[ERROR] test_nullable_with_defaults failed: {e}") + self.log_test_end("test_nullable_with_defaults", False, duration) + raise + + def test_dynamic_with_partition_key(self, upstream_client, downstream_client, sync_timeout): + """Test that dynamic fields combined with a partition key schema sync correctly via CDC.""" + start_time = time.time() + collection_name = self.gen_unique_name("test_dyn_part_key", max_length=50) + + self.log_test_start("test_dynamic_with_partition_key", "DYNAMIC_WITH_PARTITION_KEY", collection_name) + self._upstream_client = upstream_client + self.resources_to_cleanup.append(("collection", collection_name)) + + try: + self.cleanup_collection(upstream_client, collection_name) + + # Build schema: enable_dynamic_field=True + VARCHAR partition key + schema = upstream_client.create_schema(enable_dynamic_field=True) + schema.add_field("id", DataType.INT64, is_primary=True, auto_id=True) + schema.add_field("float_vector", DataType.FLOAT_VECTOR, dim=128) + schema.add_field( + "pk_field", + DataType.VARCHAR, + max_length=64, + is_partition_key=True, + ) + + self.log_operation("CREATE_COLLECTION", "collection", collection_name, "upstream") + upstream_client.create_collection( + collection_name=collection_name, + schema=schema, + ) + + # Create index and load + index_params = upstream_client.prepare_index_params() + index_params.add_index( + field_name="float_vector", + index_type="HNSW", + metric_type="L2", + params={"M": 8, "efConstruction": 64}, + ) + upstream_client.create_index(collection_name, index_params) + upstream_client.load_collection(collection_name) + + # Wait for collection creation to sync + def check_create(): + return downstream_client.has_collection(collection_name) + + assert self.wait_for_sync(check_create, sync_timeout, f"create collection {collection_name}") + + # Insert 300 rows with dynamic fields + partition key values + partitions = ["region_A", "region_B", "region_C", "region_D"] + test_data = [ + { + "float_vector": [random.random() for _ in range(128)], + "pk_field": random.choice(partitions), + # dynamic fields + "dynamic_num": random.randint(1, 10000), + "dynamic_tag": f"tag_{i % 10}", + "dynamic_score": random.uniform(0.0, 100.0), + } + for i in range(300) + ] + self.log_data_operation("INSERT", collection_name, len(test_data), "- dynamic+partition_key data") + upstream_client.insert(collection_name, test_data) + upstream_client.flush(collection_name) + + # Query dynamic_num > 0 on upstream + upstream_result = upstream_client.query( + collection_name=collection_name, + filter="dynamic_num > 0", + output_fields=["count(*)"], + ) + upstream_count = upstream_result[0]["count(*)"] if upstream_result else 0 + logger.info(f"[UPSTREAM] dynamic_num > 0 count: {upstream_count}") + + self.log_sync_verification( + "DYNAMIC_WITH_PARTITION_KEY", + collection_name, + f"dynamic_num > 0 count={upstream_count} on downstream", + ) + + # Wait for sync and verify the count matches on downstream + def check_data(): + try: + down_result = downstream_client.query( + collection_name=collection_name, + filter="dynamic_num > 0", + output_fields=["count(*)"], + ) + down_count = down_result[0]["count(*)"] if down_result else 0 + logger.info(f"[SYNC_PROGRESS] downstream dynamic_num > 0 count: {down_count}/{upstream_count}") + return down_count == upstream_count + except Exception as e: + logger.warning(f"Dynamic+partition_key sync check failed: {e}") + return False + + sync_success = self.wait_for_sync(check_data, sync_timeout, f"dynamic+partition_key sync {collection_name}") + assert sync_success, f"Dynamic+partition_key data failed to sync to downstream for {collection_name}" + + duration = time.time() - start_time + self.log_test_end("test_dynamic_with_partition_key", True, duration) + + except Exception as e: + duration = time.time() - start_time + logger.error(f"[ERROR] test_dynamic_with_partition_key failed: {e}") + self.log_test_end("test_dynamic_with_partition_key", False, duration) + raise diff --git a/tests/python_client/cdc/testcases/test_search_verification.py b/tests/python_client/cdc/testcases/test_search_verification.py new file mode 100644 index 0000000000..d710182e03 --- /dev/null +++ b/tests/python_client/cdc/testcases/test_search_verification.py @@ -0,0 +1,570 @@ +""" +CDC sync tests for search and query result verification across vector types. +""" + +import random +import time + +import pytest +from pymilvus import AnnSearchRequest, DataType, RRFRanker + +from .base import TestCDCSyncBase, logger + +# fmt: off +VECTOR_PARAMS = [ + ("FLOAT_VECTOR", "HNSW", "COSINE", 128), + ("FLOAT_VECTOR", "IVF_FLAT", "L2", 128), + ("FLOAT16_VECTOR", "HNSW", "L2", 64), + ("BFLOAT16_VECTOR", "HNSW", "L2", 64), + ("INT8_VECTOR", "HNSW", "COSINE", 64), + ("BINARY_VECTOR", "BIN_FLAT", "HAMMING", 128), + ("SPARSE_FLOAT_VECTOR", "SPARSE_INVERTED_INDEX", "IP", 0), +] +# fmt: on + + +class TestCDCSyncSearchVerification(TestCDCSyncBase): + """Test CDC sync for search and query result verification across vector types.""" + + def setup_method(self): + """Setup for each test method.""" + self.resources_to_cleanup = [] + + def teardown_method(self): + """Cleanup after each test method - only cleanup upstream, downstream will sync.""" + upstream_client = getattr(self, "_upstream_client", None) + + if upstream_client: + for resource_type, resource_name in self.resources_to_cleanup: + if resource_type == "collection": + self.cleanup_collection(upstream_client, resource_name) + + time.sleep(1) # Allow cleanup to sync to downstream + + # ------------------------------------------------------------------------- + # Internal helper + # ------------------------------------------------------------------------- + + def _setup_collection(self, client, c_name, vector_type, index_type, metric, dim): + """ + Create a single-vector-schema collection, insert 500 records, + create an index, and load. + + Returns the collection name (same as c_name). + """ + schema = self.create_single_vector_schema(client, vector_type=vector_type, dim=dim) + client.create_collection(collection_name=c_name, schema=schema) + + # Insert 500 records + data = self.generate_single_vector_data(500, vector_type=vector_type, dim=dim) + client.insert(c_name, data) + client.flush(c_name) + + # Build index + index_params = client.prepare_index_params() + if index_type == "IVF_FLAT": + idx_params = {"nlist": 64} + elif index_type == "HNSW": + idx_params = {"M": 16, "efConstruction": 200} + else: + idx_params = {} + + index_params.add_index( + field_name="vector", + index_type=index_type, + metric_type=metric, + params=idx_params, + ) + client.create_index(c_name, index_params) + client.load_collection(c_name) + + return c_name + + # ------------------------------------------------------------------------- + # Tests + # ------------------------------------------------------------------------- + + @pytest.mark.parametrize( + "vector_type,index_type,metric,dim", + VECTOR_PARAMS, + ids=[p[0] + "_" + p[1] for p in VECTOR_PARAMS], + ) + def test_search_result_consistency( + self, + upstream_client, + downstream_client, + sync_timeout, + vector_type, + index_type, + metric, + dim, + ): + """Verify that ANN search results are consistent between upstream and downstream.""" + start_time = time.time() + c_name = self.gen_unique_name(f"test_src_{vector_type[:4].lower()}", max_length=50) + + self.log_test_start( + "test_search_result_consistency", + f"SEARCH/{vector_type}/{index_type}", + c_name, + ) + self._upstream_client = upstream_client + self.resources_to_cleanup.append(("collection", c_name)) + + try: + self.cleanup_collection(upstream_client, c_name) + self._setup_collection(upstream_client, c_name, vector_type, index_type, metric, dim) + + # Wait for at least 500 records to appear on downstream + def check_sync(): + try: + res = downstream_client.query( + collection_name=c_name, + filter="", + output_fields=["count(*)"], + ) + cnt = res[0]["count(*)"] if res else 0 + logger.info(f"[SYNC_PROGRESS] downstream count: {cnt}/500") + return cnt >= 500 + except Exception as e: + logger.warning(f"Sync check failed: {e}") + return False + + assert self.wait_for_sync(check_sync, sync_timeout, f"data sync 500 records {c_name}"), ( + f"Downstream did not receive 500 records within {sync_timeout}s" + ) + + # Build 5 random query vectors + dtype = getattr(DataType, vector_type) + query_vectors = self._gen_vectors(5, dim if dim > 0 else 1000, dtype) + + avg_overlap, _, _ = self.verify_search_consistency( + upstream_client, + downstream_client, + c_name, + query_vectors, + anns_field="vector", + limit=10, + metric_type=metric, + ) + + assert avg_overlap >= self.SEARCH_OVERLAP_THRESHOLD, ( + f"Search overlap {avg_overlap:.4f} is below threshold " + f"{self.SEARCH_OVERLAP_THRESHOLD} for {vector_type}/{index_type}" + ) + + finally: + self.log_test_end( + "test_search_result_consistency", + True, + time.time() - start_time, + ) + + @pytest.mark.parametrize( + "vector_type,index_type,metric,dim", + VECTOR_PARAMS, + ids=[p[0] + "_" + p[1] for p in VECTOR_PARAMS], + ) + def test_query_data_sampling( + self, + upstream_client, + downstream_client, + sync_timeout, + vector_type, + index_type, + metric, + dim, + ): + """Verify scalar field values are identical on both sides via random sampling.""" + start_time = time.time() + c_name = self.gen_unique_name(f"test_qds_{vector_type[:4].lower()}", max_length=50) + + self.log_test_start( + "test_query_data_sampling", + f"QUERY_SAMPLE/{vector_type}/{index_type}", + c_name, + ) + self._upstream_client = upstream_client + self.resources_to_cleanup.append(("collection", c_name)) + + try: + self.cleanup_collection(upstream_client, c_name) + self._setup_collection(upstream_client, c_name, vector_type, index_type, metric, dim) + + def check_sync(): + try: + res = downstream_client.query( + collection_name=c_name, + filter="", + output_fields=["count(*)"], + ) + cnt = res[0]["count(*)"] if res else 0 + logger.info(f"[SYNC_PROGRESS] downstream count: {cnt}/500") + return cnt >= 500 + except Exception as e: + logger.warning(f"Sync check failed: {e}") + return False + + assert self.wait_for_sync(check_sync, sync_timeout, f"data sync 500 records {c_name}"), ( + f"Downstream did not receive 500 records within {sync_timeout}s" + ) + + output_fields = ["id", "int_field", "varchar_field", "float_field"] + match_count, mismatch_count, mismatch_details = self.verify_data_sampling( + upstream_client, + downstream_client, + c_name, + sample_ratio=0.2, + output_fields=output_fields, + ) + + logger.info( + f"[RESULT] Sampling — match={match_count}, mismatch={mismatch_count}, details={mismatch_details[:3]}" + ) + assert mismatch_count == 0, f"Found {mismatch_count} mismatched records: {mismatch_details[:5]}" + + finally: + self.log_test_end( + "test_query_data_sampling", + True, + time.time() - start_time, + ) + + def test_hybrid_search_consistency( + self, + upstream_client, + downstream_client, + sync_timeout, + ): + """Verify hybrid search (dense + sparse, RRF ranker) results are consistent.""" + start_time = time.time() + c_name = self.gen_unique_name("test_hybrid_srch", max_length=50) + + self.log_test_start("test_hybrid_search_consistency", "HYBRID_SEARCH", c_name) + self._upstream_client = upstream_client + self.resources_to_cleanup.append(("collection", c_name)) + + try: + self.cleanup_collection(upstream_client, c_name) + + # Build schema: dense FloatVector(128) + sparse + schema = upstream_client.create_schema(enable_dynamic_field=True) + schema.add_field("id", DataType.INT64, is_primary=True, auto_id=True) + schema.add_field("dense", DataType.FLOAT_VECTOR, dim=128) + schema.add_field("sparse", DataType.SPARSE_FLOAT_VECTOR) + schema.add_field("int_field", DataType.INT64) + schema.add_field("varchar_field", DataType.VARCHAR, max_length=256) + + upstream_client.create_collection(collection_name=c_name, schema=schema) + + # Insert 300 records + dense_vecs = self._gen_vectors(300, 128, DataType.FLOAT_VECTOR) + sparse_vecs = self._gen_vectors(300, 1000, DataType.SPARSE_FLOAT_VECTOR) + data = [ + { + "dense": dense_vecs[i], + "sparse": sparse_vecs[i], + "int_field": random.randint(0, 1000), + "varchar_field": f"hybrid_{i}_{random.randint(1000, 9999)}", + } + for i in range(300) + ] + upstream_client.insert(c_name, data) + upstream_client.flush(c_name) + + # Create indexes + index_params = upstream_client.prepare_index_params() + index_params.add_index( + field_name="dense", + index_type="HNSW", + metric_type="COSINE", + params={"M": 16, "efConstruction": 200}, + ) + index_params.add_index( + field_name="sparse", + index_type="SPARSE_INVERTED_INDEX", + metric_type="IP", + params={}, + ) + upstream_client.create_index(c_name, index_params) + upstream_client.load_collection(c_name) + + # Wait for downstream sync + def check_sync(): + try: + res = downstream_client.query( + collection_name=c_name, + filter="", + output_fields=["count(*)"], + ) + cnt = res[0]["count(*)"] if res else 0 + logger.info(f"[SYNC_PROGRESS] downstream count: {cnt}/300") + return cnt >= 300 + except Exception as e: + logger.warning(f"Sync check failed: {e}") + return False + + assert self.wait_for_sync(check_sync, sync_timeout, f"hybrid data sync {c_name}"), ( + f"Downstream did not receive 300 records within {sync_timeout}s" + ) + + # Build hybrid search requests + q_dense = self._gen_vectors(1, 128, DataType.FLOAT_VECTOR)[0] + q_sparse = self._gen_vectors(1, 1000, DataType.SPARSE_FLOAT_VECTOR)[0] + + dense_req = AnnSearchRequest( + data=[q_dense], + anns_field="dense", + param={"metric_type": "COSINE", "params": {"ef": 64}}, + limit=10, + ) + sparse_req = AnnSearchRequest( + data=[q_sparse], + anns_field="sparse", + param={"metric_type": "IP"}, + limit=10, + ) + + up_results = upstream_client.hybrid_search( + collection_name=c_name, + reqs=[dense_req, sparse_req], + ranker=RRFRanker(), + limit=10, + output_fields=["id"], + ) + down_results = downstream_client.hybrid_search( + collection_name=c_name, + reqs=[dense_req, sparse_req], + ranker=RRFRanker(), + limit=10, + output_fields=["id"], + ) + + up_pks = set(hit["id"] for hit in up_results[0]) if up_results else set() + down_pks = set(hit["id"] for hit in down_results[0]) if down_results else set() + union_size = len(up_pks | down_pks) + overlap = len(up_pks & down_pks) / union_size if union_size > 0 else 1.0 + + logger.info(f"[RESULT] Hybrid search PK overlap={overlap:.4f} (up={len(up_pks)}, down={len(down_pks)})") + assert overlap >= self.SEARCH_OVERLAP_THRESHOLD, ( + f"Hybrid search overlap {overlap:.4f} below threshold {self.SEARCH_OVERLAP_THRESHOLD}" + ) + + finally: + self.log_test_end( + "test_hybrid_search_consistency", + True, + time.time() - start_time, + ) + + def test_search_iterator_consistency( + self, + upstream_client, + downstream_client, + sync_timeout, + ): + """Verify search iterator returns the same PK set on upstream and downstream.""" + start_time = time.time() + c_name = self.gen_unique_name("test_srch_iter", max_length=50) + + self.log_test_start("test_search_iterator_consistency", "SEARCH_ITERATOR", c_name) + self._upstream_client = upstream_client + self.resources_to_cleanup.append(("collection", c_name)) + + try: + self.cleanup_collection(upstream_client, c_name) + self._setup_collection(upstream_client, c_name, "FLOAT_VECTOR", "HNSW", "COSINE", 128) + + def check_sync(): + try: + res = downstream_client.query( + collection_name=c_name, + filter="", + output_fields=["count(*)"], + ) + cnt = res[0]["count(*)"] if res else 0 + logger.info(f"[SYNC_PROGRESS] downstream count: {cnt}/500") + return cnt >= 500 + except Exception as e: + logger.warning(f"Sync check failed: {e}") + return False + + assert self.wait_for_sync(check_sync, sync_timeout, f"data sync 500 records {c_name}"), ( + f"Downstream did not receive 500 records within {sync_timeout}s" + ) + + query_vec = self._gen_vectors(1, 128, DataType.FLOAT_VECTOR)[0] + search_params = {"metric_type": "COSINE", "params": {"ef": 64}} + + def _collect_iterator_pks(client): + pks = set() + iterator = client.search_iterator( + collection_name=c_name, + data=[query_vec], + anns_field="vector", + batch_size=50, + limit=200, + param=search_params, + output_fields=["id"], + ) + while True: + batch = iterator.next() + if not batch: + iterator.close() + break + for hit in batch: + pks.add(hit["id"]) + return pks + + up_pks = _collect_iterator_pks(upstream_client) + down_pks = _collect_iterator_pks(downstream_client) + union_size = len(up_pks | down_pks) + overlap = len(up_pks & down_pks) / union_size if union_size > 0 else 1.0 + + logger.info(f"[RESULT] Search iterator overlap={overlap:.4f} (up={len(up_pks)}, down={len(down_pks)})") + assert overlap >= self.SEARCH_OVERLAP_THRESHOLD, ( + f"Search iterator PK overlap {overlap:.4f} below threshold {self.SEARCH_OVERLAP_THRESHOLD}" + ) + + finally: + self.log_test_end( + "test_search_iterator_consistency", + True, + time.time() - start_time, + ) + + def test_query_iterator_consistency( + self, + upstream_client, + downstream_client, + sync_timeout, + ): + """Verify that a query iterator retrieves identical PK sets from both sides.""" + start_time = time.time() + c_name = self.gen_unique_name("test_qry_iter", max_length=50) + + self.log_test_start("test_query_iterator_consistency", "QUERY_ITERATOR", c_name) + self._upstream_client = upstream_client + self.resources_to_cleanup.append(("collection", c_name)) + + try: + self.cleanup_collection(upstream_client, c_name) + self._setup_collection(upstream_client, c_name, "FLOAT_VECTOR", "HNSW", "COSINE", 128) + + def check_sync(): + try: + res = downstream_client.query( + collection_name=c_name, + filter="", + output_fields=["count(*)"], + ) + cnt = res[0]["count(*)"] if res else 0 + logger.info(f"[SYNC_PROGRESS] downstream count: {cnt}/500") + return cnt >= 500 + except Exception as e: + logger.warning(f"Sync check failed: {e}") + return False + + assert self.wait_for_sync(check_sync, sync_timeout, f"data sync 500 records {c_name}"), ( + f"Downstream did not receive 500 records within {sync_timeout}s" + ) + + up_count, down_count, match = self.verify_iterator_consistency( + upstream_client, + downstream_client, + c_name, + batch_size=100, + ) + + logger.info(f"[RESULT] Query iterator — upstream={up_count}, downstream={down_count}, match={match}") + assert match, f"Query iterator PK sets differ: upstream={up_count}, downstream={down_count}" + + finally: + self.log_test_end( + "test_query_iterator_consistency", + True, + time.time() - start_time, + ) + + def test_search_with_filter_consistency( + self, + upstream_client, + downstream_client, + sync_timeout, + ): + """Verify filtered search produces consistent results honoring the filter predicate.""" + start_time = time.time() + c_name = self.gen_unique_name("test_srch_filter", max_length=50) + + self.log_test_start("test_search_with_filter_consistency", "SEARCH_WITH_FILTER", c_name) + self._upstream_client = upstream_client + self.resources_to_cleanup.append(("collection", c_name)) + + try: + self.cleanup_collection(upstream_client, c_name) + self._setup_collection(upstream_client, c_name, "FLOAT_VECTOR", "HNSW", "COSINE", 128) + + def check_sync(): + try: + res = downstream_client.query( + collection_name=c_name, + filter="", + output_fields=["count(*)"], + ) + cnt = res[0]["count(*)"] if res else 0 + logger.info(f"[SYNC_PROGRESS] downstream count: {cnt}/500") + return cnt >= 500 + except Exception as e: + logger.warning(f"Sync check failed: {e}") + return False + + assert self.wait_for_sync(check_sync, sync_timeout, f"data sync 500 records {c_name}"), ( + f"Downstream did not receive 500 records within {sync_timeout}s" + ) + + filter_expr = "int_field > 500" + query_vec = self._gen_vectors(1, 128, DataType.FLOAT_VECTOR)[0] + search_params = {"metric_type": "COSINE"} + + up_results = upstream_client.search( + collection_name=c_name, + data=[query_vec], + anns_field="vector", + search_params=search_params, + filter=filter_expr, + limit=10, + output_fields=["id", "int_field"], + ) + down_results = downstream_client.search( + collection_name=c_name, + data=[query_vec], + anns_field="vector", + search_params=search_params, + filter=filter_expr, + limit=10, + output_fields=["id", "int_field"], + ) + + # Verify filter is honoured on both sides + for hit in up_results[0] if up_results else []: + assert hit["int_field"] > 500, f"Filter violated on upstream: int_field={hit['int_field']}" + for hit in down_results[0] if down_results else []: + assert hit["int_field"] > 500, f"Filter violated on downstream: int_field={hit['int_field']}" + + # Verify PK overlap + up_pks = set(hit["id"] for hit in up_results[0]) if up_results else set() + down_pks = set(hit["id"] for hit in down_results[0]) if down_results else set() + union_size = len(up_pks | down_pks) + overlap = len(up_pks & down_pks) / union_size if union_size > 0 else 1.0 + + logger.info(f"[RESULT] Filtered search overlap={overlap:.4f} (up={len(up_pks)}, down={len(down_pks)})") + assert overlap >= self.SEARCH_OVERLAP_THRESHOLD, ( + f"Filtered search overlap {overlap:.4f} below threshold {self.SEARCH_OVERLAP_THRESHOLD}" + ) + + finally: + self.log_test_end( + "test_search_with_filter_consistency", + True, + time.time() - start_time, + ) diff --git a/tests/python_client/cdc/testcases/test_setup_cdc.py b/tests/python_client/cdc/testcases/test_setup_cdc.py index 3e148523a9..d7705cd884 100644 --- a/tests/python_client/cdc/testcases/test_setup_cdc.py +++ b/tests/python_client/cdc/testcases/test_setup_cdc.py @@ -3,8 +3,12 @@ CDC topology setup and configuration test cases. """ import time + import pytest from common.common_type import CaseLabel + +from cdc.conftest import CDC_UPDATE_REPLICATE_TIMEOUT_SECONDS, apply_replicate_configuration + from .base import TestCDCSyncBase @@ -41,10 +45,7 @@ class TestCDCTopologySetup(TestCDCSyncBase): { "cluster_id": source_cluster_id, "connection_param": {"uri": upstream_uri, "token": upstream_token}, - "pchannels": [ - f"{source_cluster_id}-rootcoord-dml_{i}" - for i in range(pchannel_num) - ], + "pchannels": [f"{source_cluster_id}-rootcoord-dml_{i}" for i in range(pchannel_num)], }, { "cluster_id": target_cluster_id, @@ -52,10 +53,7 @@ class TestCDCTopologySetup(TestCDCSyncBase): "uri": downstream_uri, "token": downstream_token, }, - "pchannels": [ - f"{target_cluster_id}-rootcoord-dml_{i}" - for i in range(pchannel_num) - ], + "pchannels": [f"{target_cluster_id}-rootcoord-dml_{i}" for i in range(pchannel_num)], }, ], "cross_cluster_topology": [ @@ -67,8 +65,7 @@ class TestCDCTopologySetup(TestCDCSyncBase): } # Test normal topology setup - upstream_client.update_replicate_configuration(**config) - downstream_client.update_replicate_configuration(**config) + apply_replicate_configuration([(upstream_client, config), (downstream_client, config)]) # Wait for configuration to take effect time.sleep(3) @@ -87,9 +84,7 @@ class TestCDCTopologySetup(TestCDCSyncBase): def check_sync(): return downstream_client.has_collection(test_collection_name) - assert self.wait_for_sync( - check_sync, 30, f"collection {test_collection_name} sync" - ) + assert self.wait_for_sync(check_sync, 30, f"collection {test_collection_name} sync") # Cleanup self.cleanup_collection(upstream_client, test_collection_name) @@ -113,10 +108,7 @@ class TestCDCTopologySetup(TestCDCSyncBase): { "cluster_id": source_cluster_id, "connection_param": {"uri": upstream_uri, "token": upstream_token}, - "pchannels": [ - f"{source_cluster_id}-rootcoord-dml_{i}" - for i in range(pchannel_num) - ], + "pchannels": [f"{source_cluster_id}-rootcoord-dml_{i}" for i in range(pchannel_num)], }, { "cluster_id": target_cluster_id, @@ -124,10 +116,7 @@ class TestCDCTopologySetup(TestCDCSyncBase): "uri": downstream_uri, "token": downstream_token, }, - "pchannels": [ - f"{target_cluster_id}-rootcoord-dml_{i}" - for i in range(pchannel_num) - ], + "pchannels": [f"{target_cluster_id}-rootcoord-dml_{i}" for i in range(pchannel_num)], }, ], "cross_cluster_topology": [ @@ -138,8 +127,7 @@ class TestCDCTopologySetup(TestCDCSyncBase): ], } - upstream_client.update_replicate_configuration(**original_config) - downstream_client.update_replicate_configuration(**original_config) + apply_replicate_configuration([(upstream_client, original_config), (downstream_client, original_config)]) time.sleep(3) # Now switch the direction (target -> source) @@ -148,10 +136,7 @@ class TestCDCTopologySetup(TestCDCSyncBase): { "cluster_id": source_cluster_id, "connection_param": {"uri": upstream_uri, "token": upstream_token}, - "pchannels": [ - f"{source_cluster_id}-rootcoord-dml_{i}" - for i in range(pchannel_num) - ], + "pchannels": [f"{source_cluster_id}-rootcoord-dml_{i}" for i in range(pchannel_num)], }, { "cluster_id": target_cluster_id, @@ -159,10 +144,7 @@ class TestCDCTopologySetup(TestCDCSyncBase): "uri": downstream_uri, "token": downstream_token, }, - "pchannels": [ - f"{target_cluster_id}-rootcoord-dml_{i}" - for i in range(pchannel_num) - ], + "pchannels": [f"{target_cluster_id}-rootcoord-dml_{i}" for i in range(pchannel_num)], }, ], "cross_cluster_topology": [ @@ -174,8 +156,7 @@ class TestCDCTopologySetup(TestCDCSyncBase): } # Apply switched configuration - upstream_client.update_replicate_configuration(**switched_config) - downstream_client.update_replicate_configuration(**switched_config) + apply_replicate_configuration([(upstream_client, switched_config), (downstream_client, switched_config)]) time.sleep(3) # Test the switched topology by creating collection on downstream (now source) @@ -191,9 +172,7 @@ class TestCDCTopologySetup(TestCDCSyncBase): def check_switched_sync(): return upstream_client.has_collection(test_collection_name) - assert self.wait_for_sync( - check_switched_sync, 30, f"switched collection {test_collection_name} sync" - ) + assert self.wait_for_sync(check_switched_sync, 30, f"switched collection {test_collection_name} sync") # Cleanup self.cleanup_collection(downstream_client, test_collection_name) @@ -229,7 +208,9 @@ class TestCDCTopologySetup(TestCDCSyncBase): } with pytest.raises(Exception): - upstream_client.update_replicate_configuration(**invalid_config_1) + upstream_client.update_replicate_configuration( + timeout=CDC_UPDATE_REPLICATE_TIMEOUT_SECONDS, **invalid_config_1 + ) # Test case 4.2: Circular dependency (A -> B, B -> A) invalid_config_2 = { @@ -262,7 +243,9 @@ class TestCDCTopologySetup(TestCDCSyncBase): # This may or may not fail depending on implementation, but test it with pytest.raises(Exception): - upstream_client.update_replicate_configuration(**invalid_config_2) + upstream_client.update_replicate_configuration( + timeout=CDC_UPDATE_REPLICATE_TIMEOUT_SECONDS, **invalid_config_2 + ) # Test case 4.3: Invalid connection parameters invalid_config_3 = { @@ -292,7 +275,9 @@ class TestCDCTopologySetup(TestCDCSyncBase): ], } with pytest.raises(Exception): - upstream_client.update_replicate_configuration(**invalid_config_3) + upstream_client.update_replicate_configuration( + timeout=CDC_UPDATE_REPLICATE_TIMEOUT_SECONDS, **invalid_config_3 + ) # Test case 4.4: Empty cluster list but non-empty topology invalid_config_4 = { @@ -305,7 +290,9 @@ class TestCDCTopologySetup(TestCDCSyncBase): ], } with pytest.raises(Exception): - upstream_client.update_replicate_configuration(**invalid_config_4) + upstream_client.update_replicate_configuration( + timeout=CDC_UPDATE_REPLICATE_TIMEOUT_SECONDS, **invalid_config_4 + ) # Test case 4.5: Invalid pchannel format invalid_config_5 = { @@ -319,7 +306,9 @@ class TestCDCTopologySetup(TestCDCSyncBase): "cross_cluster_topology": [], } with pytest.raises(Exception): - upstream_client.update_replicate_configuration(**invalid_config_5) + upstream_client.update_replicate_configuration( + timeout=CDC_UPDATE_REPLICATE_TIMEOUT_SECONDS, **invalid_config_5 + ) @pytest.mark.order(-1) def test_clear_configuration_disconnect( @@ -341,10 +330,7 @@ class TestCDCTopologySetup(TestCDCSyncBase): { "cluster_id": source_cluster_id, "connection_param": {"uri": upstream_uri, "token": upstream_token}, - "pchannels": [ - f"{source_cluster_id}-rootcoord-dml_{i}" - for i in range(pchannel_num) - ], + "pchannels": [f"{source_cluster_id}-rootcoord-dml_{i}" for i in range(pchannel_num)], }, { "cluster_id": target_cluster_id, @@ -352,10 +338,7 @@ class TestCDCTopologySetup(TestCDCSyncBase): "uri": downstream_uri, "token": downstream_token, }, - "pchannels": [ - f"{target_cluster_id}-rootcoord-dml_{i}" - for i in range(pchannel_num) - ], + "pchannels": [f"{target_cluster_id}-rootcoord-dml_{i}" for i in range(pchannel_num)], }, ], "cross_cluster_topology": [ @@ -366,8 +349,7 @@ class TestCDCTopologySetup(TestCDCSyncBase): ], } - upstream_client.update_replicate_configuration(**config) - downstream_client.update_replicate_configuration(**config) + apply_replicate_configuration([(upstream_client, config), (downstream_client, config)]) time.sleep(3) # Verify topology is working @@ -380,9 +362,7 @@ class TestCDCTopologySetup(TestCDCSyncBase): def check_initial_sync(): return downstream_client.has_collection(test_collection_name) - assert self.wait_for_sync( - check_initial_sync, 30, f"initial collection {test_collection_name} sync" - ) + assert self.wait_for_sync(check_initial_sync, 30, f"initial collection {test_collection_name} sync") # Now clear the configuration (empty topology) empty_upstream_config = { @@ -390,10 +370,7 @@ class TestCDCTopologySetup(TestCDCSyncBase): { "cluster_id": source_cluster_id, "connection_param": {"uri": upstream_uri, "token": upstream_token}, - "pchannels": [ - f"{source_cluster_id}-rootcoord-dml_{i}" - for i in range(pchannel_num) - ], + "pchannels": [f"{source_cluster_id}-rootcoord-dml_{i}" for i in range(pchannel_num)], } ], "cross_cluster_topology": [], @@ -406,17 +383,15 @@ class TestCDCTopologySetup(TestCDCSyncBase): "uri": downstream_uri, "token": downstream_token, }, - "pchannels": [ - f"{target_cluster_id}-rootcoord-dml_{i}" - for i in range(pchannel_num) - ], + "pchannels": [f"{target_cluster_id}-rootcoord-dml_{i}" for i in range(pchannel_num)], } ], "cross_cluster_topology": [], } # Apply empty configuration to disconnect CDC - upstream_client.update_replicate_configuration(**empty_upstream_config) - downstream_client.update_replicate_configuration(**empty_downstream_config) + apply_replicate_configuration( + [(upstream_client, empty_upstream_config), (downstream_client, empty_downstream_config)] + ) time.sleep(3) # Test that CDC is disconnected - create new collection and verify it doesn't sync diff --git a/tests/python_client/cdc/testcases/test_switchover.py b/tests/python_client/cdc/testcases/test_switchover.py new file mode 100644 index 0000000000..159d459147 --- /dev/null +++ b/tests/python_client/cdc/testcases/test_switchover.py @@ -0,0 +1,793 @@ +""" +CDC sync tests for topology switchover and failover scenarios. +""" + +import random +import subprocess +import threading +import time + +from .base import TestCDCSyncBase, logger + + +class TestCDCSyncSwitchover(TestCDCSyncBase): + """Test CDC sync behaviour during and after topology switchover / failover.""" + + def setup_method(self): + """Setup for each test method.""" + self.resources_to_cleanup = [] + + def teardown_method(self): + """Cleanup after each test method - only cleanup upstream, downstream will sync.""" + upstream_client = getattr(self, "_upstream_client", None) + + if upstream_client: + for resource_type, resource_name in self.resources_to_cleanup: + if resource_type == "collection": + self.cleanup_collection(upstream_client, resource_name) + + time.sleep(1) # Allow cleanup to sync to downstream + + # ------------------------------------------------------------------------- + # Tests + # ------------------------------------------------------------------------- + + def test_switchover_basic( + self, + upstream_client, + downstream_client, + sync_timeout, + switchover_helper, + source_cluster_id, + target_cluster_id, + ): + """ + Basic switchover: insert 200 records, verify sync, switchover, insert 200 more + on the new source, verify count == 400 on both sides, sample verify, switch back. + """ + start_time = time.time() + c_name = self.gen_unique_name("test_sw_basic", max_length=50) + + self.log_test_start("test_switchover_basic", "SWITCHOVER_BASIC", c_name) + self._upstream_client = upstream_client + self.resources_to_cleanup.append(("collection", c_name)) + + try: + self.cleanup_collection(upstream_client, c_name) + + schema = self.create_manual_id_schema(upstream_client) + upstream_client.create_collection(collection_name=c_name, schema=schema) + + index_params = upstream_client.prepare_index_params() + index_params.add_index( + field_name="vector", + index_type="HNSW", + metric_type="L2", + params={"M": 16, "efConstruction": 200}, + ) + upstream_client.create_index(c_name, index_params) + upstream_client.load_collection(c_name) + + # Wait for collection to appear on downstream + def check_create(): + return downstream_client.has_collection(c_name) + + assert self.wait_for_sync(check_create, sync_timeout, f"create collection {c_name}") + + # Insert 200 records on upstream (original source) + batch1 = self.generate_test_data_with_id(200, start_id=0) + upstream_client.insert(c_name, batch1) + upstream_client.flush(c_name) + + def check_200(): + try: + res = downstream_client.query( + collection_name=c_name, + filter="", + output_fields=["count(*)"], + ) + cnt = res[0]["count(*)"] if res else 0 + logger.info(f"[SYNC_PROGRESS] downstream count: {cnt}/200") + return cnt >= 200 + except Exception as e: + logger.warning(f"Sync check failed: {e}") + return False + + assert self.wait_for_sync(check_200, sync_timeout, f"initial 200-record sync {c_name}") + + # Switchover: target becomes new source + logger.info("[SWITCHOVER] Initiating basic switchover...") + switchover_helper(target_cluster_id, source_cluster_id) + + # Insert 200 more records on the new source (previously downstream) + batch2 = self.generate_test_data_with_id(200, start_id=200) + downstream_client.insert(c_name, batch2) + downstream_client.flush(c_name) + + def check_400_upstream(): + try: + res = upstream_client.query( + collection_name=c_name, + filter="", + output_fields=["count(*)"], + ) + cnt = res[0]["count(*)"] if res else 0 + logger.info(f"[SYNC_PROGRESS] upstream count after switchover: {cnt}/400") + return cnt >= 400 + except Exception as e: + logger.warning(f"Sync check failed: {e}") + return False + + def check_400_downstream(): + try: + res = downstream_client.query( + collection_name=c_name, + filter="", + output_fields=["count(*)"], + ) + cnt = res[0]["count(*)"] if res else 0 + logger.info(f"[SYNC_PROGRESS] downstream count after switchover: {cnt}/400") + return cnt >= 400 + except Exception as e: + logger.warning(f"Sync check failed: {e}") + return False + + assert self.wait_for_sync(check_400_upstream, sync_timeout, f"400-record sync upstream {c_name}") + assert self.wait_for_sync(check_400_downstream, sync_timeout, f"400-record sync downstream {c_name}") + + # Sample verify + match_count, mismatch_count, details = self.verify_data_sampling( + downstream_client, # new source + upstream_client, # new target + c_name, + sample_ratio=0.2, + output_fields=["id", "vector"], + ) + assert mismatch_count == 0, f"Data mismatch after basic switchover: {details[:5]}" + + finally: + # Restore original topology + logger.info("[SWITCHOVER] Restoring original topology after test_switchover_basic...") + switchover_helper(source_cluster_id, target_cluster_id) + self.log_test_end("test_switchover_basic", True, time.time() - start_time) + + def test_switchover_during_writes( + self, + upstream_client, + downstream_client, + sync_timeout, + switchover_helper, + source_cluster_id, + target_cluster_id, + ): + """ + Switchover during concurrent writes: background thread inserts 10 batches of 100 + with 2 s between each; switchover fires at t=12 s; join thread, flush, verify + both sides have the same count >= total_inserted. + """ + start_time = time.time() + c_name = self.gen_unique_name("test_sw_writes", max_length=50) + + self.log_test_start("test_switchover_during_writes", "SWITCHOVER_DURING_WRITES", c_name) + self._upstream_client = upstream_client + self.resources_to_cleanup.append(("collection", c_name)) + + try: + self.cleanup_collection(upstream_client, c_name) + + schema = self.create_manual_id_schema(upstream_client) + upstream_client.create_collection(collection_name=c_name, schema=schema) + + index_params = upstream_client.prepare_index_params() + index_params.add_index( + field_name="vector", + index_type="HNSW", + metric_type="L2", + params={"M": 16, "efConstruction": 200}, + ) + upstream_client.create_index(c_name, index_params) + upstream_client.load_collection(c_name) + + def check_create(): + return downstream_client.has_collection(c_name) + + assert self.wait_for_sync(check_create, sync_timeout, f"create collection {c_name}") + + total_inserted = 0 + lock = threading.Lock() + write_error = [] + + def background_insert(): + nonlocal total_inserted + for batch_idx in range(10): + try: + start_id = batch_idx * 100 + data = self.generate_test_data_with_id(100, start_id=start_id) + # Cap pymilvus's retry loop at 10 (default 75). A batch + # that lands on the old primary after it flips to + # replica returns STREAMING_CODE_REPLICATE_VIOLATION; + # the default retry burns ~213 s per failed batch and + # makes the thread runtime unbounded when the role flip + # takes longer than one batch's cadence. 10 retries + # with the decorator's 3 s backoff cap bound each + # failure to ~16 s. + upstream_client.insert(c_name, data, retry_times=10) + with lock: + total_inserted += 100 + logger.info(f"[BACKGROUND] Inserted batch {batch_idx + 1}/10 (total: {total_inserted})") + except Exception as e: + logger.error(f"[BACKGROUND] Insert failed on batch {batch_idx}: {e}") + write_error.append(e) + time.sleep(2) + + insert_thread = threading.Thread(target=background_insert, daemon=True) + insert_thread.start() + + # Switchover after 12 s (mid-way through background inserts) + time.sleep(12) + logger.info("[SWITCHOVER] Initiating switchover during writes...") + switchover_helper(target_cluster_id, source_cluster_id) + + # With retry_times=10 each failed batch burns ~16 s (vs ~213 s with + # the pymilvus default of 75). Worst case: all 10 batches fail → + # ~10 * (16 + 2) = 180 s. 180 s gives ~6x margin. + insert_thread.join(timeout=180) + assert not insert_thread.is_alive(), "Background insert thread did not finish in time" + + # Transient STREAMING_CODE_REPLICATE_VIOLATION failures are EXPECTED + # during the role flip: writes that arrive on the old primary after + # it becomes a replica are rejected by design. The invariant that + # matters is that every batch which SUCCEEDED ends up replicated + # consistently (enforced by the up_cnt == down_cnt check below). + if write_error: + logger.warning( + f"Background inserts had {len(write_error)} transient failure(s) during switchover: {write_error}" + ) + + # After switchover, downstream is the new primary. Flushing the + # old primary (upstream) hits the replica-role rate limiter + # (rate=0.1/s) and exhausts pymilvus's 75 retries. Flush the + # current primary instead. + downstream_client.flush(c_name) + + expected = total_inserted + logger.info(f"[INFO] Total inserted: {expected}") + + def check_both(client, label): + def _check(): + try: + res = client.query( + collection_name=c_name, + filter="", + output_fields=["count(*)"], + ) + cnt = res[0]["count(*)"] if res else 0 + logger.info(f"[SYNC_PROGRESS] {label} count: {cnt}/{expected}") + return cnt >= expected + except Exception as e: + logger.warning(f"Sync check failed ({label}): {e}") + return False + + return _check + + assert self.wait_for_sync( + check_both(upstream_client, "upstream"), + sync_timeout, + f"upstream count>={expected} after switchover-during-writes", + ) + assert self.wait_for_sync( + check_both(downstream_client, "downstream"), + sync_timeout, + f"downstream count>={expected} after switchover-during-writes", + ) + + up_res = upstream_client.query(collection_name=c_name, filter="", output_fields=["count(*)"]) + down_res = downstream_client.query(collection_name=c_name, filter="", output_fields=["count(*)"]) + up_cnt = up_res[0]["count(*)"] if up_res else 0 + down_cnt = down_res[0]["count(*)"] if down_res else 0 + assert up_cnt == down_cnt, ( + f"Count mismatch after switchover-during-writes: upstream={up_cnt}, downstream={down_cnt}" + ) + + finally: + logger.info("[SWITCHOVER] Restoring original topology after test_switchover_during_writes...") + switchover_helper(source_cluster_id, target_cluster_id) + self.log_test_end("test_switchover_during_writes", True, time.time() - start_time) + + def test_switchover_with_all_data_types( + self, + upstream_client, + downstream_client, + sync_timeout, + switchover_helper, + source_cluster_id, + target_cluster_id, + ): + """ + Switchover with comprehensive data types: insert 100 records, verify sync, + switchover, verify scalar field sampling, switch back. + """ + start_time = time.time() + c_name = self.gen_unique_name("test_sw_dtypes", max_length=50) + + self.log_test_start( + "test_switchover_with_all_data_types", + "SWITCHOVER_ALL_DTYPES", + c_name, + ) + self._upstream_client = upstream_client + self.resources_to_cleanup.append(("collection", c_name)) + + try: + self.cleanup_collection(upstream_client, c_name) + + schema = self.create_comprehensive_schema(upstream_client) + upstream_client.create_collection(collection_name=c_name, schema=schema) + + # Index every vector field; load_collection fails otherwise. + index_params = upstream_client.prepare_index_params() + index_params.add_index(field_name="float_vector", index_type="AUTOINDEX", metric_type="L2") + index_params.add_index(field_name="float16_vector", index_type="AUTOINDEX", metric_type="L2") + index_params.add_index(field_name="binary_vector", index_type="BIN_FLAT", metric_type="HAMMING") + index_params.add_index(field_name="sparse_vector", index_type="SPARSE_INVERTED_INDEX", metric_type="IP") + upstream_client.create_index(c_name, index_params) + upstream_client.load_collection(c_name) + + def check_create(): + return downstream_client.has_collection(c_name) + + assert self.wait_for_sync(check_create, sync_timeout, f"create collection {c_name}") + + data = self.generate_comprehensive_test_data(100) + upstream_client.insert(c_name, data) + upstream_client.flush(c_name) + + def check_100(): + try: + res = downstream_client.query( + collection_name=c_name, + filter="", + output_fields=["count(*)"], + ) + cnt = res[0]["count(*)"] if res else 0 + logger.info(f"[SYNC_PROGRESS] downstream count: {cnt}/100") + return cnt >= 100 + except Exception as e: + logger.warning(f"Sync check failed: {e}") + return False + + assert self.wait_for_sync(check_100, sync_timeout, f"100-record sync {c_name}") + + # Switchover + logger.info("[SWITCHOVER] Initiating switchover with all data types...") + switchover_helper(target_cluster_id, source_cluster_id) + + # After switchover, verify scalar field sampling (new source is downstream) + scalar_fields = [ + "bool_field", + "int8_field", + "int16_field", + "int32_field", + "int64_field", + "float_field", + "double_field", + "varchar_field", + ] + match_count, mismatch_count, details = self.verify_data_sampling( + downstream_client, # new source + upstream_client, # new target + c_name, + sample_ratio=0.3, + output_fields=scalar_fields, + ) + logger.info(f"[RESULT] All-dtypes sampling — match={match_count}, mismatch={mismatch_count}") + assert mismatch_count == 0, f"Data mismatch after switchover with all types: {details[:5]}" + + finally: + logger.info("[SWITCHOVER] Restoring original topology after test_switchover_with_all_data_types...") + switchover_helper(source_cluster_id, target_cluster_id) + self.log_test_end( + "test_switchover_with_all_data_types", + True, + time.time() - start_time, + ) + + def test_switchover_with_loaded_collection( + self, + upstream_client, + downstream_client, + sync_timeout, + switchover_helper, + source_cluster_id, + target_cluster_id, + ): + """ + Switchover with a loaded collection: verify search works on downstream before + switchover; after switchover verify search works on both sides. + """ + start_time = time.time() + c_name = self.gen_unique_name("test_sw_loaded", max_length=50) + + self.log_test_start( + "test_switchover_with_loaded_collection", + "SWITCHOVER_LOADED", + c_name, + ) + self._upstream_client = upstream_client + self.resources_to_cleanup.append(("collection", c_name)) + + try: + self.cleanup_collection(upstream_client, c_name) + + schema = self.create_default_schema(upstream_client) + upstream_client.create_collection(collection_name=c_name, schema=schema) + + index_params = upstream_client.prepare_index_params() + index_params.add_index( + field_name="vector", + index_type="HNSW", + metric_type="L2", + params={"M": 16, "efConstruction": 200}, + ) + upstream_client.create_index(c_name, index_params) + upstream_client.load_collection(c_name) + + def check_create(): + return downstream_client.has_collection(c_name) + + assert self.wait_for_sync(check_create, sync_timeout, f"create collection {c_name}") + + data = self.generate_test_data(200) + upstream_client.insert(c_name, data) + upstream_client.flush(c_name) + + def check_200(): + try: + res = downstream_client.query( + collection_name=c_name, + filter="", + output_fields=["count(*)"], + ) + cnt = res[0]["count(*)"] if res else 0 + logger.info(f"[SYNC_PROGRESS] downstream count: {cnt}/200") + return cnt >= 200 + except Exception as e: + logger.warning(f"Sync check failed: {e}") + return False + + assert self.wait_for_sync(check_200, sync_timeout, f"200-record sync {c_name}") + + # Verify search works on downstream before switchover + q_vec = [random.random() for _ in range(128)] + pre_down_results = downstream_client.search( + collection_name=c_name, + data=[q_vec], + anns_field="vector", + search_params={"metric_type": "L2"}, + limit=5, + output_fields=["id"], + ) + assert pre_down_results and len(pre_down_results[0]) > 0, ( + "Search on downstream returned no results before switchover" + ) + + # Switchover + logger.info("[SWITCHOVER] Initiating switchover with loaded collection...") + switchover_helper(target_cluster_id, source_cluster_id) + + # Verify search still works on both sides after switchover + for client, label in [(upstream_client, "upstream"), (downstream_client, "downstream")]: + results = client.search( + collection_name=c_name, + data=[q_vec], + anns_field="vector", + search_params={"metric_type": "L2"}, + limit=5, + output_fields=["id"], + ) + assert results and len(results[0]) > 0, f"Search returned no results on {label} after switchover" + logger.info(f"[VERIFY] Search on {label} after switchover returned {len(results[0])} results — OK") + + finally: + logger.info("[SWITCHOVER] Restoring original topology after test_switchover_with_loaded_collection...") + switchover_helper(source_cluster_id, target_cluster_id) + self.log_test_end( + "test_switchover_with_loaded_collection", + True, + time.time() - start_time, + ) + + def test_switchover_with_index( + self, + upstream_client, + downstream_client, + sync_timeout, + switchover_helper, + source_cluster_id, + target_cluster_id, + ): + """ + Switchover after index creation: wait for index to sync, switchover, verify + list_indexes returns the same index on both sides. + """ + start_time = time.time() + c_name = self.gen_unique_name("test_sw_index", max_length=50) + + self.log_test_start("test_switchover_with_index", "SWITCHOVER_INDEX", c_name) + self._upstream_client = upstream_client + self.resources_to_cleanup.append(("collection", c_name)) + + try: + self.cleanup_collection(upstream_client, c_name) + + schema = self.create_default_schema(upstream_client) + upstream_client.create_collection(collection_name=c_name, schema=schema) + + # Insert data first + data = self.generate_test_data(200) + upstream_client.insert(c_name, data) + upstream_client.flush(c_name) + + # Create HNSW index + index_params = upstream_client.prepare_index_params() + index_params.add_index( + field_name="vector", + index_type="HNSW", + metric_type="COSINE", + params={"M": 16, "efConstruction": 200}, + ) + upstream_client.create_index(c_name, index_params) + + # Wait for index to sync to downstream + def check_index_sync(): + try: + indexes = downstream_client.list_indexes(c_name) + result = len(indexes) > 0 + logger.info(f"[SYNC_PROGRESS] downstream indexes: {indexes} (has_index={result})") + return result + except Exception as e: + logger.warning(f"Index sync check failed: {e}") + return False + + assert self.wait_for_sync(check_index_sync, sync_timeout, f"index sync {c_name}"), ( + f"Index did not sync to downstream within {sync_timeout}s" + ) + + # Switchover + logger.info("[SWITCHOVER] Initiating switchover with index...") + switchover_helper(target_cluster_id, source_cluster_id) + + # Verify list_indexes on both sides + up_indexes = upstream_client.list_indexes(c_name) + down_indexes = downstream_client.list_indexes(c_name) + + logger.info(f"[VERIFY] After switchover — upstream indexes={up_indexes}, downstream indexes={down_indexes}") + assert len(up_indexes) > 0, "No indexes on upstream after switchover" + assert len(down_indexes) > 0, "No indexes on downstream after switchover" + assert set(up_indexes) == set(down_indexes), ( + f"Index lists differ after switchover: up={up_indexes}, down={down_indexes}" + ) + + finally: + logger.info("[SWITCHOVER] Restoring original topology after test_switchover_with_index...") + switchover_helper(source_cluster_id, target_cluster_id) + self.log_test_end("test_switchover_with_index", True, time.time() - start_time) + + def test_rapid_switchover_stress( + self, + upstream_client, + downstream_client, + sync_timeout, + switchover_helper, + source_cluster_id, + target_cluster_id, + ): + """ + Rapid-switchover stress: loop 5 times — write 50 records to current source, + switchover. After loop, wait 30 s and verify counts match. Restore original topology. + """ + start_time = time.time() + c_name = self.gen_unique_name("test_sw_stress", max_length=50) + + self.log_test_start("test_rapid_switchover_stress", "RAPID_SWITCHOVER_STRESS", c_name) + self._upstream_client = upstream_client + self.resources_to_cleanup.append(("collection", c_name)) + + try: + self.cleanup_collection(upstream_client, c_name) + + schema = self.create_manual_id_schema(upstream_client) + upstream_client.create_collection(collection_name=c_name, schema=schema) + + index_params = upstream_client.prepare_index_params() + index_params.add_index( + field_name="vector", + index_type="HNSW", + metric_type="COSINE", + params={"M": 16, "efConstruction": 200}, + ) + upstream_client.create_index(c_name, index_params) + upstream_client.load_collection(c_name) + + def check_create(): + return downstream_client.has_collection(c_name) + + assert self.wait_for_sync(check_create, sync_timeout, f"create collection {c_name}") + + # Track topology state + current_source_id = source_cluster_id + current_target_id = target_cluster_id + current_source_client = upstream_client + total_inserted = 0 + + for iteration in range(5): + # Write 50 records to current source + start_id = iteration * 50 + data = self.generate_test_data_with_id(50, start_id=start_id) + current_source_client.insert(c_name, data) + current_source_client.flush(c_name) + total_inserted += 50 + + logger.info( + f"[STRESS] Iteration {iteration + 1}/5: inserted 50 records " + f"(total={total_inserted}), switching over..." + ) + + # Switchover + switchover_helper(current_target_id, current_source_id) + + # Swap topology + current_source_id, current_target_id = current_target_id, current_source_id + current_source_client = downstream_client if current_source_id == target_cluster_id else upstream_client + + logger.info( + f"[STRESS] All 5 switchovers done. Waiting 30s for final sync (total_inserted={total_inserted})..." + ) + time.sleep(30) + + # Verify counts match on both sides + up_res = upstream_client.query(collection_name=c_name, filter="", output_fields=["count(*)"]) + down_res = downstream_client.query(collection_name=c_name, filter="", output_fields=["count(*)"]) + up_cnt = up_res[0]["count(*)"] if up_res else 0 + down_cnt = down_res[0]["count(*)"] if down_res else 0 + + logger.info(f"[VERIFY] After stress — upstream={up_cnt}, downstream={down_cnt}, expected>={total_inserted}") + assert up_cnt >= total_inserted, f"Upstream count {up_cnt} < expected {total_inserted} after stress" + assert down_cnt >= total_inserted, f"Downstream count {down_cnt} < expected {total_inserted} after stress" + assert up_cnt == down_cnt, f"Count mismatch after stress: upstream={up_cnt}, downstream={down_cnt}" + + finally: + # Restore to original topology + logger.info("[SWITCHOVER] Restoring original topology after test_rapid_switchover_stress...") + switchover_helper(source_cluster_id, target_cluster_id) + self.log_test_end("test_rapid_switchover_stress", True, time.time() - start_time) + + def test_failover_source_down( + self, + upstream_client, + downstream_client, + sync_timeout, + switchover_helper, + source_cluster_id, + target_cluster_id, + milvus_ns, + ): + """ + Failover when source goes down: insert 500 records, verify sync, kill source pods, + verify target count >= 500, wait for source to recover and verify count >= 500. + """ + start_time = time.time() + c_name = self.gen_unique_name("test_failover_src", max_length=50) + + self.log_test_start("test_failover_source_down", "FAILOVER_SOURCE_DOWN", c_name) + self._upstream_client = upstream_client + self.resources_to_cleanup.append(("collection", c_name)) + + try: + self.cleanup_collection(upstream_client, c_name) + + schema = self.create_manual_id_schema(upstream_client) + upstream_client.create_collection(collection_name=c_name, schema=schema) + + index_params = upstream_client.prepare_index_params() + index_params.add_index( + field_name="vector", + index_type="HNSW", + metric_type="L2", + params={"M": 16, "efConstruction": 200}, + ) + upstream_client.create_index(c_name, index_params) + upstream_client.load_collection(c_name) + + def check_create(): + return downstream_client.has_collection(c_name) + + assert self.wait_for_sync(check_create, sync_timeout, f"create collection {c_name}") + + # Insert 500 records + data = self.generate_test_data_with_id(500, start_id=0) + upstream_client.insert(c_name, data) + upstream_client.flush(c_name) + + def check_500_downstream(): + try: + res = downstream_client.query( + collection_name=c_name, + filter="", + output_fields=["count(*)"], + ) + cnt = res[0]["count(*)"] if res else 0 + logger.info(f"[SYNC_PROGRESS] downstream count: {cnt}/500") + return cnt >= 500 + except Exception as e: + logger.warning(f"Sync check failed: {e}") + return False + + assert self.wait_for_sync(check_500_downstream, sync_timeout, f"500-record sync {c_name}"), ( + f"Downstream did not receive 500 records within {sync_timeout}s" + ) + + # Kill source pods forcefully + logger.info(f"[FAILOVER] Killing source pods (instance={source_cluster_id}, ns={milvus_ns})...") + kill_cmd = [ + "kubectl", + "delete", + "pods", + "-l", + f"app.kubernetes.io/instance={source_cluster_id}", + "-n", + milvus_ns, + "--grace-period=0", + "--force", + ] + result = subprocess.run(kill_cmd, capture_output=True, text=True) + logger.info( + f"[FAILOVER] kubectl output: stdout={result.stdout!r}, stderr={result.stderr!r}, rc={result.returncode}" + ) + + # Wait 60 s and verify target still has data + logger.info("[FAILOVER] Waiting 60s after pod kill...") + time.sleep(60) + + def check_target_intact(): + try: + res = downstream_client.query( + collection_name=c_name, + filter="", + output_fields=["count(*)"], + ) + cnt = res[0]["count(*)"] if res else 0 + logger.info(f"[FAILOVER] Target count after kill: {cnt}") + return cnt >= 500 + except Exception as e: + logger.warning(f"Target check failed: {e}") + return False + + assert self.wait_for_sync(check_target_intact, 30, f"target intact after source kill {c_name}"), ( + "Target lost data after source pod kill" + ) + + # Wait 120 s more for source to recover, then verify + logger.info("[FAILOVER] Waiting 120s for source recovery...") + time.sleep(120) + + def check_source_recovered(): + try: + res = upstream_client.query( + collection_name=c_name, + filter="", + output_fields=["count(*)"], + ) + cnt = res[0]["count(*)"] if res else 0 + logger.info(f"[FAILOVER] Source count after recovery: {cnt}") + return cnt >= 500 + except Exception as e: + logger.warning(f"Source recovery check failed: {e}") + return False + + assert self.wait_for_sync(check_source_recovered, 60, f"source recovery {c_name}"), ( + "Source did not recover with expected data count within timeout" + ) + + finally: + self.log_test_end("test_failover_source_down", True, time.time() - start_time) diff --git a/tests/python_client/common/common_type.py b/tests/python_client/common/common_type.py index 5247dc9bc4..152a25372e 100644 --- a/tests/python_client/common/common_type.py +++ b/tests/python_client/common/common_type.py @@ -24,13 +24,13 @@ max_top_k = 16384 max_nq = 16384 max_partition_num = 1024 max_role_num = 10 -default_partition_num = 16 # default num_partitions for partition key feature +default_partition_num = 16 # default num_partitions for partition key feature default_segment_row_limit = 1000 default_server_segment_row_limit = 1024 * 512 default_alias = "default" default_user = "root" default_password = "Milvus" -default_primary_field_name = 'pk' +default_primary_field_name = "pk" default_bool_field_name = "bool" default_int8_field_name = "int8" default_int16_field_name = "int16" @@ -63,13 +63,13 @@ default_reranker_field_name = "reranker_field" default_new_field_name = "field_new" all_vector_types = [ - DataType.FLOAT_VECTOR, - DataType.FLOAT16_VECTOR, - DataType.BFLOAT16_VECTOR, - DataType.SPARSE_FLOAT_VECTOR, - DataType.INT8_VECTOR, - DataType.BINARY_VECTOR, - ] + DataType.FLOAT_VECTOR, + DataType.FLOAT16_VECTOR, + DataType.BFLOAT16_VECTOR, + DataType.SPARSE_FLOAT_VECTOR, + DataType.INT8_VECTOR, + DataType.BINARY_VECTOR, +] default_metric_for_vector_type = { DataType.FLOAT_VECTOR: "COSINE", @@ -90,35 +90,51 @@ all_scalar_data_types = [ DataType.DOUBLE, DataType.VARCHAR, DataType.ARRAY, - DataType.JSON, + DataType.JSON, DataType.GEOMETRY, - DataType.TIMESTAMPTZ - ] + DataType.TIMESTAMPTZ, +] default_field_name_map = { - DataType.INT8: default_int8_field_name, - DataType.INT16: default_int16_field_name, - DataType.INT32: default_int32_field_name, - DataType.INT64: default_int64_field_name, - DataType.BOOL: default_bool_field_name, - DataType.FLOAT: default_float_field_name, - DataType.DOUBLE: default_double_field_name, - DataType.VARCHAR: default_string_field_name, - DataType.ARRAY: default_array_field_name, - DataType.JSON: default_json_field_name, - DataType.FLOAT_VECTOR: default_float_vec_field_name, - DataType.FLOAT16_VECTOR: default_float16_vec_field_name, - DataType.BFLOAT16_VECTOR: default_bfloat16_vec_field_name, - DataType.SPARSE_FLOAT_VECTOR: default_sparse_vec_field_name, - DataType.INT8_VECTOR: default_int8_vec_field_name, - DataType.BINARY_VECTOR: default_binary_vec_field_name, - } + DataType.INT8: default_int8_field_name, + DataType.INT16: default_int16_field_name, + DataType.INT32: default_int32_field_name, + DataType.INT64: default_int64_field_name, + DataType.BOOL: default_bool_field_name, + DataType.FLOAT: default_float_field_name, + DataType.DOUBLE: default_double_field_name, + DataType.VARCHAR: default_string_field_name, + DataType.ARRAY: default_array_field_name, + DataType.JSON: default_json_field_name, + DataType.FLOAT_VECTOR: default_float_vec_field_name, + DataType.FLOAT16_VECTOR: default_float16_vec_field_name, + DataType.BFLOAT16_VECTOR: default_bfloat16_vec_field_name, + DataType.SPARSE_FLOAT_VECTOR: default_sparse_vec_field_name, + DataType.INT8_VECTOR: default_int8_vec_field_name, + DataType.BINARY_VECTOR: default_binary_vec_field_name, +} -append_vector_type = [DataType.FLOAT16_VECTOR, DataType.BFLOAT16_VECTOR, DataType.SPARSE_FLOAT_VECTOR, DataType.INT8_VECTOR] -all_dense_vector_types = [DataType.FLOAT_VECTOR, DataType.FLOAT16_VECTOR, DataType.BFLOAT16_VECTOR, DataType.INT8_VECTOR] -all_float_vector_dtypes = [DataType.FLOAT_VECTOR, DataType.FLOAT16_VECTOR, DataType.BFLOAT16_VECTOR, DataType.SPARSE_FLOAT_VECTOR, DataType.INT8_VECTOR] +append_vector_type = [ + DataType.FLOAT16_VECTOR, + DataType.BFLOAT16_VECTOR, + DataType.SPARSE_FLOAT_VECTOR, + DataType.INT8_VECTOR, +] +all_dense_vector_types = [ + DataType.FLOAT_VECTOR, + DataType.FLOAT16_VECTOR, + DataType.BFLOAT16_VECTOR, + DataType.INT8_VECTOR, +] +all_float_vector_dtypes = [ + DataType.FLOAT_VECTOR, + DataType.FLOAT16_VECTOR, + DataType.BFLOAT16_VECTOR, + DataType.SPARSE_FLOAT_VECTOR, + DataType.INT8_VECTOR, +] default_partition_name = "_default" -default_resource_group_name = '__default_resource_group' +default_resource_group_name = "__default_resource_group" default_resource_group_capacity = 1000000 default_tag = "1970_01_01" row_count = "row_count" @@ -179,43 +195,37 @@ rows_all_data_type_file_path = "/tmp/rows_all_data_type" """" List of parameters used to pass """ invalid_resource_names = [ - None, # None - " ", # space - "", # empty - "12name", # start with number - "n12 ame", # contain space - "n-ame", # contain hyphen - "nam(e)", # contain special character - "name中文", # contain Chinese character - "name%$#", # contain special character - "".join("a" for i in range(max_name_length + 1))] # exceed max length + None, # None + " ", # space + "", # empty + "12name", # start with number + "n12 ame", # contain space + "n-ame", # contain hyphen + "nam(e)", # contain special character + "name中文", # contain Chinese character + "name%$#", # contain special character + "".join("a" for i in range(max_name_length + 1)), +] # exceed max length valid_resource_names = [ - "name", # valid name - "_name", # start with underline - "_12name", # start with underline and contains number - "n12ame_", # end with letter and contains number and underline - "nam_e", # contains underline - "".join("a" for i in range(max_name_length))] # max length + "name", # valid name + "_name", # start with underline + "_12name", # start with underline and contains number + "n12ame_", # end with letter and contains number and underline + "nam_e", # contains underline + "".join("a" for i in range(max_name_length)), +] # max length -invalid_dims = [min_dim-1, 32.1, -32, "vii", "十六", max_dim+1] +invalid_dims = [min_dim - 1, 32.1, -32, "vii", "十六", max_dim + 1] -get_not_string = [ - [], - {}, - None, - (1,), - 1, - 1.0, - [1, "2", 3] -] +get_not_string = [[], {}, None, (1,), 1, 1.0, [1, "2", 3]] get_invalid_vectors = [ "1*2", [1], [1, 2], [" "], - ['a'], + ["a"], [None], None, (1, 2), @@ -225,7 +235,7 @@ get_invalid_vectors = [ "String", " siede ", "中文", - "a".join("a" for i in range(256)) + "a".join("a" for i in range(256)), ] get_invalid_ints = [ @@ -239,7 +249,7 @@ get_invalid_ints = [ "String", "=c", "中文", - "a".join("a" for i in range(256)) + "a".join("a" for i in range(256)), ] get_invalid_dict = [ @@ -254,7 +264,7 @@ get_invalid_dict = [ {1: 1}, {"中文": 1}, {"%$#": ["a"]}, - {"a".join("a" for i in range(256)): "a"} + {"a".join("a" for i in range(256)): "a"}, ] get_invalid_metric_type = [ @@ -269,55 +279,105 @@ get_invalid_metric_type = [ "(mn)", "中文", "%$#", - "".join("a" for i in range(max_name_length + 1))] - -get_dict_without_host_port = [ - {"host": "host"}, - {"": ""} + "".join("a" for i in range(max_name_length + 1)), ] -get_wrong_format_dict = [ - {"host": "string_host", "port": {}}, - {"host": 0, "port": 19520} -] +get_dict_without_host_port = [{"host": "host"}, {"": ""}] + +get_wrong_format_dict = [{"host": "string_host", "port": {}}, {"host": 0, "port": 19520}] get_all_kind_data_distribution = [ - 1, np.float64(1.0), np.double(1.0), 9707199254740993.0, 9707199254740992, - '1', '123', '321', '213', True, False, None, [1, 2], [1.0, 2], {}, {"a": 1}, - {'a': 1.0}, {'a': 9707199254740993.0}, {'a': 9707199254740992}, {'a': '1'}, {'a': '123'}, - {'a': '321'}, {'a': '213'}, {'a': True}, {'a': [1, 2, 3]}, {'a': [1.0, 2, '1']}, {'a': [1.0, 2]}, - {'a': None}, {'a': {'b': 1}}, {'a': {'b': 1.0}}, {'a': [{'b': 1}, 2.0, np.double(3.0), '4', True, [1, 3.0], None]} + 1, + np.float64(1.0), + np.double(1.0), + 9707199254740993.0, + 9707199254740992, + "1", + "123", + "321", + "213", + True, + False, + None, + [1, 2], + [1.0, 2], + {}, + {"a": 1}, + {"a": 1.0}, + {"a": 9707199254740993.0}, + {"a": 9707199254740992}, + {"a": "1"}, + {"a": "123"}, + {"a": "321"}, + {"a": "213"}, + {"a": True}, + {"a": [1, 2, 3]}, + {"a": [1.0, 2, "1"]}, + {"a": [1.0, 2]}, + {"a": None}, + {"a": {"b": 1}}, + {"a": {"b": 1.0}}, + {"a": [{"b": 1}, 2.0, np.double(3.0), "4", True, [1, 3.0], None]}, ] """ Specially defined list """ L0_index_types = ["IVF_SQ8", "HNSW", "DISKANN"] -all_index_types = ["FLAT", "IVF_FLAT", "IVF_SQ8", "IVF_PQ", - "IVF_RABITQ", - "HNSW", "SCANN", "DISKANN", - "BIN_FLAT", "BIN_IVF_FLAT", - "SPARSE_INVERTED_INDEX", "SPARSE_WAND", - "GPU_IVF_FLAT", "GPU_IVF_PQ"] +all_index_types = [ + "FLAT", + "IVF_FLAT", + "IVF_SQ8", + "IVF_PQ", + "IVF_RABITQ", + "HNSW", + "SCANN", + "DISKANN", + "BIN_FLAT", + "BIN_IVF_FLAT", + "SPARSE_INVERTED_INDEX", + "SPARSE_WAND", + "GPU_IVF_FLAT", + "GPU_IVF_PQ", +] -all_dense_float_index_types = ["FLAT", "IVF_FLAT", "IVF_SQ8", "IVF_PQ", - "IVF_RABITQ", "HNSW", "SCANN", "DISKANN"] +all_dense_float_index_types = ["FLAT", "IVF_FLAT", "IVF_SQ8", "IVF_PQ", "IVF_RABITQ", "HNSW", "SCANN", "DISKANN"] -inverted_index_algo = ['TAAT_NAIVE', 'DAAT_WAND', 'DAAT_MAXSCORE'] +inverted_index_algo = ["TAAT_NAIVE", "DAAT_WAND", "DAAT_MAXSCORE"] int8_vector_index = ["HNSW"] -default_all_indexes_params = [{}, {"nlist": 128}, {"nlist": 128}, {"nlist": 128, "m": 16, "nbits": 8}, - {"nlist": 128, "refine": 'true', "refine_type": "SQ8"}, - {"M": 32, "efConstruction": 360}, {"nlist": 128}, {}, - {}, {"nlist": 64}, - {}, {"drop_ratio_build": 0.2}, - {"nlist": 64}, {"nlist": 64, "m": 16, "nbits": 8}] +default_all_indexes_params = [ + {}, + {"nlist": 128}, + {"nlist": 128}, + {"nlist": 128, "m": 16, "nbits": 8}, + {"nlist": 128, "refine": "true", "refine_type": "SQ8"}, + {"M": 32, "efConstruction": 360}, + {"nlist": 128}, + {}, + {}, + {"nlist": 64}, + {}, + {"drop_ratio_build": 0.2}, + {"nlist": 64}, + {"nlist": 64, "m": 16, "nbits": 8}, +] -default_all_search_params_params = [{}, {"nprobe": 32}, {"nprobe": 32}, {"nprobe": 32}, - {"nprobe": 8, "rbq_bits_query": 8, "refine_k": 10.0}, - {"ef": 100}, {"nprobe": 32, "reorder_k": 100}, {"search_list": 30}, - {}, {"nprobe": 32}, - {"drop_ratio_search": "0.2"}, {"drop_ratio_search": "0.2"}, - {}, {}] +default_all_search_params_params = [ + {}, + {"nprobe": 32}, + {"nprobe": 32}, + {"nprobe": 32}, + {"nprobe": 8, "rbq_bits_query": 8, "refine_k": 10.0}, + {"ef": 100}, + {"nprobe": 32, "reorder_k": 100}, + {"search_list": 30}, + {}, + {"nprobe": 32}, + {"drop_ratio_search": "0.2"}, + {"drop_ratio_search": "0.2"}, + {}, + {}, +] Handler_type = ["GRPC", "HTTP"] binary_supported_index_types = ["BIN_FLAT", "BIN_IVF_FLAT"] @@ -336,10 +396,20 @@ numeric_supported_index_types = ["STL_SORT", "INVERTED", "AUTOINDEX", ""] default_flat_index = {"index_type": "FLAT", "params": {}, "metric_type": default_L0_metric} default_bin_flat_index = {"index_type": "BIN_FLAT", "params": {}, "metric_type": "JACCARD"} -default_sparse_inverted_index = {"index_type": "SPARSE_INVERTED_INDEX", "metric_type": "IP", - "params": {"drop_ratio_build": 0.2}} -default_text_sparse_inverted_index = {"index_type": "SPARSE_INVERTED_INDEX", "metric_type": "BM25", - "params": {"drop_ratio_build": 0.2, "bm25_k1": 1.5, "bm25_b": 0.75,}} +default_sparse_inverted_index = { + "index_type": "SPARSE_INVERTED_INDEX", + "metric_type": "IP", + "params": {"drop_ratio_build": 0.2}, +} +default_text_sparse_inverted_index = { + "index_type": "SPARSE_INVERTED_INDEX", + "metric_type": "BM25", + "params": { + "drop_ratio_build": 0.2, + "bm25_k1": 1.5, + "bm25_b": 0.75, + }, +} default_search_params = {"params": {"nlist": 128}} default_search_ip_params = {"metric_type": "IP", "params": {"nlist": 128}} default_search_binary_params = {"metric_type": "JACCARD", "params": {"nprobe": 32}} @@ -349,46 +419,108 @@ default_diskann_index = {"index_type": "DISKANN", "metric_type": default_L0_metr default_diskann_search_params = {"params": {"search_list": 30}} default_sparse_search_params = {"metric_type": "IP", "params": {"drop_ratio_search": "0.2"}} default_text_sparse_search_params = {"metric_type": "BM25", "params": {}} -built_in_privilege_groups = ["CollectionReadWrite", "CollectionReadOnly", "CollectionAdmin", - "DatabaseReadWrite", "DatabaseReadOnly", "DatabaseAdmin", - "ClusterReadWrite", "ClusterReadOnly", "ClusterAdmin"] -privilege_group_privilege_dict = {"Query": False, "Search": False, "GetLoadState": False, - "GetLoadingProgress": False, "HasPartition": False, "ShowPartitions": False, - "ShowCollections": False, "ListAliases": False, "ListDatabases": False, - "DescribeDatabase": False, "DescribeAlias": False, "GetStatistics": False, - "CreateIndex": False, "DropIndex": False, "CreatePartition": False, - "DropPartition": False, "Load": False, "Release": False, - "Insert": False, "Delete": False, "Upsert": False, - "Import": False, "Flush": False, "Compaction": False, - "LoadBalance": False, "RenameCollection": False, "CreateAlias": False, - "DropAlias": False, "CreateCollection": False, "DropCollection": False, - "CreateOwnership": False, "DropOwnership": False, "SelectOwnership": False, - "ManageOwnership": False, "UpdateUser": False, "SelectUser": False, - "CreateResourceGroup": False, "DropResourceGroup": False, - "UpdateResourceGroups": False, - "DescribeResourceGroup": False, "ListResourceGroups": False, "TransferNode": False, - "TransferReplica": False, "CreateDatabase": False, "DropDatabase": False, - "AlterDatabase": False, "FlushAll": False, "ListPrivilegeGroups": False, - "CreatePrivilegeGroup": False, "DropPrivilegeGroup": False, - "OperatePrivilegeGroup": False} -all_expr_fields = [default_int8_field_name, default_int16_field_name, - default_int32_field_name, default_int64_field_name, - default_float_field_name, default_double_field_name, - default_string_field_name, default_bool_field_name, - default_int8_array_field_name, default_int16_array_field_name, - default_int32_array_field_name, default_int64_array_field_name, - default_bool_array_field_name, default_float_array_field_name, - default_double_array_field_name, default_string_array_field_name] +built_in_privilege_groups = [ + "CollectionReadWrite", + "CollectionReadOnly", + "CollectionAdmin", + "DatabaseReadWrite", + "DatabaseReadOnly", + "DatabaseAdmin", + "ClusterReadWrite", + "ClusterReadOnly", + "ClusterAdmin", +] +privilege_group_privilege_dict = { + "Query": False, + "Search": False, + "GetLoadState": False, + "GetLoadingProgress": False, + "HasPartition": False, + "ShowPartitions": False, + "ShowCollections": False, + "ListAliases": False, + "ListDatabases": False, + "DescribeDatabase": False, + "DescribeAlias": False, + "GetStatistics": False, + "CreateIndex": False, + "DropIndex": False, + "CreatePartition": False, + "DropPartition": False, + "Load": False, + "Release": False, + "Insert": False, + "Delete": False, + "Upsert": False, + "Import": False, + "Flush": False, + "Compaction": False, + "LoadBalance": False, + "RenameCollection": False, + "CreateAlias": False, + "DropAlias": False, + "CreateCollection": False, + "DropCollection": False, + "CreateOwnership": False, + "DropOwnership": False, + "SelectOwnership": False, + "ManageOwnership": False, + "UpdateUser": False, + "SelectUser": False, + "CreateResourceGroup": False, + "DropResourceGroup": False, + "UpdateResourceGroups": False, + "DescribeResourceGroup": False, + "ListResourceGroups": False, + "TransferNode": False, + "TransferReplica": False, + "CreateDatabase": False, + "DropDatabase": False, + "AlterDatabase": False, + "FlushAll": False, + "ListPrivilegeGroups": False, + "CreatePrivilegeGroup": False, + "DropPrivilegeGroup": False, + "OperatePrivilegeGroup": False, +} +all_expr_fields = [ + default_int8_field_name, + default_int16_field_name, + default_int32_field_name, + default_int64_field_name, + default_float_field_name, + default_double_field_name, + default_string_field_name, + default_bool_field_name, + default_int8_array_field_name, + default_int16_array_field_name, + default_int32_array_field_name, + default_int64_array_field_name, + default_bool_array_field_name, + default_float_array_field_name, + default_double_array_field_name, + default_string_array_field_name, +] + +not_supported_json_cast_types = [ + DataType.INT8.name, + DataType.INT16.name, + DataType.INT32.name, + DataType.INT64.name, + DataType.FLOAT.name, + DataType.ARRAY.name, + DataType.FLOAT_VECTOR.name, + DataType.FLOAT16_VECTOR.name, + DataType.BFLOAT16_VECTOR.name, + DataType.BINARY_VECTOR.name, + DataType.SPARSE_FLOAT_VECTOR.name, + DataType.INT8_VECTOR.name, +] -not_supported_json_cast_types = [DataType.INT8.name, DataType.INT16.name, DataType.INT32.name, - DataType.INT64.name, DataType.FLOAT.name, - DataType.ARRAY.name, DataType.FLOAT_VECTOR.name, - DataType.FLOAT16_VECTOR.name, DataType.BFLOAT16_VECTOR.name, - DataType.BINARY_VECTOR.name, - DataType.SPARSE_FLOAT_VECTOR.name, DataType.INT8_VECTOR.name] class CheckTasks: - """ The name of the method used to check the result """ + """The name of the method used to check the result""" + check_nothing = "check_nothing" err_res = "error_response" ccr = "check_connection_result" @@ -452,6 +584,7 @@ class CaseLabel: GPU: For GPU supported cases """ + L0 = "L0" L1 = "L1" L2 = "L2" diff --git a/tests/pyproject.toml b/tests/ruff.toml similarity index 66% rename from tests/pyproject.toml rename to tests/ruff.toml index f3b662399c..69330606db 100644 --- a/tests/pyproject.toml +++ b/tests/ruff.toml @@ -1,16 +1,3 @@ -[project] -name = "milvus-tests" -version = "0.0.0" -description = "Milvus integration/e2e test suite tooling (ruff config host)" -requires-python = ">=3.10" -dependencies = [] - -[dependency-groups] -dev = [ - "ruff>=0.6.0", -] - -[tool.ruff] line-length = 120 target-version = "py310" extend-exclude = [ @@ -20,7 +7,7 @@ extend-exclude = [ "*.ipynb_checkpoints", ] -[tool.ruff.lint] +[lint] select = [ "E", # pycodestyle errors "F", # pyflakes @@ -33,13 +20,13 @@ ignore = [ "E741", # ambiguous variable names (l, I, O) common in test code ] -[tool.ruff.lint.per-file-ignores] +[lint.per-file-ignores] "__init__.py" = ["F401", "F403"] "conftest.py" = ["F401", "F811"] "**/testcases/**/*.py" = ["E402"] "**/chaos/**/*.py" = ["E402"] -[tool.ruff.format] +[format] quote-style = "double" indent-style = "space" line-ending = "auto" diff --git a/tests/uv.lock b/tests/uv.lock deleted file mode 100644 index 466c9372ed..0000000000 --- a/tests/uv.lock +++ /dev/null @@ -1,43 +0,0 @@ -version = 1 -revision = 3 -requires-python = ">=3.10" - -[[package]] -name = "milvus-tests" -version = "0.0.0" -source = { virtual = "." } - -[package.dev-dependencies] -dev = [ - { name = "ruff" }, -] - -[package.metadata] - -[package.metadata.requires-dev] -dev = [{ name = "ruff", specifier = ">=0.6.0" }] - -[[package]] -name = "ruff" -version = "0.15.11" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e4/8d/192f3d7103816158dfd5ea50d098ef2aec19194e6cbccd4b3485bdb2eb2d/ruff-0.15.11.tar.gz", hash = "sha256:f092b21708bf0e7437ce9ada249dfe688ff9a0954fc94abab05dcea7dcd29c33", size = 4637264, upload-time = "2026-04-16T18:46:26.58Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/02/1e/6aca3427f751295ab011828e15e9bf452200ac74484f1db4be0197b8170b/ruff-0.15.11-py3-none-linux_armv6l.whl", hash = "sha256:e927cfff503135c558eb581a0c9792264aae9507904eb27809cdcff2f2c847b7", size = 10607943, upload-time = "2026-04-16T18:46:05.967Z" }, - { url = "https://files.pythonhosted.org/packages/e7/26/1341c262e74f36d4e84f3d6f4df0ac68cd53331a66bfc5080daa17c84c0b/ruff-0.15.11-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:7a1b5b2938d8f890b76084d4fa843604d787a912541eae85fd7e233398bbb73e", size = 10988592, upload-time = "2026-04-16T18:46:00.742Z" }, - { url = "https://files.pythonhosted.org/packages/03/71/850b1d6ffa9564fbb6740429bad53df1094082fe515c8c1e74b6d8d05f18/ruff-0.15.11-py3-none-macosx_11_0_arm64.whl", hash = "sha256:d4176f3d194afbdaee6e41b9ccb1a2c287dba8700047df474abfbe773825d1cb", size = 10338501, upload-time = "2026-04-16T18:46:03.723Z" }, - { url = "https://files.pythonhosted.org/packages/f2/11/cc1284d3e298c45a817a6aadb6c3e1d70b45c9b36d8d9cce3387b495a03a/ruff-0.15.11-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3b17c886fb88203ced3afe7f14e8d5ae96e9d2f4ccc0ee66aa19f2c2675a27e4", size = 10670693, upload-time = "2026-04-16T18:46:41.941Z" }, - { url = "https://files.pythonhosted.org/packages/ce/9e/f8288b034ab72b371513c13f9a41d9ba3effac54e24bfb467b007daee2ca/ruff-0.15.11-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:49fafa220220afe7758a487b048de4c8f9f767f37dfefad46b9dd06759d003eb", size = 10416177, upload-time = "2026-04-16T18:46:21.717Z" }, - { url = "https://files.pythonhosted.org/packages/85/71/504d79abfd3d92532ba6bbe3d1c19fada03e494332a59e37c7c2dabae427/ruff-0.15.11-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2ab8427e74a00d93b8bda1307b1e60970d40f304af38bccb218e056c220120d", size = 11221886, upload-time = "2026-04-16T18:46:15.086Z" }, - { url = "https://files.pythonhosted.org/packages/43/5a/947e6ab7a5ad603d65b474be15a4cbc6d29832db5d762cd142e4e3a74164/ruff-0.15.11-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:195072c0c8e1fc8f940652073df082e37a5d9cb43b4ab1e4d0566ab8977a13b7", size = 12075183, upload-time = "2026-04-16T18:46:07.944Z" }, - { url = "https://files.pythonhosted.org/packages/9f/a1/0b7bb6268775fdd3a0818aee8efd8f5b4e231d24dd4d528ced2534023182/ruff-0.15.11-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a3a0996d486af3920dec930a2e7daed4847dfc12649b537a9335585ada163e9e", size = 11516575, upload-time = "2026-04-16T18:46:31.687Z" }, - { url = "https://files.pythonhosted.org/packages/30/c3/bb5168fc4d233cc06e95f482770d0f3c87945a0cd9f614b90ea8dc2f2833/ruff-0.15.11-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1bef2cb556d509259f1fe440bb9cd33c756222cf0a7afe90d15edf0866702431", size = 11306537, upload-time = "2026-04-16T18:46:36.988Z" }, - { url = "https://files.pythonhosted.org/packages/e4/92/4cfae6441f3967317946f3b788136eecf093729b94d6561f963ed810c82e/ruff-0.15.11-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:030d921a836d7d4a12cf6e8d984a88b66094ccb0e0f17ddd55067c331191bf19", size = 11296813, upload-time = "2026-04-16T18:46:24.182Z" }, - { url = "https://files.pythonhosted.org/packages/43/26/972784c5dde8313acde8ac71ba8ac65475b85db4a2352a76c9934361f9bc/ruff-0.15.11-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:0e783b599b4577788dbbb66b9addcef87e9a8832f4ce0c19e34bf55543a2f890", size = 10633136, upload-time = "2026-04-16T18:46:39.802Z" }, - { url = "https://files.pythonhosted.org/packages/5b/53/3985a4f185020c2f367f2e08a103032e12564829742a1b417980ce1514a0/ruff-0.15.11-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:ae90592246625ba4a34349d68ec28d4400d75182b71baa196ddb9f82db025ef5", size = 10424701, upload-time = "2026-04-16T18:46:10.381Z" }, - { url = "https://files.pythonhosted.org/packages/d3/57/bf0dfb32241b56c83bb663a826133da4bf17f682ba8c096973065f6e6a68/ruff-0.15.11-py3-none-musllinux_1_2_i686.whl", hash = "sha256:1f111d62e3c983ed20e0ca2e800f8d77433a5b1161947df99a5c2a3fb60514f0", size = 10873887, upload-time = "2026-04-16T18:46:29.157Z" }, - { url = "https://files.pythonhosted.org/packages/02/05/e48076b2a57dc33ee8c7a957296f97c744ca891a8ffb4ffb1aaa3b3f517d/ruff-0.15.11-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:06f483d6646f59eaffba9ae30956370d3a886625f511a3108994000480621d1c", size = 11404316, upload-time = "2026-04-16T18:46:19.462Z" }, - { url = "https://files.pythonhosted.org/packages/88/27/0195d15fe7a897cbcba0904792c4b7c9fdd958456c3a17d2ea6093716a9a/ruff-0.15.11-py3-none-win32.whl", hash = "sha256:476a2aa56b7da0b73a3ee80b6b2f0e19cce544245479adde7baa65466664d5f3", size = 10655535, upload-time = "2026-04-16T18:46:12.47Z" }, - { url = "https://files.pythonhosted.org/packages/3a/5e/c927b325bd4c1d3620211a4b96f47864633199feed60fa936025ab27e090/ruff-0.15.11-py3-none-win_amd64.whl", hash = "sha256:8b6756d88d7e234fb0c98c91511aae3cd519d5e3ed271cae31b20f39cb2a12a3", size = 11779692, upload-time = "2026-04-16T18:46:17.268Z" }, - { url = "https://files.pythonhosted.org/packages/63/b6/aeadee5443e49baa2facd51131159fd6301cc4ccfc1541e4df7b021c37dd/ruff-0.15.11-py3-none-win_arm64.whl", hash = "sha256:063fed18cc1bbe0ee7393957284a6fe8b588c6a406a285af3ee3f46da2391ee4", size = 11032614, upload-time = "2026-04-16T18:46:34.487Z" }, -]