fix: disable SDK default checksum to fix non-AWS S3 uploads (#49978)

issue: #49977

## Summary

AWS SDK C++ 1.11.692 (bumped from 1.11.352 in #47810) changed two
defaults that together break PutObject against non-AWS S3-compatible
storage:

1. `PutObjectRequest::GetChecksumAlgorithmName()` returns `"crc64nvme"`
by default
2. `ClientConfiguration::checksumConfig::requestChecksumCalculation`
defaults to `WHEN_SUPPORTED`

`ChecksumInterceptor` then attaches a request hash to every PutObject,
and the V4 signer switches the request to `Transfer-Encoding:
aws-chunked` + `x-amz-content-sha256:
STREAMING-UNSIGNED-PAYLOAD-TRAILER`. Aliyun OSS, Tencent COS, Huawei
OBS, and GCP all reject this streaming form.

This makes any `MinioChunkManager::PutObjectBuffer` call fail in an
infinite `JobStateRetry` loop — json_stats / bson_inverted
shared_key_index builds and any non-DiskANN-style index upload (IVF_*,
HNSW, FLAT) are affected.

## Fix

Restrict `requestChecksumCalculation` to `WHEN_REQUIRED` on non-AWS
providers so the SDK only adds checksums when the operation model
demands them (PutObject does not). Same approach as
milvus-io/milvus-storage#500.

Patched in both client construction paths:
- `MinioChunkManager::MinioChunkManager` (storage_type=minio,
address-based dispatch) — gated on `storageType != S3`
- `ChunkManager.cpp` subclasses (storage_type=remote,
cloud_provider-based dispatch) — applies in `GcpChunkManager`,
`AliyunChunkManager`, `TencentCloudChunkManager`,
`HuaweiCloudChunkManager`. `AwsChunkManager` keeps the default
`WHEN_SUPPORTED`.

CARDINAL_TIERED / DiskANN indexes already work because they upload via
Arrow's `S3FileSystem` provided by milvus-storage, which was fixed in
#500.

## Test plan

- [x] Added `TEST(MinioChecksumConfig, OverridesAreWhenRequired)` in
`test_storage.cpp` — asserts the override flips both directions from
`WHEN_SUPPORTED` to `WHEN_REQUIRED`
- [ ] Build / CI green
- [ ] Manual smoke test on Aliyun OSS endpoint — index build (IVF_SQ8)
and json_stats no longer fail with `x-oss-ec=0017-00000804`. CI cannot
regression-detect this since MinIO doesn't reject `aws-chunked`; a real
Aliyun OSS endpoint is required.

## Workaround (no code change)

Setting `AWS_REQUEST_CHECKSUM_CALCULATION=when_required` and
`AWS_RESPONSE_CHECKSUM_VALIDATION=when_required` on milvus pods also
fixes this — the AWS SDK reads them at `ClientConfiguration`
default-construct time, so it works across all bundled SDK copies
without rebuilding. Useful for hotfix on running clusters until this PR
ships.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

---------

Signed-off-by: cai.zhang <cai.zhang@zilliz.com>
Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

.env

@@ -5,11 +5,11 @@ IMAGE_ARCH=amd64
 OS_NAME=ubuntu22.04
 
 # for services.builder.image in docker-compose.yml
-DATE_VERSION=20260506-b5f57d9
-LATEST_DATE_VERSION=20260506-b5f57d9
+DATE_VERSION=20260525-c5b2f49
+LATEST_DATE_VERSION=20260525-c5b2f49
 # for services.gpubuilder.image in docker-compose.yml
-GPU_DATE_VERSION=20260506-b5f57d9
-LATEST_GPU_DATE_VERSION=20260506-b5f57d9
+GPU_DATE_VERSION=20260525-c5b2f49
+LATEST_GPU_DATE_VERSION=20260525-c5b2f49
 
 # for other services in docker-compose.yml
 MINIO_ADDRESS=minio:9000

build/docker/builder/cpu/ubuntu22.04/Dockerfile

@@ -15,7 +15,7 @@ ARG TARGETARCH
 
 RUN apt-get update && apt-get install -y --no-install-recommends wget curl ca-certificates gnupg2 \
     g++ gcc gdb gdbserver ninja-build git make ccache libssl-dev zlib1g-dev zip unzip \
-    clang-format-12 clang-tidy-12 lcov libtool m4 autoconf automake python3 python3-pip \
+    clang-format-15 clang-tidy-15 lcov libtool m4 autoconf automake python3 python3-pip \
     pkg-config uuid-dev libaio-dev libopenblas-dev && \
     DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata && \
         ln -sf /usr/share/zoneinfo/Etc/UTC /etc/localtime && \

build/docker/builder/gpu/ubuntu22.04/Dockerfile

@@ -6,7 +6,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends wget curl ca-ce
     wget -qO- "https://cmake.org/files/v3.31/cmake-3.31.8-linux-`uname -m`.tar.gz" | tar --strip-components=1 -xz -C /usr/local && \
     apt-get update && apt-get install -y --no-install-recommends \
     g++ gcc gfortran git make ccache libssl-dev zlib1g-dev zip unzip \
-    clang-format-12 clang-tidy-12  lcov libtool m4 autoconf automake python3 python3-pip \
+    clang-format-15 clang-tidy-15  lcov libtool m4 autoconf automake python3 python3-pip \
     pkg-config uuid-dev libaio-dev libgoogle-perftools-dev libopenblas-dev && \
     DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata && \
         ln -sf /usr/share/zoneinfo/Etc/UTC /etc/localtime && \
@@ -35,4 +35,3 @@ RUN wget -O /tini https://github.com/krallin/tini/releases/download/v0.19.0/tini
     chmod +x /tini
 
 
-

docs/design-docs/design_docs/segcore/scripts_and_tools.md

@@ -6,7 +6,7 @@ The following scripts and commands may be used during segcore development.
 
 - under milvus/internal/core directory
   - run `./run_clang_format.sh .` to format cpp code
-    - to call clang-format-12, need to install `apt install clang-format-12` in advance
+    - to call clang-format-15, need to install `apt install clang-format-15` in advance
     - call `build-support/add_${lang}_license.sh` to add license info for cmake and cpp files
 - under milvus/ directory
   - use `make cppcheck` to check format, including

internal/core/cmake/FindClangTools.cmake

@@ -78,6 +78,7 @@ if (CLANG_FORMAT_VERSION)
 else()
     find_program(CLANG_FORMAT_BIN
       NAMES
+      clang-format-15
       clang-format-12
       clang-format
       PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin
@@ -92,4 +93,3 @@ else()
   set(CLANG_FORMAT_FOUND 1)
   message("clang-format found at ${CLANG_FORMAT_BIN}")
 endif()
-

internal/core/run_clang_format.sh

@@ -1,17 +1,31 @@
 #!/usr/bin/env bash
 
-if [ -z $1 ]; then
+set -euo pipefail
+
+if [ -z "${1:-}" ]; then
     echo "usage: $0 <path_to_core>"
-    exit -1
+    exit 1
 else
     echo start formating
 fi
 CorePath=$1
 
-CLANG_FORMAT=clang-format-12
+CLANG_FORMAT=${CLANG_FORMAT:-clang-format-15}
+
+if ! command -v "$CLANG_FORMAT" >/dev/null 2>&1; then
+    echo "ERROR: $CLANG_FORMAT is required for cpp format check but was not found in PATH" >&2
+    exit 1
+fi
 
 formatThis() {
-    find "$1" | grep -E "(\.cpp|\.c|\.h|\.cc)$" | grep -v "gen_tools/templates" | grep -v "\.pb\." | grep -v "tantivy-binding.h" | grep -v "CMakeFiles" | xargs $CLANG_FORMAT -i
+    find "$1" \
+        -type f \
+        \( -name "*.cpp" -o -name "*.c" -o -name "*.h" -o -name "*.cc" \) \
+        ! -path "*/gen_tools/templates/*" \
+        ! -name "*.pb.*" \
+        ! -name "tantivy-binding.h" \
+        ! -path "*/CMakeFiles/*" \
+        -exec "$CLANG_FORMAT" -i {} +
 }
 
 formatThis "${CorePath}/src"

internal/core/src/common/Utils.h

@@ -323,7 +323,8 @@ SparseBytesToRows(const Iterable& rows, const bool validate = false) {
 // SparseRowsToProto converts a list of knowhere::sparse::SparseRow<SparseValueType> to
 // a milvus::proto::schema::SparseFloatArray. The resulting proto is a deep copy
 // of the source data. source(i) returns the i-th row to be copied.
-inline void SparseRowsToProto(
+inline void
+SparseRowsToProto(
     const std::function<
         const knowhere::sparse::SparseRow<SparseValueType>*(size_t)>& source,
     int64_t rows,

internal/core/src/index/RTreeIndex.cpp

@@ -142,7 +142,7 @@ RTreeIndex<T>::Load(milvus::tracer::TraceContext ctx, const Config& config) {
 
     // 1. Extract and load null_offset file(s) if present
     {
-        auto find_file = [&](const std::string& target) -> auto {
+        auto find_file = [&](const std::string& target) -> auto{
             return std::find_if(
                 files.begin(), files.end(), [&](const std::string& filename) {
                     return GetFileName(filename) == target;

internal/core/src/index/VectorDiskIndex.h

@@ -95,13 +95,13 @@ class VectorDiskAnnIndex : public VectorIndex {
                   "get sparse vector not supported for disk index");
     }
 
-    knowhere::expected<knowhere::DataSetPtr> CalcDistByIDs(
-        const knowhere::DataSetPtr query_dataset,
-        const BitsetView& bitset,
-        const int64_t* labels,
-        size_t labels_len,
-        bool is_cosine,
-        milvus::OpContext* op_context = nullptr) const override;
+    knowhere::expected<knowhere::DataSetPtr>
+    CalcDistByIDs(const knowhere::DataSetPtr query_dataset,
+                  const BitsetView& bitset,
+                  const int64_t* labels,
+                  size_t labels_len,
+                  bool is_cosine,
+                  milvus::OpContext* op_context = nullptr) const override;
 
     void
     CleanLocalData() override;

internal/core/src/index/VectorMemIndex.cpp

@@ -713,7 +713,8 @@ VectorMemIndex<T>::GetSparseVector(const DatasetPtr dataset) const {
 }
 
 template <typename T>
-void VectorMemIndex<T>::LoadFromFile(const Config& config) {
+void
+VectorMemIndex<T>::LoadFromFile(const Config& config) {
     auto local_filepath =
         GetValueFromConfig<std::string>(config, MMAP_FILE_PATH);
     AssertInfo(local_filepath.has_value(),

internal/core/src/indexbuilder/ScalarIndexCreatorTest.cpp

@@ -46,7 +46,7 @@ build_index(const ScalarIndexCreatorPtr& creator,
 
     creator->Build(ds);
 
-    delete[](char*)(ds->GetTensor());
+    delete[] (char*)(ds->GetTensor());
 }
 
 template <>
@@ -59,7 +59,7 @@ build_index(const ScalarIndexCreatorPtr& creator,
 
     creator->Build(ds);
 
-    delete[](char*)(ds->GetTensor());
+    delete[] (char*)(ds->GetTensor());
 }
 
 }  // namespace

internal/core/src/segcore/IndexCTest.cpp

@@ -204,7 +204,7 @@ TEST(CBoolIndexTest, All) {
         { DeleteBinarySet(binary_set); }
     }
 
-    delete[](char*)(half_ds->GetTensor());
+    delete[] (char*)(half_ds->GetTensor());
 }
 
 // TODO: more scalar type.
@@ -321,7 +321,7 @@ TEST(CStringIndexTest, All) {
         { DeleteBinarySet(binary_set); }
     }
 
-    delete[](char*)(str_ds->GetTensor());
+    delete[] (char*)(str_ds->GetTensor());
 }
 #endif
 

internal/core/src/storage/ChunkManager.cpp

@@ -132,6 +132,7 @@ GcpChunkManager::GcpChunkManager(const StorageConfig& storage_config) {
     InitSDKAPIDefault(storage_config.log_level, storage_config.tls_min_version);
 
     Aws::Client::ClientConfiguration config = generateConfig(storage_config);
+    ApplyChecksumConfigOverrides(config);
     if (storage_config.useIAM) {
         // Using S3 client instead of google client because of compatible protocol
         client_ = std::make_shared<Aws::S3::S3Client>(
@@ -165,6 +166,7 @@ AliyunChunkManager::AliyunChunkManager(const StorageConfig& storage_config) {
     InitSDKAPIDefault(storage_config.log_level, storage_config.tls_min_version);
 
     Aws::Client::ClientConfiguration config = generateConfig(storage_config);
+    ApplyChecksumConfigOverrides(config);
 
     // For aliyun oss, support use virtual host mode
     StorageConfig mutable_config = storage_config;
@@ -212,6 +214,7 @@ TencentCloudChunkManager::TencentCloudChunkManager(
     InitSDKAPIDefault(storage_config.log_level, storage_config.tls_min_version);
 
     Aws::Client::ClientConfiguration config = generateConfig(storage_config);
+    ApplyChecksumConfigOverrides(config);
 
     StorageConfig mutable_config = storage_config;
     mutable_config.useVirtualHost = true;
@@ -257,6 +260,7 @@ HuaweiCloudChunkManager::HuaweiCloudChunkManager(
     use_crc32c_checksum_ = storage_config.use_crc32c_checksum;
     InitSDKAPIDefault(storage_config.log_level, storage_config.tls_min_version);
     Aws::Client::ClientConfiguration config = generateConfig(storage_config);
+    ApplyChecksumConfigOverrides(config);
     StorageConfig mutable_config = storage_config;
     mutable_config.useVirtualHost = true;
     if (storage_config.useIAM) {

internal/core/src/storage/minio/MinioChunkManager.cpp

@@ -355,6 +355,31 @@ MinioChunkManager::BuildGoogleCloudClient(
     }
 }
 
+void
+MinioChunkManager::ApplyChecksumConfigOverrides(
+    Aws::Client::ClientConfiguration& config) {
+    // Non-AWS S3-compatible APIs (GCP, Aliyun OSS, Tencent COS, Huawei OBS) do
+    // not accept the extra x-amz-checksum-* headers / aws-chunked streaming
+    // that AWS SDK >= 1.11.x sends by default (WHEN_SUPPORTED). The V4 signer
+    // emits Transfer-Encoding: aws-chunked + x-amz-content-sha256:
+    // STREAMING-UNSIGNED-PAYLOAD-TRAILER, which Aliyun OSS rejects with
+    // x-oss-ec=0017-00000804, Huawei OBS with XAmzContentSHA256Mismatch, etc.
+    // Restrict to WHEN_REQUIRED so the SDK only adds checksums when the
+    // operation model mandates them (PutObject does not), restoring the
+    // simple PUT behavior we had with AWS SDK 1.11.352. Same approach as
+    // milvus-storage PR #500.
+    config.checksumConfig.requestChecksumCalculation =
+        Aws::Client::RequestChecksumCalculation::WHEN_REQUIRED;
+    config.checksumConfig.responseChecksumValidation =
+        Aws::Client::ResponseChecksumValidation::WHEN_REQUIRED;
+}
+
+bool
+MinioChunkManager::NeedChecksumOverride(const std::string& cloud_provider) {
+    return cloud_provider == "gcp" || cloud_provider == "aliyun" ||
+           cloud_provider == "tencent" || cloud_provider == "huawei";
+}
+
 MinioChunkManager::MinioChunkManager(const StorageConfig& storage_config)
     : default_bucket_name_(storage_config.bucket_name),
       use_crc32c_checksum_(storage_config.use_crc32c_checksum) {
@@ -407,6 +432,10 @@ MinioChunkManager::MinioChunkManager(const StorageConfig& storage_config)
         config.region = ConvertToAwsString(storage_config.region);
     }
 
+    if (NeedChecksumOverride(storage_config.cloud_provider)) {
+        ApplyChecksumConfigOverrides(config);
+    }
+
     if (storageType == RemoteStorageType::S3) {
         BuildS3Client(storage_config, config);
     } else if (storageType == RemoteStorageType::ALIYUN_CLOUD) {

internal/core/src/storage/minio/MinioChunkManager.h

@@ -261,6 +261,24 @@ class MinioChunkManager : public ChunkManager {
     BuildAccessKeyClient(const StorageConfig& storage_config,
                          const Aws::Client::ClientConfiguration& config);
 
+    // Restrict the SDK's request/response checksum policy to WHEN_REQUIRED
+    // so the V4 signer does not switch PutObject uploads to aws-chunked /
+    // STREAMING-UNSIGNED-PAYLOAD-TRAILER. Required for non-AWS S3-compatible
+    // backends (Aliyun OSS, GCP, Tencent COS, Huawei OBS) that reject that
+    // streaming form. AWS S3 / MinIO don't need this. Mirrors milvus-storage
+    // PR #500. Exposed for derived ChunkManager subclasses and unit testing.
+    static void
+    ApplyChecksumConfigOverrides(Aws::Client::ClientConfiguration& config);
+
+    // Decide whether the checksum override is required for a given
+    // cloud_provider. Aliyun OSS, Tencent COS, Huawei OBS, and GCP all
+    // reject the streaming aws-chunked encoding the SDK adds when
+    // checksums are computed; AWS S3 / MinIO accept the default. Mirrors
+    // the cloud_provider-based dispatch used by milvus-storage PR #500.
+    // Exposed for unit testing.
+    static bool
+    NeedChecksumOverride(const std::string& cloud_provider);
+
     Aws::SDKOptions sdk_options_;
     static std::atomic<size_t> init_count_;
     static std::mutex client_mutex_;

internal/core/unittest/test_storage.cpp

@@ -20,6 +20,7 @@
 #include <optional>
 #include <string>
 
+#include "aws/core/client/ClientConfiguration.h"
 #include "common/EasyAssert.h"
 #include "common/FieldMeta.h"
 #include "common/Types.h"
@@ -35,9 +36,18 @@
 #include "storage/Types.h"
 #include "storage/Util.h"
 #include "storage/loon_ffi/property_singleton.h"
+#include "storage/minio/MinioChunkManager.h"
 #include "storage/storage_c.h"
 #include "test_utils/Constants.h"
 
+// Test-only subclass that exposes the protected ApplyChecksumConfigOverrides
+// and NeedChecksumOverride helpers so we can assert their behavior directly.
+class TestableMinioChunkManager : public milvus::storage::MinioChunkManager {
+ public:
+    using MinioChunkManager::ApplyChecksumConfigOverrides;
+    using MinioChunkManager::NeedChecksumOverride;
+};
+
 using namespace std;
 using namespace milvus;
 using namespace milvus::storage;
@@ -383,3 +393,48 @@ TEST_F(StorageUtilTest, NormalizePath) {
     EXPECT_EQ(NormalizePath(boost::filesystem::path("./.")), ".");
     EXPECT_EQ(NormalizePath(boost::filesystem::path("./..")), "..");
 }
+
+TEST(MinioChecksumConfig, OverridesAreWhenRequired) {
+    // Regression guard: AWS SDK C++ 1.11.x defaults the checksum policy to
+    // WHEN_SUPPORTED, which makes the V4 signer switch PutObject uploads to
+    // aws-chunked + STREAMING-UNSIGNED-PAYLOAD-TRAILER. Aliyun OSS rejects
+    // that combination (x-oss-ec=0017-00000804). MinioChunkManager must
+    // override both directions to WHEN_REQUIRED so the SDK only adds
+    // checksums when the operation model demands them.
+
+    // Aws::Client::ClientConfiguration's default ctor reads SDK globals
+    // (logger / http client factory) set up by Aws::InitAPI; without it the
+    // ctor segfaults. Use MinioChunkManager's idempotent init helper so we
+    // share init_count_ with any production code paths in the same binary.
+    TestableMinioChunkManager init_guard;
+    init_guard.InitSDKAPIDefault("info");
+
+    Aws::Client::ClientConfiguration config;
+    // Sanity check: the SDK defaults are WHEN_SUPPORTED for both directions.
+    EXPECT_EQ(config.checksumConfig.requestChecksumCalculation,
+              Aws::Client::RequestChecksumCalculation::WHEN_SUPPORTED);
+    EXPECT_EQ(config.checksumConfig.responseChecksumValidation,
+              Aws::Client::ResponseChecksumValidation::WHEN_SUPPORTED);
+
+    TestableMinioChunkManager::ApplyChecksumConfigOverrides(config);
+
+    EXPECT_EQ(config.checksumConfig.requestChecksumCalculation,
+              Aws::Client::RequestChecksumCalculation::WHEN_REQUIRED);
+    EXPECT_EQ(config.checksumConfig.responseChecksumValidation,
+              Aws::Client::ResponseChecksumValidation::WHEN_REQUIRED);
+}
+
+TEST(MinioChecksumConfig, NeedChecksumOverrideDispatch) {
+    using Mgr = TestableMinioChunkManager;
+
+    // Non-AWS S3-compatible backends need the override.
+    EXPECT_TRUE(Mgr::NeedChecksumOverride("gcp"));
+    EXPECT_TRUE(Mgr::NeedChecksumOverride("aliyun"));
+    EXPECT_TRUE(Mgr::NeedChecksumOverride("tencent"));
+    EXPECT_TRUE(Mgr::NeedChecksumOverride("huawei"));
+
+    // AWS S3 / MinIO accept the default WHEN_SUPPORTED behavior.
+    EXPECT_FALSE(Mgr::NeedChecksumOverride("aws"));
+    EXPECT_FALSE(Mgr::NeedChecksumOverride(""));
+    EXPECT_FALSE(Mgr::NeedChecksumOverride("unknown"));
+}

internal/core/unittest/test_utils/DataGen.h

@@ -388,7 +388,8 @@ GenerateRandomSparseFloatVector(size_t rows,
     return tensor;
 }
 
-inline SchemaPtr CreateTestSchema() {
+inline SchemaPtr
+CreateTestSchema() {
     auto schema = std::make_shared<milvus::Schema>();
     schema->AddDebugField("bool", milvus::DataType::BOOL, true);
     schema->AddDebugField("int8", milvus::DataType::INT8, true);

scripts/check_cpp_fmt.sh

@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+set -euo pipefail
 
 # Licensed to the LF AI & Data foundation under one
 # or more contributor license agreements. See the NOTICE file
@@ -26,12 +27,13 @@ ROOT_DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )"
 
 CPP_SRC_DIR="${ROOT_DIR}/internal/core"
 
-pushd $CPP_SRC_DIR
+pushd "$CPP_SRC_DIR" >/dev/null
 ./run_clang_format.sh .
-popd
+popd >/dev/null
 
 # Exclude tantivy-binding.h: it is auto-generated by cbindgen and has its own formatting style
-check_result=$(git status | grep -E "\.h|\.hpp|\.cc|\.cpp|\.c" | grep -v "tantivy-binding.h")
+git_status=$(git -C "$ROOT_DIR" status --short -- "*.h" "*.hpp" "*.cc" "*.cpp" "*.c")
+check_result=$(echo "$git_status" | grep -v "tantivy-binding.h" || true)
 echo "check_result: $check_result"
 if test -z "$check_result"; then
   exit 0

scripts/install_deps.sh

@@ -279,7 +279,8 @@ install_ubuntu_deps() {
         libopenblas-dev libgoogle-perftools-dev
     )
 
-    # Version-specific GCC and clang-format
+    # Version-specific GCC and clang-format. CI runs the cpp format checker on
+    # ubuntu22.04 with clang-format-15, so 22.04 must install that exact version.
     local gcc_packages=()
     local clang_format_package=""
 
@@ -290,11 +291,11 @@ install_ubuntu_deps() {
             ;;
         22.04)
             gcc_packages=(g++ gcc gfortran g++-11 gcc-11)
-            clang_format_package="clang-format-14 clang-tidy-14"
+            clang_format_package="clang-format-15 clang-tidy-15"
             ;;
         24.04)
             gcc_packages=(g++ gcc gfortran g++-13 gcc-13)
-            clang_format_package="clang-format-17 clang-tidy-17"
+            clang_format_package="clang-format-18 clang-tidy-18"
             ;;
         *)
             print_warn "Ubuntu ${ubuntu_version} not explicitly supported, using default packages"