enhance: bump milvus-storage and upgrade google-cloud-cpp to 2.28.0 (#47864)

## Summary
- Bump milvus-storage from f372333 to 0a9cb7ebe8
- Upgrade google-cloud-cpp from 2.5.0@milvus/2.4 to 2.28.0 (official
Conan Center package)
- Adapt storage layer code to new google-cloud-cpp 2.28.0 API changes

issue: #47863

## Changes
1. **conanfile.py**: Switch to upstream `google-cloud-cpp/2.28.0` from
custom `@milvus/2.4` package
2. **ChunkManager.cpp / MinioChunkManager.cpp**: Add `HttpClientFactory`
parameter to `ComputeEngineCredentials` constructor
3. **MinioChunkManager.h**: Add required headers, update
`AuthorizationHeader` to free function, qualify `StatusCodeToString`
4. **ffi_reader_c.cpp**: Rename `import_column_groups` →
`column_groups_import` per new milvus-storage API
5. **milvus-storage/CMakeLists.txt**: Bump version to `0a9cb7ebe8`

## Test plan
- [ ] CI build passes (C++ compilation with new dependencies)
- [ ] Unit tests pass
- [ ] Integration tests pass with GCP storage backend

---------

Signed-off-by: Wei Liu <wei.liu@zilliz.com>

internal/core/conanfile.py

@@ -32,7 +32,7 @@ class MilvusConan(ConanFile):
         "prometheus-cpp/1.1.0#ea9b101cb785943adb40ad82eda7856c",
         "re2/20230301#f8efaf45f98d0193cd0b2ea08b6b4060",
         "folly/2023.10.30.08@milvus/dev#81d7729cd4013a1b708af3340a3b04d9",
-        "google-cloud-cpp/2.5.0@milvus/2.4#c5591ab30b26b53ea6068af6f07128d3",
+        "google-cloud-cpp/2.28.0#cf2bffe2264488b6c1153fae2a8db095",
         "opentelemetry-cpp/1.8.1.1@milvus/2.4#7345034855d593047826b0c74d9a0ced",
         "librdkafka/1.9.1#e24dcbb0a1684dcf5a56d8d0692ceef3",
         "abseil/20230125.3#dad7cc4c83bbd44c1f1cc9cc4d97ac88",

internal/core/src/storage/ChunkManager.cpp

@@ -121,9 +121,15 @@ GcpChunkManager::GcpChunkManager(const StorageConfig& storage_config) {
 
     if (storage_config.useIAM) {
         sdk_options_.httpOptions.httpClientFactory_create_fn = []() {
+            auto client_factory = [](google::cloud::Options const& opts)
+                -> std::unique_ptr<google::cloud::rest_internal::RestClient> {
+                return google::cloud::rest_internal::MakeDefaultRestClient(
+                    {}, opts);
+            };
             auto credentials = std::make_shared<
                 google::cloud::oauth2_internal::GOOGLE_CLOUD_CPP_NS::
-                    ComputeEngineCredentials>();
+                    ComputeEngineCredentials>(google::cloud::Options{},
+                                              client_factory);
             return Aws::MakeShared<GoogleHttpClientFactory>(
                 GOOGLE_CLIENT_FACTORY_ALLOCATION_TAG, credentials);
         };

internal/core/src/storage/loon_ffi/ffi_reader_c.cpp

@@ -139,7 +139,7 @@ NewPackedFFIReaderWithManifest(const LoonManifest* loon_manifest,
             MakeInternalPropertiesFromStorageConfig(c_storage_config);
         auto column_groups =
             std::make_shared<milvus_storage::api::ColumnGroups>();
-        auto status = milvus_storage::import_column_groups(
+        auto status = milvus_storage::column_groups_import(
             &loon_manifest->column_groups, column_groups.get());
         AssertInfo(status.ok(),
                    "Failed to import column groups: {}",

internal/core/src/storage/minio/MinioChunkManager.cpp

@@ -127,9 +127,16 @@ MinioChunkManager::InitSDKAPI(RemoteStorageType type,
         sigaction(SIGPIPE, &psa, 0);
         if (type == RemoteStorageType::GOOGLE_CLOUD && useIAM) {
             sdk_options_.httpOptions.httpClientFactory_create_fn = []() {
+                auto client_factory = [](google::cloud::Options const& opts)
+                    -> std::unique_ptr<
+                        google::cloud::rest_internal::RestClient> {
+                    return google::cloud::rest_internal::MakeDefaultRestClient(
+                        {}, opts);
+                };
                 auto credentials = std::make_shared<
                     google::cloud::oauth2_internal::GOOGLE_CLOUD_CPP_NS::
-                        ComputeEngineCredentials>();
+                        ComputeEngineCredentials>(google::cloud::Options{},
+                                                  client_factory);
                 return Aws::MakeShared<GoogleHttpClientFactory>(
                     GOOGLE_CLIENT_FACTORY_ALLOCATION_TAG, credentials);
             };

internal/core/src/storage/minio/MinioChunkManager.h

@@ -26,6 +26,14 @@
 #include <aws/core/utils/logging/FormattedLogSystem.h>
 #include <aws/s3/S3Client.h>
 #include <fmt/core.h>
+#include <google/cloud/credentials.h>
+#include <google/cloud/internal/oauth2_credentials.h>
+#include <google/cloud/internal/oauth2_compute_engine_credentials.h>
+#include <google/cloud/internal/oauth2_google_credentials.h>
+#include <google/cloud/internal/oauth2_http_client_factory.h>
+#include <google/cloud/internal/rest_client.h>
+#include <google/cloud/storage/oauth2/compute_engine_credentials.h>
+#include <google/cloud/storage/oauth2/google_credentials.h>
 #include <google/cloud/status_or.h>
 #include <stddef.h>
 #include <stdint.h>
@@ -46,9 +54,6 @@
 #include "aws/s3/S3Errors.h"
 #include "common/EasyAssert.h"
 #include "glog/logging.h"
-#include "google/cloud/internal/oauth2_compute_engine_credentials.h"
-#include "google/cloud/internal/oauth2_credentials.h"
-#include "google/cloud/internal/oauth2_google_credentials.h"
 #include "google/cloud/status.h"
 #include "log/Log.h"
 #include "storage/ChunkManager.h"
@@ -359,12 +364,13 @@ class GoogleHttpClientFactory : public Aws::Http::HttpClientFactory {
             Aws::MakeShared<Aws::Http::Standard::StandardHttpRequest>(
                 GOOGLE_CLIENT_FACTORY_ALLOCATION_TAG, uri, method);
         request->SetResponseStreamFactory(streamFactory);
-        auto auth_header = credentials_->AuthorizationHeader();
+        auto auth_header =
+            google::cloud::oauth2_internal::AuthorizationHeader(*credentials_);
         if (!auth_header.ok()) {
-            ThrowInfo(
-                S3Error,
-                fmt::format("get authorization failed, errcode: {}",
-                            StatusCodeToString(auth_header.status().code())));
+            ThrowInfo(S3Error,
+                      fmt::format("get authorization failed, errcode: {}",
+                                  google::cloud::StatusCodeToString(
+                                      auth_header.status().code())));
         }
         request->SetHeaderValue(auth_header->first.c_str(),
                                 auth_header->second.c_str());

internal/core/thirdparty/milvus-storage/CMakeLists.txt

@@ -14,7 +14,7 @@
 # Update milvus-storage_VERSION for the first occurrence
 milvus_add_pkg_config("milvus-storage")
 set_property(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES "")
-set( milvus-storage_VERSION f372333)
+set( milvus-storage_VERSION 0a9cb7ebe8a9bcd1be15a59c7fa870aff7554593)
 set( GIT_REPOSITORY  "https://github.com/milvus-io/milvus-storage.git")
 message(STATUS "milvus-storage repo: ${GIT_REPOSITORY}")
 message(STATUS "milvus-storage version: ${milvus-storage_VERSION}")

scripts/3rdparty_build.sh

@@ -111,7 +111,7 @@ case "${unameOut}" in
     export CMAKE_CXX_COMPILER_LAUNCHER=ccache
     echo "Using CXX: $CXX"
     echo "Using CC: $CC"
-    conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s build_type=${BUILD_TYPE} -s compiler=clang -s compiler.version=${llvm_version} -s compiler.libcxx=libc++ -s compiler.cppstd=17 -r default-conan-local -u || { echo 'conan install failed'; exit 1; }
+    conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s build_type=${BUILD_TYPE} -s compiler=clang -s compiler.version=${llvm_version} -s compiler.libcxx=libc++ -s compiler.cppstd=17 -u || { echo 'conan install failed'; exit 1; }
     ;;
   Linux*)
     if [ -f /etc/os-release ]; then
@@ -123,9 +123,9 @@ case "${unameOut}" in
     export CPU_TARGET=avx
     GCC_VERSION=`gcc -dumpversion`
     if [[ `gcc -v 2>&1 | sed -n 's/.*\(--with-default-libstdcxx-abi\)=\(\w*\).*/\2/p'` == "gcc4" ]]; then
-      conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s build_type=${BUILD_TYPE} -s compiler.version=${GCC_VERSION} -r default-conan-local -u || { echo 'conan install failed'; exit 1; }
+      conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s build_type=${BUILD_TYPE} -s compiler.version=${GCC_VERSION} -u || { echo 'conan install failed'; exit 1; }
     else
-      conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s build_type=${BUILD_TYPE} -s compiler.version=${GCC_VERSION} -s compiler.libcxx=libstdc++11 -r default-conan-local -u || { echo 'conan install failed'; exit 1; }
+      conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s build_type=${BUILD_TYPE} -s compiler.version=${GCC_VERSION} -s compiler.libcxx=libstdc++11 -u || { echo 'conan install failed'; exit 1; }
     fi
     ;;
   *)

scripts/core_build.sh

@@ -254,6 +254,17 @@ echo "CC $CC"
 echo ${CMAKE_CMD}
 ${CMAKE_CMD} -G "${CMAKE_GENERATOR}"
 
+# Export PROTOC for Rust prost-build (used by lance-encoding).
+# Conan imports() copies the protoc binary into cmake_build/bin/.
+# We prefer that over a system protoc to match the Conan protobuf version.
+if [ -z "${PROTOC}" ] && [ -x "${BUILD_OUTPUT_DIR}/bin/protoc" ]; then
+  export PROTOC="${BUILD_OUTPUT_DIR}/bin/protoc"
+  echo "Using Conan protoc: ${PROTOC}"
+elif [ -z "${PROTOC}" ] && command -v protoc &> /dev/null; then
+  export PROTOC="$(command -v protoc)"
+  echo "Using system protoc: ${PROTOC}"
+fi
+
 if [[ ${RUN_CPPLINT} == "ON" ]]; then
   if [ "$CMAKE_GENERATOR" = "Ninja" ]; then
     BUILD_CMD="ninja"