enhance: improve HuaweiCloud credential provider robustness and observability (#48046)

## Summary

- **Go layer**: Replace `sync.Once` with recoverable `sync.Mutex` init;
add process-level singleton; set `duration_seconds=7200` (was defaulting
to 15min); add `refreshMu` to deduplicate concurrent STS calls and
prevent data race on `IsExpired()`; validate credential completeness
(AK/SK/Token); add comprehensive logging with masked AK
- **C++ layer**: Default-initialize `STSCallResult.success{false}` (was
UB); validate AK/SK/Token completeness before updating cached
credentials in `Reload()`; mask access key in DEBUG log

related: #48045

🤖 Generated with [Claude Code](https://claude.com/claude-code)

---------

Signed-off-by: MrPresent-Han <chun.han@gmail.com>
Co-authored-by: MrPresent-Han <chun.han@gmail.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

internal/core/src/storage/huawei/HuaweiCloudCredentialsProvider.cpp

@@ -119,6 +119,8 @@ HuaweiCloudSTSAssumeRoleWebIdentityCredentialsProvider::
     Aws::Client::ClientConfiguration config;
     config.scheme = Aws::Http::Scheme::HTTPS;
     config.region = m_region;
+    config.requestTimeoutMs =
+        30000;  // 30s timeout to prevent indefinite hang on STS calls
 
     Aws::Vector<Aws::String> retryableErrors;
     retryableErrors.push_back("IDPCommunicationError");
@@ -168,8 +170,13 @@ HuaweiCloudSTSAssumeRoleWebIdentityCredentialsProvider::Reload() {
         }
         m_token = token;
     } else {
-        AWS_LOGSTREAM_ERROR(STS_ASSUME_ROLE_WEB_IDENTITY_LOG_TAG,
-                            "Can't open token file: " << m_tokenFile);
+        m_stsFailureCount++;
+        LOG_WARN(
+            "HuaweiCloud credential provider: can't open token file: {}, "
+            "sts_success={}, sts_failure={}",
+            m_tokenFile,
+            m_stsSuccessCount.load(),
+            m_stsFailureCount.load());
         m_lastReloadFailed = true;
         m_lastFailedReloadTime = std::chrono::steady_clock::now();
         return;
@@ -182,26 +189,46 @@ HuaweiCloudSTSAssumeRoleWebIdentityCredentialsProvider::Reload() {
     // GetAssumeRoleWithWebIdentityCredentials catches all exceptions internally
     // and returns result.success=false on any failure.
     auto result = m_client->GetAssumeRoleWithWebIdentityCredentials(request);
+    const auto& creds = result.creds;
 
-    if (!result.success || result.creds.GetAWSAccessKeyId().empty() ||
-        result.creds.GetAWSSecretKey().empty()) {
-        AWS_LOGSTREAM_WARN(
-            STS_ASSUME_ROLE_WEB_IDENTITY_LOG_TAG,
-            "STS credential retrieval failed. Retaining existing credentials.");
+    if (!result.success) {
+        m_stsFailureCount++;
+        LOG_WARN(
+            "HuaweiCloud credential provider: STS call failed, keeping "
+            "existing credentials, sts_success={}, sts_failure={}",
+            m_stsSuccessCount.load(),
+            m_stsFailureCount.load());
         m_lastReloadFailed = true;
         m_lastFailedReloadTime = std::chrono::steady_clock::now();
         return;
     }
 
-    m_credentials = result.creds;
+    if (creds.GetAWSAccessKeyId().empty() || creds.GetAWSSecretKey().empty() ||
+        creds.GetSessionToken().empty()) {
+        m_stsFailureCount++;
+        LOG_WARN(
+            "HuaweiCloud credential provider: STS returned incomplete "
+            "credentials (missing ak/sk/token), keeping existing credentials, "
+            "sts_success={}, sts_failure={}",
+            m_stsSuccessCount.load(),
+            m_stsFailureCount.load());
+        m_lastReloadFailed = true;
+        m_lastFailedReloadTime = std::chrono::steady_clock::now();
+        return;
+    }
+
+    m_stsSuccessCount++;
+    auto akId = creds.GetAWSAccessKeyId();
+    auto maskedAk = akId.length() > 4 ? akId.substr(0, 4) + "***" : akId;
+    m_credentials = creds;
     m_lastReloadFailed = false;
-    AWS_LOGSTREAM_DEBUG(
-        STS_ASSUME_ROLE_WEB_IDENTITY_LOG_TAG,
-        "Successfully retrieved credentials with AWS_ACCESS_KEY: "
-            << result.creds.GetAWSAccessKeyId()
-            << ", expiration_count_diff_ms: "
-            << (result.creds.GetExpiration() - Aws::Utils::DateTime::Now())
-                   .count());
+    LOG_INFO(
+        "HuaweiCloud credential provider: credentials retrieved successfully, "
+        "ak={}, expiration_diff_ms={}, sts_success={}, sts_failure={}",
+        maskedAk,
+        (creds.GetExpiration() - Aws::Utils::DateTime::Now()).count(),
+        m_stsSuccessCount.load(),
+        m_stsFailureCount.load());
 }
 
 bool
@@ -239,9 +266,9 @@ HuaweiCloudSTSAssumeRoleWebIdentityCredentialsProvider::RefreshIfExpired() {
     }
 
     if (IsInCooldown()) {
-        AWS_LOGSTREAM_DEBUG(
-            STS_ASSUME_ROLE_WEB_IDENTITY_LOG_TAG,
-            "Skipping credential reload — in cooldown after previous failure.");
+        LOG_WARN(
+            "HuaweiCloud credential provider: skipping reload, in cooldown "
+            "after previous failure");
         return;
     }
 

internal/core/src/storage/huawei/HuaweiCloudCredentialsProvider.h

@@ -9,6 +9,7 @@
 // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 // or implied. See the License for the specific language governing permissions and limitations under the License
 
+#include <atomic>
 #include <chrono>
 #include <aws/core/auth/AWSCredentialsProvider.h>
 #include <memory>
@@ -38,9 +39,6 @@ class HuaweiCloudSTSAssumeRoleWebIdentityCredentialsProvider
  private:
     void
     RefreshIfExpired();
-    Aws::String
-    CalculateQueryString() const;
-
     Aws::UniquePtr<Aws::Internal::HuaweiCloudSTSCredentialsClient> m_client;
     Aws::Auth::AWSCredentials m_credentials;
     Aws::String m_region;
@@ -54,6 +52,8 @@ class HuaweiCloudSTSAssumeRoleWebIdentityCredentialsProvider
     std::chrono::steady_clock::time_point m_lastFailedReloadTime;
     static constexpr int RELOAD_COOLDOWN_SECONDS = 30;
     static constexpr int RELOAD_COOLDOWN_SECONDS_URGENT = 5;
+    std::atomic<uint64_t> m_stsSuccessCount{0};
+    std::atomic<uint64_t> m_stsFailureCount{0};
 
     bool
     ExpiresSoon() const;
@@ -61,4 +61,4 @@ class HuaweiCloudSTSAssumeRoleWebIdentityCredentialsProvider
     IsInCooldown() const;
 };
 }  // namespace Auth
-}  // namespace Aws
+}  // namespace Aws

internal/core/src/storage/huawei/HuaweiCloudCredentialsProviderTest.cpp

@@ -118,6 +118,16 @@ class HuaweiCloudCredentialsProviderTestHelper {
     lastReloadFailed(const Provider& p) {
         return p.m_lastReloadFailed;
     }
+
+    static uint64_t
+    stsSuccessCount(const Provider& p) {
+        return p.m_stsSuccessCount.load();
+    }
+
+    static uint64_t
+    stsFailureCount(const Provider& p) {
+        return p.m_stsFailureCount.load();
+    }
 };
 
 using Helper = HuaweiCloudCredentialsProviderTestHelper;
@@ -384,6 +394,30 @@ TEST_F(HuaweiCloudCredentialsProviderTest, ReloadRetainsCredsOnEmptyKeys) {
               oldCreds.GetAWSAccessKeyId());
 }
 
+TEST_F(HuaweiCloudCredentialsProviderTest,
+       ReloadRetainsCredsOnEmptySessionToken) {
+    CreateProviderWithMock();
+    auto oldCreds = MakeFreshCredentials();
+    Helper::setCredentials(*provider_, oldCreds);
+    Helper::setTokenFile(*provider_, tokenFilePath_);
+
+    auto incompleteResult =
+        MakeSuccessfulSTSResult("PARTIAL_AKID", "PARTIAL_SECRET");
+    incompleteResult.creds.SetSessionToken("");
+    EXPECT_CALL(*mock_, GetAssumeRoleWithWebIdentityCredentials(testing::_))
+        .WillOnce(testing::Return(incompleteResult));
+
+    Helper::callReload(*provider_);
+
+    EXPECT_TRUE(Helper::lastReloadFailed(*provider_));
+    EXPECT_EQ(Helper::getCredentials(*provider_).GetAWSAccessKeyId(),
+              oldCreds.GetAWSAccessKeyId());
+    EXPECT_EQ(Helper::getCredentials(*provider_).GetSessionToken(),
+              oldCreds.GetSessionToken());
+    EXPECT_EQ(Helper::stsSuccessCount(*provider_), 0);
+    EXPECT_EQ(Helper::stsFailureCount(*provider_), 1);
+}
+
 // ============================================================================
 // Group 4: RefreshIfExpired + cooldown integration tests
 // ============================================================================
@@ -454,3 +488,47 @@ TEST_F(HuaweiCloudCredentialsProviderTest,
     EXPECT_EQ(Helper::getCredentials(*provider_).GetAWSAccessKeyId(),
               "NEW_AKID");
 }
+
+TEST_F(HuaweiCloudCredentialsProviderTest,
+       RefreshIfExpiredRetriesAfterNormalCooldownWhenCredsStillValid) {
+    CreateProviderWithMock();
+    Helper::setCredentials(*provider_, MakeExpiringSoonCredentials(60));
+    Helper::setLastReloadFailed(
+        *provider_,
+        true,
+        std::chrono::steady_clock::now() - std::chrono::seconds(31));
+    Helper::setTokenFile(*provider_, tokenFilePath_);
+
+    auto stsResult =
+        MakeSuccessfulSTSResult("REFRESHED_AKID", "REFRESHED_SECRET");
+    EXPECT_CALL(*mock_, GetAssumeRoleWithWebIdentityCredentials(testing::_))
+        .WillOnce(testing::Return(stsResult));
+
+    Helper::callRefreshIfExpired(*provider_);
+
+    EXPECT_FALSE(Helper::lastReloadFailed(*provider_));
+    EXPECT_EQ(Helper::getCredentials(*provider_).GetAWSAccessKeyId(),
+              "REFRESHED_AKID");
+    EXPECT_EQ(Helper::stsSuccessCount(*provider_), 1);
+    EXPECT_EQ(Helper::stsFailureCount(*provider_), 0);
+}
+
+TEST_F(HuaweiCloudCredentialsProviderTest,
+       RefreshIfExpiredMarksFailureWhenReloadGetsIncompleteResult) {
+    CreateProviderWithMock();
+    Helper::setLastReloadFailed(*provider_, false);
+    Helper::setTokenFile(*provider_, tokenFilePath_);
+
+    auto incompleteResult =
+        MakeSuccessfulSTSResult("PARTIAL_AKID", "PARTIAL_SECRET");
+    incompleteResult.creds.SetSessionToken("");
+    EXPECT_CALL(*mock_, GetAssumeRoleWithWebIdentityCredentials(testing::_))
+        .WillOnce(testing::Return(incompleteResult));
+
+    Helper::callRefreshIfExpired(*provider_);
+
+    EXPECT_TRUE(Helper::lastReloadFailed(*provider_));
+    EXPECT_TRUE(Helper::getCredentials(*provider_).IsEmpty());
+    EXPECT_EQ(Helper::stsSuccessCount(*provider_), 0);
+    EXPECT_EQ(Helper::stsFailureCount(*provider_), 1);
+}

internal/core/src/storage/huawei/HuaweiCloudSTSClient.cpp

@@ -151,6 +151,60 @@ HuaweiCloudSTSCredentialsClient::GetAssumeRoleWithWebIdentityCredentials(
     return result;
 }
 
+HuaweiCloudSTSCredentialsClient::STSCallResult
+HuaweiCloudSTSCredentialsClient::parseSTSResponse(
+    Aws::Http::HttpResponseCode httpResponseCode,
+    const Aws::String& responseBody) {
+    STSCallResult result;
+    if (httpResponseCode != Aws::Http::HttpResponseCode::OK &&
+        httpResponseCode != Aws::Http::HttpResponseCode::CREATED) {
+        result.errorMessage =
+            "Huawei Cloud STS security token request failed with HTTP code: " +
+            std::to_string(static_cast<int>(httpResponseCode)) +
+            ", body: " + responseBody.substr(0, 200);
+        return result;
+    }
+    if (responseBody.empty()) {
+        result.errorMessage = "Get an empty credential from Huawei Cloud STS";
+        return result;
+    }
+    Aws::Utils::Json::JsonValue jsonValue(responseBody);
+    if (!jsonValue.WasParseSuccessful()) {
+        result.errorMessage = "Failed to parse STS response as JSON: " +
+                              std::string(responseBody.substr(0, 200).c_str());
+        return result;
+    }
+    auto json = jsonValue.View();
+    auto rootNode = json.GetObject("credential");
+
+    if (rootNode.IsNull()) {
+        result.errorMessage = "Get credential from STS result failed";
+        return result;
+    }
+    result.credentials.SetAWSAccessKeyId(rootNode.GetString("access"));
+    result.credentials.SetAWSSecretKey(rootNode.GetString("secret"));
+    result.credentials.SetSessionToken(rootNode.GetString("securitytoken"));
+
+    auto expiresAt = rootNode.GetString("expires_at");
+    if (expiresAt.empty()) {
+        result.errorMessage =
+            "STS response missing 'expires_at' field, rejecting credentials";
+        return result;
+    }
+    auto parsedExpiration = Aws::Utils::DateTime(
+        Aws::Utils::StringUtils::Trim(expiresAt.c_str()).c_str(),
+        Aws::Utils::DateFormat::ISO_8601);
+    if (!parsedExpiration.WasParseSuccessful()) {
+        result.errorMessage =
+            "STS response 'expires_at' field has invalid format: " +
+            std::string(expiresAt.c_str());
+        return result;
+    }
+    result.credentials.SetExpiration(parsedExpiration);
+    result.success = true;
+    return result;
+}
+
 HuaweiCloudSTSCredentialsClient::STSCallResult
 HuaweiCloudSTSCredentialsClient::callHuaweiCloudSTS(
     const Aws::String& userToken,
@@ -187,48 +241,17 @@ HuaweiCloudSTSCredentialsClient::callHuaweiCloudSTS(
     req->AddContentBody(body);
 
     auto resp = m_httpClient->MakeRequest(req);
-    STSCallResult result;
     if (!resp) {
+        STSCallResult result;
         result.errorMessage =
             "Null response from Huawei Cloud STS HTTP request";
         return result;
     }
     auto httpResponseCode = resp->GetResponseCode();
-    if (httpResponseCode != Aws::Http::HttpResponseCode::OK &&
-        httpResponseCode != Aws::Http::HttpResponseCode::CREATED) {
-        result.errorMessage =
-            "Huawei Cloud STS security token request failed with HTTP code: " +
-            std::to_string(static_cast<int>(httpResponseCode));
-        return result;
-    }
     std::ostringstream oss;
     oss << resp->GetResponseBody().rdbuf();
-    Aws::String credentialsStr = oss.str();
-    if (credentialsStr.empty()) {
-        result.errorMessage = "Get an empty credential from Huawei Cloud STS";
-        return result;
-    }
-    Aws::Utils::Json::JsonValue jsonValue(credentialsStr);
-    auto json = jsonValue.View();
-    auto rootNode = json.GetObject("credential");
-
-    if (rootNode.IsNull()) {
-        result.errorMessage = "Get credential from STS result failed";
-        return result;
-    }
-    result.credentials.SetAWSAccessKeyId(rootNode.GetString("access"));
-    result.credentials.SetAWSSecretKey(rootNode.GetString("secret"));
-    result.credentials.SetSessionToken(rootNode.GetString("securitytoken"));
-
-    auto expiresAt = rootNode.GetString("expires_at");
-    if (!expiresAt.empty()) {
-        result.credentials.SetExpiration(Aws::Utils::DateTime(
-            Aws::Utils::StringUtils::Trim(expiresAt.c_str()).c_str(),
-            Aws::Utils::DateFormat::ISO_8601));
-    }
-    result.success = true;
-    return result;
+    return parseSTSResponse(httpResponseCode, oss.str());
 }
 
 }  // namespace Internal
-}  // namespace Aws
+}  // namespace Aws

internal/core/src/storage/huawei/HuaweiCloudSTSClient.h

@@ -25,8 +25,11 @@ enum class HttpResponseCode;
 }  // namespace Http
 
 namespace Internal {
+class HuaweiCloudSTSClientTestHelper;
 class AWS_CORE_API HuaweiCloudSTSCredentialsClient
     : public AWSHttpResourceClient {
+    friend class ::Aws::Internal::HuaweiCloudSTSClientTestHelper;
+
  public:
     explicit HuaweiCloudSTSCredentialsClient(
         const Aws::Client::ClientConfiguration& clientConfiguration);
@@ -72,6 +75,10 @@ class AWS_CORE_API HuaweiCloudSTSCredentialsClient
     STSCallResult
     callHuaweiCloudSTS(const Aws::String& userToken,
                        const STSAssumeRoleWithWebIdentityRequest& request);
+
+    static STSCallResult
+    parseSTSResponse(Aws::Http::HttpResponseCode httpResponseCode,
+                     const Aws::String& responseBody);
 };
 }  // namespace Internal
-}  // namespace Aws
+}  // namespace Aws

internal/core/src/storage/huawei/HuaweiCloudSTSClientTest.cpp

@@ -0,0 +1,146 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include <string>
+
+#include "aws/core/Aws.h"
+#include "aws/core/http/HttpTypes.h"
+#include "storage/huawei/HuaweiCloudSTSClient.h"
+
+namespace Aws {
+namespace Internal {
+
+class HuaweiCloudSTSClientTestHelper {
+ public:
+    using Client = HuaweiCloudSTSCredentialsClient;
+
+    struct CallResultView {
+        bool success;
+        Aws::Auth::AWSCredentials credentials;
+        Aws::String errorMessage;
+    };
+
+    static CallResultView
+    parseSTSResponse(Aws::Http::HttpResponseCode httpResponseCode,
+                     const Aws::String& responseBody) {
+        auto result = Client::parseSTSResponse(httpResponseCode, responseBody);
+        return {result.success, result.credentials, result.errorMessage};
+    }
+};
+
+}  // namespace Internal
+}  // namespace Aws
+
+namespace {
+class HuaweiCloudSTSClientTest : public ::testing::Test {
+ protected:
+    static Aws::SDKOptions sdkOptions_;
+
+    static void
+    SetUpTestSuite() {
+        sdkOptions_.loggingOptions.logLevel =
+            Aws::Utils::Logging::LogLevel::Off;
+        Aws::InitAPI(sdkOptions_);
+    }
+
+    static void
+    TearDownTestSuite() {
+        Aws::ShutdownAPI(sdkOptions_);
+    }
+
+    using Helper = Aws::Internal::HuaweiCloudSTSClientTestHelper;
+};
+
+Aws::SDKOptions HuaweiCloudSTSClientTest::sdkOptions_;
+
+TEST_F(HuaweiCloudSTSClientTest,
+       CallHuaweiCloudSTSReturnsErrorWithHttpBodyOnNon200) {
+    const std::string body = R"({"error":"access denied"})";
+    auto result =
+        Helper::parseSTSResponse(Aws::Http::HttpResponseCode::FORBIDDEN, body);
+
+    EXPECT_FALSE(result.success);
+    EXPECT_THAT(result.errorMessage, testing::HasSubstr("HTTP code: 403"));
+    EXPECT_THAT(result.errorMessage, testing::HasSubstr("access denied"));
+}
+
+TEST_F(HuaweiCloudSTSClientTest, CallHuaweiCloudSTSRejectsInvalidJson) {
+    auto result = Helper::parseSTSResponse(Aws::Http::HttpResponseCode::OK,
+                                           "{invalid-json");
+
+    EXPECT_FALSE(result.success);
+    EXPECT_THAT(result.errorMessage,
+                testing::HasSubstr("Failed to parse STS response as JSON"));
+}
+
+TEST_F(HuaweiCloudSTSClientTest, CallHuaweiCloudSTSRejectsMissingExpiresAt) {
+    const std::string body = R"({
+        "credential": {
+            "access": "AKID_TEST",
+            "secret": "SECRET_TEST",
+            "securitytoken": "SESSION_TOKEN"
+        }
+    })";
+    auto result =
+        Helper::parseSTSResponse(Aws::Http::HttpResponseCode::OK, body);
+
+    EXPECT_FALSE(result.success);
+    EXPECT_THAT(result.errorMessage,
+                testing::HasSubstr("missing 'expires_at' field"));
+}
+
+TEST_F(HuaweiCloudSTSClientTest,
+       CallHuaweiCloudSTSRejectsInvalidExpiresAtFormat) {
+    const std::string body = R"({
+        "credential": {
+            "access": "AKID_TEST",
+            "secret": "SECRET_TEST",
+            "securitytoken": "SESSION_TOKEN",
+            "expires_at": "not-a-timestamp"
+        }
+    })";
+    auto result =
+        Helper::parseSTSResponse(Aws::Http::HttpResponseCode::OK, body);
+
+    EXPECT_FALSE(result.success);
+    EXPECT_THAT(result.errorMessage,
+                testing::HasSubstr("invalid format: not-a-timestamp"));
+}
+
+TEST_F(HuaweiCloudSTSClientTest,
+       CallHuaweiCloudSTSAcceptsValidExpiresAtAndReturnsCredentials) {
+    const std::string body = R"({
+        "credential": {
+            "access": "AKID_TEST",
+            "secret": "SECRET_TEST",
+            "securitytoken": "SESSION_TOKEN",
+            "expires_at": "2026-03-15T12:34:56Z"
+        }
+    })";
+    auto result =
+        Helper::parseSTSResponse(Aws::Http::HttpResponseCode::OK, body);
+
+    EXPECT_TRUE(result.success);
+    EXPECT_EQ(result.credentials.GetAWSAccessKeyId(), "AKID_TEST");
+    EXPECT_EQ(result.credentials.GetAWSSecretKey(), "SECRET_TEST");
+    EXPECT_EQ(result.credentials.GetSessionToken(), "SESSION_TOKEN");
+    EXPECT_FALSE(result.credentials.GetExpiration()
+                     .ToGmtString(Aws::Utils::DateFormat::ISO_8601)
+                     .empty());
+}
+
+}  // namespace

pkg/objectstorage/huawei/huawei.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"os"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/cockroachdb/errors"
@@ -32,6 +33,15 @@ import (
 	iamRegion "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/iam/v3/region"
 	"github.com/minio/minio-go/v7"
 	minioCred "github.com/minio/minio-go/v7/pkg/credentials"
+	"go.uber.org/zap"
+
+	"github.com/milvus-io/milvus/pkg/v2/log"
+)
+
+const (
+	reloadCooldownNormal  = 30 * time.Second // cooldown when valid credentials exist
+	reloadCooldownUrgent  = 5 * time.Second  // cooldown when credentials are empty/expired
+	expirationGracePeriod = 3 * time.Minute  // refresh credentials before expiration, matching C++ layer's 180s
 )
 
 func NewMinioClient(address string, opts *minio.Options) (*minio.Client, error) {
@@ -49,8 +59,24 @@ func NewMinioClient(address string, opts *minio.Options) (*minio.Client, error)
 	return minio.New(address, opts)
 }
 
+var (
+	globalCredProvider   *HuaweiCredentialProvider
+	globalCredProviderMu sync.Mutex
+)
+
 func NewCredentialProvider() minioCred.Provider {
-	return &HuaweiCredentialProvider{}
+	globalCredProviderMu.Lock()
+	defer globalCredProviderMu.Unlock()
+	if globalCredProvider == nil {
+		globalCredProvider = &HuaweiCredentialProvider{}
+	}
+	return globalCredProvider
+}
+
+// iamTokenCreator is the subset of *iam.IamClient used by HuaweiCredentialProvider.
+// It is defined as an interface to allow substitution in tests.
+type iamTokenCreator interface {
+	CreateTemporaryAccessKeyByToken(request *model.CreateTemporaryAccessKeyByTokenRequest) (*model.CreateTemporaryAccessKeyByTokenResponse, error)
 }
 
 type HuaweiCredentialProvider struct {
@@ -59,57 +85,119 @@ type HuaweiCredentialProvider struct {
 
 	basicCred auth.ICredential
 	regionObj *region.Region
-	iamClient *iam.IamClient
+	iamClient iamTokenCreator
 
-	initOnce sync.Once
-	initErr  error
+	mu     sync.Mutex
+	inited bool
+
+	refreshMu            sync.RWMutex // serializes STS refresh calls; RLock for IsExpired, Lock for Retrieve
+	lastReloadFailed     bool
+	lastFailedReloadTime time.Time
+	stsSuccessCount      atomic.Int64
+	stsFailureCount      atomic.Int64
 }
 
-func (p *HuaweiCredentialProvider) initClients() {
-	p.initOnce.Do(func() {
-		basicChain := provider.BasicCredentialProviderChain()
-		basicCred, err := basicChain.GetCredentials()
-		if err != nil {
-			p.initErr = errors.Wrap(err, "failed to get basic credentials")
-			return
-		}
-		p.basicCred = basicCred
+func (p *HuaweiCredentialProvider) initClients() error {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if p.inited {
+		return nil
+	}
 
-		regionName := os.Getenv("HUAWEICLOUD_SDK_REGION")
-		if regionName == "" {
-			regionName = "cn-east-3"
-		}
+	basicChain := provider.BasicCredentialProviderChain()
+	basicCred, err := basicChain.GetCredentials()
+	if err != nil {
+		log.Warn("HuaweiCloud credential provider: failed to get basic credentials", zap.Error(err))
+		return errors.Wrap(err, "failed to get basic credentials")
+	}
+	p.basicCred = basicCred
 
-		regionObj, err := iamRegion.SafeValueOf(regionName)
-		if err != nil {
-			regionObj, _ = iamRegion.SafeValueOf("cn-east-3")
-		}
-		p.regionObj = regionObj
+	regionName := os.Getenv("HUAWEICLOUD_SDK_REGION")
+	if regionName == "" {
+		regionName = "cn-east-3"
+	}
 
-		hcClient, err := iam.IamClientBuilder().
-			WithRegion(p.regionObj).
-			WithCredential(p.basicCred).
-			WithHttpConfig(config.DefaultHttpConfig()).
-			SafeBuild()
-		if err != nil {
-			p.initErr = errors.Wrap(err, "failed to build IAM client")
-			return
-		}
-		p.iamClient = iam.NewIamClient(hcClient)
-	})
+	regionObj, err := iamRegion.SafeValueOf(regionName)
+	if err != nil {
+		endpoint := fmt.Sprintf("https://iam.%s.myhuaweicloud.com", regionName)
+		regionObj = region.NewRegion(regionName, endpoint)
+		log.Warn("HuaweiCloud credential provider: region not in SDK, using constructed endpoint",
+			zap.String("region", regionName), zap.String("endpoint", endpoint))
+	}
+	p.regionObj = regionObj
+
+	hcClient, err := iam.IamClientBuilder().
+		WithRegion(p.regionObj).
+		WithCredential(p.basicCred).
+		WithHttpConfig(config.DefaultHttpConfig().WithTimeout(30 * time.Second)).
+		SafeBuild()
+	if err != nil {
+		log.Warn("HuaweiCloud credential provider: failed to build IAM client", zap.Error(err))
+		return errors.Wrap(err, "failed to build IAM client")
+	}
+	p.iamClient = iam.NewIamClient(hcClient)
+	p.inited = true
+	log.Info("HuaweiCloud credential provider: IAM client initialized successfully",
+		zap.String("region", regionName))
+	return nil
+}
+
+// isInCooldown returns true if a recent reload failure occurred and the cooldown
+// period has not yet elapsed. Uses shorter cooldown when credentials are
+// empty/expired (urgent) vs when valid credentials still exist (normal).
+// Must be called with refreshMu held.
+func (p *HuaweiCredentialProvider) isInCooldown() bool {
+	if !p.lastReloadFailed {
+		return false
+	}
+	cooldown := reloadCooldownNormal
+	if p.expiration.IsZero() || time.Now().UTC().After(p.expiration) {
+		cooldown = reloadCooldownUrgent
+	}
+	return time.Since(p.lastFailedReloadTime) < cooldown
+}
+
+// hasValidCachedCredentials returns true if cached credentials exist and haven't fully expired yet.
+// Must be called with refreshMu held.
+func (p *HuaweiCredentialProvider) hasValidCachedCredentials() bool {
+	return p.credentials.AccessKeyID != "" && !p.expiration.IsZero() && time.Now().UTC().Before(p.expiration)
 }
 
 func (p *HuaweiCredentialProvider) Retrieve() (minioCred.Value, error) {
-	p.initClients()
-	if p.initErr != nil {
-		return minioCred.Value{}, p.initErr
+	if err := p.initClients(); err != nil {
+		return minioCred.Value{}, err
 	}
 
+	// Multiple minio Credentials wrappers share this singleton provider.
+	// Each wrapper independently calls Retrieve() when its own cache expires.
+	// Use refreshMu + cached check to deduplicate concurrent STS calls.
+	p.refreshMu.Lock()
+	defer p.refreshMu.Unlock()
+
+	if !p.expiration.IsZero() && time.Now().UTC().Before(p.expiration.Add(-expirationGracePeriod)) {
+		return p.credentials, nil
+	}
+
+	// Throttle retries after STS failures to avoid hammering the service.
+	if p.isInCooldown() {
+		if p.hasValidCachedCredentials() {
+			log.Warn("HuaweiCloud credential provider: in cooldown after failure, returning cached credentials",
+				zap.Time("cached_expiration", p.expiration))
+			return p.credentials, nil
+		}
+		log.Warn("HuaweiCloud credential provider: in cooldown after failure, no valid cached credentials available")
+		return minioCred.Value{}, errors.New("STS refresh in cooldown, no valid cached credentials available")
+	}
+
+	durationSeconds := int32(2 * 60 * 60) // 2 hours, matching C++ layer's duration
 	request := &model.CreateTemporaryAccessKeyByTokenRequest{
 		Body: &model.CreateTemporaryAccessKeyByTokenRequestBody{
 			Auth: &model.TokenAuth{
 				Identity: &model.TokenAuthIdentity{
 					Methods: []model.TokenAuthIdentityMethods{model.GetTokenAuthIdentityMethodsEnum().TOKEN},
+					Token: &model.IdentityToken{
+						DurationSeconds: &durationSeconds,
+					},
 				},
 			},
 		},
@@ -117,31 +205,93 @@ func (p *HuaweiCredentialProvider) Retrieve() (minioCred.Value, error) {
 
 	response, err := p.iamClient.CreateTemporaryAccessKeyByToken(request)
 	if err != nil {
+		p.stsFailureCount.Add(1)
+		p.lastReloadFailed = true
+		p.lastFailedReloadTime = time.Now()
+		if p.hasValidCachedCredentials() {
+			log.Warn("HuaweiCloud credential provider: STS refresh failed, falling back to cached credentials",
+				zap.Time("cached_expiration", p.expiration),
+				zap.Int64("sts_success", p.stsSuccessCount.Load()),
+				zap.Int64("sts_failure", p.stsFailureCount.Load()),
+				zap.Error(err))
+			return p.credentials, nil
+		}
+		log.Warn("HuaweiCloud credential provider: failed to create temporary access key",
+			zap.Int64("sts_success", p.stsSuccessCount.Load()),
+			zap.Int64("sts_failure", p.stsFailureCount.Load()),
+			zap.Error(err))
 		return minioCred.Value{}, errors.Wrap(err, "failed to create temporary access key")
 	}
 
-	if response.Credential == nil {
-		return minioCred.Value{}, errors.New("no credential returned from Huawei Cloud")
+	if response.Credential == nil ||
+		response.Credential.Access == "" || response.Credential.Secret == "" || response.Credential.Securitytoken == "" {
+		p.stsFailureCount.Add(1)
+		p.lastReloadFailed = true
+		p.lastFailedReloadTime = time.Now()
+		if p.hasValidCachedCredentials() {
+			log.Warn("HuaweiCloud credential provider: STS returned incomplete credentials, falling back to cached credentials",
+				zap.Time("cached_expiration", p.expiration),
+				zap.Int64("sts_success", p.stsSuccessCount.Load()),
+				zap.Int64("sts_failure", p.stsFailureCount.Load()))
+			return p.credentials, nil
+		}
+		log.Warn("HuaweiCloud credential provider: STS returned nil or incomplete credentials",
+			zap.Int64("sts_success", p.stsSuccessCount.Load()),
+			zap.Int64("sts_failure", p.stsFailureCount.Load()))
+		return minioCred.Value{}, errors.New("incomplete credential returned from Huawei Cloud (missing ak/sk/token)")
 	}
 
-	expiration, err := time.Parse("2006-01-02T15:04:05Z", response.Credential.ExpiresAt)
+	expiration, err := time.Parse(time.RFC3339, response.Credential.ExpiresAt)
 	if err != nil {
+		p.stsFailureCount.Add(1)
+		log.Warn("HuaweiCloud credential provider: failed to parse expiration time",
+			zap.String("expires_at", response.Credential.ExpiresAt),
+			zap.Int64("sts_success", p.stsSuccessCount.Load()),
+			zap.Int64("sts_failure", p.stsFailureCount.Load()),
+			zap.Error(err))
+		p.lastReloadFailed = true
+		p.lastFailedReloadTime = time.Now()
+		if p.hasValidCachedCredentials() {
+			return p.credentials, nil
+		}
 		return minioCred.Value{}, errors.Wrap(err, "failed to parse expiration time")
 	}
 
+	p.stsSuccessCount.Add(1)
 	credentials := minioCred.Value{
 		AccessKeyID:     response.Credential.Access,
 		SecretAccessKey: response.Credential.Secret,
 		SessionToken:    response.Credential.Securitytoken,
+		Expiration:      expiration,
 		SignerType:      minioCred.SignatureV4,
 	}
 
 	p.credentials = credentials
 	p.expiration = expiration
+	p.lastReloadFailed = false
+
+	akPrefix := response.Credential.Access
+	if len(akPrefix) > 4 {
+		akPrefix = akPrefix[:4] + "***"
+	}
+	log.Info("HuaweiCloud credential provider: credentials retrieved successfully",
+		zap.String("ak_prefix", akPrefix), zap.Time("expiration", expiration),
+		zap.Int64("sts_success", p.stsSuccessCount.Load()),
+		zap.Int64("sts_failure", p.stsFailureCount.Load()))
 
 	return credentials, nil
 }
 
+// IsExpired always returns true to force minio's Credentials.Get() to call
+// Retrieve() on every S3 request. This is necessary because multiple minio
+// clients share this singleton provider — if IsExpired() returned false based
+// on expiration time, a minio client whose Credentials.Get() missed the brief
+// IsExpired()=true window would cache stale credentials in its own c.creds
+// and never refresh them until the next 2-hour expiration cycle.
+//
+// Retrieve() has its own cache-hit fast path (expiration check + mutex),
+// so the overhead of calling it on every request is negligible (one time
+// comparison + lock/unlock).
 func (p *HuaweiCredentialProvider) IsExpired() bool {
-	return time.Now().UTC().After(p.expiration.Add(-5 * time.Minute))
+	return true
 }

pkg/objectstorage/huawei/huawei_test.go

@@ -1,15 +1,39 @@
 package huawei
 
 import (
+	"sync"
 	"testing"
 	"time"
 
 	"github.com/cockroachdb/errors"
+	"github.com/huaweicloud/huaweicloud-sdk-go-v3/services/iam/v3/model"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
+// mockIAMClient implements iamTokenCreator for unit tests.
+type mockIAMClient struct {
+	response *model.CreateTemporaryAccessKeyByTokenResponse
+	err      error
+}
+
+func (m *mockIAMClient) CreateTemporaryAccessKeyByToken(
+	_ *model.CreateTemporaryAccessKeyByTokenRequest,
+) (*model.CreateTemporaryAccessKeyByTokenResponse, error) {
+	return m.response, m.err
+}
+
+func makeFullCredential(ak, sk, token, expiresAt string) *model.Credential {
+	return &model.Credential{
+		Access:        ak,
+		Secret:        sk,
+		Securitytoken: token,
+		ExpiresAt:     expiresAt,
+	}
+}
+
 const OBSDefaultAddress = "obs.cn-east-3.myhuaweicloud.com"
 
 func TestNewMinioClient(t *testing.T) {
@@ -31,18 +55,89 @@ func TestNewMinioClient(t *testing.T) {
 	})
 }
 
-func TestHuaweiCredentialProvider_Retrieve(t *testing.T) {
-	// Skip detailed mocking tests for now, as they require complex setup
-	// This test focuses on the basic functionality
-	t.Run("init error", func(t *testing.T) {
-		c := &HuaweiCredentialProvider{
-			initErr: errors.New("init failed"),
-		}
-		c.initOnce.Do(func() {}) // Mark as initialized
+func TestNewCredentialProvider_Singleton(t *testing.T) {
+	// Reset the global singleton for this test
+	globalCredProviderMu.Lock()
+	globalCredProvider = nil
+	globalCredProviderMu.Unlock()
 
+	p1 := NewCredentialProvider()
+	p2 := NewCredentialProvider()
+	// Both calls should return the same singleton instance
+	assert.Same(t, p1, p2)
+
+	// Clean up
+	globalCredProviderMu.Lock()
+	globalCredProvider = nil
+	globalCredProviderMu.Unlock()
+}
+
+func TestNewCredentialProvider_ConcurrentAccess(t *testing.T) {
+	globalCredProviderMu.Lock()
+	globalCredProvider = nil
+	globalCredProviderMu.Unlock()
+
+	var wg sync.WaitGroup
+	results := make([]credentials.Provider, 10)
+	for i := 0; i < 10; i++ {
+		wg.Add(1)
+		go func(idx int) {
+			defer wg.Done()
+			results[idx] = NewCredentialProvider()
+		}(i)
+	}
+	wg.Wait()
+
+	// All goroutines should get the same instance
+	for i := 1; i < 10; i++ {
+		assert.Same(t, results[0], results[i])
+	}
+
+	globalCredProviderMu.Lock()
+	globalCredProvider = nil
+	globalCredProviderMu.Unlock()
+}
+
+func TestHuaweiCredentialProvider_Retrieve(t *testing.T) {
+	t.Run("not initialized", func(t *testing.T) {
+		c := &HuaweiCredentialProvider{}
+		// Without proper env vars, initClients will fail and Retrieve returns error
 		_, err := c.Retrieve()
 		assert.Error(t, err)
-		assert.Contains(t, err.Error(), "init failed")
+	})
+
+	t.Run("returns cached credentials when not expired", func(t *testing.T) {
+		c := &HuaweiCredentialProvider{
+			inited: true,
+			credentials: credentials.Value{
+				AccessKeyID:     "CACHED_AK",
+				SecretAccessKey: "CACHED_SK",
+				SessionToken:    "CACHED_TOKEN",
+				SignerType:      credentials.SignatureV4,
+			},
+			expiration: time.Now().UTC().Add(1 * time.Hour),
+		}
+
+		val, err := c.Retrieve()
+		assert.NoError(t, err)
+		assert.Equal(t, "CACHED_AK", val.AccessKeyID)
+		assert.Equal(t, "CACHED_SK", val.SecretAccessKey)
+		assert.Equal(t, "CACHED_TOKEN", val.SessionToken)
+	})
+
+	t.Run("re-init allowed after previous init failure", func(t *testing.T) {
+		c := &HuaweiCredentialProvider{
+			inited: false,
+		}
+		// First call fails (no env vars)
+		_, err := c.Retrieve()
+		assert.Error(t, err)
+		// inited should still be false, allowing retry
+		assert.False(t, c.inited)
+
+		// Second call also fails but demonstrates retryability
+		_, err = c.Retrieve()
+		assert.Error(t, err)
 	})
 }
 
@@ -54,17 +149,283 @@ func TestHuaweiCredentialProvider_IsExpired(t *testing.T) {
 	})
 
 	t.Run("expired - past time", func(t *testing.T) {
+		c.refreshMu.Lock()
 		c.expiration = time.Now().UTC().Add(-10 * time.Minute)
+		c.refreshMu.Unlock()
 		assert.True(t, c.IsExpired())
 	})
 
 	t.Run("expired - within refresh window", func(t *testing.T) {
-		c.expiration = time.Now().UTC().Add(3 * time.Minute)
+		c.refreshMu.Lock()
+		c.expiration = time.Now().UTC().Add(2 * time.Minute) // within 3min grace period
+		c.refreshMu.Unlock()
 		assert.True(t, c.IsExpired())
 	})
 
-	t.Run("not expired", func(t *testing.T) {
+	t.Run("always expired - forces minio to call Retrieve", func(t *testing.T) {
+		c.refreshMu.Lock()
 		c.expiration = time.Now().UTC().Add(10 * time.Minute)
-		assert.False(t, c.IsExpired())
+		c.refreshMu.Unlock()
+		// IsExpired() always returns true so minio always calls Retrieve(),
+		// which has its own cache-hit fast path.
+		assert.True(t, c.IsExpired())
 	})
 }
+
+func TestHuaweiCredentialProvider_InitClientsIdempotent(t *testing.T) {
+	c := &HuaweiCredentialProvider{}
+	// First call fails (no env vars)
+	err1 := c.initClients()
+	assert.Error(t, err1)
+	assert.False(t, c.inited)
+
+	// Second call also runs (not blocked by sync.Once), allowing retry
+	err2 := c.initClients()
+	assert.Error(t, err2)
+	assert.False(t, c.inited)
+}
+
+func TestHuaweiCredentialProvider_IsInCooldown(t *testing.T) {
+	t.Run("not in cooldown when no failure", func(t *testing.T) {
+		c := &HuaweiCredentialProvider{
+			lastReloadFailed: false,
+		}
+		assert.False(t, c.isInCooldown())
+	})
+
+	t.Run("urgent cooldown with empty credentials", func(t *testing.T) {
+		c := &HuaweiCredentialProvider{
+			lastReloadFailed:     true,
+			lastFailedReloadTime: time.Now(),
+			// expiration is zero → urgent cooldown (5s)
+		}
+		assert.True(t, c.isInCooldown())
+	})
+
+	t.Run("urgent cooldown expires after 5s", func(t *testing.T) {
+		c := &HuaweiCredentialProvider{
+			lastReloadFailed:     true,
+			lastFailedReloadTime: time.Now().Add(-6 * time.Second),
+			// expiration is zero → urgent cooldown (5s), 6s elapsed → expired
+		}
+		assert.False(t, c.isInCooldown())
+	})
+
+	t.Run("normal cooldown with valid credentials", func(t *testing.T) {
+		c := &HuaweiCredentialProvider{
+			lastReloadFailed:     true,
+			lastFailedReloadTime: time.Now(),
+			expiration:           time.Now().UTC().Add(1 * time.Hour), // valid creds → normal cooldown (30s)
+		}
+		assert.True(t, c.isInCooldown())
+	})
+
+	t.Run("normal cooldown expires after 30s", func(t *testing.T) {
+		c := &HuaweiCredentialProvider{
+			lastReloadFailed:     true,
+			lastFailedReloadTime: time.Now().Add(-31 * time.Second),
+			expiration:           time.Now().UTC().Add(1 * time.Hour), // valid creds → normal cooldown (30s)
+		}
+		assert.False(t, c.isInCooldown())
+	})
+
+	t.Run("urgent cooldown when credentials expired", func(t *testing.T) {
+		c := &HuaweiCredentialProvider{
+			lastReloadFailed:     true,
+			lastFailedReloadTime: time.Now(),
+			expiration:           time.Now().UTC().Add(-10 * time.Minute), // expired → urgent cooldown
+		}
+		assert.True(t, c.isInCooldown())
+	})
+}
+
+func TestHuaweiCredentialProvider_RetrieveCooldown(t *testing.T) {
+	t.Run("returns cached creds during cooldown", func(t *testing.T) {
+		c := &HuaweiCredentialProvider{
+			inited:               true,
+			lastReloadFailed:     true,
+			lastFailedReloadTime: time.Now(),
+			credentials: credentials.Value{
+				AccessKeyID:     "OLD_AK",
+				SecretAccessKey: "OLD_SK",
+				SessionToken:    "OLD_TOKEN",
+				SignerType:      credentials.SignatureV4,
+			},
+			expiration: time.Now().UTC().Add(1 * time.Minute), // within 3min grace period, triggers refresh
+		}
+
+		val, err := c.Retrieve()
+		assert.NoError(t, err)
+		assert.Equal(t, "OLD_AK", val.AccessKeyID)
+	})
+
+	t.Run("returns error during cooldown with no cached creds", func(t *testing.T) {
+		c := &HuaweiCredentialProvider{
+			inited:               true,
+			lastReloadFailed:     true,
+			lastFailedReloadTime: time.Now(),
+			// no cached credentials, expiration is zero
+		}
+
+		_, err := c.Retrieve()
+		assert.Error(t, err)
+		assert.Contains(t, err.Error(), "cooldown")
+	})
+}
+
+// TestNewMinioClient_NilOpts covers the opts==nil branch in NewMinioClient.
+func TestNewMinioClient_NilOpts(t *testing.T) {
+	client, err := NewMinioClient(OBSDefaultAddress+":443", nil)
+	require.NoError(t, err)
+	assert.Equal(t, OBSDefaultAddress+":443", client.EndpointURL().Host)
+}
+
+func TestHuaweiCredentialProvider_Retrieve_STSError_WithCachedCreds(t *testing.T) {
+	c := &HuaweiCredentialProvider{
+		inited: true,
+		iamClient: &mockIAMClient{
+			err: errors.New("STS network error"),
+		},
+		credentials: credentials.Value{
+			AccessKeyID:     "CACHED_AK",
+			SecretAccessKey: "CACHED_SK",
+			SessionToken:    "CACHED_TOKEN",
+			SignerType:      credentials.SignatureV4,
+		},
+		expiration: time.Now().UTC().Add(1 * time.Minute),
+	}
+
+	val, err := c.Retrieve()
+	require.NoError(t, err)
+	assert.Equal(t, "CACHED_AK", val.AccessKeyID)
+	assert.True(t, c.lastReloadFailed)
+}
+
+func TestHuaweiCredentialProvider_Retrieve_STSError_NoCachedCreds(t *testing.T) {
+	c := &HuaweiCredentialProvider{
+		inited: true,
+		iamClient: &mockIAMClient{
+			err: errors.New("STS network error"),
+		},
+	}
+
+	_, err := c.Retrieve()
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "failed to create temporary access key")
+	assert.True(t, c.lastReloadFailed)
+}
+
+func TestHuaweiCredentialProvider_Retrieve_NilCredential_NoCachedCreds(t *testing.T) {
+	c := &HuaweiCredentialProvider{
+		inited: true,
+		iamClient: &mockIAMClient{
+			response: &model.CreateTemporaryAccessKeyByTokenResponse{
+				Credential: nil,
+			},
+		},
+	}
+
+	_, err := c.Retrieve()
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "incomplete credential")
+}
+
+func TestHuaweiCredentialProvider_Retrieve_NilCredential_WithCachedCreds(t *testing.T) {
+	c := &HuaweiCredentialProvider{
+		inited: true,
+		iamClient: &mockIAMClient{
+			response: &model.CreateTemporaryAccessKeyByTokenResponse{
+				Credential: nil,
+			},
+		},
+		credentials: credentials.Value{
+			AccessKeyID:     "CACHED_AK",
+			SecretAccessKey: "CACHED_SK",
+			SessionToken:    "CACHED_TOKEN",
+			SignerType:      credentials.SignatureV4,
+		},
+		expiration: time.Now().UTC().Add(1 * time.Minute),
+	}
+
+	val, err := c.Retrieve()
+	require.NoError(t, err)
+	assert.Equal(t, "CACHED_AK", val.AccessKeyID)
+}
+
+func TestHuaweiCredentialProvider_Retrieve_BadExpiration_NoCachedCreds(t *testing.T) {
+	c := &HuaweiCredentialProvider{
+		inited: true,
+		iamClient: &mockIAMClient{
+			response: &model.CreateTemporaryAccessKeyByTokenResponse{
+				Credential: makeFullCredential("AK", "SK", "TOKEN", "NOT-A-DATE"),
+			},
+		},
+	}
+
+	_, err := c.Retrieve()
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "failed to parse expiration time")
+}
+
+func TestHuaweiCredentialProvider_Retrieve_BadExpiration_WithCachedCreds(t *testing.T) {
+	c := &HuaweiCredentialProvider{
+		inited: true,
+		iamClient: &mockIAMClient{
+			response: &model.CreateTemporaryAccessKeyByTokenResponse{
+				Credential: makeFullCredential("AK", "SK", "TOKEN", "NOT-A-DATE"),
+			},
+		},
+		credentials: credentials.Value{
+			AccessKeyID:     "CACHED_AK",
+			SecretAccessKey: "CACHED_SK",
+			SessionToken:    "CACHED_TOKEN",
+			SignerType:      credentials.SignatureV4,
+		},
+		expiration: time.Now().UTC().Add(1 * time.Minute),
+	}
+
+	val, err := c.Retrieve()
+	require.NoError(t, err)
+	assert.Equal(t, "CACHED_AK", val.AccessKeyID)
+}
+
+func TestHuaweiCredentialProvider_Retrieve_Success(t *testing.T) {
+	futureExpiry := time.Now().UTC().Add(2 * time.Hour).Format(time.RFC3339)
+	c := &HuaweiCredentialProvider{
+		inited: true,
+		iamClient: &mockIAMClient{
+			response: &model.CreateTemporaryAccessKeyByTokenResponse{
+				Credential: makeFullCredential("NEW_AK", "NEW_SK", "NEW_TOKEN", futureExpiry),
+			},
+		},
+	}
+
+	val, err := c.Retrieve()
+	require.NoError(t, err)
+	assert.Equal(t, "NEW_AK", val.AccessKeyID)
+	assert.Equal(t, "NEW_SK", val.SecretAccessKey)
+	assert.Equal(t, "NEW_TOKEN", val.SessionToken)
+	assert.Equal(t, credentials.SignatureV4, val.SignerType)
+	assert.Equal(t, "NEW_AK", c.credentials.AccessKeyID)
+	assert.False(t, c.expiration.IsZero())
+	assert.False(t, c.lastReloadFailed)
+	assert.Equal(t, int64(1), c.stsSuccessCount.Load())
+}
+
+func TestHuaweiCredentialProvider_Retrieve_CacheHitAfterSuccess(t *testing.T) {
+	futureExpiry := time.Now().UTC().Add(2 * time.Hour).Format(time.RFC3339)
+	mc := &mockIAMClient{
+		response: &model.CreateTemporaryAccessKeyByTokenResponse{
+			Credential: makeFullCredential("AK", "SK", "TOKEN", futureExpiry),
+		},
+	}
+
+	c := &HuaweiCredentialProvider{inited: true, iamClient: mc}
+
+	_, err := c.Retrieve()
+	require.NoError(t, err)
+
+	// Second call: expiration is 2h away, well past the 3min grace — should hit cache
+	_, err = c.Retrieve()
+	require.NoError(t, err)
+}