diff --git a/ggml/src/CMakeLists.txt b/ggml/src/CMakeLists.txt
index 48848e4c6..a128c0c06 100644
--- a/ggml/src/CMakeLists.txt
+++ b/ggml/src/CMakeLists.txt
@@ -1100,11 +1100,11 @@ if (CMAKE_OSX_ARCHITECTURES      STREQUAL "arm64" OR
         if (GGML_SVE)
             list(APPEND ARCH_FLAGS -march=armv8.6-a+sve)
         endif()
-        if (GGML_NATIVE)
-            list(APPEND ARCH_FLAGS -march=native)
-        endif()
         if (CMAKE_CXX_COMPILER_ID MATCHES "GNU")
             # else we fail on Gravitons and such
+            if (GGML_NATIVE)
+                list(APPEND ARCH_FLAGS -march=native)
+            endif()
             list(APPEND ARCH_FLAGS -flax-vector-conversions)
         endif()
     endif()