diff --git a/cmake/onnxruntime_mlas.cmake b/cmake/onnxruntime_mlas.cmake
index e79e1ed985..6292a667d4 100644
--- a/cmake/onnxruntime_mlas.cmake
+++ b/cmake/onnxruntime_mlas.cmake
@@ -7,7 +7,17 @@ set(MLAS_SRC_DIR ${ONNXRUNTIME_ROOT}/core/mlas/lib)
 set(MLAS_AMX_SUPPORTED FALSE)
 
 if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_C_COMPILER_VERSION VERSION_GREATER_EQUAL 11)
-  set(MLAS_AMX_SUPPORTED TRUE)
+  # match assembler version, AMX instructions are supported from 2.40
+  if (CMAKE_ASM-ATT_COMPILER_ID STREQUAL "GNU")
+    execute_process(
+        COMMAND ${CMAKE_ASM-ATT_COMPILER} --version
+        OUTPUT_VARIABLE _gas_version
+    )
+    # 2.40 or later
+    if (_gas_version MATCHES "GNU.[Aa]ssembler.*(2\\.[4-9][0-9])")
+        set(MLAS_AMX_SUPPORTED TRUE)
+    endif()
+  endif()
 endif()
 
 if(CMAKE_CXX_COMPILER_ID MATCHES "MSVC")