diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt
index b0f4ab5ab7..7c85de67e3 100644
--- a/cmake/CMakeLists.txt
+++ b/cmake/CMakeLists.txt
@@ -504,7 +504,7 @@ else()
   endif()
 endif()
 
-if(UNIX AND onnxruntime_ENABLE_LTO)
+if(UNIX AND onnxruntime_ENABLE_LTO AND NOT onnxruntime_PREFER_SYSTEM_LIB)
   #https://github.com/protocolbuffers/protobuf/issues/5923
   target_link_options(protoc PRIVATE "-Wl,--no-as-needed")
 endif()