diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt
index 9d3c94f91d..0a00b0835f 100644
--- a/cmake/CMakeLists.txt
+++ b/cmake/CMakeLists.txt
@@ -262,7 +262,7 @@ if (onnxruntime_USE_ROCM)
   endif()
 
   if (NOT CMAKE_HIP_ARCHITECTURES)
-    set(CMAKE_HIP_ARCHITECTURES "gfx906;gfx908;gfx90a;gfx1030")
+    set(CMAKE_HIP_ARCHITECTURES "gfx906;gfx908;gfx90a")
   endif()
 
   file(GLOB rocm_cmake_components ${onnxruntime_ROCM_HOME}/lib/cmake/*)
diff --git a/onnxruntime/contrib_ops/rocm/bert/layer_norm.cuh b/onnxruntime/contrib_ops/rocm/bert/layer_norm.cuh
index 7386851647..169d0ed3b0 100644
--- a/onnxruntime/contrib_ops/rocm/bert/layer_norm.cuh
+++ b/onnxruntime/contrib_ops/rocm/bert/layer_norm.cuh
@@ -23,7 +23,7 @@ limitations under the License.
 #pragma once
 
 #include <hip/hip_fp16.h>
-#include <hipblas.h>
+#include <hipblas/hipblas.h>
 #include <hipcub/hipcub.hpp>
 #include "core/providers/rocm/rocm_common.h"
 #include "core/providers/rocm/cu_inc/common.cuh"
diff --git a/tools/ci_build/amd_hipify.py b/tools/ci_build/amd_hipify.py
index 063f066d6d..e6a5f8f8cc 100644
--- a/tools/ci_build/amd_hipify.py
+++ b/tools/ci_build/amd_hipify.py
@@ -123,7 +123,7 @@ def hipify(hipify_perl_path, src_file_path, dst_file_path):
 
     # NCCL -> RCCL
     # s = s.replace('NCCL_CALL', 'RCCL_CALL')
-    s = s.replace("#include <nccl.h>", "#include <rccl.h>")
+    s = s.replace("#include <nccl.h>", "#include <rccl/rccl.h>")
 
     # CUDNN -> MIOpen
     s = s.replace("CUDNN", "MIOPEN")
@@ -161,6 +161,11 @@ def hipify(hipify_perl_path, src_file_path, dst_file_path):
     # Deletions
     s = s.replace('#include "device_atomic_functions.h"', "")  # HIP atomics in main hip header already
 
+    # Fix warnings due to incorrect header paths, intentionally after all other hipify steps.
+    s = s.replace("#include <hiprand_kernel.h>", "#include <hiprand/hiprand_kernel.h>")
+    s = s.replace("#include <rocblas.h>", "#include <rocblas/rocblas.h>")
+    s = s.replace("#include <hipblas.h>", "#include <hipblas/hipblas.h>")
+
     with open(dst_file_path, "w") as f:
         f.write(s)