diff --git a/onnxruntime/test/common/cuda_op_test_utils.h b/onnxruntime/test/common/cuda_op_test_utils.h
index cf72f66e32..043e3059c3 100644
--- a/onnxruntime/test/common/cuda_op_test_utils.h
+++ b/onnxruntime/test/common/cuda_op_test_utils.h
@@ -44,5 +44,13 @@ inline bool NeedSkipIfCudaArchLowerThan(int min_cuda_architecture) {
   }
   return false;
 }
+
+inline bool NeedSkipIfCudaArchGreaterEqualThan(int max_cuda_architecture) {
+  // only skip when CUDA ep is enabled.
+  if (DefaultCudaExecutionProvider().get() != nullptr) {
+    return HasCudaEnvironment(max_cuda_architecture);
+  }
+  return false;
+}
 }  // namespace test
 }  // namespace onnxruntime
diff --git a/onnxruntime/test/contrib_ops/layer_norm_op_test.cc b/onnxruntime/test/contrib_ops/layer_norm_op_test.cc
index 12534e283a..84bbee35ee 100644
--- a/onnxruntime/test/contrib_ops/layer_norm_op_test.cc
+++ b/onnxruntime/test/contrib_ops/layer_norm_op_test.cc
@@ -20,7 +20,7 @@ namespace onnxruntime {
 namespace test {
 
 TEST(LayerNormTest, BERTLayerNorm) {
-  OpTester tester("LayerNormalization", 1 /*opset_version*/);
+  OpTester tester("LayerNormalization", 17 /*opset_version*/);
   tester.AddAttribute<int64_t>("axis", -1);
   tester.AddAttribute<float>("epsilon", 1e-12f);
 
@@ -40,12 +40,11 @@ TEST(LayerNormTest, BERTLayerNorm) {
   tester.AddInput<float>("B", B_dims, B_data);
 
   tester.AddReferenceOutputs("testdata/layernorm.onnx");
-
   tester.Run();
 }
 
 TEST(LayerNormTest, BERTLayerNorm_NoBias) {
-  OpTester tester("LayerNormalization", 1 /*opset_version*/);
+  OpTester tester("LayerNormalization", 17 /*opset_version*/);
   tester.AddAttribute<int64_t>("axis", -1);
   tester.AddAttribute<float>("epsilon", 1e-12f);
 
@@ -60,8 +59,6 @@ TEST(LayerNormTest, BERTLayerNorm_NoBias) {
   std::vector<float> scale_data = random.Uniform<float>(scale_dims, 0.0f, 1.0f);
   tester.AddInput<float>("Scale", scale_dims, scale_data);
 
-  tester.AddOptionalInputEdge<float>();
-
   tester.AddReferenceOutputs("testdata/layernorm_no_bias.onnx");
 
   tester.Run();
@@ -227,7 +224,7 @@ TEST(LayerNormTest, LayerNorm_InvalidScaleBias) {
   // implementation for which we don't control the check or error message.
   test.Run(OpTester::ExpectResult::kExpectFailure,
            "Size of X.shape()[axis:] == 6. Size of scale and bias (if provided) must match this",
-           {kDnnlExecutionProvider, kDmlExecutionProvider});
+           {kDnnlExecutionProvider, kDmlExecutionProvider, kTensorrtExecutionProvider});
 }
 
 #if defined(USE_DNNL)
diff --git a/onnxruntime/test/contrib_ops/qordered_attention_test.cc b/onnxruntime/test/contrib_ops/qordered_attention_test.cc
index 5257fbdb08..72bae6fd12 100644
--- a/onnxruntime/test/contrib_ops/qordered_attention_test.cc
+++ b/onnxruntime/test/contrib_ops/qordered_attention_test.cc
@@ -246,8 +246,8 @@ TEST(QOrderedTest, Attention_WithData_ROW_ORDER) {
     return;
   }
 
-  // Needs Turing or higher architecture
-  if (NeedSkipIfCudaArchLowerThan(750)) {
+  // Needs Turing architecture
+  if (NeedSkipIfCudaArchLowerThan(750) || NeedSkipIfCudaArchGreaterEqualThan(800)) {
     return;
   }
 
diff --git a/onnxruntime/test/contrib_ops/qordered_longformer_attention_op_test.cc b/onnxruntime/test/contrib_ops/qordered_longformer_attention_op_test.cc
index fa32536e4d..4549e67327 100644
--- a/onnxruntime/test/contrib_ops/qordered_longformer_attention_op_test.cc
+++ b/onnxruntime/test/contrib_ops/qordered_longformer_attention_op_test.cc
@@ -41,8 +41,8 @@ static void run_qordered_longformer_attention_op_test(
     return;
   }
 
-  // Needs Turing or higher architecture
-  if (NeedSkipIfCudaArchLowerThan(750)) {
+  // Needs Turing architecture
+  if (NeedSkipIfCudaArchLowerThan(750) || NeedSkipIfCudaArchGreaterEqualThan(800)) {
     return;
   }
 
diff --git a/onnxruntime/test/contrib_ops/qordered_matmul_op_test.cc b/onnxruntime/test/contrib_ops/qordered_matmul_op_test.cc
index d5a8948445..ec9f2f7d26 100644
--- a/onnxruntime/test/contrib_ops/qordered_matmul_op_test.cc
+++ b/onnxruntime/test/contrib_ops/qordered_matmul_op_test.cc
@@ -27,8 +27,8 @@ static void RunQOrdered_MatMul_Test(
     return;
   }
 
-  // Needs Turing or higher architecture
-  if (NeedSkipIfCudaArchLowerThan(750)) {
+  // Needs Turing architecture
+  if (NeedSkipIfCudaArchLowerThan(750) || NeedSkipIfCudaArchGreaterEqualThan(800)) {
     return;
   }