diff --git a/onnxruntime/contrib_ops/rocm/bert/fast_gelu.cc b/onnxruntime/contrib_ops/rocm/bert/fast_gelu.cc
index 57ffcb6b4b..e42d2bbbe1 100644
--- a/onnxruntime/contrib_ops/rocm/bert/fast_gelu.cc
+++ b/onnxruntime/contrib_ops/rocm/bert/fast_gelu.cc
@@ -33,12 +33,6 @@ REGISTER_KERNEL_TYPED(BFloat16)
 
 using namespace ONNX_NAMESPACE;
 
-template <typename T>
-FastGelu<T>::FastGelu(const OpKernelInfo& op_kernel_info) : RocmKernel(op_kernel_info) {
-  const TransformerOptions* options = TransformerOptions::GetInstance();
-  tuning_ = options->IsTuningEnabled();
-}
-
 template <typename T>
 Status FastGelu<T>::ComputeInternal(OpKernelContext* context) const {
   ORT_RETURN_IF_ERROR(bias_gelu_helper::CheckInputs(context));
@@ -54,13 +48,13 @@ Status FastGelu<T>::ComputeInternal(OpKernelContext* context) const {
   int64_t bias_length = (nullptr == bias) ? 0 : bias->Shape().Size();
   typedef typename ToHipType<T>::MappedType HipT;
 
-  return LaunchFastGeluKernel<HipT>(Stream(),
-                                  static_cast<int>(input_length),
-                                  static_cast<int>(bias_length),
-                                  reinterpret_cast<const HipT*>(input->Data<T>()),
-                                  (nullptr != bias) ? reinterpret_cast<const HipT*>(bias->Data<T>()) : nullptr,
-                                  reinterpret_cast<HipT*>(output->MutableData<T>()),
-                                  tuning_);
+  return LaunchFastGeluKernel<HipT>(IsTunableOpEnabled(),
+                                    Stream(),
+                                    static_cast<int>(input_length),
+                                    static_cast<int>(bias_length),
+                                    reinterpret_cast<const HipT*>(input->Data<T>()),
+                                    (nullptr != bias) ? reinterpret_cast<const HipT*>(bias->Data<T>()) : nullptr,
+                                    reinterpret_cast<HipT*>(output->MutableData<T>()));
 }
 
 }  // namespace rocm
diff --git a/onnxruntime/contrib_ops/rocm/bert/fast_gelu.h b/onnxruntime/contrib_ops/rocm/bert/fast_gelu.h
index 6286473820..1bef59334c 100644
--- a/onnxruntime/contrib_ops/rocm/bert/fast_gelu.h
+++ b/onnxruntime/contrib_ops/rocm/bert/fast_gelu.h
@@ -16,11 +16,8 @@ using namespace onnxruntime::rocm;
 template <typename T>
 class FastGelu final : public RocmKernel {
  public:
-  FastGelu(const OpKernelInfo& op_kernel_info);
+  FastGelu(const OpKernelInfo& op_kernel_info) : RocmKernel(op_kernel_info) {}
   Status ComputeInternal(OpKernelContext* ctx) const override;
-
- private:
-  bool tuning_;
 };
 
 }  // namespace rocm
diff --git a/onnxruntime/contrib_ops/rocm/bert/fast_gelu_impl.cu b/onnxruntime/contrib_ops/rocm/bert/fast_gelu_impl.cu
index 7982fdd497..009ea9e0fa 100644
--- a/onnxruntime/contrib_ops/rocm/bert/fast_gelu_impl.cu
+++ b/onnxruntime/contrib_ops/rocm/bert/fast_gelu_impl.cu
@@ -40,10 +40,9 @@ namespace contrib {
 namespace rocm {
 
 template <typename T>
-Status LaunchFastGeluKernel(hipStream_t stream, int input_length, int bias_length,
-                            const T* input, const T* bias, T* output, bool tuning) {
+Status LaunchFastGeluKernel(bool tuning, hipStream_t stream, int input_length, int bias_length,
+                            const T* input, const T* bias, T* output) {
   FastGeluParams<T> params(stream, input, bias, output, input_length, bias_length);
-
   if (tuning) {
     static FastGeluTunableOp<T> op;
     op.EnableTuning();
@@ -53,14 +52,14 @@ Status LaunchFastGeluKernel(hipStream_t stream, int input_length, int bias_lengt
   return FastGeluStaticSelection<T>(&params);
 }
 
-template Status LaunchFastGeluKernel<float>(hipStream_t stream, int input_length, int bias_length,
-                                            const float* input, const float* bias, float* output, bool tuning);
+template Status LaunchFastGeluKernel<float>(bool tuning, hipStream_t stream, int input_length, int bias_length,
+                                            const float* input, const float* bias, float* output);
 
-template Status LaunchFastGeluKernel<BFloat16>(hipStream_t stream, int input_length, int bias_length,
-                                               const BFloat16* input, const BFloat16* bias, BFloat16* output, bool tuning);
+template Status LaunchFastGeluKernel<BFloat16>(bool tuning, hipStream_t stream, int input_length, int bias_length,
+                                               const BFloat16* input, const BFloat16* bias, BFloat16* output);
 
-template Status LaunchFastGeluKernel<half>(hipStream_t stream, int input_length, int bias_length,
-                                           const half* input, const half* bias, half* output, bool tuning);
+template Status LaunchFastGeluKernel<half>(bool tuning, hipStream_t stream, int input_length, int bias_length,
+                                           const half* input, const half* bias, half* output);
 
 }  // namespace rocm
 }  // namespace contrib
diff --git a/onnxruntime/contrib_ops/rocm/bert/fast_gelu_impl.h b/onnxruntime/contrib_ops/rocm/bert/fast_gelu_impl.h
index 5c97066c77..7f6475e3d6 100644
--- a/onnxruntime/contrib_ops/rocm/bert/fast_gelu_impl.h
+++ b/onnxruntime/contrib_ops/rocm/bert/fast_gelu_impl.h
@@ -13,8 +13,8 @@ namespace contrib {
 namespace rocm {
 
 template <typename T>
-Status LaunchFastGeluKernel(hipStream_t stream, int input_length, int bias_length,
-                            const T* input, const T* bias, T* output, bool tuning);
+Status LaunchFastGeluKernel(bool tuning, hipStream_t stream, int input_length, int bias_length,
+                            const T* input, const T* bias, T* output);
 
 }  // namespace rocm
 }  // namespace contrib
diff --git a/onnxruntime/contrib_ops/rocm/bert/gemm_fast_gelu_tunable_op.h b/onnxruntime/contrib_ops/rocm/bert/gemm_fast_gelu_tunable_op.h
index 2ce6040b5c..c6181afa26 100644
--- a/onnxruntime/contrib_ops/rocm/bert/gemm_fast_gelu_tunable_op.h
+++ b/onnxruntime/contrib_ops/rocm/bert/gemm_fast_gelu_tunable_op.h
@@ -57,13 +57,13 @@ Status GemmFastGeluUnfused(const GemmFastGeluParams<T>* params) {
   int64_t bias_length = (params->bias != nullptr) ? params->n : 0;
 
   // inplace computation
-  return LaunchFastGeluKernel<T>(params->stream,
+  return LaunchFastGeluKernel<T>(params->tuning,
+                                 params->stream,
                                  static_cast<int>(fast_gelu_input_length),
                                  static_cast<int>(bias_length),
                                  params->c,
                                  params->bias,
-                                 params->c,
-                                 params->tuning);
+                                 params->c);
 }
 
 template <typename T>
diff --git a/onnxruntime/contrib_ops/rocm/bert/skip_layer_norm.cc b/onnxruntime/contrib_ops/rocm/bert/skip_layer_norm.cc
index c1cc1a20e2..2476659682 100644
--- a/onnxruntime/contrib_ops/rocm/bert/skip_layer_norm.cc
+++ b/onnxruntime/contrib_ops/rocm/bert/skip_layer_norm.cc
@@ -31,8 +31,6 @@ template <typename T>
 SkipLayerNorm<T>::SkipLayerNorm(const OpKernelInfo& op_kernel_info) : RocmKernel(op_kernel_info) {
   ORT_ENFORCE(op_kernel_info.GetAttr<float>("epsilon", &epsilon_).IsOK());
   ORT_ENFORCE(epsilon_ >= 0);
-  const TransformerOptions* options = TransformerOptions::GetInstance();
-  tuning_ = options->IsTuningEnabled();
 }
 
 template <typename T>
@@ -100,6 +98,7 @@ Status SkipLayerNorm<T>::ComputeInternal(OpKernelContext* ctx) const {
   typedef typename ToHipType<T>::MappedType HipT;
 
   return LaunchSkipLayerNormKernel<HipT>(
+      IsTunableOpEnabled(),
       Stream(),
       reinterpret_cast<HipT*>(output->MutableData<T>()),
       reinterpret_cast<const HipT*>(input->Data<T>()),
@@ -109,8 +108,7 @@ Status SkipLayerNorm<T>::ComputeInternal(OpKernelContext* ctx) const {
       (bias != nullptr) ? reinterpret_cast<const HipT*>(bias->Data<T>()) : nullptr,
       epsilon_,
       hidden_size,
-      static_cast<int>(element_count),
-      tuning_);
+      static_cast<int>(element_count));
 }
 
 }  // namespace rocm
diff --git a/onnxruntime/contrib_ops/rocm/bert/skip_layer_norm.h b/onnxruntime/contrib_ops/rocm/bert/skip_layer_norm.h
index 5e43f193f6..07d7037227 100644
--- a/onnxruntime/contrib_ops/rocm/bert/skip_layer_norm.h
+++ b/onnxruntime/contrib_ops/rocm/bert/skip_layer_norm.h
@@ -19,7 +19,6 @@ class SkipLayerNorm final : public RocmKernel {
 
  private:
   float epsilon_;
-  bool tuning_;
 };
 
 }  // namespace rocm
diff --git a/onnxruntime/contrib_ops/rocm/bert/skip_layer_norm_impl.cu b/onnxruntime/contrib_ops/rocm/bert/skip_layer_norm_impl.cu
index d54166c2c1..13baf5ec1e 100644
--- a/onnxruntime/contrib_ops/rocm/bert/skip_layer_norm_impl.cu
+++ b/onnxruntime/contrib_ops/rocm/bert/skip_layer_norm_impl.cu
@@ -41,8 +41,8 @@ namespace rocm {
 
 template <typename T>
 Status LaunchSkipLayerNormKernel(
-    hipStream_t stream, T* output, const T* input, const T* skip, const T* gamma,
-    const T* beta, const T* bias, float epsilon, int ld, int element_count, bool tuning) {
+    bool tuning, hipStream_t stream, T* output, const T* input, const T* skip, const T* gamma,
+    const T* beta, const T* bias, float epsilon, int ld, int element_count) {
   // this must be true because element_count is the total size of the tensor
   assert(element_count % ld == 0);
 
@@ -57,15 +57,15 @@ Status LaunchSkipLayerNormKernel(
   return SkipLayerNormStaticSelection<T>(&params);
 }
 
-template Status LaunchSkipLayerNormKernel<float>(hipStream_t stream, float* output, const float* input,
+template Status LaunchSkipLayerNormKernel<float>(bool tuning, hipStream_t stream, float* output, const float* input,
                                                  const float* skip, const float* gamma, const float* beta,
                                                  const float* bias, float epsilon, int ld,
-                                                 int element_count, bool tuning);
+                                                 int element_count);
 
-template Status LaunchSkipLayerNormKernel<half>(hipStream_t stream, half* output, const half* input,
+template Status LaunchSkipLayerNormKernel<half>(bool tuning, hipStream_t stream, half* output, const half* input,
                                                 const half* skip, const half* gamma, const half* beta,
                                                 const half* bias, float epsilon, int ld,
-                                                int element_count, bool tuning);
+                                                int element_count);
 
 }  // namespace rocm
 }  // namespace contrib
diff --git a/onnxruntime/contrib_ops/rocm/bert/skip_layer_norm_impl.h b/onnxruntime/contrib_ops/rocm/bert/skip_layer_norm_impl.h
index 9758988bad..c32b6c48a8 100644
--- a/onnxruntime/contrib_ops/rocm/bert/skip_layer_norm_impl.h
+++ b/onnxruntime/contrib_ops/rocm/bert/skip_layer_norm_impl.h
@@ -10,17 +10,18 @@ namespace rocm {
 
 template <typename T>
 Status LaunchSkipLayerNormKernel(
+    bool tuning,
     hipStream_t stream,
-    T* output,          // output tensor
-    const T* input,     // input tensor
-    const T* skip,      // skip tensor
-    const T* gamma,     // Layer normalization gamma tensor
-    const T* beta,      // Layer normalization beta tensor
-    const T* bias,      // Layer normalization beta tensor
-    float epsilon,      // Layer normalization epsilon
-    int hidden_size,    // hidden size, it is the leading dimension (ld)
-    int element_count,  // number of elements in input tensor
-    bool tuning);
+    T* output,         // output tensor
+    const T* input,    // input tensor
+    const T* skip,     // skip tensor
+    const T* gamma,    // Layer normalization gamma tensor
+    const T* beta,     // Layer normalization beta tensor
+    const T* bias,     // Layer normalization beta tensor
+    float epsilon,     // Layer normalization epsilon
+    int hidden_size,   // hidden size, it is the leading dimension (ld)
+    int element_count  // number of elements in input tensor
+);
 
 }  // namespace rocm
 }  // namespace contrib
diff --git a/onnxruntime/contrib_ops/rocm/bert/transformer_common.h b/onnxruntime/contrib_ops/rocm/bert/transformer_common.h
index 5984958eda..6816b5b9d0 100644
--- a/onnxruntime/contrib_ops/rocm/bert/transformer_common.h
+++ b/onnxruntime/contrib_ops/rocm/bert/transformer_common.h
@@ -19,13 +19,10 @@ class TransformerOptions {
 
   bool DisableHalf2() const { return disable_half2_; }
 
-  bool IsTuningEnabled() const { return tuning_; }
-
   void Initialize(int value) {
     is_precision_mode_ = (value & 0x01) > 0;
     disable_persistent_softmax_ = (value & 0x02) > 0;
     disable_half2_ = (value & 0x04) > 0;
-    tuning_ = (value & 0x08) > 0;
     initialized_ = true;
   }
 
@@ -40,8 +37,6 @@ class TransformerOptions {
   bool disable_half2_{false};
 
   bool initialized_{false};
- 
-  bool tuning_{false};
 
   static TransformerOptions instance;
 };