diff --git a/onnxruntime/contrib_ops/cuda/bert/fast_gelu.cc b/onnxruntime/contrib_ops/cuda/bert/fast_gelu.cc
index 864c614b99..a547089232 100644
--- a/onnxruntime/contrib_ops/cuda/bert/fast_gelu.cc
+++ b/onnxruntime/contrib_ops/cuda/bert/fast_gelu.cc
@@ -34,7 +34,7 @@ using namespace ONNX_NAMESPACE;
 template <typename T>
 FastGelu<T>::FastGelu(const OpKernelInfo& op_kernel_info) : CudaKernel(op_kernel_info) {
   const TransformerOptions* options = TransformerOptions::GetInstance();
-  use_half2_ = !options->DisableHalf2();
+  use_half2_ = options->EnableHalf2();
 }
 
 template <typename T>
diff --git a/onnxruntime/contrib_ops/cuda/bert/transformer_common.cc b/onnxruntime/contrib_ops/cuda/bert/transformer_common.cc
index 847ad11cbf..bac94463aa 100644
--- a/onnxruntime/contrib_ops/cuda/bert/transformer_common.cc
+++ b/onnxruntime/contrib_ops/cuda/bert/transformer_common.cc
@@ -25,7 +25,7 @@ const TransformerOptions* TransformerOptions::GetInstance() {
     if (value > 0)
       std::cout << "ORT_TRANSFORMER_OPTIONS: IsPrecisionMode=" << instance.IsPrecisionMode()
                 << ",DisablePersistentSoftmax=" << instance.DisablePersistentSoftmax()
-                << ",DisableHalf2=" << instance.DisableHalf2()
+                << ",EnableHalf2=" << instance.EnableHalf2()
                 << std::endl;
   }
 
diff --git a/onnxruntime/contrib_ops/cuda/bert/transformer_common.h b/onnxruntime/contrib_ops/cuda/bert/transformer_common.h
index 74c1b2522d..1ef75639c1 100644
--- a/onnxruntime/contrib_ops/cuda/bert/transformer_common.h
+++ b/onnxruntime/contrib_ops/cuda/bert/transformer_common.h
@@ -15,12 +15,12 @@ class TransformerOptions {
 
   bool DisablePersistentSoftmax() const { return disable_persistent_softmax_; }
 
-  bool DisableHalf2() const { return disable_half2_; }
+  bool EnableHalf2() const { return enable_half2_; }
 
   void Initialize(int value) {
     is_precision_mode_ = (value & 0x01) > 0;
     disable_persistent_softmax_ = (value & 0x02) > 0;
-    disable_half2_ = (value & 0x04) > 0;
+    enable_half2_ = (value & 0x04) > 0;
     initialized_ = true;
   }
 
@@ -31,8 +31,8 @@ class TransformerOptions {
   // Disable persistent softmax.
   bool disable_persistent_softmax_{false};
 
-  // Disable half2 kernel.
-  bool disable_half2_{false};
+  // Enable half2 kernel.
+  bool enable_half2_{false};
 
   bool initialized_{false};