diff --git a/cmake/onnxruntime_rocm_hipify.cmake b/cmake/onnxruntime_rocm_hipify.cmake
index 8a9c54359e..ebd6229204 100644
--- a/cmake/onnxruntime_rocm_hipify.cmake
+++ b/cmake/onnxruntime_rocm_hipify.cmake
@@ -90,8 +90,8 @@ set(contrib_ops_excluded_files
   "cuda_contrib_kernels.h"
   "inverse.cc"
   "fused_conv.cc"
-  "decoder/decoder_masked_multihead_attention.h"
-  "decoder/decoder_masked_multihead_attention.cc"
+  "decoder/decoder_masked_self_attention.h"
+  "decoder/decoder_masked_self_attention.cc"
   "decoder/fastertransformer_decoder_attention/decoder_masked_multihead_attention_impl.h"
   "decoder/fastertransformer_decoder_attention/decoder_masked_multihead_attention.h"
   "decoder/fastertransformer_decoder_attention/decoder_masked_multihead_attention_impl.cu"
diff --git a/onnxruntime/contrib_ops/cpu/transformers/beam_search_impl_gpt.h b/onnxruntime/contrib_ops/cpu/transformers/beam_search_impl_gpt.h
index b822938719..e90738a263 100644
--- a/onnxruntime/contrib_ops/cpu/transformers/beam_search_impl_gpt.h
+++ b/onnxruntime/contrib_ops/cpu/transformers/beam_search_impl_gpt.h
@@ -51,7 +51,7 @@ class BeamSearchGpt : public BeamSearchBase<T> {
         cuda_device_arch_(cuda_device_arch) {
     if (gpt_subgraph_.has_decoder_masked_self_attention_) {
       ORT_ENFORCE(cuda_device_arch_ >= 530,
-                  "Decoder masked multihead attention can only be used on "
+                  "Decoder masked self attention can only be used on "
                   "GPU cards of compute capability 5.3 or higher. "
                   "This card has compute capability ",
                   cuda_device_arch_);
diff --git a/onnxruntime/contrib_ops/cpu/transformers/greedy_search_impl_gpt.h b/onnxruntime/contrib_ops/cpu/transformers/greedy_search_impl_gpt.h
index 4f5271ff46..ddbf86dd82 100644
--- a/onnxruntime/contrib_ops/cpu/transformers/greedy_search_impl_gpt.h
+++ b/onnxruntime/contrib_ops/cpu/transformers/greedy_search_impl_gpt.h
@@ -67,7 +67,7 @@ class GreedySearchGpt : public GreedySearchBase<T, ParametersT> {
         cuda_device_arch_(cuda_device_arch) {
     if (gpt_subgraph_.has_decoder_masked_self_attention_) {
       ORT_ENFORCE(cuda_device_arch_ >= 530,
-                  "Decoder masked multihead attention can only be used on "
+                  "Decoder masked self attention can only be used on "
                   "GPU cards of compute capability 5.3 or higher. "
                   "This card has compute capability ",
                   cuda_device_arch_);
diff --git a/onnxruntime/contrib_ops/cuda/decoder/decoder_masked_self_attention.cc b/onnxruntime/contrib_ops/cuda/decoder/decoder_masked_self_attention.cc
index e56dfeeec3..bba7649703 100644
--- a/onnxruntime/contrib_ops/cuda/decoder/decoder_masked_self_attention.cc
+++ b/onnxruntime/contrib_ops/cuda/decoder/decoder_masked_self_attention.cc
@@ -73,10 +73,7 @@ Status DecoderMaskedSelfAttention<T1, T2>::ComputeInternal(OpKernelContext* cont
     return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input sequence length should be 1 to use DecoderMaskedSelfAttention");
   }
 
-  // TODO(hasesh): In future, we may support CrossAttention. Currently, this kernel only supports SelfAttention.
-  if (parameters.sequence_length != parameters.kv_sequence_length) {
-    return ORT_MAKE_STATUS(ONNXRUNTIME, NOT_IMPLEMENTED, "DecoderMaskedSelfAttention only supports self attention currently");
-  }
+  ORT_ENFORCE(parameters.sequence_length == parameters.kv_sequence_length);
 
   // TODO(hasesh): If there is a need, we will support this later
   if (parameters.head_size != parameters.v_head_size) {