diff --git a/onnxruntime/contrib_ops/cpu/bert/attention_cpu_base.h b/onnxruntime/contrib_ops/cpu/bert/attention_cpu_base.h
index 8ae7b4589d..9eb9038961 100644
--- a/onnxruntime/contrib_ops/cpu/bert/attention_cpu_base.h
+++ b/onnxruntime/contrib_ops/cpu/bert/attention_cpu_base.h
@@ -9,6 +9,7 @@
 #include "core/common/common.h"
 #include "core/common/safeint.h"
 #include "core/framework/op_kernel.h"
+#include "contrib_ops/cpu/utils/dump_tensor.h"
 
 namespace onnxruntime {
 namespace contrib {
@@ -204,12 +205,18 @@ class AttentionCPUBase : public AttentionBase {
       });
     }
 
+    DUMP_CPU_TENSOR_INIT();
+    DUMP_CPU_TENSOR("Q", Q, batch_size, num_heads_, sequence_length, head_size);
+    DUMP_CPU_TENSOR("QK (scaled)", attention_probs, batch_size, num_heads_, sequence_length, total_sequence_length);
+
     // attention_probs(B, N, S, T) = Softmax(attention_probs)
     {
       const int N = batch_size * num_heads_ * sequence_length;
       const int D = total_sequence_length;
       ComputeAttentionSoftmaxInplace(attention_probs, N, D, tp);
     }
+
+    DUMP_CPU_TENSOR("Softmax(QK)", attention_probs, batch_size, num_heads_, sequence_length, total_sequence_length);
   }
 
   template <typename T>
diff --git a/onnxruntime/contrib_ops/cpu/transformers/beam_search.cc b/onnxruntime/contrib_ops/cpu/transformers/beam_search.cc
index 93cda00e5a..688b7d6341 100644
--- a/onnxruntime/contrib_ops/cpu/transformers/beam_search.cc
+++ b/onnxruntime/contrib_ops/cpu/transformers/beam_search.cc
@@ -30,7 +30,7 @@
 #include "contrib_ops/cpu/transformers/beam_search.h"
 #include "contrib_ops/cpu/transformers/logits_processor.h"
 #include "contrib_ops/cpu/transformers/sequences.h"
-#include "contrib_ops/cpu/transformers/dump_tensor.h"
+#include "contrib_ops/cpu/utils/dump_tensor.h"
 #include "contrib_ops/cpu/transformers/beam_search_scorer.h"
 #include "contrib_ops/cpu/transformers/beam_search_impl_gpt.h"
 #include "contrib_ops/cpu/transformers/beam_search_impl_t5.h"
diff --git a/onnxruntime/contrib_ops/cpu/transformers/generation_device_helper.cc b/onnxruntime/contrib_ops/cpu/transformers/generation_device_helper.cc
index 927d3a58e5..8c1ceec62f 100644
--- a/onnxruntime/contrib_ops/cpu/transformers/generation_device_helper.cc
+++ b/onnxruntime/contrib_ops/cpu/transformers/generation_device_helper.cc
@@ -288,7 +288,7 @@ Status ProcessLogits(const OrtValue& logits,                                 //
                      const transformers::IGenerationParameters* parameters,  // parameters
                      int step,                                               // iteration counter
                      Stream* stream,                                         // cuda stream (for CUDA only)
-                     const transformers::IConsoleDumper* dumper) {           // tensor dumper
+                     const IConsoleDumper* dumper) {                         // tensor dumper
 #ifndef DEBUG_GENERATION
   ORT_UNUSED_PARAMETER(dumper);
 #endif
@@ -450,7 +450,7 @@ Status GreedySearchProcessLogits(
     bool do_sampling,                                       // whether to do sampling
     int step,                                               // iteration counter
     Stream* stream,                                         // cuda stream (for CUDA only)
-    const transformers::IConsoleDumper* dumper) {           // tensor dumper
+    const IConsoleDumper* dumper) {                         // tensor dumper
 
   int batch_size = parameters->batch_size;
   int vocab_size = parameters->vocab_size;
@@ -810,7 +810,7 @@ Status UpdateDecoderFeeds(
     bool past_present_share_buffer,
     bool need_cache_indir,
     transformers::Sequences& sequences,
-    const transformers::IConsoleDumper* dumper) {
+    const IConsoleDumper* dumper) {
   ORT_UNUSED_PARAMETER(stream);
   ORT_UNUSED_PARAMETER(beam_indices_gpu);
   ORT_UNUSED_PARAMETER(input_sequence_len);
@@ -952,7 +952,7 @@ template Status ProcessLogits<float>(
     const transformers::IGenerationParameters* parameters,
     int step,
     Stream* stream,
-    const transformers::IConsoleDumper* dumper);
+    const IConsoleDumper* dumper);
 
 template Status GreedySearchProcessLogits<float>(
     const OrtValue& logits,
@@ -966,7 +966,7 @@ template Status GreedySearchProcessLogits<float>(
     bool do_sampling,
     int step,
     Stream* ort_stream,
-    const transformers::IConsoleDumper* dumper);
+    const IConsoleDumper* dumper);
 
 template Status DeviceCopy<float>(
     gsl::span<float> target,
@@ -1017,7 +1017,7 @@ template Status UpdateDecoderFeeds<float>(
     bool past_present_share_buffer,
     bool need_cache_indir,
     transformers::Sequences& sequences,
-    const transformers::IConsoleDumper* dumper);
+    const IConsoleDumper* dumper);
 
 template Status UpdateDecoderFeeds<MLFloat16>(
     AllocatorPtr allocator,
@@ -1037,7 +1037,7 @@ template Status UpdateDecoderFeeds<MLFloat16>(
     bool past_present_share_buffer,
     bool need_cache_indir,
     transformers::Sequences& sequences,
-    const transformers::IConsoleDumper* dumper);
+    const IConsoleDumper* dumper);
 
 template void ExpandInputs<int32_t>(const OrtValue& input, int num_beams, AllocatorPtr allocator, OrtValue& expanded);
 
diff --git a/onnxruntime/contrib_ops/cpu/transformers/generation_device_helper.h b/onnxruntime/contrib_ops/cpu/transformers/generation_device_helper.h
index 6dfdc6b027..8f778c57bb 100644
--- a/onnxruntime/contrib_ops/cpu/transformers/generation_device_helper.h
+++ b/onnxruntime/contrib_ops/cpu/transformers/generation_device_helper.h
@@ -106,7 +106,7 @@ using ProcessLogitsFunc = std::function<Status(
     const transformers::IGenerationParameters* parameters,  // parameters
     int step,                                               // iteration counter
     Stream* stream,                                         // cuda stream (for CUDA only)
-    const transformers::IConsoleDumper* dumper)>;           // tensor dumper
+    const IConsoleDumper* dumper)>;                         // tensor dumper
 
 template <typename T>
 using GreedySearchProcessLogitsFunc = std::function<Status(
@@ -121,7 +121,7 @@ using GreedySearchProcessLogitsFunc = std::function<Status(
     bool do_sampling,                                       // whether to do sampling
     int step,                                               // iteration counter
     Stream* ort_stream,                                     // cuda stream (for CUDA only)
-    const transformers::IConsoleDumper* dumper)>;           // tensor dumper
+    const IConsoleDumper* dumper)>;                         // tensor dumper
 
 template <typename T>
 using DeviceCopyFunc = std::function<Status(
@@ -182,7 +182,7 @@ using UpdateDecoderFeedsFunc = std::function<Status(
     bool past_present_share_buffer,
     bool need_cache_indir,
     transformers::Sequences& sequences,
-    const transformers::IConsoleDumper* dumper)>;
+    const IConsoleDumper* dumper)>;
 
 //------------------------------------------------
 //  Modified functions for Whisper Model
@@ -277,7 +277,7 @@ Status ProcessLogits(const OrtValue& logits,                                 //
                      const transformers::IGenerationParameters* parameters,  // parameters
                      int step,                                               // iteration counter
                      Stream* stream,                                         // cuda stream (for CUDA only)
-                     const transformers::IConsoleDumper* dumper);            // tensor dumper
+                     const IConsoleDumper* dumper);                          // tensor dumper
 
 template <typename T>
 Status GreedySearchProcessLogits(const OrtValue& logits,                                 // logits output of subgraph
@@ -291,7 +291,7 @@ Status GreedySearchProcessLogits(const OrtValue& logits,
                                  bool do_sampling,                                       // whether to do sampling
                                  int step,                                               // iteration counter
                                  Stream* stream,                                         // cuda stream (for CUDA only)
-                                 const transformers::IConsoleDumper* dumper);            // tensor dumper
+                                 const IConsoleDumper* dumper);                          // tensor dumper
 
 template <typename T>
 Status DeviceCopy(gsl::span<T> target,
@@ -367,7 +367,7 @@ Status UpdateDecoderFeeds(
     bool past_present_share_buffer,
     bool need_cache_indir,
     transformers::Sequences& sequences,
-    const transformers::IConsoleDumper* dumper);
+    const IConsoleDumper* dumper);
 
 // ---------------------------------------------------------------
 // Functions for encoder-decoder model with float input like Whisper
diff --git a/onnxruntime/contrib_ops/cpu/transformers/generation_shared.h b/onnxruntime/contrib_ops/cpu/transformers/generation_shared.h
index b1dd55eb20..2b8b26f0a0 100644
--- a/onnxruntime/contrib_ops/cpu/transformers/generation_shared.h
+++ b/onnxruntime/contrib_ops/cpu/transformers/generation_shared.h
@@ -7,7 +7,8 @@
 #include <random>
 #include "core/common/gsl.h"
 #include "core/framework/allocator.h"
-#include "contrib_ops/cpu/utils/console_dumper.h"
+#include "core/framework/ort_value.h"
+#include "contrib_ops/cpu/utils/debug_macros.h"
 
 namespace onnxruntime {
 
diff --git a/onnxruntime/contrib_ops/cpu/transformers/greedy_search.cc b/onnxruntime/contrib_ops/cpu/transformers/greedy_search.cc
index 78395e347b..788eab1b67 100644
--- a/onnxruntime/contrib_ops/cpu/transformers/greedy_search.cc
+++ b/onnxruntime/contrib_ops/cpu/transformers/greedy_search.cc
@@ -30,7 +30,7 @@
 #include "contrib_ops/cpu/transformers/greedy_search.h"
 #include "contrib_ops/cpu/transformers/logits_processor.h"
 #include "contrib_ops/cpu/transformers/sequences.h"
-#include "contrib_ops/cpu/transformers/dump_tensor.h"
+#include "contrib_ops/cpu/utils/dump_tensor.h"
 #include "contrib_ops/cpu/transformers/greedy_search_impl_gpt.h"
 
 using namespace ONNX_NAMESPACE;
diff --git a/onnxruntime/contrib_ops/cpu/transformers/logits_processor.cc b/onnxruntime/contrib_ops/cpu/transformers/logits_processor.cc
index c74e9160cc..d28aae02ab 100644
--- a/onnxruntime/contrib_ops/cpu/transformers/logits_processor.cc
+++ b/onnxruntime/contrib_ops/cpu/transformers/logits_processor.cc
@@ -8,7 +8,7 @@
 #include "core/common/span_utils.h"
 #include "core/providers/cpu/math/softmax_shared.h"
 #include "contrib_ops/cpu/transformers/logits_processor.h"
-#include "contrib_ops/cpu/transformers/dump_tensor.h"
+#include "contrib_ops/cpu/utils/dump_tensor.h"
 #include <vector>
 #include <numeric>
 #include <algorithm>
diff --git a/onnxruntime/contrib_ops/cpu/transformers/logits_processor.h b/onnxruntime/contrib_ops/cpu/transformers/logits_processor.h
index 231eb17d1a..6e157d8315 100644
--- a/onnxruntime/contrib_ops/cpu/transformers/logits_processor.h
+++ b/onnxruntime/contrib_ops/cpu/transformers/logits_processor.h
@@ -6,7 +6,7 @@
 #include "core/common/inlined_containers.h"
 #include "contrib_ops/cpu/transformers/sequences.h"
 #include "contrib_ops/cpu/transformers/beam_search_parameters.h"
-#include "contrib_ops/cpu/transformers/dump_tensor.h"
+#include "contrib_ops/cpu/utils/dump_tensor.h"
 #include "contrib_ops/cpu/transformers/greedy_search_parameters.h"
 #include "contrib_ops/cpu/transformers/sampling_parameters.h"
 #include "contrib_ops/cpu/transformers/generation_shared.h"
diff --git a/onnxruntime/contrib_ops/cpu/transformers/sampling.cc b/onnxruntime/contrib_ops/cpu/transformers/sampling.cc
index 101b059848..4a13331386 100644
--- a/onnxruntime/contrib_ops/cpu/transformers/sampling.cc
+++ b/onnxruntime/contrib_ops/cpu/transformers/sampling.cc
@@ -14,7 +14,7 @@
 #include "contrib_ops/cpu/transformers/sampling.h"
 #include "contrib_ops/cpu/transformers/logits_processor.h"
 #include "contrib_ops/cpu/transformers/sequences.h"
-#include "contrib_ops/cpu/transformers/dump_tensor.h"
+#include "contrib_ops/cpu/utils/dump_tensor.h"
 #include "contrib_ops/cpu/transformers/greedy_search_impl_gpt.h"
 
 using namespace ONNX_NAMESPACE;
diff --git a/onnxruntime/contrib_ops/cpu/transformers/sampling_cpu_helper.h b/onnxruntime/contrib_ops/cpu/transformers/sampling_cpu_helper.h
index b05342382d..413ef596cd 100644
--- a/onnxruntime/contrib_ops/cpu/transformers/sampling_cpu_helper.h
+++ b/onnxruntime/contrib_ops/cpu/transformers/sampling_cpu_helper.h
@@ -61,7 +61,7 @@ Status Sample(AllocatorPtr& allocator,
               transformers::ISamplingState<T>* sampling_state,
               transformers::IGreedySearchState<T>* greedy_state,
               const transformers::IGenerationParameters* parameters,
-              const transformers::IConsoleDumper* dumper) {
+              const IConsoleDumper* dumper) {
   ORT_UNUSED_PARAMETER(dumper);
 
   gsl::span<T>& sorted_scores = sampling_state->sorted_scores;
diff --git a/onnxruntime/contrib_ops/cpu/transformers/subgraph_base.cc b/onnxruntime/contrib_ops/cpu/transformers/subgraph_base.cc
index 5f9d48fe50..83aa99ff4d 100644
--- a/onnxruntime/contrib_ops/cpu/transformers/subgraph_base.cc
+++ b/onnxruntime/contrib_ops/cpu/transformers/subgraph_base.cc
@@ -9,7 +9,7 @@
 #include "core/providers/cpu/tensor/utils.h"
 #include "core/common/gsl.h"
 #include "contrib_ops/cpu/transformers/subgraph_base.h"
-#include "contrib_ops/cpu/transformers/dump_tensor.h"
+#include "contrib_ops/cpu/utils/dump_tensor.h"
 
 namespace onnxruntime {
 namespace contrib {
diff --git a/onnxruntime/contrib_ops/cpu/transformers/subgraph_gpt.cc b/onnxruntime/contrib_ops/cpu/transformers/subgraph_gpt.cc
index b07d485577..443d69d494 100644
--- a/onnxruntime/contrib_ops/cpu/transformers/subgraph_gpt.cc
+++ b/onnxruntime/contrib_ops/cpu/transformers/subgraph_gpt.cc
@@ -8,7 +8,7 @@
 #include "core/providers/cpu/tensor/utils.h"
 #include "core/common/gsl.h"
 #include "contrib_ops/cpu/transformers/subgraph_gpt.h"
-#include "contrib_ops/cpu/transformers/dump_tensor.h"
+#include "contrib_ops/cpu/utils/dump_tensor.h"
 
 namespace onnxruntime {
 namespace contrib {
diff --git a/onnxruntime/contrib_ops/cpu/transformers/subgraph_t5_decoder.cc b/onnxruntime/contrib_ops/cpu/transformers/subgraph_t5_decoder.cc
index 4d61ce71c6..4264ceff04 100644
--- a/onnxruntime/contrib_ops/cpu/transformers/subgraph_t5_decoder.cc
+++ b/onnxruntime/contrib_ops/cpu/transformers/subgraph_t5_decoder.cc
@@ -8,7 +8,7 @@
 #include "core/providers/cpu/tensor/utils.h"
 #include "core/common/gsl.h"
 #include "contrib_ops/cpu/transformers/subgraph_t5_decoder.h"
-#include "contrib_ops/cpu/transformers/dump_tensor.h"
+#include "contrib_ops/cpu/utils/dump_tensor.h"
 #include "contrib_ops/cpu/transformers/generation_device_helper.h"
 #include "contrib_ops/cpu/transformers/sequences.h"
 
diff --git a/onnxruntime/contrib_ops/cpu/transformers/subgraph_whisper_decoder.cc b/onnxruntime/contrib_ops/cpu/transformers/subgraph_whisper_decoder.cc
index 7d0c62b618..f3da01c952 100644
--- a/onnxruntime/contrib_ops/cpu/transformers/subgraph_whisper_decoder.cc
+++ b/onnxruntime/contrib_ops/cpu/transformers/subgraph_whisper_decoder.cc
@@ -9,7 +9,7 @@
 #include "core/common/gsl.h"
 #include "contrib_ops/cpu/transformers/subgraph_t5_decoder.h"
 #include "contrib_ops/cpu/transformers/subgraph_whisper_decoder.h"
-#include "contrib_ops/cpu/transformers/dump_tensor.h"
+#include "contrib_ops/cpu/utils/dump_tensor.h"
 #include "contrib_ops/cpu/transformers/generation_device_helper.h"
 #include "contrib_ops/cpu/transformers/sequences.h"
 
diff --git a/onnxruntime/contrib_ops/cpu/utils/console_dumper.h b/onnxruntime/contrib_ops/cpu/utils/console_dumper.h
index 85ba53b9ed..3c255879df 100644
--- a/onnxruntime/contrib_ops/cpu/utils/console_dumper.h
+++ b/onnxruntime/contrib_ops/cpu/utils/console_dumper.h
@@ -4,12 +4,11 @@
 #pragma once
 #include <string>
 #include "core/framework/ort_value.h"
-
-// #define DEBUG_GENERATION 1  // uncomment it for debugging generation (like beam search etc)
+#include "core/framework/float16.h"
+#include "contrib_ops/cpu/utils/debug_macros.h"
 
 namespace onnxruntime {
 namespace contrib {
-namespace transformers {
 
 class IConsoleDumper {
  public:
@@ -22,10 +21,17 @@ class IConsoleDumper {
   virtual void Print(const char* name, const size_t* tensor, int dim0, int dim1) const = 0;
   virtual void Print(const char* name, const int64_t* tensor, int dim0, int dim1) const = 0;
   virtual void Print(const char* name, const int32_t* tensor, int dim0, int dim1) const = 0;
+
   virtual void Print(const char* name, const float* tensor, int dim0, int dim1, int dim2) const = 0;
   virtual void Print(const char* name, const MLFloat16* tensor, int dim0, int dim1, int dim2) const = 0;
   virtual void Print(const char* name, const int64_t* tensor, int dim0, int dim1, int dim2) const = 0;
   virtual void Print(const char* name, const int32_t* tensor, int dim0, int dim1, int dim2) const = 0;
+
+  virtual void Print(const char* name, const float* tensor, int dim0, int dim1, int dim2, int dim3) const = 0;
+  virtual void Print(const char* name, const MLFloat16* tensor, int dim0, int dim1, int dim2, int dim3) const = 0;
+  virtual void Print(const char* name, const int64_t* tensor, int dim0, int dim1, int dim2, int dim3) const = 0;
+  virtual void Print(const char* name, const int32_t* tensor, int dim0, int dim1, int dim2, int dim3) const = 0;
+
   virtual void Print(const char* name, const Tensor& value) const = 0;
   virtual void Print(const char* name, const OrtValue& value) const = 0;
   virtual void Print(const char* name, int index, bool end_line) const = 0;
@@ -35,6 +41,5 @@ class IConsoleDumper {
   bool is_enabled_;
 };
 
-}  // namespace transformers
 }  // namespace contrib
 }  // namespace onnxruntime
diff --git a/onnxruntime/contrib_ops/cpu/utils/debug_macros.h b/onnxruntime/contrib_ops/cpu/utils/debug_macros.h
new file mode 100644
index 0000000000..37a9b0160a
--- /dev/null
+++ b/onnxruntime/contrib_ops/cpu/utils/debug_macros.h
@@ -0,0 +1,41 @@
+#pragma once
+
+// #define DEBUG_GENERATION 1  // uncomment it for debugging generation (like beam search etc)
+
+#ifdef DEBUG_GENERATION
+#define DUMP_TENSOR_LEVEL 2
+#else
+#define DUMP_TENSOR_LEVEL 0  // change it to 1 or 2 if want to enable dumping for code not in generation.
+#endif
+
+#define DUMP_CPU_TENSOR_LEVEL DUMP_TENSOR_LEVEL
+
+// For CPU tensor dumping.
+#if DUMP_CPU_TENSOR_LEVEL > 0
+#define DUMP_CPU_TENSOR_INIT() onnxruntime::contrib::CpuTensorConsoleDumper cpu_dumper
+#define DUMP_CPU_TENSOR(...) cpu_dumper.Print(__VA_ARGS__)
+#else
+#define DUMP_CPU_TENSOR_INIT()
+#define DUMP_CPU_TENSOR(...)
+#endif
+
+#if DUMP_CPU_TENSOR_LEVEL > 1
+#define DUMP_CPU_TENSOR_D(...) cpu_dumper.Print(__VA_ARGS__)
+#else
+#define DUMP_CPU_TENSOR_D(...)
+#endif
+
+// For GPU tensor dumping.
+#if DUMP_TENSOR_LEVEL > 0
+#define DUMP_TENSOR_INIT() onnxruntime::contrib::cuda::CudaTensorConsoleDumper dumper
+#define DUMP_TENSOR(...) dumper.Print(__VA_ARGS__)
+#else
+#define DUMP_TENSOR_INIT()
+#define DUMP_TENSOR(...)
+#endif
+
+#if DUMP_TENSOR_LEVEL > 1
+#define DUMP_TENSOR_D(...) dumper.Print(__VA_ARGS__)
+#else
+#define DUMP_TENSOR_D(...)
+#endif
diff --git a/onnxruntime/contrib_ops/cpu/transformers/dump_tensor.cc b/onnxruntime/contrib_ops/cpu/utils/dump_tensor.cc
similarity index 83%
rename from onnxruntime/contrib_ops/cpu/transformers/dump_tensor.cc
rename to onnxruntime/contrib_ops/cpu/utils/dump_tensor.cc
index d389e4ad76..3a5deef35d 100644
--- a/onnxruntime/contrib_ops/cpu/transformers/dump_tensor.cc
+++ b/onnxruntime/contrib_ops/cpu/utils/dump_tensor.cc
@@ -1,15 +1,15 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
-#include "dump_tensor.h"
 #include <iomanip>
+#include "contrib_ops/cpu/utils/dump_tensor.h"
 #include "core/framework/print_tensor_utils.h"
+#include "contrib_ops/cpu/utils/debug_macros.h"
 
 namespace onnxruntime {
 namespace contrib {
-namespace transformers {
 
-#ifdef DEBUG_GENERATION
+#if DUMP_CPU_TENSOR_LEVEL > 0
 
 template <typename T>
 void DumpCpuTensor(const char* name, const T* tensor, int dim0, int dim1) {
@@ -147,6 +147,30 @@ void CpuTensorConsoleDumper::Print(const char* name, const int32_t* tensor, int
   DumpCpuTensor<int32_t>(name, tensor, dim0, dim1, dim2);
 }
 
+void CpuTensorConsoleDumper::Print(const char* name, const float* tensor, int dim0, int dim1, int dim2, int dim3) const {
+  if (!is_enabled_)
+    return;
+  DumpCpuTensor<float>(name, tensor, dim0 * dim1, dim2, dim3);
+}
+
+void CpuTensorConsoleDumper::Print(const char* name, const MLFloat16* tensor, int dim0, int dim1, int dim2, int dim3) const {
+  if (!is_enabled_)
+    return;
+  DumpCpuTensor<MLFloat16>(name, tensor, dim0 * dim1, dim2, dim3);
+}
+
+void CpuTensorConsoleDumper::Print(const char* name, const int64_t* tensor, int dim0, int dim1, int dim2, int dim3) const {
+  if (!is_enabled_)
+    return;
+  DumpCpuTensor<int64_t>(name, tensor, dim0 * dim1, dim2, dim3);
+}
+
+void CpuTensorConsoleDumper::Print(const char* name, const int32_t* tensor, int dim0, int dim1, int dim2, int dim3) const {
+  if (!is_enabled_)
+    return;
+  DumpCpuTensor<int32_t>(name, tensor, dim0 * dim1, dim2, dim3);
+}
+
 void CpuTensorConsoleDumper::Print(const char* name, const Tensor& tensor) const {
   if (!is_enabled_)
     return;
@@ -207,6 +231,18 @@ void CpuTensorConsoleDumper::Print(const char*, const int64_t*, int, int, int) c
 void CpuTensorConsoleDumper::Print(const char*, const int32_t*, int, int, int) const {
 }
 
+void CpuTensorConsoleDumper::Print(const char*, const float*, int, int, int, int) const {
+}
+
+void CpuTensorConsoleDumper::Print(const char*, const MLFloat16*, int, int, int, int) const {
+}
+
+void CpuTensorConsoleDumper::Print(const char*, const int64_t*, int, int, int, int) const {
+}
+
+void CpuTensorConsoleDumper::Print(const char*, const int32_t*, int, int, int, int) const {
+}
+
 void CpuTensorConsoleDumper::Print(const char*, const Tensor&) const {
 }
 
@@ -221,6 +257,5 @@ void CpuTensorConsoleDumper::Print(const char*, const std::string&, bool) const
 
 #endif
 
-}  // namespace transformers
 }  // namespace contrib
 }  // namespace onnxruntime
diff --git a/onnxruntime/contrib_ops/cpu/transformers/dump_tensor.h b/onnxruntime/contrib_ops/cpu/utils/dump_tensor.h
similarity index 74%
rename from onnxruntime/contrib_ops/cpu/transformers/dump_tensor.h
rename to onnxruntime/contrib_ops/cpu/utils/dump_tensor.h
index 78b1e6cf9a..d902806fd0 100644
--- a/onnxruntime/contrib_ops/cpu/transformers/dump_tensor.h
+++ b/onnxruntime/contrib_ops/cpu/utils/dump_tensor.h
@@ -3,12 +3,11 @@
 
 #pragma once
 #include <string>
-#include "core/framework/tensorprotoutils.h"
-#include "contrib_ops/cpu/transformers/generation_shared.h"
+#include "core/framework/ort_value.h"
+#include "contrib_ops/cpu/utils/console_dumper.h"
 
 namespace onnxruntime {
 namespace contrib {
-namespace transformers {
 
 class CpuTensorConsoleDumper : public IConsoleDumper {
  public:
@@ -19,16 +18,22 @@ class CpuTensorConsoleDumper : public IConsoleDumper {
   void Print(const char* name, const size_t* tensor, int dim0, int dim1) const override;
   void Print(const char* name, const int64_t* tensor, int dim0, int dim1) const override;
   void Print(const char* name, const int32_t* tensor, int dim0, int dim1) const override;
+
   void Print(const char* name, const float* tensor, int dim0, int dim1, int dim2) const override;
   void Print(const char* name, const MLFloat16* tensor, int dim0, int dim1, int dim2) const override;
   void Print(const char* name, const int64_t* tensor, int dim0, int dim1, int dim2) const override;
   void Print(const char* name, const int32_t* tensor, int dim0, int dim1, int dim2) const override;
+
+  void Print(const char* name, const float* tensor, int dim0, int dim1, int dim2, int dim3) const override;
+  void Print(const char* name, const MLFloat16* tensor, int dim0, int dim1, int dim2, int dim3) const override;
+  void Print(const char* name, const int64_t* tensor, int dim0, int dim1, int dim2, int dim3) const override;
+  void Print(const char* name, const int32_t* tensor, int dim0, int dim1, int dim2, int dim3) const override;
+
   void Print(const char* name, const Tensor& value) const override;
   void Print(const char* name, const OrtValue& value) const override;
   void Print(const char* name, int index, bool end_line) const override;
   void Print(const char* name, const std::string& value, bool end_line) const override;
 };
 
-}  // namespace transformers
 }  // namespace contrib
 }  // namespace onnxruntime
diff --git a/onnxruntime/contrib_ops/cuda/bert/attention_impl.cu b/onnxruntime/contrib_ops/cuda/bert/attention_impl.cu
index a93fdf74dc..5c13bb731c 100644
--- a/onnxruntime/contrib_ops/cuda/bert/attention_impl.cu
+++ b/onnxruntime/contrib_ops/cuda/bert/attention_impl.cu
@@ -35,7 +35,7 @@ limitations under the License.
 #include "contrib_ops/cuda/bert/tensorrt_fused_multihead_attention/cross_attention/fmha_cross_attention.h"
 #include "contrib_ops/cpu/bert/attention_base.h"
 #include "contrib_ops/cuda/bert/bert_padding.h"
-#include "contrib_ops/cuda/transformers/dump_cuda_tensor.h"
+#include "contrib_ops/cuda/utils/dump_cuda_tensor.h"
 #include "contrib_ops/cuda/bert/cutlass_fmha/memory_efficient_attention.h"
 #include "contrib_ops/cuda/bert/flash_attention/flash_api.h"
 #include "contrib_ops/cuda/bert/attention_impl.h"
diff --git a/onnxruntime/contrib_ops/cuda/bert/attention_prepare_qkv.cu b/onnxruntime/contrib_ops/cuda/bert/attention_prepare_qkv.cu
index ba335c1f4b..040d6124e7 100644
--- a/onnxruntime/contrib_ops/cuda/bert/attention_prepare_qkv.cu
+++ b/onnxruntime/contrib_ops/cuda/bert/attention_prepare_qkv.cu
@@ -4,7 +4,7 @@
 #include "contrib_ops/cuda/bert/attention_impl.h"
 #include "core/providers/cuda/cu_inc/common.cuh"
 #include "contrib_ops/cuda/bert/add_bias_transpose.h"
-#include "contrib_ops/cuda/transformers/dump_cuda_tensor.h"
+#include "contrib_ops/cuda/utils/dump_cuda_tensor.h"
 
 using namespace onnxruntime::cuda;
 
diff --git a/onnxruntime/contrib_ops/cuda/bert/group_query_attention_impl.cu b/onnxruntime/contrib_ops/cuda/bert/group_query_attention_impl.cu
index 2dd6e0acfd..62974d1200 100644
--- a/onnxruntime/contrib_ops/cuda/bert/group_query_attention_impl.cu
+++ b/onnxruntime/contrib_ops/cuda/bert/group_query_attention_impl.cu
@@ -36,7 +36,7 @@ limitations under the License.
 #include "contrib_ops/cuda/bert/add_bias_transpose.h"
 #include "contrib_ops/cpu/bert/attention_base.h"
 #include "contrib_ops/cuda/bert/bert_padding.h"
-#include "contrib_ops/cuda/transformers/dump_cuda_tensor.h"
+#include "contrib_ops/cuda/utils/dump_cuda_tensor.h"
 #include "contrib_ops/cuda/bert/cutlass_fmha/memory_efficient_attention.h"
 #include "contrib_ops/cuda/bert/flash_attention/flash_api.h"
 #include "contrib_ops/cuda/bert/group_query_attention_impl.h"
diff --git a/onnxruntime/contrib_ops/cuda/bert/packed_attention_impl.cu b/onnxruntime/contrib_ops/cuda/bert/packed_attention_impl.cu
index a84a310b46..db9f30c25c 100644
--- a/onnxruntime/contrib_ops/cuda/bert/packed_attention_impl.cu
+++ b/onnxruntime/contrib_ops/cuda/bert/packed_attention_impl.cu
@@ -13,7 +13,7 @@
 #include "contrib_ops/cuda/bert/tensorrt_fused_multihead_attention/mha_runner.h"
 #include "contrib_ops/cuda/bert/tensorrt_fused_multihead_attention/cross_attention/fmha_cross_attention.h"
 #include "contrib_ops/cuda/bert/bert_padding.h"
-#include "contrib_ops/cuda/transformers/dump_cuda_tensor.h"
+#include "contrib_ops/cuda/utils/dump_cuda_tensor.h"
 #include "contrib_ops/cuda/bert/cutlass_fmha/memory_efficient_attention.h"
 #include "contrib_ops/cuda/bert/rotary_embedding_util.h"
 #include "contrib_ops/cuda/bert/flash_attention/flash_api.h"
diff --git a/onnxruntime/contrib_ops/cuda/bert/packed_multihead_attention_impl.cu b/onnxruntime/contrib_ops/cuda/bert/packed_multihead_attention_impl.cu
index 982c7eaa2c..a18744d29b 100644
--- a/onnxruntime/contrib_ops/cuda/bert/packed_multihead_attention_impl.cu
+++ b/onnxruntime/contrib_ops/cuda/bert/packed_multihead_attention_impl.cu
@@ -14,7 +14,7 @@
 #include "contrib_ops/cuda/bert/tensorrt_fused_multihead_attention/mha_runner.h"
 #include "contrib_ops/cuda/bert/tensorrt_fused_multihead_attention/cross_attention/fmha_cross_attention.h"
 #include "contrib_ops/cuda/bert/bert_padding.h"
-#include "contrib_ops/cuda/transformers/dump_cuda_tensor.h"
+#include "contrib_ops/cuda/utils/dump_cuda_tensor.h"
 #include "contrib_ops/cuda/bert/cutlass_fmha/memory_efficient_attention.h"
 #include "contrib_ops/cuda/bert/rotary_embedding_util.h"
 #include "contrib_ops/cuda/bert/flash_attention/flash_api.h"
diff --git a/onnxruntime/contrib_ops/cuda/diffusion/group_norm_impl.cu b/onnxruntime/contrib_ops/cuda/diffusion/group_norm_impl.cu
index 4909dc5e38..25c550874e 100644
--- a/onnxruntime/contrib_ops/cuda/diffusion/group_norm_impl.cu
+++ b/onnxruntime/contrib_ops/cuda/diffusion/group_norm_impl.cu
@@ -26,7 +26,7 @@
 #include "core/providers/cuda/cuda_common.h"
 #include "core/providers/cuda/cu_inc/common.cuh"
 #include "contrib_ops/cuda/diffusion/group_norm_impl.h"
-#include "contrib_ops/cuda/transformers/dump_cuda_tensor.h"
+#include "contrib_ops/cuda/utils/dump_cuda_tensor.h"
 #include "contrib_ops/cuda/diffusion/group_norm_common_base.h"
 #include "contrib_ops/cuda/diffusion/group_norm_impl_kernel.cuh"
 
diff --git a/onnxruntime/contrib_ops/cuda/sparse/sparse_attention_impl.cu b/onnxruntime/contrib_ops/cuda/sparse/sparse_attention_impl.cu
index d833a7cf02..4cb25af970 100644
--- a/onnxruntime/contrib_ops/cuda/sparse/sparse_attention_impl.cu
+++ b/onnxruntime/contrib_ops/cuda/sparse/sparse_attention_impl.cu
@@ -2,7 +2,7 @@
 // Licensed under the MIT License.
 
 #include "contrib_ops/cuda/sparse/sparse_attention_impl.h"
-#include "contrib_ops/cuda/transformers/dump_cuda_tensor.h"
+#include "contrib_ops/cuda/utils/dump_cuda_tensor.h"
 #include "contrib_ops/cuda/bert/rotary_embedding_impl.h"
 #include "contrib_ops/cuda/bert/group_query_attention_impl.h"
 #include "contrib_ops/cpu/bert/attention_common.h"
diff --git a/onnxruntime/contrib_ops/cuda/sparse/sparse_attention_v1/sparse_attention_common.h b/onnxruntime/contrib_ops/cuda/sparse/sparse_attention_v1/sparse_attention_common.h
index a90c603d7d..222c215a50 100644
--- a/onnxruntime/contrib_ops/cuda/sparse/sparse_attention_v1/sparse_attention_common.h
+++ b/onnxruntime/contrib_ops/cuda/sparse/sparse_attention_v1/sparse_attention_common.h
@@ -3,7 +3,7 @@
 
 #pragma once
 #include "core/providers/cuda/cuda_common.h"
-#include "contrib_ops/cuda/transformers/dump_cuda_tensor.h"
+#include "contrib_ops/cuda/utils/dump_cuda_tensor.h"
 #include "contrib_ops/cuda/bert/tensorrt_fused_multihead_attention/cudaDriverWrapper.h"
 
 #define CU_CHECK(expr, driver) cuErrCheck(expr, *driver)
diff --git a/onnxruntime/contrib_ops/cuda/transformers/beam_search.cc b/onnxruntime/contrib_ops/cuda/transformers/beam_search.cc
index 08cbb145a6..bec78d081e 100644
--- a/onnxruntime/contrib_ops/cuda/transformers/beam_search.cc
+++ b/onnxruntime/contrib_ops/cuda/transformers/beam_search.cc
@@ -5,7 +5,7 @@
 #include "core/providers/cuda/cuda_execution_provider.h"
 #include "contrib_ops/cuda/transformers/beam_search.h"
 #include "contrib_ops/cuda/transformers/generation_device_helper.h"
-#include "contrib_ops/cuda/transformers/dump_cuda_tensor.h"
+#include "contrib_ops/cuda/utils/dump_cuda_tensor.h"
 
 namespace onnxruntime {
 namespace contrib {
@@ -56,7 +56,7 @@ ONNX_OPERATOR_KERNEL_EX(
                               DataTypeImpl::GetTensorType<MLFloat16>()}),
     WhisperBeamSearch);
 
-transformers::CudaTensorConsoleDumper g_cuda_dumper;
+CudaTensorConsoleDumper g_cuda_dumper;
 
 BeamSearch::BeamSearch(const OpKernelInfo& info)
     : onnxruntime::contrib::transformers::BeamSearch(info) {
diff --git a/onnxruntime/contrib_ops/cuda/transformers/generation_device_helper.cc b/onnxruntime/contrib_ops/cuda/transformers/generation_device_helper.cc
index 7adc2fe0a6..09d2dba7d2 100644
--- a/onnxruntime/contrib_ops/cuda/transformers/generation_device_helper.cc
+++ b/onnxruntime/contrib_ops/cuda/transformers/generation_device_helper.cc
@@ -12,7 +12,7 @@
 #include "contrib_ops/cuda/bert/transformer_cuda_common.h"
 #include <cuda_runtime.h>
 #include "contrib_ops/cuda/transformers/generation_cuda_impl.h"
-#include "contrib_ops/cuda/transformers/dump_cuda_tensor.h"
+#include "contrib_ops/cuda/utils/dump_cuda_tensor.h"
 #include "contrib_ops/cpu/transformers/logits_processor.h"
 #include "contrib_ops/cpu/transformers/generation_shared.h"
 #include "contrib_ops/cpu/transformers/subgraph_t5_decoder.h"
@@ -332,7 +332,7 @@ Status ProcessLogits(const OrtValue& logits,                                 //
                      const transformers::IGenerationParameters* parameters,  // parameters
                      int step,                                               // iteration counter
                      Stream* ort_stream,                                     // cuda stream (for CUDA only)
-                     const transformers::IConsoleDumper* dumper) {           // tensor dumper
+                     const IConsoleDumper* dumper) {                         // tensor dumper
 
 #ifdef ENABLE_NVTX_PROFILE
   profile::NvtxNestedRangeCreator processLogitsRange("ProcessLogits", profile::Color::Red);
@@ -824,7 +824,7 @@ Status GreedySearchProcessLogits(
     bool do_sampling,                                       // whether to do sampling
     int step,                                               // iteration counter
     Stream* stream,                                         // cuda stream (for CUDA only)
-    const transformers::IConsoleDumper* dumper) {           // tensor dumper
+    const IConsoleDumper* dumper) {                         // tensor dumper
 
 #ifdef ENABLE_NVTX_PROFILE
   profile::NvtxNestedRangeCreator processLogitsRange("ProcessLogits", profile::Color::Red);
@@ -1242,7 +1242,7 @@ Status UpdateDecoderFeeds(
     bool past_present_share_buffer,
     bool need_cache_indir,
     transformers::Sequences& sequences,
-    const transformers::IConsoleDumper* dumper) {
+    const IConsoleDumper* dumper) {
   // last_outputs: logits, present_key_self_0, present_value_self_0, ...
   // next_inputs: input_ids,
   //              encoder_attention_mask, encoder_hidden_states,
@@ -1448,7 +1448,7 @@ template Status ProcessLogits<float>(
     const transformers::IGenerationParameters* parameters,
     int step,
     Stream* ort_stream,
-    const transformers::IConsoleDumper* dumper);
+    const IConsoleDumper* dumper);
 
 template Status GreedySearchProcessLogits<float>(
     const OrtValue& logits,
@@ -1462,7 +1462,7 @@ template Status GreedySearchProcessLogits<float>(
     bool do_sampling,
     int step,
     Stream* ort_stream,
-    const transformers::IConsoleDumper* dumper);
+    const IConsoleDumper* dumper);
 
 template Status DeviceCopy<float>(
     gsl::span<float> target,
@@ -1519,7 +1519,7 @@ template Status ProcessLogits<MLFloat16>(
     const transformers::IGenerationParameters* parameters,
     int step,
     Stream* ort_stream,
-    const transformers::IConsoleDumper* dumper);
+    const IConsoleDumper* dumper);
 
 template Status GreedySearchProcessLogits<MLFloat16>(
     const OrtValue& logits,
@@ -1533,7 +1533,7 @@ template Status GreedySearchProcessLogits<MLFloat16>(
     bool do_sampling,
     int step,
     Stream* ort_stream,
-    const transformers::IConsoleDumper* dumper);
+    const IConsoleDumper* dumper);
 
 template Status UpdateGptFeeds<MLFloat16>(
     AllocatorPtr allocator,
@@ -1572,7 +1572,7 @@ template Status UpdateDecoderFeeds<float>(
     bool past_present_share_buffer,
     bool need_cache_indir,
     transformers::Sequences& sequences,
-    const transformers::IConsoleDumper* dumper);
+    const IConsoleDumper* dumper);
 
 template Status UpdateDecoderFeeds<MLFloat16>(
     AllocatorPtr allocator,
@@ -1592,7 +1592,7 @@ template Status UpdateDecoderFeeds<MLFloat16>(
     bool past_present_share_buffer,
     bool need_cache_indir,
     transformers::Sequences& sequences,
-    const transformers::IConsoleDumper* dumper);
+    const IConsoleDumper* dumper);
 
 template Status ExpandBuffer<int32_t>(
     Stream* ort_stream,
diff --git a/onnxruntime/contrib_ops/cuda/transformers/generation_device_helper.h b/onnxruntime/contrib_ops/cuda/transformers/generation_device_helper.h
index 7a718eb9f6..6b712ccfbe 100644
--- a/onnxruntime/contrib_ops/cuda/transformers/generation_device_helper.h
+++ b/onnxruntime/contrib_ops/cuda/transformers/generation_device_helper.h
@@ -72,7 +72,7 @@ Status ProcessLogits(const OrtValue& logits,                                 //
                      const transformers::IGenerationParameters* parameters,  // parameters
                      int step,                                               // iteration counter
                      Stream* stream,                                         // cuda stream (for CUDA only)
-                     const transformers::IConsoleDumper* dumper);            // tensor dumper
+                     const IConsoleDumper* dumper);                          // tensor dumper
 
 template <typename T>
 Status GreedySearchProcessLogits(const OrtValue& logits,                                 // logits output of subgraph
@@ -86,7 +86,7 @@ Status GreedySearchProcessLogits(const OrtValue& logits,
                                  bool do_sampling,                                       // whether to do sampling
                                  int step,                                               // iteration counter
                                  Stream* stream,                                         // cuda stream (for CUDA only)
-                                 const transformers::IConsoleDumper* dumper);            // tensor dumper
+                                 const IConsoleDumper* dumper);                          // tensor dumper
 
 template <typename T>
 Status DeviceCopy(gsl::span<T> target,
@@ -138,7 +138,7 @@ Status UpdateDecoderFeeds(
     bool past_present_share_buffer,
     bool need_cache_indir,
     transformers::Sequences& sequences,
-    const transformers::IConsoleDumper* dumper);
+    const IConsoleDumper* dumper);
 
 template <typename T>
 Status ExpandBuffer(
diff --git a/onnxruntime/contrib_ops/cuda/transformers/greedy_search.cc b/onnxruntime/contrib_ops/cuda/transformers/greedy_search.cc
index 812ab0b1bc..cf623ab360 100644
--- a/onnxruntime/contrib_ops/cuda/transformers/greedy_search.cc
+++ b/onnxruntime/contrib_ops/cuda/transformers/greedy_search.cc
@@ -5,7 +5,7 @@
 #include "core/providers/cuda/cuda_execution_provider.h"
 #include "contrib_ops/cuda/transformers/greedy_search.h"
 #include "contrib_ops/cuda/transformers/generation_device_helper.h"
-#include "contrib_ops/cuda/transformers/dump_cuda_tensor.h"
+#include "contrib_ops/cuda/utils/dump_cuda_tensor.h"
 
 namespace onnxruntime {
 namespace contrib {
@@ -27,7 +27,7 @@ ONNX_OPERATOR_KERNEL_EX(
                               DataTypeImpl::GetTensorType<MLFloat16>()}),
     GreedySearch);
 
-transformers::CudaTensorConsoleDumper g_cuda_dumper_greedysearch;
+CudaTensorConsoleDumper g_cuda_dumper_greedysearch;
 
 GreedySearch::GreedySearch(const OpKernelInfo& info)
     : onnxruntime::contrib::transformers::GreedySearch(info) {
diff --git a/onnxruntime/contrib_ops/cuda/transformers/sampling.cc b/onnxruntime/contrib_ops/cuda/transformers/sampling.cc
index 49c7bee28d..a9cbdfd324 100644
--- a/onnxruntime/contrib_ops/cuda/transformers/sampling.cc
+++ b/onnxruntime/contrib_ops/cuda/transformers/sampling.cc
@@ -5,7 +5,7 @@
 #include "core/providers/cuda/cuda_execution_provider.h"
 #include "contrib_ops/cuda/transformers/sampling.h"
 #include "contrib_ops/cuda/transformers/generation_device_helper.h"
-#include "contrib_ops/cuda/transformers/dump_cuda_tensor.h"
+#include "contrib_ops/cuda/utils/dump_cuda_tensor.h"
 
 namespace onnxruntime {
 namespace contrib {
@@ -28,7 +28,7 @@ ONNX_OPERATOR_KERNEL_EX(
                               DataTypeImpl::GetTensorType<MLFloat16>()}),
     Sampling);
 
-transformers::CudaTensorConsoleDumper g_cuda_dumper_sampling;
+CudaTensorConsoleDumper g_cuda_dumper_sampling;
 
 Sampling::Sampling(const OpKernelInfo& info)
     : onnxruntime::contrib::transformers::Sampling(info) {
diff --git a/onnxruntime/contrib_ops/cuda/transformers/sampling_cuda_helper.h b/onnxruntime/contrib_ops/cuda/transformers/sampling_cuda_helper.h
index 339c07dbc5..d1c904987e 100644
--- a/onnxruntime/contrib_ops/cuda/transformers/sampling_cuda_helper.h
+++ b/onnxruntime/contrib_ops/cuda/transformers/sampling_cuda_helper.h
@@ -26,7 +26,7 @@ Status Sample(AllocatorPtr& allocator,
               transformers::IGreedySearchState<T>* greedy_state,
               const transformers::IGenerationParameters* parameters,
               int step,
-              const transformers::IConsoleDumper* dumper) {
+              const IConsoleDumper* dumper) {
   ORT_UNUSED_PARAMETER(dumper);
   typedef typename ToCudaType<T>::MappedType CudaT;
 
diff --git a/onnxruntime/contrib_ops/cuda/transformers/dump_cuda_tensor.cc b/onnxruntime/contrib_ops/cuda/utils/dump_cuda_tensor.cc
similarity index 95%
rename from onnxruntime/contrib_ops/cuda/transformers/dump_cuda_tensor.cc
rename to onnxruntime/contrib_ops/cuda/utils/dump_cuda_tensor.cc
index 4cfa89a4d5..fb7af3cfdd 100644
--- a/onnxruntime/contrib_ops/cuda/transformers/dump_cuda_tensor.cc
+++ b/onnxruntime/contrib_ops/cuda/utils/dump_cuda_tensor.cc
@@ -4,13 +4,12 @@
 #include <cuda_runtime_api.h>
 #include "core/providers/cuda/cuda_common.h"
 #include "core/framework/print_tensor_utils.h"
-#include "contrib_ops/cuda/transformers/dump_cuda_tensor.h"
+#include "contrib_ops/cuda/utils/dump_cuda_tensor.h"
 #include "core/platform/env_var_utils.h"
 
 namespace onnxruntime {
 namespace contrib {
 namespace cuda {
-namespace transformers {
 
 #if DUMP_TENSOR_LEVEL > 0
 
@@ -218,6 +217,11 @@ void CudaTensorConsoleDumper::Print(const char* name, const int32_t* tensor, int
     DumpGpuTensor<int32_t>(name, tensor, dim0, dim1, dim2, true);
 }
 
+void CudaTensorConsoleDumper::Print(const char* name, const int32_t* tensor, int dim0, int dim1, int dim2, int dim3) const {
+  if (is_enabled_)
+    DumpGpuTensor<int32_t>(name, tensor, dim0, dim1, dim2, dim3, true);
+}
+
 void CudaTensorConsoleDumper::Print(const char* name, const int64_t* tensor, int dim0, int dim1) const {
   if (is_enabled_)
     DumpGpuTensor<int64_t>(name, tensor, dim0, dim1, true);
@@ -228,6 +232,11 @@ void CudaTensorConsoleDumper::Print(const char* name, const int64_t* tensor, int
     DumpGpuTensor<int64_t>(name, tensor, dim0, dim1, dim2, true);
 }
 
+void CudaTensorConsoleDumper::Print(const char* name, const int64_t* tensor, int dim0, int dim1, int dim2, int dim3) const {
+  if (is_enabled_)
+    DumpGpuTensor<int64_t>(name, tensor, dim0, dim1, dim2, dim3, true);
+}
+
 void CudaTensorConsoleDumper::Print(const char* name, const float* tensor, int dim0, int dim1) const {
   if (is_enabled_)
     DumpGpuTensor<float>(name, tensor, dim0, dim1, true);
@@ -325,12 +334,18 @@ void CudaTensorConsoleDumper::Print(const char*, const int32_t*, int, int) const
 void CudaTensorConsoleDumper::Print(const char*, const int32_t*, int, int, int) const {
 }
 
+void CudaTensorConsoleDumper::Print(const char*, const int32_t*, int, int, int, int) const {
+}
+
 void CudaTensorConsoleDumper::Print(const char*, const int64_t*, int, int) const {
 }
 
 void CudaTensorConsoleDumper::Print(const char*, const int64_t*, int, int, int) const {
 }
 
+void CudaTensorConsoleDumper::Print(const char*, const int64_t*, int, int, int, int) const {
+}
+
 void CudaTensorConsoleDumper::Print(const char*, const float*, int, int) const {
 }
 
@@ -380,7 +395,6 @@ void CudaTensorConsoleDumper::Print(const char*, const std::string&, bool) const
 }
 #endif
 
-}  // namespace transformers
 }  // namespace cuda
 }  // namespace contrib
 }  // namespace onnxruntime
diff --git a/onnxruntime/contrib_ops/cuda/transformers/dump_cuda_tensor.h b/onnxruntime/contrib_ops/cuda/utils/dump_cuda_tensor.h
similarity index 76%
rename from onnxruntime/contrib_ops/cuda/transformers/dump_cuda_tensor.h
rename to onnxruntime/contrib_ops/cuda/utils/dump_cuda_tensor.h
index b4d6cb80e3..0f25e85bb9 100644
--- a/onnxruntime/contrib_ops/cuda/transformers/dump_cuda_tensor.h
+++ b/onnxruntime/contrib_ops/cuda/utils/dump_cuda_tensor.h
@@ -7,28 +7,11 @@
 #include "core/framework/ort_value.h"
 #include "contrib_ops/cpu/utils/console_dumper.h"
 
-#define DUMP_TENSOR_LEVEL 0  // change it to 1 or 2 if want to enable dumping for code not in generation.
-
-#if DUMP_TENSOR_LEVEL > 0
-#define DUMP_TENSOR_INIT() onnxruntime::contrib::cuda::transformers::CudaTensorConsoleDumper dumper
-#define DUMP_TENSOR(...) dumper.Print(__VA_ARGS__)
-#else
-#define DUMP_TENSOR_INIT()
-#define DUMP_TENSOR(...)
-#endif
-
-#if DUMP_TENSOR_LEVEL > 1
-#define DUMP_TENSOR_D(...) dumper.Print(__VA_ARGS__)
-#else
-#define DUMP_TENSOR_D(...)
-#endif
-
 namespace onnxruntime {
 namespace contrib {
 namespace cuda {
-namespace transformers {
 
-class CudaTensorConsoleDumper : public onnxruntime::contrib::transformers::IConsoleDumper {
+class CudaTensorConsoleDumper : public onnxruntime::contrib::IConsoleDumper {
  public:
   CudaTensorConsoleDumper() = default;
   virtual ~CudaTensorConsoleDumper() {}
@@ -37,13 +20,15 @@ class CudaTensorConsoleDumper : public onnxruntime::contrib::transformers::ICons
 
   void Print(const char* name, const int32_t* tensor, int dim0, int dim1) const override;
   void Print(const char* name, const int32_t* tensor, int dim0, int dim1, int dim2) const override;
+  void Print(const char* name, const int32_t* tensor, int dim0, int dim1, int dim2, int dim3) const override;
 
   void Print(const char* name, const int64_t* tensor, int dim0, int dim1) const override;
   void Print(const char* name, const int64_t* tensor, int dim0, int dim1, int dim2) const override;
+  void Print(const char* name, const int64_t* tensor, int dim0, int dim1, int dim2, int dim3) const override;
 
   void Print(const char* name, const float* tensor, int dim0, int dim1) const override;
   void Print(const char* name, const float* tensor, int dim0, int dim1, int dim2) const override;
-  void Print(const char* name, const float* tensor, int dim0, int dim1, int dim2, int dim3) const;
+  void Print(const char* name, const float* tensor, int dim0, int dim1, int dim2, int dim3) const override;
 
   void Print(const char* name, const half* tensor, int dim0, int dim1) const;
   void Print(const char* name, const half* tensor, int dim0, int dim1, int dim2) const;
@@ -51,7 +36,7 @@ class CudaTensorConsoleDumper : public onnxruntime::contrib::transformers::ICons
 
   void Print(const char* name, const MLFloat16* tensor, int dim0, int dim1) const override;
   void Print(const char* name, const MLFloat16* tensor, int dim0, int dim1, int dim2) const override;
-  void Print(const char* name, const MLFloat16* tensor, int dim0, int dim1, int dim2, int dim3) const;
+  void Print(const char* name, const MLFloat16* tensor, int dim0, int dim1, int dim2, int dim3) const override;
 
   void Print(const char* name, const BFloat16* tensor, int dim0, int dim1) const;
   void Print(const char* name, const BFloat16* tensor, int dim0, int dim1, int dim2) const;
@@ -63,7 +48,6 @@ class CudaTensorConsoleDumper : public onnxruntime::contrib::transformers::ICons
   void Print(const char* name, const std::string& value, bool end_line) const override;
 };
 
-}  // namespace transformers
 }  // namespace cuda
 }  // namespace contrib
 }  // namespace onnxruntime
diff --git a/onnxruntime/core/framework/print_tensor_utils.h b/onnxruntime/core/framework/print_tensor_utils.h
index b8c50a266b..e6af5e9e58 100644
--- a/onnxruntime/core/framework/print_tensor_utils.h
+++ b/onnxruntime/core/framework/print_tensor_utils.h
@@ -8,7 +8,6 @@
 
 namespace onnxruntime {
 namespace utils {
-
 constexpr int64_t kDefaultSnippetEdgeItems = 3;
 constexpr int64_t kDefaultSnippetThreshold = 200;
 
@@ -40,12 +39,12 @@ inline void PrintValue(const T& value) {
 
 // Explicit specialization
 template <>
-inline void PrintValue(const MLFloat16& value) {
+inline void PrintValue(const onnxruntime::MLFloat16& value) {
   std::cout << std::setprecision(8) << value.ToFloat();
 }
 
 template <>
-inline void PrintValue(const BFloat16& value) {
+inline void PrintValue(const onnxruntime::BFloat16& value) {
   std::cout << std::setprecision(8) << value.ToFloat();
 }
 
@@ -220,7 +219,9 @@ DEF_PRINT_CPU_TENSOR_FULL_3D_INT4(Int4x2)
 DEF_PRINT_CPU_TENSOR_FULL_3D_INT4(UInt4x2)
 
 template <typename T>
-void PrintCpuTensor(const Tensor& tensor, int threshold = kDefaultSnippetThreshold, int edge_items = kDefaultSnippetEdgeItems) {
+void PrintCpuTensor(const onnxruntime::Tensor& tensor,
+                    int threshold = kDefaultSnippetThreshold,
+                    int edge_items = kDefaultSnippetEdgeItems) {
   const auto& shape = tensor.Shape();
   auto num_items = shape.Size();
   if (num_items == 0) {