Add more node debug dump functionality. (#4921)

Add ability to dump node inputs/outputs to files, filter nodes, configure behavior with environment variables.

BUILD.md

@@ -616,26 +616,35 @@ The Vitis-AI execution provider is only supported on Linux.
 OnnxRuntime supports build options for enabling debugging of intermediate tensor shapes and data.
 
 #### Build Instructions
-Set onnxruntime_DEBUG_NODE_INPUTS_OUTPUT to one of the values below.
+Set onnxruntime_DEBUG_NODE_INPUTS_OUTPUT to build with this enabled.
 
 **Linux**
 ```
-./build.sh --cmake_extra_defines onnxruntime_DEBUG_NODE_INPUTS_OUTPUTS=VALUE
+./build.sh --cmake_extra_defines onnxruntime_DEBUG_NODE_INPUTS_OUTPUTS=1
 ```
 
 **Windows**
 ```
-.\build.bat --cmake_extra_defines onnxruntime_DEBUG_NODE_INPUTS_OUTPUTS=VALUE
+.\build.bat --cmake_extra_defines onnxruntime_DEBUG_NODE_INPUTS_OUTPUTS=1
 ```
 
-Values:
-* **0**: Disables this functionality if previously enabled; alternatively, delete CMakeCache.txt instead of setting this to 0
-* **1**: Dump tensor input/output shapes for all nodes to stdout
-* **2**: Dump tensor input/output shapes and output data for all nodes to stdout
-
+#### Configuration
+The debug dump behavior can be controlled with several environment variables.
+See [onnxruntime/core/framework/debug_node_inputs_outputs_utils.h](./onnxruntime/core/framework/debug_node_inputs_outputs_utils.h) for details.
 
+##### Examples
 
+To specify that node output data should be dumped (to stdout by default), set this environment variable:
+```
+ORT_DEBUG_NODE_IO_DUMP_OUTPUT_DATA=1
+```
 
+To specify that node output data should be dumped to files for nodes with name "Foo" or "Bar", set these environment variables:
+```
+ORT_DEBUG_NODE_IO_DUMP_OUTPUT_DATA=1
+ORT_DEBUG_NODE_IO_NODE_NAME_FILTER="Foo;Bar"
+ORT_DEBUG_NODE_IO_DUMP_DATA_TO_FILES=1
+```
 
 ---
 

cmake/CMakeLists.txt

@@ -89,7 +89,7 @@ option(onnxruntime_DISABLE_ML_OPS "Disable traditional ML ops" OFF)
 option(onnxruntime_DISABLE_RTTI "Disable RTTI" OFF)
 option(tensorflow_C_PACKAGE_PATH "Path to tensorflow C package installation dir")
 option(onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS "Enable operator implemented in language other than cpp" OFF)
-option(onnxruntime_DEBUG_NODE_INPUTS_OUTPUTS "Dump node input shapes and output data to standard output when executing the model." OFF)
+option(onnxruntime_DEBUG_NODE_INPUTS_OUTPUTS "Dump debug information about node inputs and outputs when executing the model." OFF)
 option(onnxruntime_USE_DML "Build with DirectML support" OFF)
 option(onnxruntime_USE_MIGRAPHX "Build with AMDMIGraphX support" OFF)
 option(onnxruntime_USE_WINML "Build with WinML support" OFF)

cmake/onnxruntime_framework.cmake

@@ -36,7 +36,7 @@ set_target_properties(onnxruntime_framework PROPERTIES FOLDER "ONNXRuntime")
 add_dependencies(onnxruntime_framework ${onnxruntime_EXTERNAL_DEPENDENCIES})
 
 if (onnxruntime_DEBUG_NODE_INPUTS_OUTPUTS)
-  target_compile_definitions(onnxruntime_framework PRIVATE DEBUG_NODE_INPUTS_OUTPUTS=${onnxruntime_DEBUG_NODE_INPUTS_OUTPUTS})
+  target_compile_definitions(onnxruntime_framework PRIVATE DEBUG_NODE_INPUTS_OUTPUTS)
 endif()
 
 

cmake/onnxruntime_unittests.cmake

@@ -557,6 +557,9 @@ set(all_dependencies ${onnxruntime_test_providers_dependencies} )
   if(onnxruntime_RUN_MODELTEST_IN_DEBUG_MODE)
     target_compile_definitions(onnxruntime_test_all PUBLIC -DRUN_MODELTEST_IN_DEBUG_MODE)
   endif()
+  if (onnxruntime_DEBUG_NODE_INPUTS_OUTPUTS)
+    target_compile_definitions(onnxruntime_test_all PRIVATE DEBUG_NODE_INPUTS_OUTPUTS)
+  endif()
   if (onnxruntime_USE_FEATURIZERS)
     target_include_directories(onnxruntime_test_all PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/external/FeaturizersLibrary/src)
   endif()

onnxruntime/core/framework/debug_node_inputs_outputs_utils.cc

@@ -0,0 +1,312 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#ifdef DEBUG_NODE_INPUTS_OUTPUTS
+
+#include "core/framework/debug_node_inputs_outputs_utils.h"
+
+#include <iomanip>
+
+#include "core/common/path_utils.h"
+#include "core/framework/tensorprotoutils.h"
+#include "core/platform/env.h"
+
+namespace onnxruntime {
+namespace utils {
+
+namespace {
+
+bool FilterNode(const NodeDumpOptions& dump_options, const Node& node) {
+  auto match_pattern =
+      [](const std::string& value, const std::string& delimited_patterns) {
+        // match all if empty
+        if (delimited_patterns.empty()) return true;
+
+        // search for exact match against delimited patterns
+        auto pattern_begin = delimited_patterns.begin();
+        while (true) {
+          const auto pattern_end = std::find(
+              pattern_begin, delimited_patterns.end(), kFilterPatternDelimiter);
+
+          if (std::equal(value.begin(), value.end(), pattern_begin, pattern_end)) return true;
+
+          if (pattern_end == delimited_patterns.end()) break;
+
+          pattern_begin = pattern_end + 1;
+        }
+
+        return false;
+      };
+
+  return match_pattern(node.Name(), dump_options.filter.name_pattern) &&
+         match_pattern(node.OpType(), dump_options.filter.op_type_pattern);
+}
+
+std::ostream& operator<<(std::ostream& out, const BFloat16& value) {
+  return out << value.ToFloat();
+}
+
+std::ostream& operator<<(std::ostream& out, const MLFloat16& value) {
+  return out << static_cast<float>(value);
+}
+
+template <typename T>
+void DumpTensorToStdOut(const Tensor& tensor) {
+  const auto& shape = tensor.Shape();
+  auto num_items = shape.Size();
+
+  if (num_items == 0) {
+    std::cout << "no data";
+    return;
+  }
+
+  size_t num_dims = shape.NumDimensions();
+  size_t num_rows = 1;
+  if (num_dims > 1) {
+    num_rows = static_cast<size_t>(shape[0]);
+  }
+
+  size_t row_size = num_items / num_rows;
+
+  auto data = tensor.DataAsSpan<T>();
+
+  auto print_val = [](const T& value) {
+    if (std::is_floating_point<T>::value)
+      std::cout << std::setprecision(8) << value;
+    else
+      std::cout << value;
+  };
+
+  for (size_t row = 0; row < num_rows; ++row) {
+    print_val(data[row * row_size]);
+    for (size_t i = 1; i < row_size; ++i) {
+      std::cout << ", ";
+      print_val(data[row * row_size + i]);
+    }
+    std::cout << "\n";
+  }
+
+  std::cout << std::endl;
+}
+
+PathString MakeTensorFileName(const std::string& tensor_name) {
+  auto make_valid_name = [](std::string name) {
+    std::replace_if(
+        name.begin(), name.end(), [](char c) { return !std::isalnum(c); }, '_');
+    return name;
+  };
+
+  return path_utils::MakePathString(make_valid_name(tensor_name), ".tensorproto");
+}
+
+void DumpTensorToFile(const Tensor& tensor, const std::string& tensor_name, const Path& file_path) {
+  auto tensor_proto = utils::TensorToTensorProto(tensor, tensor_name);
+  const PathString file_path_str = file_path.ToPathString();
+  int output_fd;
+  ORT_THROW_IF_ERROR(Env::Default().FileOpenWr(file_path_str, output_fd));
+  try {
+    ORT_ENFORCE(
+        tensor_proto.SerializeToFileDescriptor(output_fd),
+        "Failed to write tensor to file - tensor: ", tensor_name, ", file: ", ToMBString(file_path_str));
+  } catch (...) {
+    ORT_IGNORE_RETURN_VALUE(Env::Default().FileClose(output_fd));
+    throw;
+  }
+  ORT_THROW_IF_ERROR(Env::Default().FileClose(output_fd));
+}
+
+void DumpCpuTensor(
+    const NodeDumpOptions& dump_options,
+    const Tensor& tensor, const std::string& tensor_name) {
+  switch (dump_options.data_destination) {
+    case NodeDumpOptions::DataDestination::StdOut: {
+      DispatchOnTensorType(tensor.DataType(), DumpTensorToStdOut, tensor);
+      break;
+    }
+    case NodeDumpOptions::DataDestination::TensorProtoFiles: {
+      const Path tensor_file = dump_options.output_dir / Path::Parse(MakeTensorFileName(tensor_name));
+      DumpTensorToFile(tensor, tensor_name, tensor_file);
+      break;
+    }
+    default:
+      ORT_THROW("Unsupported data destination type: ", static_cast<int>(dump_options.data_destination));
+  }
+}
+
+void DumpTensor(
+    const NodeDumpOptions& dump_options,
+    const Tensor& tensor, const std::string& tensor_name,
+    const SessionState& session_state) {
+  // check tensor is on CPU before dumping it
+  auto& tensor_location = tensor.Location();
+  const auto data_type = tensor.DataType();
+  if (tensor_location.device.Type() == OrtDevice::CPU ||
+      tensor_location.mem_type == OrtMemTypeCPUInput ||
+      tensor_location.mem_type == OrtMemTypeCPUOutput) {
+    DumpCpuTensor(dump_options, tensor, tensor_name);
+  } else {
+    std::cout << tensor_location << "\n";
+
+#ifdef USE_CUDA
+    // Dumping GPU only when cuda is enabled.
+    if (tensor_location.device.Type() == OrtDevice::GPU) {
+      const auto& execution_providers = session_state.GetExecutionProviders();
+      const auto* cpu_execution_provider = execution_providers.Get(onnxruntime::kCpuExecutionProvider);
+      auto cpu_allocator = cpu_execution_provider->GetAllocator(0, OrtMemTypeDefault);
+      Tensor cpu_tensor{data_type, tensor.Shape(), cpu_allocator};
+      const auto& data_transfer_mgr = session_state.GetDataTransferMgr();
+      auto status = data_transfer_mgr.CopyTensor(tensor, cpu_tensor);
+      if (status == common::Status::OK()) {
+        DumpCpuTensor(dump_options, cpu_tensor, tensor_name);
+      } else {
+        std::cout << " failed to transfer data to cpu.\n";
+      }
+    }
+#else
+    ORT_UNUSED_PARAMETER(session_state);
+#endif
+  }
+}
+
+}  // namespace
+
+const NodeDumpOptions& NodeDumpOptionsFromEnvironmentVariables() {
+  static const NodeDumpOptions node_dump_options = []() {
+    namespace env_vars = debug_node_inputs_outputs_env_vars;
+
+    auto get_bool_env_var = [](const char* env_var) {
+      const auto val = Env::Default().GetEnvironmentVar(env_var);
+      if (val.empty()) return false;
+      std::istringstream s{val};
+      int i;
+      ORT_ENFORCE(
+          s >> i && s.eof(),
+          "Failed to parse environment variable ", env_var, ": ", val);
+      return i != 0;
+    };
+
+    NodeDumpOptions opts{};
+
+    opts.dump_flags = NodeDumpOptions::DumpFlags::ShapeOnly;
+    if (get_bool_env_var(env_vars::kDumpInputData)) {
+      opts.dump_flags |= NodeDumpOptions::DumpFlags::InputData;
+    }
+    if (get_bool_env_var(env_vars::kDumpOutputData)) {
+      opts.dump_flags |= NodeDumpOptions::DumpFlags::OutputData;
+    }
+
+    opts.filter.name_pattern = Env::Default().GetEnvironmentVar(env_vars::kNameFilter);
+    opts.filter.op_type_pattern = Env::Default().GetEnvironmentVar(env_vars::kOpTypeFilter);
+
+    if (get_bool_env_var(env_vars::kDumpDataToFiles)) {
+      opts.data_destination = NodeDumpOptions::DataDestination::TensorProtoFiles;
+    }
+
+    opts.output_dir = Path::Parse(ToPathString(Env::Default().GetEnvironmentVar(env_vars::kOutputDir)));
+
+    // check for confirmation for dumping data to files for all nodes
+    if (opts.dump_flags != NodeDumpOptions::DumpFlags::ShapeOnly &&
+        opts.data_destination == NodeDumpOptions::DataDestination::TensorProtoFiles &&
+        opts.filter.name_pattern.empty() && opts.filter.op_type_pattern.empty()) {
+      ORT_ENFORCE(
+          get_bool_env_var(env_vars::kDumpingDataToFilesForAllNodesIsOk),
+          "The current environment variable configuration will dump node input or output data to files for every node. "
+          "This may cause a lot of files to be generated. Set the environment variable ",
+          env_vars::kDumpingDataToFilesForAllNodesIsOk, " to confirm this is what you want.");
+    }
+
+    return opts;
+  }();
+
+  return node_dump_options;
+}
+
+void DumpNodeInputs(
+    const NodeDumpOptions& dump_options,
+    const OpKernelContext& context, const Node& node, const SessionState& session_state) {
+  if (!FilterNode(dump_options, node)) return;
+
+  std::cout << "-----------\n";
+  std::cout << node.OpType() << " node: " << node.Name() << "\n";
+
+  const auto& input_defs = node.InputDefs();
+
+  for (auto i = 0, end = context.InputCount(); i < end; ++i) {
+    if (input_defs[i]->Exists()) {
+      std::cout << "Input " << i << " Name: " << input_defs[i]->Name();
+
+      const auto* type = context.InputType(i);
+
+      if (type) {
+        if (type->IsTensorType()) {
+          const auto& tensor = *context.Input<Tensor>(i);
+          const auto& shape = tensor.Shape();
+
+          std::cout << " Shape: " << shape << "\n";
+
+          if ((dump_options.dump_flags & NodeDumpOptions::DumpFlags::InputData) != 0) {
+            DumpTensor(dump_options, tensor, input_defs[i]->Name(), session_state);
+          }
+        } else {
+          std::cout << " is non-tensor type.\n";
+        }
+      } else {
+        // should never happen...
+        std::cout << " was missing data type\n";
+      }
+    } else {
+      std::cout << "Input " << i << " is optional and was not provided.\n";
+    }
+  }
+}
+
+void DumpNodeInputs(
+    const OpKernelContext& context, const Node& node, const SessionState& session_state) {
+  DumpNodeInputs(NodeDumpOptionsFromEnvironmentVariables(), context, node, session_state);
+}
+
+void DumpNodeOutputs(const NodeDumpOptions& dump_options, OpKernelContext& context, const Node& node, const SessionState& session_state) {
+  if (!FilterNode(dump_options, node)) return;
+
+  std::cout << "-----------\n";
+  const auto& output_defs = node.OutputDefs();
+
+  for (auto i = 0, end = context.OutputCount(); i < end; ++i) {
+    if (output_defs[i]->Exists()) {
+      std::cout << "Output " << i << " Name: " << output_defs[i]->Name();
+
+      const auto* type = context.OutputType(i);
+      if (type) {
+        if (type->IsTensorType()) {
+          const auto& tensor = *context.Output<Tensor>(i);
+          const auto& shape = tensor.Shape();
+
+          std::cout << " Shape: " << shape << "\n";
+
+          if ((dump_options.dump_flags & NodeDumpOptions::DumpFlags::OutputData) != 0) {
+            DumpTensor(dump_options, tensor, output_defs[i]->Name(), session_state);
+          }
+        } else {
+          std::cout << " is non-tensor type.\n";
+        }
+      } else {
+        // should never happen...
+        std::cout << "missing data type\n";
+      }
+    } else {
+      std::cout << "Output " << i << " is optional and was not produced.\n";
+    }
+
+    std::cout << std::endl;
+  }
+}
+
+void DumpNodeOutputs(
+    OpKernelContext& context, const Node& node, const SessionState& session_state) {
+  DumpNodeOutputs(NodeDumpOptionsFromEnvironmentVariables(), context, node, session_state);
+}
+
+}  // namespace utils
+}  // namespace onnxruntime
+
+#endif

onnxruntime/core/framework/debug_node_inputs_outputs_utils.h

@@ -0,0 +1,104 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+// to create a build with these enabled run the build script with:
+//   --cmake_extra_defines onnxruntime_DEBUG_NODE_INPUTS_OUTPUTS=1
+
+#ifdef DEBUG_NODE_INPUTS_OUTPUTS
+
+#pragma once
+
+#include "core/common/path.h"
+#include "core/framework/op_kernel.h"
+#include "core/framework/session_state.h"
+#include "core/graph/graph.h"
+
+namespace onnxruntime {
+namespace utils {
+
+// environment variables that control debug node dumping behavior
+namespace debug_node_inputs_outputs_env_vars {
+// set to non-zero to dump node input data
+constexpr const char* kDumpInputData = "ORT_DEBUG_NODE_IO_DUMP_INPUT_DATA";
+// set to non-zero to dump node output data
+constexpr const char* kDumpOutputData = "ORT_DEBUG_NODE_IO_DUMP_OUTPUT_DATA";
+// specify a node name filter to limit the nodes that are dumped
+// see NodeDumpOptions::FilterOptions
+constexpr const char* kNameFilter = "ORT_DEBUG_NODE_IO_NAME_FILTER";
+// specify a node op type filter to limit the nodes that are dumped
+// see NodeDumpOptions::FilterOptions
+constexpr const char* kOpTypeFilter = "ORT_DEBUG_NODE_IO_OP_TYPE_FILTER";
+// set to non-zero to dump data to files instead of stdout
+constexpr const char* kDumpDataToFiles = "ORT_DEBUG_NODE_IO_DUMP_DATA_TO_FILES";
+// specify the output directory for any data files produced
+constexpr const char* kOutputDir = "ORT_DEBUG_NODE_IO_OUTPUT_DIR";
+// set to non-zero to confirm that dumping data files for all nodes is acceptable
+constexpr const char* kDumpingDataToFilesForAllNodesIsOk =
+    "ORT_DEBUG_NODE_IO_DUMPING_DATA_TO_FILES_FOR_ALL_NODES_IS_OK";
+}  // namespace debug_node_inputs_outputs_env_vars
+
+constexpr char kFilterPatternDelimiter = ';';
+
+struct NodeDumpOptions {
+  enum DumpFlags {
+    ShapeOnly = 0,
+    InputData = 1 << 0,
+    OutputData = 1 << 1,
+    AllData = InputData | OutputData,
+  };
+
+  // specifies the information to dump per node
+  // see DumpFlags
+  // Note:
+  // When dumping every node, dumping both input and output may be redundant.
+  // Doing that may be more useful when dumping a subset of all nodes.
+  int dump_flags{DumpFlags::ShapeOnly};
+
+  // filters the nodes that are dumped
+  // Note:
+  // Pattern strings are substrings (individual patterns) delimited by ';'.
+  // For a given pattern type (e.g. Name), a pattern string matches a value if one of the following is true:
+  // - the pattern string is empty
+  // - one of the delimited substrings matches exactly
+  // For a node to be dumped, it must match each pattern type.
+  // By default (with empty pattern strings), all nodes will match and be dumped.
+  struct FilterOptions {
+    std::string name_pattern{};
+    std::string op_type_pattern{};
+  } filter{};
+
+  // the destination for dumped data
+  enum class DataDestination {
+    // write to stdout
+    StdOut,
+    // write to one file per tensor input/output as a TensorProto
+    TensorProtoFiles,
+  } data_destination{DataDestination::StdOut};
+
+  // the output directory for dumped data files
+  Path output_dir;
+};
+
+// gets NodeDumpOptions instance configured from environment variable values
+const NodeDumpOptions& NodeDumpOptionsFromEnvironmentVariables();
+
+// dumps inputs for a node
+void DumpNodeInputs(
+    const NodeDumpOptions& dump_options,
+    const OpKernelContext& context, const Node& node, const SessionState& session_state);
+
+void DumpNodeInputs(
+    const OpKernelContext& context, const Node& node, const SessionState& session_state);
+
+// dumps outputs for a node
+void DumpNodeOutputs(
+    const NodeDumpOptions& dump_options,
+    OpKernelContext& context, const Node& node, const SessionState& session_state);
+
+void DumpNodeOutputs(
+    OpKernelContext& context, const Node& node, const SessionState& session_state);
+
+}  // namespace utils
+}  // namespace onnxruntime
+
+#endif

onnxruntime/core/framework/sequential_executor.cc

@@ -15,7 +15,7 @@
 #include "core/framework/op_kernel_context_internal.h"
 
 #if defined DEBUG_NODE_INPUTS_OUTPUTS
-#include "core/framework/utils.h"
+#include "core/framework/debug_node_inputs_outputs_utils.h"
 #endif
 
 #ifdef ENABLE_NVTX_PROFILE
@@ -269,8 +269,8 @@ Status SequentialExecutor::Execute(const SessionState& session_state, const std:
         }
       }
     }
-#if defined DEBUG_NODE_INPUTS_OUTPUTS
-    utils::DumpNodeInputs(op_kernel_context, p_op_kernel->Node());
+#ifdef DEBUG_NODE_INPUTS_OUTPUTS
+    utils::DumpNodeInputs(op_kernel_context, p_op_kernel->Node(), session_state);
 #endif
 
     const std::string node_name_for_profiling = [&]() -> std::string {
@@ -400,7 +400,7 @@ Status SequentialExecutor::Execute(const SessionState& session_state, const std:
                                                      {{"op_name", p_op_kernel->KernelDef().OpName()}});
     }
 
-#if defined(DEBUG_NODE_INPUTS_OUTPUTS)
+#ifdef DEBUG_NODE_INPUTS_OUTPUTS
     utils::DumpNodeOutputs(op_kernel_context, p_op_kernel->Node(), session_state);
 #endif
 

onnxruntime/core/framework/utils.cc

@@ -527,145 +527,6 @@ common::Status ExecuteSubgraph(const SessionState& session_state, const FeedsFet
   return status;
 }
 
-#if defined(DEBUG_NODE_INPUTS_OUTPUTS)
-std::ostream& operator<<(std::ostream& out, const BFloat16& value) {
-  return out << value.ToFloat();
-}
-
-std::ostream& operator<<(std::ostream& out, const MLFloat16& value) {
-  return out << value.val;
-}
-
-template <typename T>
-static void DumpTensor(const Tensor& tensor, const TensorShape& shape) {
-  auto num_items = shape.Size();
-
-  if (num_items == 0) {
-    std::cout << "no data";
-    return;
-  }
-
-  size_t num_dims = shape.NumDimensions();
-  size_t num_rows = 1;
-  if (num_dims > 1) {
-    num_rows = static_cast<size_t>(shape[0]);
-  }
-
-  size_t row_size = num_items / num_rows;
-
-  auto data = tensor.DataAsSpan<T>();
-
-  auto print_val = [](const T& value) {
-    if (std::is_floating_point<T>::value)
-      std::cout << std::setprecision(8) << value;
-    else
-      std::cout << value;
-  };
-
-  for (size_t row = 0; row < num_rows; ++row) {
-    print_val(data[row * row_size]);
-    for (size_t i = 1; i < row_size; ++i) {
-      std::cout << ", ";
-      print_val(data[row * row_size + i]);
-    }
-    std::cout << "\n";
-  }
-
-  std::cout << std::endl;
-}
-
-void DumpNodeInputs(const OpKernelContext& context, const Node& node) {
-  std::cout << "-----------\n";
-  std::cout << node.OpType() << " node: " << node.Name() << "\n";
-
-  const auto& input_defs = node.InputDefs();
-
-  for (auto i = 0, end = context.InputCount(); i < end; ++i) {
-    if (input_defs[i]->Exists()) {
-      std::cout << "Input " << i << " Name: " << input_defs[i]->Name();
-
-      const auto* type = context.InputType(i);
-
-      if (type) {
-        if (type->IsTensorType()) {
-          const auto& tensor = *context.Input<Tensor>(i);
-          const auto& shape = tensor.Shape();
-
-          std::cout << " Shape: " << shape << "\n";
-        } else {
-          std::cout << " is non-tensor type.\n";
-        }
-      } else {
-        // should never happen...
-        std::cout << " was missing data type\n";
-      }
-    } else {
-      std::cout << "Input " << i << " is optional and was not provided.\n";
-    }
-  }
-}
-
-void DumpNodeOutputs(OpKernelContext& context, const Node& node, const SessionState& session_state) {
-  std::cout << "-----------\n";
-  const auto& output_defs = node.OutputDefs();
-
-  for (auto i = 0, end = context.OutputCount(); i < end; ++i) {
-    if (output_defs[i]->Exists()) {
-      std::cout << "Output " << i << " Name: " << output_defs[i]->Name();
-
-      const auto* type = context.OutputType(i);
-      if (type) {
-        if (type->IsTensorType()) {
-          const auto& tensor = *context.Output<Tensor>(i);
-          const auto& shape = tensor.Shape();
-
-          std::cout << " Shape: " << shape << "\n";
-
-          if (DEBUG_NODE_INPUTS_OUTPUTS > 1) {
-            // check tensor is on CPU before dumping it
-            auto& tensor_location = tensor.Location();
-            const auto data_type = tensor.DataType();
-            if (tensor_location.device.Type() == OrtDevice::CPU || tensor_location.mem_type == OrtMemTypeCPUOutput) {
-              DispatchOnTensorType(data_type, DumpTensor, tensor, shape);
-            } else {
-              std::cout << tensor_location << "\n";
-
-#ifdef USE_CUDA
-              // Dumping GPU only when cuda is enabled. Most op has only one output, so put GPU related code here to get best performance.
-              if (tensor_location.device.Type() == OrtDevice::GPU) {
-                const auto& execution_providers = session_state.GetExecutionProviders();
-                const auto* cpu_execution_provider = execution_providers.Get(onnxruntime::kCpuExecutionProvider);
-                auto cpu_allocator = cpu_execution_provider->GetAllocator(0, OrtMemTypeDefault);
-                std::unique_ptr<Tensor> cpu_tensor = onnxruntime::make_unique<Tensor>(data_type, shape, cpu_allocator);
-                const auto& data_transfer_mgr = session_state.GetDataTransferMgr();
-                auto status = data_transfer_mgr.CopyTensor(tensor, *cpu_tensor.get(), 0);
-                if (status == common::Status::OK()) {
-                  DispatchOnTensorType(data_type, DumpTensor, *cpu_tensor.get(), shape);
-                } else {
-                  std::cout << " failed to transfer data to cpu.\n";
-                }
-              }
-#else
-              ORT_UNUSED_PARAMETER(session_state);
-#endif
-            }
-          }
-        } else {
-          std::cout << " is non-tensor type.\n";
-        }
-      } else {
-        // should never happen...
-        std::cout << "missing data type\n";
-      }
-    } else {
-      std::cout << "Output " << i << " is optional and was not produced.\n";
-    }
-
-    std::cout << std::endl;
-  }
-}
-#endif
-
 int32_t ONNXTensorElementDataTypeToProtoTensorType(ONNXTensorElementDataType onnx_enum) {
   switch (onnx_enum) {
     case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT:

onnxruntime/core/framework/utils.h

@@ -72,16 +72,6 @@ common::Status ExecuteSubgraph(const SessionState& session_state, const FeedsFet
                                const std::unordered_map<size_t, IExecutor::CustomAllocator>& fetch_allocators,
                                ExecutionMode execution_mode, const bool& terminate_flag, const logging::Logger& logger);
 
-#if defined(DEBUG_NODE_INPUTS_OUTPUTS)
-// to create a build with these enabled run the build script with 1 to dump just shapes, or 2 to dump shapes and data
-// e.g.
-//   --cmake_extra_defines onnxruntime_DEBUG_NODE_INPUTS_OUTPUTS=1
-// To unset you'll need to either delete CMakeCache.txt or run with
-//   --cmake_extra_defines onnxruntime_DEBUG_NODE_INPUTS_OUTPUTS=0
-void DumpNodeInputs(const OpKernelContext& context, const Node& node);
-void DumpNodeOutputs(OpKernelContext& context, const Node& node, const SessionState& session_state);
-#endif
-
 template <typename T>
 constexpr ONNXTensorElementDataType GetONNXTensorElementDataType() {
   return ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED;

onnxruntime/test/framework/debug_node_inputs_outputs_utils_test.cc

@@ -0,0 +1,77 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#ifdef DEBUG_NODE_INPUTS_OUTPUTS
+
+#include "core/framework/debug_node_inputs_outputs_utils.h"
+
+#include <fstream>
+
+#include "gtest/gtest.h"
+
+#include "core/framework/tensorprotoutils.h"
+#include "core/platform/env.h"
+#include "core/platform/path_lib.h"
+#include "test/providers/provider_test_utils.h"
+#include "test/util/include/scoped_env_vars.h"
+#include "test/util/include/temp_dir.h"
+
+namespace onnxruntime {
+namespace test {
+
+namespace {
+template <typename T>
+void VerifyTensorProtoFileData(const PathString& tensor_proto_path, gsl::span<const T> expected_data) {
+  std::ifstream tensor_proto_stream{tensor_proto_path};
+
+  ONNX_NAMESPACE::TensorProto tensor_proto{};
+  ASSERT_TRUE(tensor_proto.ParseFromIstream(&tensor_proto_stream));
+
+  std::vector<T> actual_data{};
+  actual_data.resize(expected_data.size());
+  ASSERT_STATUS_OK(utils::UnpackTensor(tensor_proto, actual_data.data(), actual_data.size()));
+
+  ASSERT_EQ(gsl::make_span(actual_data), expected_data);
+}
+}  // namespace
+
+namespace env_vars = utils::debug_node_inputs_outputs_env_vars;
+
+TEST(DebugNodeInputsOutputs, BasicFileOutput) {
+  TemporaryDirectory temp_dir{ORT_TSTR("debug_node_inputs_outputs_utils_test")};
+  ScopedEnvironmentVariables scoped_env_vars{
+      EnvVarMap{
+          {env_vars::kDumpInputData, {"1"}},
+          {env_vars::kDumpOutputData, {"1"}},
+          {env_vars::kDumpDataToFiles, {"1"}},
+          {env_vars::kOutputDir, {ToMBString(temp_dir.Path())}},
+          {env_vars::kDumpingDataToFilesForAllNodesIsOk, {"1"}},
+      }};
+
+  OpTester tester{"Round", 11, kOnnxDomain};
+  const std::vector<float> input{0.9f, 1.8f};
+  tester.AddInput<float>("x", {static_cast<int64_t>(input.size())}, input);
+  const std::vector<float> output{1.0f, 2.0f};
+  tester.AddOutput<float>("y", {static_cast<int64_t>(output.size())}, output);
+
+  auto verify_file_data =
+      [&temp_dir, &input, &output](
+          const std::vector<OrtValue>& /*fetches*/,
+          const std::string& /*provider_type*/) {
+        VerifyTensorProtoFileData(
+            temp_dir.Path() + ORT_TSTR("/x.tensorproto"),
+            gsl::make_span(input));
+        VerifyTensorProtoFileData(
+            temp_dir.Path() + ORT_TSTR("/y.tensorproto"),
+            gsl::make_span(output));
+      };
+
+  tester.Run(
+      OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, nullptr,
+      ExecutionMode::ORT_SEQUENTIAL, verify_file_data);
+}
+
+}  // namespace test
+}  // namespace onnxruntime
+
+#endif

onnxruntime/test/util/include/scoped_env_vars.h

@@ -0,0 +1,31 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#include <unordered_map>
+
+#include "core/common/common.h"
+#include "core/common/optional.h"
+
+namespace onnxruntime {
+namespace test {
+
+// map of environment variable name to optional value
+// no value means the variable is not defined
+using EnvVarMap = std::unordered_map<std::string, optional<std::string>>;
+
+// Sets the given environment variables to their given values while in scope.
+// The original values are reset on destruction.
+class ScopedEnvironmentVariables {
+ public:
+  explicit ScopedEnvironmentVariables(const EnvVarMap& new_environment_variables);
+
+  ~ScopedEnvironmentVariables();
+
+ private:
+  ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(ScopedEnvironmentVariables);
+
+  EnvVarMap original_environment_variables_;
+};
+
+}  // namespace test
+}  // namespace onnxruntime

onnxruntime/test/util/scoped_env_vars.cc

@@ -0,0 +1,97 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#include "test/util/include/scoped_env_vars.h"
+
+#ifndef WIN32
+#include <stdlib.h>
+#else  // WIN32
+#include <Windows.h>
+#endif
+
+namespace onnxruntime {
+namespace test {
+
+namespace {
+#ifndef WIN32
+Status SetEnvironmentVar(const std::string& name, const optional<std::string>& value) {
+  if (value.has_value()) {
+    ORT_RETURN_IF_NOT(
+        setenv(name.c_str(), value.value().c_str(), 1) == 0,
+        "setenv() failed: ", errno);
+  } else {
+    ORT_RETURN_IF_NOT(
+        unsetenv(name.c_str()) == 0,
+        "unsetenv() failed: ", errno);
+  }
+  return Status::OK();
+}
+
+Status GetEnvironmentVar(const std::string& name, optional<std::string>& value) {
+  const char* val = getenv(name.c_str());
+  value = val == nullptr ? optional<std::string>{} : optional<std::string>{std::string{val}};
+  return Status::OK();
+}
+#else  // WIN32
+Status SetEnvironmentVar(const std::string& name, const optional<std::string>& value) {
+  ORT_RETURN_IF_NOT(
+      SetEnvironmentVariableA(name.c_str(), value.has_value() ? value.value().c_str() : nullptr) != 0,
+      "SetEnvironmentVariableA() failed: ", GetLastError());
+  return Status::OK();
+}
+
+Status GetEnvironmentVar(const std::string& name, optional<std::string>& value) {
+  constexpr DWORD kBufferSize = 32767;
+
+  char buffer[kBufferSize];
+
+  const auto char_count = GetEnvironmentVariableA(name.c_str(), buffer, kBufferSize);
+  if (char_count > 0) {
+    value = std::string{buffer, buffer + char_count};
+    return Status::OK();
+  }
+
+  if (GetLastError() == ERROR_ENVVAR_NOT_FOUND) {
+    value = optional<std::string>{};
+    return Status::OK();
+  }
+
+  return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "GetEnvironmentVariableA() failed: ", GetLastError());
+}
+#endif
+
+void SetEnvironmentVars(const EnvVarMap& env_vars) {
+  for (const auto& env_var : env_vars) {
+    ORT_THROW_IF_ERROR(SetEnvironmentVar(env_var.first, env_var.second));
+  }
+}
+
+EnvVarMap GetEnvironmentVars(const std::vector<std::string>& env_var_names) {
+  EnvVarMap result{};
+  for (const auto& env_var_name : env_var_names) {
+    // TODO update Env::GetEnvironmentVar() to distinguish between empty and undefined variables and use that instead
+    optional<std::string> env_var_value{};
+    ORT_THROW_IF_ERROR(GetEnvironmentVar(env_var_name, env_var_value));
+    result.insert({env_var_name, env_var_value});
+  }
+  return result;
+}
+}  // namespace
+
+ScopedEnvironmentVariables::ScopedEnvironmentVariables(const EnvVarMap& new_env_vars) {
+  std::vector<std::string> new_env_var_names{};
+  std::transform(
+      new_env_vars.begin(), new_env_vars.end(), std::back_inserter(new_env_var_names),
+      [](const EnvVarMap::value_type& new_env_var) { return new_env_var.first; });
+
+  original_environment_variables_ = GetEnvironmentVars(new_env_var_names);
+
+  SetEnvironmentVars(new_env_vars);
+}
+
+ScopedEnvironmentVariables::~ScopedEnvironmentVariables() {
+  SetEnvironmentVars(original_environment_variables_);
+}
+
+}  // namespace test
+}  // namespace onnxruntime