Update onnxruntime_perf_test.exe to accept free dimension overrides (#6962)

Co-authored-by: Ori Levari <orlevari@microsoft.com>

onnxruntime/test/perftest/command_args_parser.cc

@@ -44,6 +44,10 @@ namespace perftest {
       "\t-v: Show verbose information.\n"
       "\t-x [intra_op_num_threads]: Sets the number of threads used to parallelize the execution within nodes, A value of 0 means ORT will pick a default. Must >=0.\n"
       "\t-y [inter_op_num_threads]: Sets the number of threads used to parallelize the execution of the graph (across nodes), A value of 0 means ORT will pick a default. Must >=0.\n"
+      "\t-f [free_dimension_override]: Specifies a free dimension by name to override to a specific value for performance optimization. "
+      "Syntax is [dimension_name:override_value]. override_value must > 0\n"
+      "\t-F [free_dimension_override]: Specifies a free dimension by denotation to override to a specific value for performance optimization. "
+      "Syntax is [dimension_denotation:override_value]. override_value must > 0\n"
       "\t-P: Use parallel executor instead of sequential executor.\n"
       "\t-o [optimization level]: Default is 1. Valid values are 0 (disable), 1 (basic), 2 (extended), 99 (all).\n"
       "\t\tPlease see onnxruntime_c_api.h (enum GraphOptimizationLevel) for the full list of all optimization levels.\n"
@@ -60,11 +64,52 @@ namespace perftest {
       "\t [Example] [For OpenVINO EP] -e openvino -i 'device_type|CPU_FP32 enable_vpu_fast_compile|true num_of_threads|5'\n"
       "\t-h: help\n");
 }
+#ifdef _WIN32
+static const ORTCHAR_T* overrideDelimiter = L":";
+#else
+static const ORTCHAR_T* overrideDelimiter = ":";
+#endif
+static bool ParseDimensionOverride(std::basic_string<ORTCHAR_T>& dim_identifier, int64_t& override_val) {
+  std::basic_string<ORTCHAR_T> free_dim_str(optarg);
+  size_t delimiter_location = free_dim_str.find(overrideDelimiter);
+  if (delimiter_location >= free_dim_str.size() - 1) {
+    return false;
+  }
+  dim_identifier = free_dim_str.substr(0, delimiter_location);
+  std::basic_string<ORTCHAR_T> override_val_str = free_dim_str.substr(delimiter_location + 1, std::wstring::npos);
+  ORT_TRY {
+    override_val = std::stoll(override_val_str.c_str());
+    if (override_val <= 0) {
+      return false;
+    }
+  } ORT_CATCH (...) {
+    return false;
+  }
+  return true;
+}
 
 /*static*/ bool CommandLineParser::ParseArguments(PerformanceTestConfig& test_config, int argc, ORTCHAR_T* argv[]) {
   int ch;
-  while ((ch = getopt(argc, argv, ORT_TSTR("b:m:e:r:t:p:x:y:c:d:o:u:i:AMPIvhsqz"))) != -1) {
+  while ((ch = getopt(argc, argv, ORT_TSTR("b:m:e:r:t:p:x:y:c:d:o:u:i:f:F:AMPIvhsqz"))) != -1) {
     switch (ch) {
+      case 'f': {
+        std::basic_string<ORTCHAR_T> dim_name;
+        int64_t override_val;
+        if (!ParseDimensionOverride(dim_name, override_val)) {
+          return false;
+        }
+        test_config.run_config.free_dim_name_overrides[dim_name] = override_val;
+        break;
+      }
+      case 'F': {
+        std::basic_string<ORTCHAR_T> dim_denotation;
+        int64_t override_val;
+        if (!ParseDimensionOverride(dim_denotation, override_val)) {
+          return false;
+        }
+        test_config.run_config.free_dim_denotation_overrides[dim_denotation] = override_val;
+        break;
+      }
       case 'm':
         if (!CompareCString(optarg, ORT_TSTR("duration"))) {
           test_config.run_config.test_mode = TestMode::kFixDurationMode;

onnxruntime/test/perftest/ort_test_session.cc

@@ -8,6 +8,7 @@
 #ifdef _WIN32
 #define strdup _strdup
 #endif
+extern const OrtApi* g_ort;
 
 namespace onnxruntime {
 namespace perftest {
@@ -201,6 +202,24 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device
     session_options.SetOptimizedModelFilePath(performance_test_config.run_config.optimized_model_path.c_str());
   if (performance_test_config.run_config.set_denormal_as_zero)
     session_options.AddConfigEntry(kOrtSessionOptionsConfigSetDenormalAsZero, "1");
+  if (!performance_test_config.run_config.free_dim_name_overrides.empty()) {
+    for (auto const& dim_override : performance_test_config.run_config.free_dim_name_overrides) {
+      if (g_ort->AddFreeDimensionOverrideByName(session_options, ToMBString(dim_override.first).c_str(), dim_override.second) != nullptr) {
+        fprintf(stderr, "AddFreeDimensionOverrideByName failed for named dimension: %s\n", ToMBString(dim_override.first).c_str());
+      } else {
+        fprintf(stdout, "Overriding dimension with name, %s, to %d\n", ToMBString(dim_override.first).c_str(), (int) dim_override.second);
+      }
+    }
+  }
+  if (!performance_test_config.run_config.free_dim_denotation_overrides.empty()) {
+    for (auto const& dim_override : performance_test_config.run_config.free_dim_denotation_overrides) {
+      if (g_ort->AddFreeDimensionOverride(session_options, ToMBString(dim_override.first).c_str(), dim_override.second) != nullptr) {
+        fprintf(stderr, "AddFreeDimensionOverride failed for dimension denotation: %s\n", ToMBString(dim_override.first).c_str());
+      } else {
+        fprintf(stdout, "Overriding dimension with denotation, %s, to %d\n", ToMBString(dim_override.first).c_str(), (int) dim_override.second);
+      }
+    }
+  }
 
   session_ = Ort::Session(env, performance_test_config.model_info.model_file_path.c_str(), session_options);
 
@@ -233,7 +252,7 @@ bool OnnxRuntimeTestSession::PopulateGeneratedInputTestData() {
       auto tensor_info = type_info.GetTensorTypeAndShapeInfo();
       std::vector<int64_t> input_node_dim = tensor_info.GetShape();
 
-      // free dimensions are treated as 1
+      // free dimensions are treated as 1 if not overriden
       for (int64_t& dim : input_node_dim) {
         if (dim == -1) {
           dim = 1;

onnxruntime/test/perftest/test_configuration.h

@@ -3,6 +3,7 @@
 
 #pragma once
 
+#include <map>
 #include <cstdint>
 #include <string>
 
@@ -54,6 +55,8 @@ struct RunConfig {
   bool do_cuda_copy_in_separate_stream{false};
   bool set_denormal_as_zero{false};
   std::basic_string<ORTCHAR_T> ep_runtime_config_string;
+  std::map<std::basic_string<ORTCHAR_T>, int64_t> free_dim_name_overrides;
+  std::map<std::basic_string<ORTCHAR_T>, int64_t> free_dim_denotation_overrides;
 };
 
 struct PerformanceTestConfig {