From 596b505f6a3a27359786a8513576e5efbb0cf1df Mon Sep 17 00:00:00 2001
From: RandySheriffH <48490400+RandySheriffH@users.noreply.github.com>
Date: Tue, 31 Jan 2023 14:29:53 -0800
Subject: [PATCH] Add spin args for perf test (#14507)

Add two args for spinning control for onnxruntime_perf_test:

1. Stop spinning entirely for threads in intra-op thread pool.
2. Stop spinning only between ort runs.

Co-authored-by: Randy Shuai <rashuai@microsoft.com>
---
 onnxruntime/test/perftest/command_args_parser.cc | 12 ++++++++++--
 onnxruntime/test/perftest/ort_test_session.cc    | 12 +++++++++++-
 onnxruntime/test/perftest/test_configuration.h   |  2 ++
 3 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/onnxruntime/test/perftest/command_args_parser.cc b/onnxruntime/test/perftest/command_args_parser.cc
index 39337a8d21..a2ddb1fe9f 100644
--- a/onnxruntime/test/perftest/command_args_parser.cc
+++ b/onnxruntime/test/perftest/command_args_parser.cc
@@ -94,10 +94,12 @@ namespace perftest {
       "\t [Usage]: -e <provider_name> -i '<key1>|<value1> <key2>|<value2>' \n\n"
       "\t [Example] [For SNPE EP] -e snpe -i \"runtime|CPU priority|low\" \n\n"
       "\t-T [Set intra op thread affinities]: Specify intra op thread affinity string\n"
-      "\t [Example]: -T 1,2;3,4;5,6 or -T 1-2;3-4;5-6' \n"
+      "\t [Example]: -T 1,2;3,4;5,6 or -T 1-2;3-4;5-6 \n"
       "\t\t Use semicolon to separate configuration between threads.\n"
       "\t\t E.g. 1,2;3,4;5,6 specifies affinities for three threads, the first thread will be attached to the first and second logical processor.\n"
       "\t\t The number of affinities must be equal to intra_op_num_threads - 1\n\n"
+      "\t-D [Disable thread spinning]: disable spinning entirely for thread owned by onnxruntime intra-op thread pool.\n"
+      "\t-Z [Force thread to stop spinning between runs]: disallow thread from spinning during runs to reduce cpu usage.\n"
       "\t-h: help\n");
 }
 #ifdef _WIN32
@@ -127,7 +129,7 @@ static bool ParseDimensionOverride(std::basic_string<ORTCHAR_T>& dim_identifier,
 
 /*static*/ bool CommandLineParser::ParseArguments(PerformanceTestConfig& test_config, int argc, ORTCHAR_T* argv[]) {
   int ch;
-  while ((ch = getopt(argc, argv, ORT_TSTR("b:m:e:r:t:p:x:y:c:d:o:u:i:f:F:S:T:AMPIvhsqz"))) != -1) {
+  while ((ch = getopt(argc, argv, ORT_TSTR("b:m:e:r:t:p:x:y:c:d:o:u:i:f:F:S:T:AMPIDZvhsqz"))) != -1) {
     switch (ch) {
       case 'f': {
         std::basic_string<ORTCHAR_T> dim_name;
@@ -294,6 +296,12 @@ static bool ParseDimensionOverride(std::basic_string<ORTCHAR_T>& dim_identifier,
       case 'T':
         test_config.run_config.intra_op_thread_affinities = ToUTF8String(optarg);
         break;
+      case 'D':
+        test_config.run_config.disable_spinning = true;
+        break;
+      case 'Z':
+        test_config.run_config.disable_spinning_between_run = true;
+        break;
       case '?':
       case 'h':
       default:
diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc
index 96019a13d5..e123acf150 100644
--- a/onnxruntime/test/perftest/ort_test_session.cc
+++ b/onnxruntime/test/perftest/ort_test_session.cc
@@ -515,7 +515,17 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)");
 
   if (!performance_test_config.run_config.intra_op_thread_affinities.empty()) {
     fprintf(stdout, "Setting intra op thread affinity as %s\n", performance_test_config.run_config.intra_op_thread_affinities.c_str());
-    session_options.AddConfigEntry("session.intra_op_thread_affinities", performance_test_config.run_config.intra_op_thread_affinities.c_str());
+    session_options.AddConfigEntry(kOrtSessionOptionsConfigIntraOpThreadAffinities, performance_test_config.run_config.intra_op_thread_affinities.c_str());
+  }
+
+  if (performance_test_config.run_config.disable_spinning) {
+    fprintf(stdout, "Disabling intra-op thread spinning entirely\n");
+    session_options.AddConfigEntry(kOrtSessionOptionsConfigAllowIntraOpSpinning, "0");
+  }
+
+  if (performance_test_config.run_config.disable_spinning_between_run) {
+    fprintf(stdout, "Disabling intra-op thread spinning between runs\n");
+    session_options.AddConfigEntry(kOrtSessionOptionsConfigForceSpinningStop, "1");
   }
 
   if (performance_test_config.run_config.execution_mode == ExecutionMode::ORT_PARALLEL && performance_test_config.run_config.inter_op_num_threads > 0) {
diff --git a/onnxruntime/test/perftest/test_configuration.h b/onnxruntime/test/perftest/test_configuration.h
index 81b47d963b..43ad556247 100644
--- a/onnxruntime/test/perftest/test_configuration.h
+++ b/onnxruntime/test/perftest/test_configuration.h
@@ -59,6 +59,8 @@ struct RunConfig {
   std::map<std::basic_string<ORTCHAR_T>, int64_t> free_dim_name_overrides;
   std::map<std::basic_string<ORTCHAR_T>, int64_t> free_dim_denotation_overrides;
   std::string intra_op_thread_affinities;
+  bool disable_spinning = false;
+  bool disable_spinning_between_run = false;
 };
 
 struct PerformanceTestConfig {