Unbreak microbenchmark build (#5710)

Minor updates to the microbenchmarks built optionally with "--build_micro_benchmarks". These are not built as part of CI, and builds started to fail. There are three changes:

- I updated the threading-related benchmarks to use the static-method ThreadPool API, and to expose control over the thread pool configuration via constexpr int variables.

- Disable GCC warnings seen with recent compiler versions when including parts of the Eigen headers in batchnorm.cc and eigen.cc files.

- Flush std::cerr on error conditions to avoid buffered messages being lost.

I tested manual builds with Linux (GCC) and Windows (MSVC).

onnxruntime/test/onnx/microbenchmark/activation.cc

@@ -1,5 +1,6 @@
 #include "common.h"
 
+#include "core/common/status.h"
 #include "core/session/ort_env.h"
 #include "core/graph/model.h"
 #include "core/graph/graph.h"
@@ -15,6 +16,7 @@
 #include <benchmark/benchmark.h>
 #include <random>
 
+using namespace onnxruntime::common;
 using namespace onnxruntime;
 using namespace onnx;
 extern OrtEnv* env;
@@ -380,7 +382,7 @@ static void BM_Powx(benchmark::State& state) {
   f.input2 = input2;
   f.output = output;
   for (auto _ : state) {
-    tp->ParallelFor(batch_size, TensorOpCost{2, 1, static_cast<double>(cost)}, f);
+    concurrency::ThreadPool::TryParallelFor(tp.get(), batch_size, TensorOpCost{2, 1, static_cast<double>(cost)}, f);
   }
   aligned_free(input1);
   aligned_free(input2);

onnxruntime/test/onnx/microbenchmark/batchnorm.cc

@@ -1,8 +1,18 @@
 #include "core/platform/threadpool.h"
-#include "core/common/eigen_common_wrapper.h"
 #include "core/util/thread_utils.h"
 #include <benchmark/benchmark.h>
 
+#if defined(__GNUC__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
+#endif
+
+#include "core/common/eigen_common_wrapper.h"
+
+#if defined(__GNUC__)
+#pragma GCC diagnostic pop
+#endif
+
 using namespace onnxruntime;
 #if 0
 static void BM_BatchNormEigenTensor(benchmark::State& state) {
@@ -99,4 +109,4 @@ BENCHMARK(BM_BatchNormEigenTensorSingleThread)
     ->Arg(16)
     ->Arg(64)
     ->UseRealTime()
-    ->Unit(benchmark::TimeUnit::kMicrosecond);
+    ->Unit(benchmark::TimeUnit::kMicrosecond);

onnxruntime/test/onnx/microbenchmark/eigen.cc

@@ -4,6 +4,7 @@
 #pragma GCC diagnostic ignored "-Wignored-attributes"
 #endif
 #pragma GCC diagnostic ignored "-Wunused-parameter"
+#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
 #elif defined(_MSC_VER)
 #pragma warning(push)
 #pragma warning(disable : 4267)

onnxruntime/test/onnx/microbenchmark/gelu.cc

@@ -9,6 +9,7 @@
 #include <mlas.h>
 
 using namespace onnxruntime;
+using namespace onnxruntime::concurrency;
 
 //naive implementation of Gelu
 static void BM_GeluSingleThreadPlainLoop(benchmark::State& state) {
@@ -72,7 +73,7 @@ static void BM_GeluParallelFor(benchmark::State& state) {
   std::unique_ptr<concurrency::ThreadPool> tp(
       concurrency::CreateThreadPool(&onnxruntime::Env::Default(), tpo, concurrency::ThreadPoolType::INTRA_OP));
   for (auto _ : state) {
-    tp->ParallelFor(batch_size, cost, [data, output](ptrdiff_t first, ptrdiff_t last) {
+    ThreadPool::TryParallelFor(tp.get(), batch_size, cost, [data, output](ptrdiff_t first, ptrdiff_t last) {
       ptrdiff_t len = last - first;
       float* output_ptr = output + first;
       onnxruntime::ConstEigenVectorArrayMap<float> xm(data + first, len);
@@ -138,7 +139,8 @@ static void BM_ScaledTanhParallelFor(benchmark::State& state) {
   const float alpha_ = 0.3f;
   const float beta_ = 0.6f;
   for (auto _ : state) {
-    tp->ParallelFor(batch_size, cost, [alpha_, beta_, data, output](ptrdiff_t first, ptrdiff_t last) {
+    ThreadPool::TryParallelFor(tp.get(), batch_size, cost,
+                            [alpha_, beta_, data, output](ptrdiff_t first, ptrdiff_t last) {
       ptrdiff_t len = last - first;
       float* output_ptr = output + first;
       onnxruntime::ConstEigenVectorArrayMap<float> xm(data + first, len);
@@ -207,7 +209,7 @@ static void BM_GeluBatchParallelFor(benchmark::State& state) {
       concurrency::CreateThreadPool(&onnxruntime::Env::Default(), tpo, concurrency::ThreadPoolType::INTRA_OP));
   const int num_batches = 4;
   for (auto _ : state) {
-    tp->SimpleParallelFor(num_batches, [&](std::ptrdiff_t batch_index) {
+    ThreadPool::TrySimpleParallelFor(tp.get(), num_batches, [&](std::ptrdiff_t batch_index) {
       std::ptrdiff_t start, work_remaining;
       TestPartitionWork(batch_index, num_batches, batch_size, &start, &work_remaining);
       float* output_ptr = output + start;
@@ -327,4 +329,4 @@ BENCHMARK(BM_GeluBatchParallelFor3)
     ->Arg(20000)
     ->Arg(40000)
     ->Arg(98304)
-    ->Arg(1572864);
+    ->Arg(1572864);

onnxruntime/test/onnx/microbenchmark/main.cc

@@ -18,6 +18,7 @@
 #include <core/session/ort_env.h>
 #include <core/util/thread_utils.h>
 
+#include <iostream>
 #include <unordered_map>
 
 const OrtApi* g_ort = OrtGetApiBase()->GetApi(ORT_API_VERSION);
@@ -43,7 +44,7 @@ static void BM_ResolveGraph(benchmark::State& state) {
   auto st =
       onnxruntime::Model::Load(ORT_TSTR("../models/opset8/test_tiny_yolov2/model.onnx"), model_copy, nullptr, *logger);
   if (!st.IsOK()) {
-    printf("Parse model failed: %s", st.ErrorMessage().c_str());
+    ::std::cerr << "Parse model failed: " << st.ErrorMessage().c_str() << ::std::endl;
     abort();
   }
   auto proto = model_copy->ToProto();
@@ -55,7 +56,7 @@ static void BM_ResolveGraph(benchmark::State& state) {
     state.ResumeTiming();
     st = graph.Resolve();
     if (!st.IsOK()) {
-      printf("Resolve graph failed: %s", st.ErrorMessage().c_str());
+      ::std::cerr << "Resolve graph failed: " << st.ErrorMessage().c_str() << ::std::endl;
       abort();
     }
   }
@@ -67,7 +68,7 @@ BENCHMARK(BM_ResolveGraph);
     OrtStatus* onnx_status = (expr);                         \
     if (onnx_status != NULL) {                               \
       const char* msg = g_ort->GetErrorMessage(onnx_status); \
-      fprintf(stderr, "%s\n", msg);                          \
+      ::std::cerr << msg << ::std::endl;                     \
       g_ort->ReleaseStatus(onnx_status);                     \
       abort();                                               \
     }                                                        \

onnxruntime/test/onnx/microbenchmark/tptest.cc

@@ -10,9 +10,17 @@
 using namespace onnxruntime;
 using namespace onnxruntime::concurrency;
 
+// Thread pool configuration to test.
+constexpr int NUM_THREADS = 8;
+constexpr bool ALLOW_SPINNING = true;
+
 static void BM_CreateThreadPool(benchmark::State& state) {
   for (auto _ : state) {
-    ThreadPool tp(&onnxruntime::Env::Default(), onnxruntime::ThreadOptions(), ORT_TSTR(""), 48, true);
+    ThreadPool tp(&onnxruntime::Env::Default(),
+                  onnxruntime::ThreadOptions(),
+                  ORT_TSTR(""),
+                  NUM_THREADS,
+                  ALLOW_SPINNING);
   }
 }
 BENCHMARK(BM_CreateThreadPool)
@@ -42,10 +50,12 @@ static void BM_ThreadPoolParallelFor(benchmark::State& state) {
   const size_t len = state.range(0);
   const int cost = static_cast<int>(state.range(1));
   OrtThreadPoolParams tpo;
-  std::unique_ptr<concurrency::ThreadPool> tp(
-      concurrency::CreateThreadPool(&onnxruntime::Env::Default(), tpo, ThreadPoolType::INTRA_OP));
+  auto tp = onnxruntime::make_unique<ThreadPool>(&onnxruntime::Env::Default(),
+                                                 onnxruntime::ThreadOptions(),
+                                                 nullptr,
+                                                 NUM_THREADS, ALLOW_SPINNING);
   for (auto _ : state) {
-    tp->ParallelFor(len, cost, SimpleForLoop);
+    ThreadPool::TryParallelFor(tp.get(), len, cost, SimpleForLoop);
   }
 }
 BENCHMARK(BM_ThreadPoolParallelFor)
@@ -115,7 +125,7 @@ static void BM_SimpleScheduleWait(benchmark::State& state) {
   for (auto _ : state) {
     onnxruntime::Barrier barrier(static_cast<unsigned int>(threads));
     for (std::ptrdiff_t id = 0; id < threads; ++id) {
-      tp->Schedule([id, threads, len, &barrier]() {
+      ThreadPool::Schedule(tp.get(), [id, threads, len, &barrier]() {
         std::ptrdiff_t start, work_remaining;
         TestPartitionWork(id, threads, len, &start, &work_remaining);
         SimpleForLoop(start, start + work_remaining);