From 4c7fd499496c20e1a2ad9c1588c607e5ff2ccf2e Mon Sep 17 00:00:00 2001
From: Changming Sun <chasun@microsoft.com>
Date: Fri, 22 Feb 2019 13:19:54 -0800
Subject: [PATCH] Add a few benchmarks (#512)

* Add a few benchmarks
---
 onnxruntime/test/onnx/microbenchmark/main.cc  | 23 ++++++----
 .../test/onnx/microbenchmark/modeltest.cc     | 44 +++++++++++++++++++
 2 files changed, 59 insertions(+), 8 deletions(-)

diff --git a/onnxruntime/test/onnx/microbenchmark/main.cc b/onnxruntime/test/onnx/microbenchmark/main.cc
index d5cb722f99..0e11f4e652 100644
--- a/onnxruntime/test/onnx/microbenchmark/main.cc
+++ b/onnxruntime/test/onnx/microbenchmark/main.cc
@@ -11,6 +11,7 @@
 #include <core/graph/model.h>
 #include <core/graph/graph.h>
 #include <core/framework/kernel_def_builder.h>
+#include <core/session/onnxruntime_c_api.h>
 #include <unordered_map>
 
 using namespace onnxruntime;
@@ -48,18 +49,24 @@ static void BM_ResolveGraph(benchmark::State& state) {
 }
 
 BENCHMARK(BM_ResolveGraph);
+#define ORT_ABORT_ON_ERROR(expr)                         \
+  do {                                                   \
+    OrtStatus* onnx_status = (expr);                     \
+    if (onnx_status != NULL) {                           \
+      const char* msg = OrtGetErrorMessage(onnx_status); \
+      fprintf(stderr, "%s\n", msg);                      \
+      OrtReleaseStatus(onnx_status);                     \
+      abort();                                           \
+    }                                                    \
+  } while (0);
+
+OrtEnv* env = nullptr;
 
 int main(int argc, char** argv) {
   ::benchmark::Initialize(&argc, argv);
   if (::benchmark::ReportUnrecognizedArguments(argc, argv)) return -1;
-  std::string default_logger_id{"Default"};
-  logging::LoggingManager default_logging_manager{std::unique_ptr<logging::ISink>{new logging::CLogSink{}},
-                                                  logging::Severity::kWARNING, false,
-                                                  logging::LoggingManager::InstanceType::Default,
-                                                  &default_logger_id};
-
-  std::unique_ptr<Environment> env;
-  auto status = Environment::Create(env);
+  ORT_ABORT_ON_ERROR(OrtCreateEnv(ORT_LOGGING_LEVEL_WARNING, "test", &env));
   ::benchmark::RunSpecifiedBenchmarks();
+  OrtReleaseEnv(env);
   return 0;
 }
diff --git a/onnxruntime/test/onnx/microbenchmark/modeltest.cc b/onnxruntime/test/onnx/microbenchmark/modeltest.cc
index e021d1daec..3266a02843 100644
--- a/onnxruntime/test/onnx/microbenchmark/modeltest.cc
+++ b/onnxruntime/test/onnx/microbenchmark/modeltest.cc
@@ -3,6 +3,8 @@
 
 #include <benchmark/benchmark.h>
 #include <core/graph/model.h>
+#include <core/session/onnxruntime_c_api.h>
+#include "providers.h"
 
 static void BM_LoadModel(benchmark::State& state) {
   for (auto _ : state) {
@@ -16,3 +18,45 @@ static void BM_LoadModel(benchmark::State& state) {
 }
 
 BENCHMARK(BM_LoadModel);
+
+extern OrtEnv* env;
+
+#define ORT_BREAK_ON_ERROR(expr)                         \
+  do {                                                   \
+    OrtStatus* onnx_status = (expr);                     \
+    if (onnx_status != NULL) {                           \
+      state.SkipWithError(OrtGetErrorMessage(onnx_status)); \
+      OrtReleaseStatus(onnx_status);                     \
+    }                                                    \
+  } while (0);
+
+#ifdef USE_CUDA
+static void BM_CreateSession_WithGPU(benchmark::State& state) {
+  const char* model_path = "../models/opset8/test_bvlc_alexnet/model.onnx";
+  OrtSessionOptions* session_option = OrtCreateSessionOptions();
+  ORT_BREAK_ON_ERROR(OrtSessionOptionsAppendExecutionProvider_CUDA(session_option, 0));
+  for (auto _ : state) {
+    OrtSession* session;
+    ORT_BREAK_ON_ERROR(OrtCreateSession(env, model_path, session_option, &session));
+    state.PauseTiming();
+    OrtReleaseSession(session);
+    state.ResumeTiming();
+  }
+  OrtReleaseSessionOptions(session_option);
+}
+BENCHMARK(BM_CreateSession_WithGPU);
+#endif
+
+static void BM_CreateSession(benchmark::State& state) {
+  const char* model_path = "../models/opset8/test_bvlc_alexnet/model.onnx";
+  OrtSessionOptions* session_option = OrtCreateSessionOptions();
+  for (auto _ : state) {
+    OrtSession* session;
+    ORT_BREAK_ON_ERROR(OrtCreateSession(env, model_path, session_option, &session));
+    state.PauseTiming();
+    OrtReleaseSession(session);
+    state.ResumeTiming();
+  }
+  OrtReleaseSessionOptions(session_option);
+}
+BENCHMARK(BM_CreateSession);