saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-24 19:43:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Integrate tensorflow into onnxruntime_perf_test tool

Browse source
This commit is contained in:
Changming Sun 2019-04-08 10:53:47 -07:00
parent 9467c5f967
commit 0d4055def4
17 changed files with 471 additions and 176 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
cmake/CMakeLists.txt
View file

@ -71,6 +71,7 @@ option(onnxruntime_ENABLE_LTO "Enable link time optimization, which is not stabl
option(onnxruntime_CROSS_COMPILING "Cross compiling onnx runtime" OFF)
option(onnxruntime_USE_FULL_PROTOBUF "Use full protobuf" OFF)
option(onnxruntime_DISABLE_CONTRIB_OPS "Disable contrib ops" OFF)
option(tensorflow_C_PACKAGE_PATH "Path to tensorflow C package installation dir")
set(protobuf_BUILD_TESTS OFF CACHE BOOL "Build protobuf tests" FORCE)
#nsync tests failed on Mac Build

14
cmake/onnxruntime_unittests.cmake
View file

@ -460,18 +460,28 @@ else ()
endif()
file(GLOB onnxruntime_perf_test_src ${onnxruntime_perf_test_src_patterns})
add_executable(onnxruntime_perf_test ${onnxruntime_perf_test_src})
add_executable(onnxruntime_perf_test ${onnxruntime_perf_test_src} ${ONNXRUNTIME_ROOT}/core/framework/path_lib.cc)
target_include_directories(onnxruntime_perf_test PRIVATE ${onnx_test_runner_src_dir} ${ONNXRUNTIME_ROOT}
${eigen_INCLUDE_DIRS} ${extra_includes} ${onnxruntime_graph_header} ${onnxruntime_exec_src_dir}
${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}/onnx)
if (WIN32)
target_compile_options(onnxruntime_perf_test PRIVATE ${disabled_warnings})
SET(SYS_PATH_LIB shlwapi)
endif()
onnxruntime_add_include_to_target(onnxruntime_perf_test gsl)
target_link_libraries(onnxruntime_perf_test PRIVATE onnx_test_runner_common ${GETOPT_LIB_WIDE} ${onnx_test_libs})
target_link_libraries(onnxruntime_perf_test PRIVATE onnxruntime_test_utils onnx_test_runner_common onnxruntime_common
onnx_test_data_proto onnx_proto libprotobuf ${GETOPT_LIB_WIDE} onnxruntime
${SYS_PATH_LIB} ${CMAKE_DL_LIBS} Threads::Threads)
set_target_properties(onnxruntime_perf_test PROPERTIES FOLDER "ONNXRuntimeTest")
if(tensorflow_C_PACKAGE_PATH)
target_include_directories(onnxruntime_perf_test PRIVATE ${tensorflow_C_PACKAGE_PATH}/include)
target_link_directories(onnxruntime_perf_test PRIVATE ${tensorflow_C_PACKAGE_PATH}/lib)
target_link_libraries(onnxruntime_perf_test PRIVATE tensorflow)
target_compile_definitions(onnxruntime_perf_test PRIVATE HAVE_TENSORFLOW)
endif()
# shared lib
if (onnxruntime_BUILD_SHARED_LIB)
add_library(onnxruntime_mocked_allocator ${ONNXRUNTIME_ROOT}/test/util/test_allocator.cc)

54
onnxruntime/core/platform/posix/env.cc
View file

@ -145,16 +145,6 @@ class PosixEnv : public Env {
return Status::OK();
}
static bool GetFileSizeIfUnknown(int fd, size_t& len) {
if(len > 0) return true;
struct stat stbuf;
if ((fstat(fd, &stbuf) != 0) || (!S_ISREG(stbuf.st_mode))) {
return false;
}
len = static_cast<size_t>(stbuf.st_size);
return true;
}
common::Status ReadFileAsString(const char* fname, off_t offset, void*& p, size_t& len,
OrtCallback& deleter) const override {
if (!fname) {
@ -169,13 +159,21 @@ class PosixEnv : public Env {
deleter.param = nullptr;
int fd = open(fname, O_RDONLY);
if (fd < 0) {
int err = errno;
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "open file ", fname, " fail, errcode =", err);
return ReportSystemError("open", fname);
}
if (!GetFileSizeIfUnknown(fd, len)) {
(void)close(fd);
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Get file '", fname, "' size fail");
if (len <= 0) {
struct stat stbuf;
if (fstat(fd, &stbuf) != 0) {
return ReportSystemError("fstat", fname);
}
if (!S_ISREG(stbuf.st_mode)) {
return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT,
"ReadFileAsString: input is not a regular file");
}
len = static_cast<size_t>(stbuf.st_size);
}
if (len == 0) {
p = nullptr;
} else {
@ -199,10 +197,30 @@ class PosixEnv : public Env {
return common::Status::OK();
}
static common::Status ReportSystemError(const char* operation_name, const std::string& path) {
auto e = errno;
char buf[1024];
const char* msg = "";
if (e > 0) {
#if defined(_GNU_SOURCE) && !defined(__APPLE__)
msg = strerror_r(e, buf, sizeof(buf));
#else
// for Mac OS X
if (strerror_r(e, buf, sizeof(buf)) != 0) {
buf[0] = '\0';
}
msg = buf;
#endif
}
std::ostringstream oss;
oss << operation_name << " file \"" << path << "\" failed: " << msg;
return common::Status(common::SYSTEM, e, oss.str());
}
common::Status FileOpenRd(const std::string& path, /*out*/ int& fd) const override {
fd = open(path.c_str(), O_RDONLY);
if (0 > fd) {
return common::Status(common::SYSTEM, errno);
return ReportSystemError("open", path);
}
return Status::OK();
}
@ -210,7 +228,7 @@ class PosixEnv : public Env {
common::Status FileOpenWr(const std::string& path, /*out*/ int& fd) const override {
fd = open(path.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644);
if (0 > fd) {
return common::Status(common::SYSTEM, errno);
return ReportSystemError("open", path);
}
return Status::OK();
}
@ -218,7 +236,7 @@ class PosixEnv : public Env {
common::Status FileClose(int fd) const override {
int ret = close(fd);
if (0 != ret) {
return common::Status(common::SYSTEM, errno);
return ReportSystemError("close", "");
}
return Status::OK();
}

8
onnxruntime/test/onnx/TestCase.cc
View file

@ -222,14 +222,6 @@ class OnnxModelInfo : public TestModelInfo {
const PATH_CHAR_TYPE* GetModelUrl() const override { return model_url_.c_str(); }
std::basic_string<PATH_CHAR_TYPE> GetDir() const override {
std::basic_string<PATH_CHAR_TYPE> test_case_dir;
auto st = GetDirNameFromFilePath(model_url_, test_case_dir);
if (!st.IsOK()) {
ORT_THROW("GetDirNameFromFilePath failed");
}
return test_case_dir;
}
const std::string& GetNodeName() const override { return node_name_; }
const ONNX_NAMESPACE::ValueInfoProto* GetOutputInfoFromModel(size_t i) const override {
return &output_value_info_[i];

9
onnxruntime/test/onnx/TestCase.h
View file

@ -37,7 +37,14 @@ class ITestCase {
class TestModelInfo {
public:
virtual const PATH_CHAR_TYPE* GetModelUrl() const = 0;
virtual std::basic_string<PATH_CHAR_TYPE> GetDir() const = 0;
virtual std::basic_string<PATH_CHAR_TYPE> GetDir() const {
std::basic_string<PATH_CHAR_TYPE> test_case_dir;
auto st = onnxruntime::GetDirNameFromFilePath(GetModelUrl(), test_case_dir);
if (!st.IsOK()) {
ORT_THROW("GetDirNameFromFilePath failed");
}
return test_case_dir;
}
virtual const std::string& GetNodeName() const = 0;
virtual const ONNX_NAMESPACE::ValueInfoProto* GetOutputInfoFromModel(size_t i) const = 0;
virtual int GetInputCount() const = 0;

2
onnxruntime/test/onnx/runner.cc
View file

@ -317,7 +317,7 @@ EXECUTE_RESULT DataRunner::RunTaskImpl(size_t task_id) {
c_->LoadTestData(task_id, holder, feeds, true);
// Create output feed
size_t output_count;
size_t output_count = 0;
ORT_THROW_ON_ERROR(OrtSessionGetOutputCount(session, &output_count));
std::vector<std::string> output_names(output_count);
for (size_t i = 0; i != output_count; ++i) {

51
onnxruntime/test/perftest/TFModelInfo.cc Normal file
View file

@ -0,0 +1,51 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "TFModelInfo.h"
#include <core/platform/env.h>
TestModelInfo* TFModelInfo::Create(_In_ const PATH_CHAR_TYPE* model_url) {
TFModelInfo* ret = new TFModelInfo();
ret->model_url_ = model_url;
std::basic_string<PATH_CHAR_TYPE> meta_file_path = model_url;
meta_file_path.append(ORT_TSTR(".meta"));
void* p = nullptr;
size_t len = 0;
OrtCallback b;
auto st = onnxruntime::Env::Default().ReadFileAsString(meta_file_path.c_str(), 0, p, len, b);
if (!st.IsOK()) {
ORT_THROW(st.ErrorMessage());
}
// this string is not null terminated
std::string filecontent(reinterpret_cast<char*>(p), len);
std::istringstream is(filecontent);
std::string line;
while (std::getline(is, line)) {
size_t line_len = 0;
if (!line.empty() && line.back() == '\n') {
line_len = line.length() - 1;
if (line_len > 0 && line[line_len - 1] == '\r') {
--line_len;
}
line.resize(line_len);
}
if (line.empty()) continue;
if (line.compare(0, 6, "input=") == 0) {
ret->input_names_.push_back(line.substr(6));
} else if (line.compare(0, 7, "output=") == 0) {
ret->output_names_.push_back(line.substr(7));
} else {
ORT_THROW("unknow line:", line.size());
}
}
if (b.f) b.f(b.param);
return ret;
}
int TFModelInfo::GetInputCount() const { return static_cast<int>(input_names_.size()); }
int TFModelInfo::GetOutputCount() const { return static_cast<int>(output_names_.size()); }
const std::string& TFModelInfo::GetInputName(size_t i) const { return input_names_[i]; }
const std::string& TFModelInfo::GetOutputName(size_t i) const { return output_names_[i]; }

31
onnxruntime/test/perftest/TFModelInfo.h Normal file
View file

@ -0,0 +1,31 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#pragma once
#include "TestCase.h"
#include <string>
#include <vector>
class TFModelInfo : public TestModelInfo {
public:
const PATH_CHAR_TYPE* GetModelUrl() const override { return model_url_.c_str(); }
const std::string& GetNodeName() const override { return node_name_; }
const ONNX_NAMESPACE::ValueInfoProto* GetOutputInfoFromModel(size_t) const override { return nullptr; }
int GetInputCount() const override;
int GetOutputCount() const override;
const std::string& GetInputName(size_t i) const override;
const std::string& GetOutputName(size_t i) const override;
~TFModelInfo() override = default;
static TestModelInfo* Create(_In_ const PATH_CHAR_TYPE* model_url);
private:
TFModelInfo() = default;
std::basic_string<PATH_CHAR_TYPE> model_url_;
std::vector<std::string> input_names_;
std::vector<std::string> output_names_;
std::string node_name_;
};

17
onnxruntime/test/perftest/command_args_parser.cc
View file

@ -27,8 +27,10 @@ namespace perftest {
"perf_test [options...] model_path result_file\n"
"Options:\n"
"\t-m [test_mode]: Specifies the test mode. Value coulde be 'duration' or 'times'.\n"
"\t\tProvide 'duration' to run the test for a fix duration, and 'times' to repeated for a certain times. Default:'duration'.\n"
"\t\tProvide 'duration' to run the test for a fix duration, and 'times' to repeated for a certain times. "
"Default:'duration'.\n"
"\t-e [cpu|cuda|mkldnn|tensorrt]: Specifies the provider 'cpu','cuda','mkldnn' or 'tensorrt'. Default:'cpu'.\n"
"\t-b [tf|ort]: backend to use. Default:ort\n"
"\t-r [repeated_times]: Specifies the repeated times if running in 'times' test mode.Default:1000.\n"
"\t-t [seconds_to_run]: Specifies the seconds to run for 'duration' mode. Default:600.\n"
"\t-p [profile_file]: Specifies the profile name to enable profiling and dump the profile data to the file.\n"
@ -41,7 +43,7 @@ namespace perftest {
/*static*/ bool CommandLineParser::ParseArguments(PerformanceTestConfig& test_config, int argc, ORTCHAR_T* argv[]) {
int ch;
while ((ch = getopt(argc, argv, ORT_TSTR("m:e:r:t:p:x:o:vhs"))) != -1) {
while ((ch = getopt(argc, argv, ORT_TSTR("b:m:e:r:t:p:x:o:vhs"))) != -1) {
switch (ch) {
case 'm':
if (!CompareCString(optarg, ORT_TSTR("duration"))) {
@ -52,6 +54,9 @@ namespace perftest {
return false;
}
break;
case 'b':
test_config.backend = optarg;
break;
case 'p':
test_config.run_config.profile_file = optarg;
break;
@ -71,14 +76,14 @@ namespace perftest {
}
break;
case 'r':
test_config.run_config.repeated_times = static_cast<int>(OrtStrtol<PATH_CHAR_TYPE>(optarg, nullptr));
test_config.run_config.repeated_times = static_cast<size_t>(OrtStrtol<PATH_CHAR_TYPE>(optarg, nullptr));
if (test_config.run_config.repeated_times <= 0) {
return false;
}
test_config.run_config.test_mode = TestMode::KFixRepeatedTimesMode;
break;
case 't':
test_config.run_config.duration_in_seconds = static_cast<int>(OrtStrtol<PATH_CHAR_TYPE>(optarg, nullptr));
test_config.run_config.duration_in_seconds = static_cast<size_t>(OrtStrtol<PATH_CHAR_TYPE>(optarg, nullptr));
if (test_config.run_config.repeated_times <= 0) {
return false;
}
@ -100,8 +105,8 @@ namespace perftest {
case 'o':
test_config.run_config.optimization_level = static_cast<uint32_t>(OrtStrtol<PATH_CHAR_TYPE>(optarg, nullptr));
// Valid values are: 0, 1, 2.
if (test_config.run_config.optimization_level > 2 ) {
return false;
if (test_config.run_config.optimization_level > 2) {
return false;
}
break;
case '?':

9
onnxruntime/test/perftest/main.cc
View file

@ -2,10 +2,7 @@
// Licensed under the MIT License.
// onnxruntime dependencies
#include <core/common/logging/sinks/clog_sink.h>
#include <core/common/logging/logging.h>
#include <core/framework/environment.h>
#include <core/platform/env.h>
#include <core/session/onnxruntime_c_api.h>
#include "command_args_parser.h"
#include "performance_runner.h"
@ -36,7 +33,7 @@ int real_main(int argc, char* argv[], OrtEnv** p_env) {
perftest::PerformanceRunner perf_runner(env, test_config);
auto status = perf_runner.Run();
if (!status.IsOK()) {
LOGF_DEFAULT(ERROR, "Run failed:%s", status.ErrorMessage().c_str());
printf("Run failed:%s\n", status.ErrorMessage().c_str());
return -1;
}
@ -60,8 +57,6 @@ int main(int argc, char* argv[]) {
}
if (env) {
OrtReleaseEnv(env);
} else {
::google::protobuf::ShutdownProtobufLibrary();
}
return retval;
}

105
onnxruntime/test/perftest/ort_test_session.cc Normal file
View file

@ -0,0 +1,105 @@
#include "ort_test_session.h"
#include <core/session/onnxruntime_cxx_api.h>
#include <assert.h>
#include "providers.h"
#include "TestCase.h"
#ifdef _WIN32
#define strdup _strdup
#endif
namespace onnxruntime {
namespace perftest {
std::chrono::duration<double> OnnxRuntimeTestSession::Run(const OrtValue* const* input) {
auto start = std::chrono::high_resolution_clock::now();
ORT_THROW_ON_ERROR(OrtRun(session_object_, nullptr, input_names_.data(), input, input_names_.size(),
output_names_raw_ptr.data(), output_names_raw_ptr.size(), output_values_.data()));
auto end = std::chrono::high_resolution_clock::now();
std::chrono::duration<double> duration_seconds = end - start;
for (size_t i = 0; i != output_values_.size(); ++i) {
OrtReleaseValue(output_values_[i]);
output_values_[i] = nullptr;
}
return duration_seconds;
}
OnnxRuntimeTestSession::OnnxRuntimeTestSession(OrtEnv* env, PerformanceTestConfig& performance_test_config,
const TestModelInfo* m)
: input_names_(m->GetInputCount()) {
SessionOptionsWrapper sf(env);
const bool enable_cpu_mem_arena = true;
const std::string& provider_name = performance_test_config.machine_config.provider_type_name;
if (provider_name == onnxruntime::kMklDnnExecutionProvider) {
#ifdef USE_MKLDNN
ORT_THROW_ON_ERROR(OrtSessionOptionsAppendExecutionProvider_Mkldnn(sf, enable_cpu_mem_arena ? 1 : 0));
#else
ORT_THROW("MKL-DNN is not supported in this build\n");
#endif
} else if (provider_name == onnxruntime::kCudaExecutionProvider) {
#ifdef USE_CUDA
ORT_THROW_ON_ERROR(OrtSessionOptionsAppendExecutionProvider_CUDA(sf, 0));
#else
ORT_THROW("CUDA is not supported in this build\n");
#endif
} else if (provider_name == onnxruntime::kNupharExecutionProvider) {
#ifdef USE_NUPHAR
ORT_THROW_ON_ERROR(OrtSessionOptionsAppendExecutionProvider_Nuphar(sf, 0, ""));
#else
ORT_THROW("Nuphar is not supported in this build\n");
#endif
} else if (provider_name == onnxruntime::kTensorrtExecutionProvider) {
#ifdef USE_TENSORRT
ORT_THROW_ON_ERROR(OrtSessionOptionsAppendExecutionProvider_Tensorrt(sf));
ORT_THROW_ON_ERROR(OrtSessionOptionsAppendExecutionProvider_CUDA(sf, 0));
#else
ORT_THROW("TensorRT is not supported in this build\n");
#endif
} else if (!provider_name.empty() && provider_name != onnxruntime::kCpuExecutionProvider) {
ORT_THROW("This backend is not included in perf test runner.\n");
}
if (enable_cpu_mem_arena)
sf.EnableCpuMemArena();
else
sf.DisableCpuMemArena();
if (performance_test_config.run_config.enable_sequential_execution)
sf.EnableSequentialExecution();
else
sf.DisableSequentialExecution();
fprintf(stdout, "Setting thread pool size to %d\n", performance_test_config.run_config.session_thread_pool_size);
sf.SetSessionThreadPoolSize(performance_test_config.run_config.session_thread_pool_size);
// Set optimization level.
sf.SetSessionGraphOptimizationLevel(performance_test_config.run_config.optimization_level);
if (!performance_test_config.run_config.profile_file.empty())
sf.EnableProfiling(performance_test_config.run_config.profile_file.c_str());
session_object_ = sf.OrtCreateSession(performance_test_config.model_info.model_file_path.c_str());
size_t output_count;
ORT_THROW_ON_ERROR(OrtSessionGetOutputCount(session_object_, &output_count));
output_names_.resize(output_count);
OrtAllocator* a;
ORT_THROW_ON_ERROR(OrtCreateDefaultAllocator(&a));
for (size_t i = 0; i != output_count; ++i) {
char* output_name = nullptr;
ORT_THROW_ON_ERROR(OrtSessionGetOutputName(session_object_, i, a, &output_name));
assert(output_name != nullptr);
output_names_[i] = output_name;
a->Free(a, output_name);
}
output_names_raw_ptr.resize(output_count);
for (size_t i = 0; i != output_count; ++i) {
output_names_raw_ptr[i] = output_names_[i].c_str();
}
OrtReleaseAllocator(a);
output_values_.resize(output_count);
size_t input_count = static_cast<size_t>(m->GetInputCount());
for (size_t i = 0; i != input_count; ++i) {
input_names_[i] = strdup(m->GetInputName(i).c_str());
}
}
} // namespace perftest
} // namespace onnxruntime

36
onnxruntime/test/perftest/ort_test_session.h Normal file
View file

@ -0,0 +1,36 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#pragma once
#include <core/session/onnxruntime_cxx_api.h>
#include "test_configuration.h"
#include "test_session.h"
class TestModelInfo;
namespace onnxruntime {
namespace perftest {
class OnnxRuntimeTestSession : public TestSession {
public:
OnnxRuntimeTestSession(OrtEnv* env, PerformanceTestConfig& performance_test_config, const TestModelInfo* m);
~OnnxRuntimeTestSession() override {
if (session_object_ != nullptr) OrtReleaseSession(session_object_);
for (char* p : input_names_) {
free(p);
}
}
std::chrono::duration<double> Run(const OrtValue* const* input) override;
ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(OnnxRuntimeTestSession);
private:
OrtSession* session_object_ = nullptr;
std::vector<std::string> output_names_;
// The same size with output_names_.
// TODO: implement a customized allocator, then we can remove output_names_ to simplify this code
std::vector<const char*> output_names_raw_ptr;
std::vector<OrtValue*> output_values_;
std::vector<char*> input_names_;
};
} // namespace perftest
} // namespace onnxruntime

154
onnxruntime/test/perftest/performance_runner.cc
View file

@ -2,13 +2,15 @@
// Licensed under the MIT License.
#include "performance_runner.h"
#include "TestCase.h"
#include "core/graph/graph_viewer.h" //for onnxruntime::NodeArg
#include "core/session/inference_session.h"
#include "utils.h"
#include "testenv.h"
#include "providers.h"
#include <iostream>
#include "TestCase.h"
#include "TFModelInfo.h"
#include "utils.h"
#include "ort_test_session.h"
#ifdef HAVE_TENSORFLOW
#include "tf_test_session.h"
#endif
using onnxruntime::Status;
namespace onnxruntime {
@ -20,10 +22,9 @@ Status PerformanceRunner::Run() {
// warm up
RunOneIteration(true /*isWarmup*/);
InferenceSession* session_object = (InferenceSession*)session_object_;
if (!performance_test_config_.run_config.profile_file.empty())
session_object->StartProfiling(performance_test_config_.run_config.profile_file);
// TODO: start profiling
// if (!performance_test_config_.run_config.profile_file.empty())
std::unique_ptr<utils::ICPUUsage> p_ICPUUsage = utils::CreateICPUUsage();
switch (performance_test_config_.run_config.test_mode) {
@ -39,7 +40,8 @@ Status PerformanceRunner::Run() {
performance_result_.average_CPU_usage = p_ICPUUsage->GetUsage();
performance_result_.peak_workingset_size = utils::GetPeakWorkingSetSize();
if (!performance_test_config_.run_config.profile_file.empty()) session_object->EndProfiling();
// TODO: end profiling
// if (!performance_test_config_.run_config.profile_file.empty()) session_object->EndProfiling();
std::cout << "Total time cost:" << performance_result_.total_time_cost << std::endl
<< "Total iterations:" << performance_result_.time_costs.size() << std::endl
@ -48,18 +50,8 @@ Status PerformanceRunner::Run() {
}
Status PerformanceRunner::RunOneIteration(bool isWarmup) {
auto start = std::chrono::high_resolution_clock::now();
OrtRunOptions run_options;
ORT_THROW_ON_ERROR(OrtRun(session_object_, nullptr, input_names_.data(), input_values_.data(), input_names_.size(),
output_names_raw_ptr.data(), output_names_raw_ptr.size(), output_values_.data()));
auto end = std::chrono::high_resolution_clock::now();
for (size_t i = 0; i != output_values_.size(); ++i) {
OrtReleaseValue(output_values_[i]);
output_values_[i] = nullptr;
}
std::chrono::duration<double> duration_seconds = session_->Run(input_values_.data());
if (!isWarmup) {
std::chrono::duration<double> duration_seconds = end - start;
performance_result_.time_costs.emplace_back(duration_seconds.count());
performance_result_.total_time_cost += duration_seconds.count();
if (performance_test_config_.run_config.f_verbose) {
@ -70,16 +62,16 @@ Status PerformanceRunner::RunOneIteration(bool isWarmup) {
return Status::OK();
}
PerformanceRunner::~PerformanceRunner() = default;
PerformanceRunner::PerformanceRunner(OrtEnv* env, const PerformanceTestConfig& test_config)
: env_(env), performance_test_config_(test_config) {}
bool PerformanceRunner::Initialize() {
bool has_valid_extension = HasExtensionOf(performance_test_config_.model_info.model_file_path, ORT_TSTR("onnx"));
if (!has_valid_extension) {
LOGF_DEFAULT(ERROR, "input path is not a valid model");
return false;
}
std::basic_string<PATH_CHAR_TYPE> test_case_dir;
auto st = GetDirNameFromFilePath(performance_test_config_.model_info.model_file_path, test_case_dir);
if (!st.IsOK()) {
LOGF_DEFAULT(ERROR, "input path is not a valid model");
printf("input path is not a valid model\n");
return false;
}
std::basic_string<PATH_CHAR_TYPE> model_name = GetLastComponent(test_case_dir);
@ -90,102 +82,40 @@ bool PerformanceRunner::Initialize() {
std::string narrow_model_name = ToMBString(model_name);
performance_result_.model_name = narrow_model_name;
auto p_model = TestModelInfo::LoadOnnxModel(performance_test_config_.model_info.model_file_path.c_str());
std::unique_ptr<ITestCase> test_case(CreateOnnxTestCase(narrow_model_name, p_model, 0.0, 0.0));
TestModelInfo* p_model;
if (CompareCString(performance_test_config_.backend.c_str(), ORT_TSTR("ort")) == 0) {
p_model = TestModelInfo::LoadOnnxModel(performance_test_config_.model_info.model_file_path.c_str());
} else if (CompareCString(performance_test_config_.backend.c_str(), ORT_TSTR("tf")) == 0) {
p_model = TFModelInfo::Create(performance_test_config_.model_info.model_file_path.c_str());
} else {
ORT_NOT_IMPLEMENTED(ToMBString(performance_test_config_.backend), " is not supported");
}
test_case_.reset(CreateOnnxTestCase(narrow_model_name, p_model, 0.0, 0.0));
SessionOptionsWrapper sf(env_);
const bool enable_cpu_mem_arena = true;
const std::string& provider_name = performance_test_config_.machine_config.provider_type_name;
if (provider_name == onnxruntime::kMklDnnExecutionProvider) {
#ifdef USE_MKLDNN
ORT_THROW_ON_ERROR(OrtSessionOptionsAppendExecutionProvider_Mkldnn(sf, enable_cpu_mem_arena ? 1 : 0));
#else
fprintf(stderr, "MKL-DNN is not supported in this build");
return false;
#endif
} else if (provider_name == onnxruntime::kCudaExecutionProvider) {
#ifdef USE_CUDA
ORT_THROW_ON_ERROR(OrtSessionOptionsAppendExecutionProvider_CUDA(sf, 0));
#else
fprintf(stderr, "CUDA is not supported in this build");
return false;
#endif
} else if (provider_name == onnxruntime::kNupharExecutionProvider) {
#ifdef USE_NUPHAR
ORT_THROW_ON_ERROR(OrtSessionOptionsAppendExecutionProvider_Nuphar(sf, 0, ""));
#else
fprintf(stderr, "Nuphar is not supported in this build");
return false;
#endif
} else if (provider_name == onnxruntime::kTensorrtExecutionProvider) {
#ifdef USE_TENSORRT
ORT_THROW_ON_ERROR(OrtSessionOptionsAppendExecutionProvider_Tensorrt(sf));
ORT_THROW_ON_ERROR(OrtSessionOptionsAppendExecutionProvider_CUDA(sf, 0));
#else
fprintf(stderr, "TensorRT is not supported in this build");
return false;
#endif
} else if (!provider_name.empty() && provider_name != onnxruntime::kCpuExecutionProvider) {
fprintf(stderr, "This backend is not included in perf test runner.");
// TODO: Place input tensor on cpu memory if mkldnn provider type to avoid CopyTensor logic in CopyInputAcrossDevices
if (test_case_->GetDataCount() <= 0) {
std::cout << "there is no test data for model " << test_case_->GetTestCaseName() << std::endl;
return false;
}
if (enable_cpu_mem_arena)
sf.EnableCpuMemArena();
else
sf.DisableCpuMemArena();
if (performance_test_config_.run_config.enable_sequential_execution)
sf.EnableSequentialExecution();
else
sf.DisableSequentialExecution();
fprintf(stdout, "Setting thread pool size to %d\n", performance_test_config_.run_config.session_thread_pool_size);
sf.SetSessionThreadPoolSize(performance_test_config_.run_config.session_thread_pool_size);
// Set optimization level.
sf.SetSessionGraphOptimizationLevel(performance_test_config_.run_config.optimization_level);
session_object_ = sf.OrtCreateSession(test_case->GetModelUrl());
auto provider_type = performance_test_config_.machine_config.provider_type_name;
// Place input tensor on cpu memory if mkldnn provider type to avoid CopyTensor logic in CopyInputAcrossDevices
// TODO: find a better way to do this.
if (provider_type == onnxruntime::kMklDnnExecutionProvider) {
provider_type = onnxruntime::kCpuExecutionProvider;
}
if (test_case->GetDataCount() <= 0) {
LOGS_DEFAULT(ERROR) << "there is no test data for model " << test_case->GetTestCaseName();
return false;
}
test_case->LoadTestData(0 /* id */, b_, feeds_, true);
input_names_.resize(feeds_.size());
test_case_->LoadTestData(0 /* id */, b_, feeds_, true);
input_values_.resize(feeds_.size());
size_t input_index = 0;
for (auto& kvp : feeds_) {
input_names_[input_index] = kvp.first.c_str();
input_values_[input_index] = kvp.second;
++input_index;
}
size_t output_count;
ORT_THROW_ON_ERROR(OrtSessionGetOutputCount(session_object_, &output_count));
output_names_.resize(output_count);
OrtAllocator* a;
ORT_THROW_ON_ERROR(OrtCreateDefaultAllocator(&a));
for (size_t i = 0; i != output_count; ++i) {
char* output_name = nullptr;
ORT_THROW_ON_ERROR(OrtSessionGetOutputName(session_object_, i, a, &output_name));
assert(output_name != nullptr);
output_names_[i] = output_name;
a->Free(a, output_name);
if (CompareCString(performance_test_config_.backend.c_str(), ORT_TSTR("ort")) == 0) {
session_ = new OnnxRuntimeTestSession(env_, performance_test_config_, p_model);
#ifdef HAVE_TENSORFLOW
} else if (CompareCString(performance_test_config_.backend.c_str(), ORT_TSTR("tf")) == 0) {
session_ = new TensorflowTestSession(performance_test_config_, p_model);
#endif
} else {
ORT_NOT_IMPLEMENTED(ToMBString(performance_test_config_.backend), " is not supported");
}
output_names_raw_ptr.resize(output_count);
for (size_t i = 0; i != output_count; ++i) {
output_names_raw_ptr[i] = output_names_[i].c_str();
}
OrtReleaseAllocator(a);
output_values_.resize(output_count);
return true;
}

31
onnxruntime/test/perftest/performance_runner.h
View file

@ -10,17 +10,14 @@
// onnxruntime dependencies
#include <core/common/common.h>
#include <core/common/logging/sinks/clog_sink.h>
#include <core/common/logging/logging.h>
#include <core/common/status.h>
#include <core/graph/onnx_protobuf.h>
#include <core/framework/environment.h>
#include <core/session/inference_session.h>
#include <core/platform/env.h>
#include <core/session/IOBinding.h>
#include <core/session/onnxruntime_cxx_api.h>
#include "test_configuration.h"
#include "heap_buffer.h"
#include "test_session.h"
class ITestCase;
namespace onnxruntime {
namespace perftest {
@ -36,7 +33,7 @@ struct PerformanceResult {
std::ofstream outfile;
outfile.open(path, std::ofstream::out | std::ofstream::app);
if (!outfile.good()) {
LOGF_DEFAULT(ERROR, "failed to open result file");
printf("failed to open result file");
return;
}
@ -44,7 +41,7 @@ struct PerformanceResult {
outfile << model_name << "," << time_costs[runs] << "," << peak_workingset_size << "," << average_CPU_usage << "," << runs << std::endl;
}
if (time_costs.size() > 0 && f_include_statistics) {
if (!time_costs.empty() && f_include_statistics) {
std::vector<double> sorted_time = time_costs;
size_t total = sorted_time.size();
@ -70,9 +67,9 @@ struct PerformanceResult {
class PerformanceRunner {
public:
PerformanceRunner(OrtEnv* env, const PerformanceTestConfig& test_config)
: env_(env), performance_test_config_(test_config) {}
PerformanceRunner(OrtEnv* env, const PerformanceTestConfig& test_config);
~PerformanceRunner();
Status Run();
inline const PerformanceResult& GetResult() const { return performance_result_; }
@ -81,9 +78,6 @@ class PerformanceRunner {
performance_result_.DumpToFile(performance_test_config_.model_info.result_file_path,
performance_test_config_.run_config.f_dump_statistics);
}
~PerformanceRunner() {
if (session_object_ != nullptr) OrtReleaseSession(session_object_);
}
ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(PerformanceRunner);
private:
@ -108,17 +102,12 @@ class PerformanceRunner {
OrtEnv* env_;
PerformanceResult performance_result_;
PerformanceTestConfig performance_test_config_;
// not owned
OrtSession* session_object_ = nullptr;
std::vector<const char*> input_names_;
std::unordered_map<std::string, OrtValue*> feeds_;
std::vector<OrtValue*> input_values_;
HeapBuffer b_;
std::vector<std::string> output_names_;
// The same size with output_names_.
// TODO: implement a customized allocator, then we can remove output_names_ to simplify this code
std::vector<const char*> output_names_raw_ptr;
std::vector<OrtValue*> output_values_;
std::unique_ptr<ITestCase> test_case_;
TestSession* session_;
};
} // namespace perftest
} // namespace onnxruntime

1
onnxruntime/test/perftest/test_configuration.h
View file

@ -49,6 +49,7 @@ struct PerformanceTestConfig {
ModelInfo model_info;
MachineConfig machine_config;
RunConfig run_config;
std::basic_string<ORTCHAR_T> backend = ORT_TSTR("ort");
};
} // namespace perftest

13
onnxruntime/test/perftest/test_session.h Normal file
View file

@ -0,0 +1,13 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#pragma once
namespace onnxruntime {
namespace perftest {
class TestSession {
public:
virtual std::chrono::duration<double> Run(const OrtValue* const* input) = 0;
virtual ~TestSession() = default;
};
} // namespace perftest
} // namespace onnxruntime

111
onnxruntime/test/perftest/tf_test_session.h Normal file
View file

@ -0,0 +1,111 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#pragma once
#include <core/session/onnxruntime_cxx_api.h>
#include <core/platform/env.h>
#include "test_configuration.h"
#include "tensorflow/c/c_api.h"
#include "test_session.h"
namespace onnxruntime {
namespace perftest {
class TensorflowTestSession : public TestSession {
private:
OrtCallback model_deleter;
std::vector<TF_Output> feed_;
std::vector<TF_Output> fetches_;
TF_Session* sess_;
TF_Graph* tf_graph_;
// This function is for both graph inputs and outputs
static TF_Output GetOutputFromGraph(const char* tensor_name, TF_Graph* tf_graph) {
TF_Output ret;
const char* start = tensor_name;
const char* sep = strchr(start, ':');
if (sep == nullptr) {
ORT_THROW("invalid name:", tensor_name);
}
size_t name_len = sep - start;
std::string name(name_len, '\0');
memcpy(const_cast<char*>(name.data()), start, name_len);
ret.oper = TF_GraphOperationByName(tf_graph, name.c_str());
if (ret.oper == nullptr) ORT_THROW("input name: \"", name, "\" can not be find in the graph");
start = sep + 1;
char* end;
ret.index = static_cast<int>(strtol(start, &end, 10));
if (start == end) {
ORT_THROW("invalid name:", tensor_name);
}
return ret;
}
public:
TensorflowTestSession(PerformanceTestConfig& performance_test_config, const TestModelInfo* m) {
TF_Status* s = TF_NewStatus();
tf_graph_ = TF_NewGraph();
TF_ImportGraphDefOptions* opts = TF_NewImportGraphDefOptions();
TF_ImportGraphDefOptionsSetPrefix(opts, "");
TF_Buffer* graph_def = TF_NewBuffer();
void* model_data;
auto st = Env::Default().ReadFileAsString(performance_test_config.model_info.model_file_path.c_str(), 0, model_data,
graph_def->length, model_deleter);
if (!st.IsOK())
ORT_THROW("read file ", performance_test_config.model_info.model_file_path, " failed:", st.ErrorMessage());
graph_def->data = model_data;
TF_GraphImportGraphDef(tf_graph_, graph_def, opts, s);
if (TF_GetCode(s) != TF_OK) ORT_THROW("load TF model failed:", TF_Message(s));
TF_SessionOptions* session_opts = TF_NewSessionOptions();
sess_ = TF_NewSession(tf_graph_, session_opts, s);
if (TF_GetCode(s) != TF_OK) ORT_THROW("load TF model failed:", TF_Message(s));
feed_.resize(static_cast<size_t>(m->GetInputCount()));
for (size_t i = 0; i != feed_.size(); ++i) {
feed_[i] = GetOutputFromGraph(m->GetInputName(i).c_str(), tf_graph_);
}
fetches_.resize(static_cast<size_t>(m->GetOutputCount()));
for (size_t i = 0; i != fetches_.size(); ++i) {
fetches_[i] = GetOutputFromGraph(m->GetOutputName(i).c_str(), tf_graph_);
}
}
ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(TensorflowTestSession);
std::chrono::duration<double> Run(const OrtValue* const* input) override {
size_t input_len = feed_.size();
std::vector<TF_Tensor*> feed_tensors(input_len);
for (size_t i = 0; i != input_len; ++i) {
void* input_buffer = nullptr;
ORT_THROW_ON_ERROR(OrtGetTensorMutableData(const_cast<OrtValue*>(input[i]), &input_buffer));
assert(input_buffer != nullptr);
OrtTensorTypeAndShapeInfo* shape;
ORT_THROW_ON_ERROR(OrtGetTensorShapeAndType(input[i], &shape));
size_t dim_count = OrtGetNumOfDimensions(shape);
std::vector<int64_t> dims(dim_count);
OrtGetDimensions(shape, dims.data(), dim_count);
int64_t ele_count = OrtGetTensorShapeElementCount(shape);
size_t buffer_length = ele_count * sizeof(float);
TF_Tensor* t = TF_AllocateTensor(TF_FLOAT, dims.data(), static_cast<int>(dims.size()), buffer_length);
assert(t != nullptr);
feed_tensors[i] = t;
assert(TF_TensorByteSize(t) == buffer_length);
memcpy(TF_TensorData(t), input_buffer, buffer_length);
}
std::vector<TF_Tensor*> output_tensors(fetches_.size());
TF_Status* s = TF_NewStatus();
auto start = std::chrono::high_resolution_clock::now();
TF_SessionRun(sess_, nullptr, feed_.data(), feed_tensors.data(), static_cast<int>(feed_.size()), fetches_.data(),
output_tensors.data(), static_cast<int>(fetches_.size()), nullptr, 0, nullptr, s);
auto end = std::chrono::high_resolution_clock::now();
if (TF_GetCode(s) != TF_OK) ORT_THROW("run TF model failed:", TF_Message(s));
TF_DeleteStatus(s);
return end - start;
}
~TensorflowTestSession() override {
if (model_deleter.f != nullptr) {
model_deleter.f(model_deleter.param);
}
TF_Status* s = TF_NewStatus();
TF_DeleteSession(sess_, s);
TF_DeleteStatus(s);
}
};
} // namespace perftest
} // namespace onnxruntime