mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-06-30 03:37:44 +00:00
Merge remote-tracking branch 'upstream/master' into DmlDev
This commit is contained in:
ISS Build Account
commit
42a5a4bff1
11 changed files with 1039 additions and 203 deletions
|
|
@ -439,6 +439,58 @@ target_include_directories(onnxruntime_test_utils PUBLIC "${TEST_SRC_DIR}/util/i
|
|||
${eigen_INCLUDE_DIRS} ${ONNXRUNTIME_ROOT})
|
||||
set_target_properties(onnxruntime_test_utils PROPERTIES FOLDER "ONNXRuntimeTest")
|
||||
|
||||
set(onnx_test_runner_src_dir ${TEST_SRC_DIR}/onnx)
|
||||
set(onnx_test_runner_common_srcs
|
||||
${onnx_test_runner_src_dir}/TestResultStat.cc
|
||||
${onnx_test_runner_src_dir}/TestResultStat.h
|
||||
${onnx_test_runner_src_dir}/testenv.h
|
||||
${onnx_test_runner_src_dir}/FixedCountFinishCallback.h
|
||||
${onnx_test_runner_src_dir}/TestCaseResult.cc
|
||||
${onnx_test_runner_src_dir}/TestCaseResult.h
|
||||
${onnx_test_runner_src_dir}/testenv.cc
|
||||
${onnx_test_runner_src_dir}/heap_buffer.h
|
||||
${onnx_test_runner_src_dir}/heap_buffer.cc
|
||||
${onnx_test_runner_src_dir}/OrtValueList.h
|
||||
${onnx_test_runner_src_dir}/runner.h
|
||||
${onnx_test_runner_src_dir}/runner.cc
|
||||
${onnx_test_runner_src_dir}/TestCase.cc
|
||||
${onnx_test_runner_src_dir}/TestCase.h
|
||||
${onnx_test_runner_src_dir}/onnxruntime_event.h
|
||||
${onnx_test_runner_src_dir}/sync_api.h
|
||||
${onnx_test_runner_src_dir}/sync_api.cc
|
||||
${onnx_test_runner_src_dir}/callback.h
|
||||
${onnx_test_runner_src_dir}/callback.cc
|
||||
${onnx_test_runner_src_dir}/pb_helper.h
|
||||
${onnx_test_runner_src_dir}/pb_helper.cc
|
||||
${onnx_test_runner_src_dir}/mem_buffer.h
|
||||
${onnx_test_runner_src_dir}/tensorprotoutils.h
|
||||
${onnx_test_runner_src_dir}/tensorprotoutils.cc
|
||||
${onnx_test_runner_src_dir}/onnx_model_info.h
|
||||
${onnx_test_runner_src_dir}/onnx_model_info.cc)
|
||||
|
||||
|
||||
add_library(onnx_test_runner_common ${onnx_test_runner_common_srcs})
|
||||
if(MSVC)
|
||||
target_compile_options(onnx_test_runner_common PRIVATE "$<$<COMPILE_LANGUAGE:CUDA>:SHELL:--compiler-options /utf-8>"
|
||||
"$<$<NOT:$<COMPILE_LANGUAGE:CUDA>>:/utf-8>")
|
||||
else()
|
||||
target_compile_definitions(onnx_test_runner_common PUBLIC -DNSYNC_ATOMIC_CPP11)
|
||||
target_include_directories(onnx_test_runner_common PRIVATE ${CMAKE_CURRENT_BINARY_DIR} ${ONNXRUNTIME_ROOT}
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/external/nsync/public")
|
||||
endif()
|
||||
if (MSVC AND NOT CMAKE_SIZEOF_VOID_P EQUAL 8)
|
||||
#TODO: fix the warnings, they are dangerous
|
||||
target_compile_options(onnx_test_runner_common PRIVATE "/wd4244")
|
||||
endif()
|
||||
onnxruntime_add_include_to_target(onnx_test_runner_common onnxruntime_common onnxruntime_framework
|
||||
onnxruntime_test_utils onnx onnx_proto re2::re2)
|
||||
|
||||
add_dependencies(onnx_test_runner_common onnx_test_data_proto ${onnxruntime_EXTERNAL_DEPENDENCIES})
|
||||
target_include_directories(onnx_test_runner_common PRIVATE ${eigen_INCLUDE_DIRS} ${RE2_INCLUDE_DIR}
|
||||
${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}/onnx ${ONNXRUNTIME_ROOT})
|
||||
|
||||
set_target_properties(onnx_test_runner_common PROPERTIES FOLDER "ONNXRuntimeTest")
|
||||
|
||||
set(all_tests ${onnxruntime_test_common_src} ${onnxruntime_test_ir_src} ${onnxruntime_test_optimizer_src}
|
||||
${onnxruntime_test_framework_src} ${onnxruntime_test_providers_src})
|
||||
if(NOT TARGET onnxruntime)
|
||||
|
|
@ -474,7 +526,7 @@ set(all_dependencies ${onnxruntime_test_providers_dependencies} )
|
|||
AddTest(
|
||||
TARGET onnxruntime_test_all
|
||||
SOURCES ${all_tests}
|
||||
LIBS ${onnxruntime_test_providers_libs} ${onnxruntime_test_common_libs}
|
||||
LIBS onnx_test_runner_common ${onnxruntime_test_providers_libs} ${onnxruntime_test_common_libs} re2::re2 onnx_test_data_proto
|
||||
DEPENDS ${all_dependencies}
|
||||
)
|
||||
|
||||
|
|
@ -484,7 +536,6 @@ set(all_dependencies ${onnxruntime_test_providers_dependencies} )
|
|||
if (onnxruntime_USE_FEATURIZERS)
|
||||
target_include_directories(onnxruntime_test_all PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/external/FeaturizersLibrary/src)
|
||||
endif()
|
||||
|
||||
if (onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS)
|
||||
target_link_libraries(onnxruntime_test_all PRIVATE onnxruntime_language_interop onnxruntime_pyop)
|
||||
endif()
|
||||
|
|
@ -559,32 +610,7 @@ target_include_directories(onnx_test_data_proto PRIVATE ${CMAKE_CURRENT_BINARY_D
|
|||
set_target_properties(onnx_test_data_proto PROPERTIES FOLDER "ONNXRuntimeTest")
|
||||
onnxruntime_protobuf_generate(APPEND_PATH IMPORT_DIRS ${ONNXRUNTIME_ROOT}/core/protobuf TARGET onnx_test_data_proto)
|
||||
|
||||
set(onnx_test_runner_src_dir ${TEST_SRC_DIR}/onnx)
|
||||
set(onnx_test_runner_common_srcs
|
||||
${onnx_test_runner_src_dir}/TestResultStat.cc
|
||||
${onnx_test_runner_src_dir}/TestResultStat.h
|
||||
${onnx_test_runner_src_dir}/testenv.h
|
||||
${onnx_test_runner_src_dir}/FixedCountFinishCallback.h
|
||||
${onnx_test_runner_src_dir}/TestCaseResult.cc
|
||||
${onnx_test_runner_src_dir}/TestCaseResult.h
|
||||
${onnx_test_runner_src_dir}/testenv.cc
|
||||
${onnx_test_runner_src_dir}/heap_buffer.h
|
||||
${onnx_test_runner_src_dir}/heap_buffer.cc
|
||||
${onnx_test_runner_src_dir}/OrtValueList.h
|
||||
${onnx_test_runner_src_dir}/runner.h
|
||||
${onnx_test_runner_src_dir}/runner.cc
|
||||
${onnx_test_runner_src_dir}/TestCase.cc
|
||||
${onnx_test_runner_src_dir}/TestCase.h
|
||||
${onnx_test_runner_src_dir}/onnxruntime_event.h
|
||||
${onnx_test_runner_src_dir}/sync_api.h
|
||||
${onnx_test_runner_src_dir}/sync_api.cc
|
||||
${onnx_test_runner_src_dir}/callback.h
|
||||
${onnx_test_runner_src_dir}/callback.cc
|
||||
${onnx_test_runner_src_dir}/pb_helper.h
|
||||
${onnx_test_runner_src_dir}/pb_helper.cc
|
||||
${onnx_test_runner_src_dir}/mem_buffer.h
|
||||
${onnx_test_runner_src_dir}/tensorprotoutils.h
|
||||
${onnx_test_runner_src_dir}/tensorprotoutils.cc)
|
||||
|
||||
|
||||
if(WIN32)
|
||||
set(wide_get_opt_src_dir ${TEST_SRC_DIR}/win_getopt/wide)
|
||||
|
|
@ -595,27 +621,6 @@ if(WIN32)
|
|||
set(GETOPT_LIB_WIDE win_getopt_wide)
|
||||
endif()
|
||||
|
||||
add_library(onnx_test_runner_common ${onnx_test_runner_common_srcs})
|
||||
if(MSVC)
|
||||
target_compile_options(onnx_test_runner_common PRIVATE "$<$<COMPILE_LANGUAGE:CUDA>:SHELL:--compiler-options /utf-8>"
|
||||
"$<$<NOT:$<COMPILE_LANGUAGE:CUDA>>:/utf-8>")
|
||||
else()
|
||||
target_compile_definitions(onnx_test_runner_common PUBLIC -DNSYNC_ATOMIC_CPP11)
|
||||
target_include_directories(onnx_test_runner_common PRIVATE ${CMAKE_CURRENT_BINARY_DIR} ${ONNXRUNTIME_ROOT}
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/external/nsync/public")
|
||||
endif()
|
||||
if (MSVC AND NOT CMAKE_SIZEOF_VOID_P EQUAL 8)
|
||||
#TODO: fix the warnings, they are dangerous
|
||||
target_compile_options(onnx_test_runner_common PRIVATE "/wd4244")
|
||||
endif()
|
||||
onnxruntime_add_include_to_target(onnx_test_runner_common onnxruntime_common onnxruntime_framework
|
||||
onnxruntime_test_utils onnx onnx_proto re2::re2)
|
||||
|
||||
add_dependencies(onnx_test_runner_common onnx_test_data_proto ${onnxruntime_EXTERNAL_DEPENDENCIES})
|
||||
target_include_directories(onnx_test_runner_common PRIVATE ${eigen_INCLUDE_DIRS} ${RE2_INCLUDE_DIR}
|
||||
${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}/onnx ${ONNXRUNTIME_ROOT})
|
||||
|
||||
set_target_properties(onnx_test_runner_common PROPERTIES FOLDER "ONNXRuntimeTest")
|
||||
|
||||
set(onnx_test_libs
|
||||
onnxruntime_test_utils
|
||||
|
|
@ -723,7 +728,7 @@ if (WIN32)
|
|||
endif()
|
||||
|
||||
if (onnxruntime_BUILD_SHARED_LIB)
|
||||
set(onnxruntime_perf_test_libs onnxruntime_test_utils onnx_test_runner_common onnxruntime_common re2::re2
|
||||
set(onnxruntime_perf_test_libs onnx_test_runner_common onnxruntime_test_utils onnxruntime_common re2::re2
|
||||
onnx_test_data_proto onnx_proto ${PROTOBUF_LIB} ${GETOPT_LIB_WIDE} onnxruntime ${SYS_PATH_LIB}
|
||||
${CMAKE_DL_LIBS})
|
||||
if(NOT WIN32)
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@
|
|||
#include <map>
|
||||
#include <regex>
|
||||
#include "OrtValueList.h"
|
||||
#include "onnx_model_info.h"
|
||||
|
||||
#include "pb_helper.h"
|
||||
|
||||
|
|
@ -152,85 +153,7 @@ static int ExtractFileNo(const std::basic_string<CHAR_T>& name) {
|
|||
}
|
||||
using PATH_STRING_TYPE = std::basic_string<PATH_CHAR_TYPE>;
|
||||
|
||||
class OnnxModelInfo : public TestModelInfo {
|
||||
private:
|
||||
std::string node_name_;
|
||||
std::string onnx_commit_tag_;
|
||||
std::vector<ONNX_NAMESPACE::ValueInfoProto> input_value_info_;
|
||||
std::vector<ONNX_NAMESPACE::ValueInfoProto> output_value_info_;
|
||||
|
||||
template <typename T>
|
||||
static void RepeatedPtrFieldToVector(const ::google::protobuf::RepeatedPtrField<T>& input_value_info,
|
||||
std::vector<T>& out) {
|
||||
for (int i = 0; i != input_value_info.size(); ++i) {
|
||||
out.push_back(input_value_info[i]);
|
||||
}
|
||||
}
|
||||
const std::basic_string<PATH_CHAR_TYPE> model_url_;
|
||||
|
||||
public:
|
||||
OnnxModelInfo(_In_ const PATH_CHAR_TYPE* model_url) : model_url_(model_url) {
|
||||
// parse model
|
||||
int model_fd;
|
||||
auto st = Env::Default().FileOpenRd(model_url, model_fd);
|
||||
if (!st.IsOK()) {
|
||||
ORT_THROW(st.ErrorMessage());
|
||||
}
|
||||
|
||||
ONNX_NAMESPACE::ModelProto model_pb;
|
||||
::google::protobuf::io::FileInputStream input(model_fd, protobuf_block_size_in_bytes);
|
||||
const bool parse_result = model_pb.ParseFromZeroCopyStream(&input) && input.GetErrno() == 0;
|
||||
if (!parse_result) {
|
||||
(void)Env::Default().FileClose(model_fd);
|
||||
ORT_THROW("Failed to load model because protobuf parsing failed.");
|
||||
}
|
||||
(void)Env::Default().FileClose(model_fd);
|
||||
{
|
||||
const RE2::Anchor re2_anchor = RE2::UNANCHORED;
|
||||
const std::string model_url_string = ToMBString(model_url);
|
||||
re2::StringPiece text(model_url_string);
|
||||
re2::StringPiece submatch;
|
||||
re2::RE2 regex("onnx[0-9a-z]{3}", re2::RE2::Options()); //e.g. onnx141, onnx150, onnxtip
|
||||
if (!regex.ok()) {
|
||||
ORT_THROW("Failed to parse regex: onnx[0-9a-z]{3}");
|
||||
}
|
||||
bool match = regex.Match(text, 0, text.length(), re2_anchor, &submatch, 1);
|
||||
if (match) {
|
||||
onnx_commit_tag_.assign(submatch.data(), submatch.length());
|
||||
} else {
|
||||
onnx_commit_tag_ = TestModelInfo::unknown_version;
|
||||
}
|
||||
}
|
||||
const ONNX_NAMESPACE::GraphProto& graph = model_pb.graph();
|
||||
if (graph.node().size() == 1) {
|
||||
node_name_ = graph.node()[0].op_type();
|
||||
}
|
||||
std::unordered_set<std::string> initializer_names;
|
||||
for (const auto& init : graph.initializer()) {
|
||||
if (!init.has_name()) continue;
|
||||
initializer_names.insert(init.name());
|
||||
}
|
||||
//Ignore the inputs that are already in initializers
|
||||
for (const auto& p : graph.input()) {
|
||||
if (!p.has_name()) ORT_THROW("input without name??");
|
||||
if (initializer_names.find(p.name()) == initializer_names.end()) input_value_info_.push_back(p);
|
||||
}
|
||||
RepeatedPtrFieldToVector(graph.output(), output_value_info_);
|
||||
}
|
||||
|
||||
const PATH_CHAR_TYPE* GetModelUrl() const override { return model_url_.c_str(); }
|
||||
std::string GetModelVersion() const override { return onnx_commit_tag_; }
|
||||
|
||||
const std::string& GetNodeName() const override { return node_name_; }
|
||||
const ONNX_NAMESPACE::ValueInfoProto* GetOutputInfoFromModel(size_t i) const override {
|
||||
return &output_value_info_[i];
|
||||
}
|
||||
int GetInputCount() const override { return static_cast<int>(input_value_info_.size()); }
|
||||
int GetOutputCount() const override { return static_cast<int>(output_value_info_.size()); }
|
||||
const std::string& GetInputName(size_t i) const override { return input_value_info_[i].name(); }
|
||||
|
||||
const std::string& GetOutputName(size_t i) const override { return output_value_info_[i].name(); }
|
||||
};
|
||||
|
||||
static void SortTensorFileNames(std::vector<std::basic_string<PATH_CHAR_TYPE>>& input_pb_files) {
|
||||
if (input_pb_files.size() <= 1) return;
|
||||
|
|
|
|||
72
onnxruntime/test/onnx/onnx_model_info.cc
Normal file
72
onnxruntime/test/onnx/onnx_model_info.cc
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
#include "onnx_model_info.h"
|
||||
#include "core/platform/env.h"
|
||||
#include "re2/re2.h"
|
||||
#include "pb_helper.h"
|
||||
|
||||
using namespace onnxruntime;
|
||||
static constexpr int protobuf_block_size_in_bytes = 4 * 1024 * 1024;
|
||||
|
||||
template <typename T>
|
||||
static void RepeatedPtrFieldToVector(const ::google::protobuf::RepeatedPtrField<T>& input_value_info,
|
||||
std::vector<T>& out) {
|
||||
for (int i = 0; i != input_value_info.size(); ++i) {
|
||||
out.push_back(input_value_info[i]);
|
||||
}
|
||||
}
|
||||
|
||||
OnnxModelInfo::OnnxModelInfo(_In_ const PATH_CHAR_TYPE* model_url) : model_url_(model_url) {
|
||||
// parse model
|
||||
int model_fd;
|
||||
auto st = Env::Default().FileOpenRd(model_url, model_fd);
|
||||
if (!st.IsOK()) {
|
||||
ORT_THROW(st.ErrorMessage());
|
||||
}
|
||||
|
||||
ONNX_NAMESPACE::ModelProto model_pb;
|
||||
::google::protobuf::io::FileInputStream input(model_fd, protobuf_block_size_in_bytes);
|
||||
const bool parse_result = model_pb.ParseFromZeroCopyStream(&input) && input.GetErrno() == 0;
|
||||
if (!parse_result) {
|
||||
(void)Env::Default().FileClose(model_fd);
|
||||
ORT_THROW("Failed to load model because protobuf parsing failed.");
|
||||
}
|
||||
(void)Env::Default().FileClose(model_fd);
|
||||
{
|
||||
const RE2::Anchor re2_anchor = RE2::UNANCHORED;
|
||||
const std::string model_url_string = ToMBString(model_url);
|
||||
re2::StringPiece text(model_url_string);
|
||||
re2::StringPiece submatch;
|
||||
re2::RE2 regex("onnx[0-9a-z]{3}", re2::RE2::Options()); //e.g. onnx141, onnx150, onnxtip
|
||||
if (!regex.ok()) {
|
||||
ORT_THROW("Failed to parse regex: onnx[0-9a-z]{3}");
|
||||
}
|
||||
bool match = regex.Match(text, 0, text.length(), re2_anchor, &submatch, 1);
|
||||
if (match) {
|
||||
onnx_commit_tag_.assign(submatch.data(), submatch.length());
|
||||
} else {
|
||||
onnx_commit_tag_ = TestModelInfo::unknown_version;
|
||||
}
|
||||
}
|
||||
for (const auto& opset : model_pb.opset_import()) {
|
||||
std::string s = opset.domain();
|
||||
if (s == "ai.onnx") s = "";
|
||||
domain_to_version_[s] = opset.version();
|
||||
}
|
||||
const ONNX_NAMESPACE::GraphProto& graph = model_pb.graph();
|
||||
if (graph.node().size() == 1) {
|
||||
node_name_ = graph.node()[0].op_type();
|
||||
}
|
||||
std::unordered_set<std::string> initializer_names;
|
||||
for (const auto& init : graph.initializer()) {
|
||||
if (!init.has_name()) continue;
|
||||
initializer_names.insert(init.name());
|
||||
}
|
||||
//Ignore the inputs that are already in initializers
|
||||
for (const auto& p : graph.input()) {
|
||||
if (!p.has_name()) ORT_THROW("input without name??");
|
||||
if (initializer_names.find(p.name()) == initializer_names.end()) input_value_info_.push_back(p);
|
||||
}
|
||||
RepeatedPtrFieldToVector(graph.output(), output_value_info_);
|
||||
}
|
||||
41
onnxruntime/test/onnx/onnx_model_info.h
Normal file
41
onnxruntime/test/onnx/onnx_model_info.h
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License.
|
||||
#pragma once
|
||||
|
||||
#include "core/graph/onnx_protobuf.h"
|
||||
#include "TestCase.h"
|
||||
|
||||
class OnnxModelInfo : public TestModelInfo {
|
||||
private:
|
||||
std::string node_name_;
|
||||
std::string onnx_commit_tag_;
|
||||
std::vector<ONNX_NAMESPACE::ValueInfoProto> input_value_info_;
|
||||
std::vector<ONNX_NAMESPACE::ValueInfoProto> output_value_info_;
|
||||
std::unordered_map<std::string, int64_t> domain_to_version_;
|
||||
const std::basic_string<PATH_CHAR_TYPE> model_url_;
|
||||
|
||||
public:
|
||||
OnnxModelInfo(_In_ const PATH_CHAR_TYPE* model_url);
|
||||
|
||||
bool HasDomain(const std::string& name) const {
|
||||
return domain_to_version_.find(name) != domain_to_version_.end();
|
||||
}
|
||||
|
||||
int64_t GetONNXOpSetVersion() const {
|
||||
auto iter = domain_to_version_.find("");
|
||||
return iter == domain_to_version_.end() ? -1 : iter->second;
|
||||
}
|
||||
|
||||
const PATH_CHAR_TYPE* GetModelUrl() const override { return model_url_.c_str(); }
|
||||
std::string GetModelVersion() const override { return onnx_commit_tag_; }
|
||||
|
||||
const std::string& GetNodeName() const override { return node_name_; }
|
||||
const ONNX_NAMESPACE::ValueInfoProto* GetOutputInfoFromModel(size_t i) const override {
|
||||
return &output_value_info_[i];
|
||||
}
|
||||
int GetInputCount() const override { return static_cast<int>(input_value_info_.size()); }
|
||||
int GetOutputCount() const override { return static_cast<int>(output_value_info_.size()); }
|
||||
const std::string& GetInputName(size_t i) const override { return input_value_info_[i].name(); }
|
||||
|
||||
const std::string& GetOutputName(size_t i) const override { return output_value_info_[i].name(); }
|
||||
};
|
||||
831
onnxruntime/test/providers/cpu/model_tests.cc
Normal file
831
onnxruntime/test/providers/cpu/model_tests.cc
Normal file
|
|
@ -0,0 +1,831 @@
|
|||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
#include "core/session/onnxruntime_c_api.h"
|
||||
#include "core/session/onnxruntime_cxx_api.h"
|
||||
#include "core/session/inference_session.h"
|
||||
#include "core/session/ort_env.h"
|
||||
#include "asserts.h"
|
||||
#include <iterator>
|
||||
#include "gtest/gtest.h"
|
||||
#include <core/platform/path_lib.h>
|
||||
#include "test/onnx/TestCase.h"
|
||||
#include "test/onnx/runner.h"
|
||||
#include "test/compare_ortvalue.h"
|
||||
#include "default_providers.h"
|
||||
#include "test/onnx/onnx_model_info.h"
|
||||
|
||||
extern std::unique_ptr<Ort::Env> ort_env;
|
||||
|
||||
namespace onnxruntime {
|
||||
namespace test {
|
||||
// parameter is provider_name + "_" + model_path
|
||||
class ModelTest : public testing::TestWithParam<std::basic_string<ORTCHAR_T>> {};
|
||||
namespace {
|
||||
struct BrokenTest {
|
||||
std::string test_name_;
|
||||
std::string reason_;
|
||||
std::set<std::string> broken_versions_ = {}; // apply to all versions if empty
|
||||
BrokenTest(std::string name, std::string reason) : test_name_(std::move(name)), reason_(std::move(reason)) {
|
||||
}
|
||||
|
||||
BrokenTest(std::string name, std::string reason, const std::initializer_list<std::string>& versions)
|
||||
: test_name_(std::move(name)), reason_(std::move(reason)), broken_versions_(versions) {
|
||||
}
|
||||
|
||||
bool operator<(const struct BrokenTest& test) const {
|
||||
return strcmp(test_name_.c_str(), test.test_name_.c_str()) < 0;
|
||||
}
|
||||
};
|
||||
} // namespace
|
||||
TEST_P(ModelTest, Run) {
|
||||
std::basic_string<ORTCHAR_T> param = GetParam();
|
||||
size_t pos = param.find(ORT_TSTR("_"));
|
||||
ASSERT_NE(pos, std::string::npos);
|
||||
std::string provider_name = ToMBString(param.substr(0, pos));
|
||||
std::basic_string<ORTCHAR_T> model_path = param.substr(pos + 1);
|
||||
double per_sample_tolerance = 1e-3;
|
||||
// when cuda is enabled, set it to a larger value for resolving random MNIST test failure
|
||||
// when openvino is enabled, set it to a larger value for resolving MNIST accuracy mismatch
|
||||
double relative_per_sample_tolerance = 1e-3;
|
||||
if (provider_name == "openvino") {
|
||||
relative_per_sample_tolerance = 0.009;
|
||||
}
|
||||
|
||||
std::unique_ptr<OnnxModelInfo> model_info = onnxruntime::make_unique<OnnxModelInfo>(model_path.c_str());
|
||||
if (model_info->GetONNXOpSetVersion() != 8 && provider_name == "tensorrt") {
|
||||
// TensorRT can run most of the model tests, but only part of
|
||||
// them is enabled here to save CI build time.
|
||||
return;
|
||||
}
|
||||
#ifndef ENABLE_TRAINING
|
||||
if (model_info->HasDomain(ONNX_NAMESPACE::AI_ONNX_TRAINING_DOMAIN) ||
|
||||
model_info->HasDomain(ONNX_NAMESPACE::AI_ONNX_PREVIEW_TRAINING_DOMAIN)) {
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
// TODO: filter model based on opset
|
||||
std::set<BrokenTest> broken_tests = {
|
||||
{"mnist", "Input data isn't in valid range"},
|
||||
{"BERT_Squad", "test data bug"},
|
||||
{"constantofshape_float_ones", "test data bug", {"onnx141", "onnx150"}},
|
||||
{"constantofshape_int_zeros", "test data bug", {"onnx141", "onnx150"}},
|
||||
{"cast_STRING_to_FLOAT", "Linux CI has old ONNX python package with bad test data", {"onnx141"}},
|
||||
// Numpy float to string has unexpected rounding for some results given numpy default precision is meant to be 8.
|
||||
// "e.g. 0.296140194 -> '0.2961402' not '0.29614019'. ORT produces the latter with precision set to 8,
|
||||
// which doesn't match the expected output that was generated with numpy.
|
||||
{"cast_FLOAT_to_STRING", "Numpy float to string has unexpected rounding for some results."},
|
||||
{"tf_nasnet_large", "disable temporarily"},
|
||||
{"tf_nasnet_mobile", "disable temporarily"},
|
||||
{"tf_pnasnet_large", "disable temporarily"},
|
||||
{"shrink", "test case is wrong", {"onnx141"}},
|
||||
{"maxpool_with_argmax_2d_precomputed_strides", "ShapeInferenceError"},
|
||||
{"tf_inception_v2", "result mismatch"},
|
||||
{"tf_resnet_v1_50", "result mismatch when Conv BN Fusion is applied"},
|
||||
{"tf_resnet_v1_101", "result mismatch when Conv BN Fusion is applied"},
|
||||
{"tf_resnet_v1_152", "result mismatch when Conv BN Fusion is applied"},
|
||||
{"mxnet_arcface", "Model is an invalid ONNX model"},
|
||||
{"unique_not_sorted_without_axis", "Expected data for 'Y' is incorrect and in sorted order."},
|
||||
{"cumsum_1d_reverse_exclusive", "only failing linux GPU CI. Likely build error."},
|
||||
{"resize_downsample_scales_cubic_align_corners", "results mismatch with onnx tests"},
|
||||
{"resize_downsample_scales_linear_align_corners", "results mismatch with onnx tests"},
|
||||
{"resize_tf_crop_and_resize", "Bad onnx test output. Needs test fix."},
|
||||
{"resize_upsample_sizes_nearest_ceil_half_pixel", "Bad onnx test output. Needs test fix."},
|
||||
{"resize_upsample_sizes_nearest_floor_align_corners", "Bad onnx test output. Needs test fix."},
|
||||
{"resize_upsample_sizes_nearest_round_prefer_ceil_asymmetric", "Bad onnx test output. Needs test fix."},
|
||||
{"bitshift_right_uint16", "BitShift(11) uint16 support not enabled currently"},
|
||||
{"bitshift_left_uint16", "BitShift(11) uint16 support not enabled currently"},
|
||||
{"maxunpool_export_with_output_shape",
|
||||
"Invalid output in ONNX test. See https://github.com/onnx/onnx/issues/2398"},
|
||||
{"training_dropout", "result differs", {}}, // Temporary, subsequent PR will remove this.
|
||||
{"training_dropout_default", "result differs", {}}, // Temporary, subsequent PR will remove this.
|
||||
{"training_dropout_default_mask", "result differs", {}}, // Temporary, subsequent PR will remove this.
|
||||
{"training_dropout_mask", "result differs", {}}, // Temporary, subsequent PR will remove this.
|
||||
#ifdef ENABLE_TRAINING
|
||||
{"adagrad", "not a registered function/op", {}}, // Op not registered.
|
||||
{"adagrad_multiple", "not a registered function/op", {}}, // Op not registered.
|
||||
{"adam", "not a registered function/op", {}}, // Op not registered.
|
||||
{"adam_multiple", "not a registered function/op", {}}, // Op not registered.
|
||||
{"gradient_of_add", "not a registered function/op", {}}, // Op not registered.
|
||||
{"gradient_of_add_and_mul", "not a registered function/op", {}}, // Op not registered.
|
||||
{"momentum", "not a registered function/op", {}}, // Op not registered.
|
||||
{"momentum_multiple", "not a registered function/op", {}}, // Op not registered.
|
||||
{"nesterov_momentum", "not a registered function/op", {}}, // Op not registered.
|
||||
#endif
|
||||
{"mask_rcnn_keras", "this model currently has an invalid contrib op version set to 10", {}}};
|
||||
|
||||
if (provider_name == "ngraph") {
|
||||
broken_tests.insert({"qlinearconv", "ambiguity in scalar dimensions [] vs [1]"});
|
||||
broken_tests.insert({"clip_splitbounds", "not implemented yet for opset 11"});
|
||||
broken_tests.insert({"clip_outbounds", "not implemented yet for opset 11"});
|
||||
broken_tests.insert({"clip_example", "not implemented yet for opset 11"});
|
||||
broken_tests.insert({"clip_default_min", "not implemented yet for opset 11"});
|
||||
broken_tests.insert({"clip_default_max", "not implemented yet for opset 11"});
|
||||
broken_tests.insert({"clip", "not implemented yet for opset 11"});
|
||||
broken_tests.insert({"depthtospace_crd_mode_example", "NGraph does not support CRD mode"});
|
||||
broken_tests.insert({"depthtospace_crd_mode", "NGraph does not support CRD mode"});
|
||||
broken_tests.insert({"gemm_default_no_bias", "not implemented yet for opset 11"});
|
||||
broken_tests.insert({"quantizelinear", "ambiguity in scalar dimensions [] vs [1]", {"onnx150"}});
|
||||
broken_tests.insert({"dequantizelinear", "ambiguity in scalar dimensions [] vs [1]", {"onnx150"}});
|
||||
broken_tests.insert({"mlperf_ssd_resnet34_1200", "Results mismatch"});
|
||||
broken_tests.insert({"BERT_Squad", "Invalid Feed Input Name:input4"});
|
||||
broken_tests.insert({"candy", "Results mismatch: 2 of 150528"});
|
||||
broken_tests.insert({"tf_mobilenet_v1_1.0_224", "Results mismatch"});
|
||||
broken_tests.insert({"tf_mobilenet_v2_1.0_224", "Results mismatch"});
|
||||
broken_tests.insert({"tf_mobilenet_v2_1.4_224", "Results mismatch"});
|
||||
broken_tests.insert({"convtranspose_1d", "1d convtranspose not supported yet"});
|
||||
broken_tests.insert({"convtranspose_3d", "3d convtranspose not supported yet"});
|
||||
}
|
||||
|
||||
if (provider_name == "nnapi") {
|
||||
broken_tests.insert({"scan9_sum", "Error with the extra graph"});
|
||||
broken_tests.insert({"scan_sum", "Error with the extra graph"});
|
||||
broken_tests.insert({"mvn_expanded", "Failed to find kernel for MemcpyFromHost(1) (node Memcpy_1)"});
|
||||
broken_tests.insert({"dynamicquantizelinear_expanded", "Temporarily disabled pending investigation"});
|
||||
broken_tests.insert({"dynamicquantizelinear_max_adjusted_expanded", "Temporarily disabled pending investigation"});
|
||||
broken_tests.insert({"dynamicquantizelinear_min_adjusted_expanded", "Temporarily disabled pending investigation"});
|
||||
broken_tests.insert({"gemm_transposeB", "Temporarily disabled pending investigation"});
|
||||
broken_tests.insert({"range_float_type_positive_delta_expanded", "Temporarily disabled pending investigation"});
|
||||
broken_tests.insert({"range_int32_type_negative_delta_expanded", "Temporarily disabled pending investigation"});
|
||||
broken_tests.insert({"convtranspose_1d", "1d convtranspose not supported yet"});
|
||||
broken_tests.insert({"convtranspose_3d", "3d convtranspose not supported yet"});
|
||||
broken_tests.insert({"maxpool_2d_uint8", "result mismatch"});
|
||||
broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NC_expanded", "shape mismatch"});
|
||||
broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1d2_expanded", "shape mismatch"});
|
||||
broken_tests.insert(
|
||||
{"negative_log_likelihood_loss_input_shape_is_NCd1d2_reduction_mean_expanded", "shape mismatch"});
|
||||
broken_tests.insert(
|
||||
{"negative_log_likelihood_loss_input_shape_is_NCd1d2_reduction_sum_expanded", "shape mismatch"});
|
||||
broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_expanded", "shape mismatch"});
|
||||
broken_tests.insert(
|
||||
{"negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_reduction_mean_expanded", "shape mismatch"});
|
||||
broken_tests.insert(
|
||||
{"negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_reduction_sum_expanded", "shape mismatch"});
|
||||
// Disable based on George Wu's recommendation.
|
||||
broken_tests.insert(
|
||||
{"negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_reduction_sum_ignore_index_expanded",
|
||||
"shape mismatch"});
|
||||
broken_tests.insert({"negative_log_likelihood_loss_iinput_shape_is_NCd1_weight_ignore_index", "Shape mismatch"});
|
||||
broken_tests.insert(
|
||||
{"negative_log_likelihood_loss_iinput_shape_is_NCd1_weight_ignore_index_expanded", "Shape mismatch"});
|
||||
broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NC", "Shape mismatch"});
|
||||
broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1", "Shape mismatch"});
|
||||
broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1_expanded", "Shape mismatch"});
|
||||
broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1_ignore_index", "Shape mismatch"});
|
||||
broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1_ignore_index_expanded", "Shape mismatch"});
|
||||
broken_tests.insert(
|
||||
{"negative_log_likelihood_loss_input_shape_is_NCd1_mean_weight_negative_ignore_index", "Shape mismatch"});
|
||||
broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1_mean_weight_negative_ignore_index_expanded",
|
||||
"Shape mismatch"});
|
||||
broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1_weight", "Shape mismatch"});
|
||||
broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1_weight_expanded", "Shape mismatch"});
|
||||
broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1d2", "Shape mismatch"});
|
||||
broken_tests.insert(
|
||||
{"negative_log_likelihood_loss_input_shape_is_NCd1d2_no_weight_reduction_mean_ignore_index", "Shape mismatch"});
|
||||
broken_tests.insert(
|
||||
{"negative_log_likelihood_loss_input_shape_is_NCd1d2_no_weight_reduction_mean_ignore_index_expanded",
|
||||
"Shape mismatch"});
|
||||
broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1d2_reduction_mean", "Shape mismatch"});
|
||||
broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1d2_reduction_sum", "Shape mismatch"});
|
||||
broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight", "Shape mismatch"});
|
||||
broken_tests.insert(
|
||||
{"negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_reduction_mean", "Shape mismatch"});
|
||||
broken_tests.insert(
|
||||
{"negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_reduction_sum", "Shape mismatch"});
|
||||
broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_reduction_sum_ignore_index",
|
||||
"Shape mismatch"});
|
||||
broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index",
|
||||
"Shape mismatch"});
|
||||
broken_tests.insert(
|
||||
{"negative_log_likelihood_loss_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index_expanded",
|
||||
"Shape mismatch"});
|
||||
broken_tests.insert(
|
||||
{"negative_log_likelihood_loss_input_shape_is_NCd1d2d3_sum_weight_high_ignore_index", "Shape mismatch"});
|
||||
broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1d2d3_sum_weight_high_ignore_index_expanded",
|
||||
"Shape mismatch"});
|
||||
broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1d2d3d4d5_mean_weight", "Shape mismatch"});
|
||||
broken_tests.insert(
|
||||
{"negative_log_likelihood_loss_input_shape_is_NCd1d2d3d4d5_mean_weight_expanded", "Shape mismatch"});
|
||||
broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1d2d3d4d5_none_no_weight", "Shape mismatch"});
|
||||
broken_tests.insert(
|
||||
{"negative_log_likelihood_loss_input_shape_is_NCd1d2d3d4d5_none_no_weight_expanded", "Shape mismatch"});
|
||||
broken_tests.insert(
|
||||
{"softmax_cross_entropy_input_shape_is_NCd1_mean_weight_negative_ignore_index", "Shape mismatch"});
|
||||
broken_tests.insert(
|
||||
{"softmax_cross_entropy_input_shape_is_NCd1_mean_weight_negative_ignore_index_expanded", "Shape mismatch"});
|
||||
broken_tests.insert(
|
||||
{"softmax_cross_entropy_input_shape_is_NCd1_mean_weight_negative_ignore_index_log_prob", "Shape mismatch"});
|
||||
broken_tests.insert(
|
||||
{"softmax_cross_entropy_input_shape_is_NCd1_mean_weight_negative_ignore_index_log_prob_expanded",
|
||||
"Shape mismatch"});
|
||||
broken_tests.insert(
|
||||
{"softmax_cross_entropy_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index_expanded",
|
||||
"Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index_log_prob",
|
||||
"Shape mismatch"});
|
||||
broken_tests.insert(
|
||||
{"softmax_cross_entropy_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index_log_prob_expanded",
|
||||
"Shape mismatch"});
|
||||
broken_tests.insert(
|
||||
{"softmax_cross_entropy_input_shape_is_NCd1d2d3_sum_weight_high_ignore_index", "Shape mismatch"});
|
||||
broken_tests.insert(
|
||||
{"softmax_cross_entropy_input_shape_is_NCd1d2d3_sum_weight_high_ignore_index_expanded", "Shape mismatch"});
|
||||
broken_tests.insert(
|
||||
{"softmax_cross_entropy_input_shape_is_NCd1d2d3_sum_weight_high_ignore_index_log_prob", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_input_shape_is_NCd1d2d3_sum_weight_high_ignore_index_log_prob_expanded",
|
||||
"Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_mean_weight", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_mean_weight_expanded", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_mean_weight_log_prob", "Shape mismatch"});
|
||||
broken_tests.insert(
|
||||
{"softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_mean_weight_log_prob_expanded", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_none_no_weight", "Shape mismatch"});
|
||||
broken_tests.insert(
|
||||
{"softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_none_no_weight_expanded", "Shape mismatch"});
|
||||
broken_tests.insert(
|
||||
{"softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_none_no_weight_log_prob", "Shape mismatch"});
|
||||
broken_tests.insert(
|
||||
{"softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_none_no_weight_log_prob_expanded", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_mean", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_mean_3d", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_mean_3d_expanded", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_mean_3d_log_prob", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_mean_3d_log_prob_expanded", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_mean_expanded", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_mean_log_prob", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_mean_log_prob_expanded", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_mean_no_weight_ignore_index", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_mean_no_weight_ignore_index_3d", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_mean_no_weight_ignore_index_3d_expanded", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_mean_no_weight_ignore_index_3d_log_prob", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_mean_no_weight_ignore_index_3d_log_prob_expanded", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_mean_no_weight_ignore_index_4d", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_mean_no_weight_ignore_index_4d_expanded", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_mean_no_weight_ignore_index_4d_log_prob", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_mean_no_weight_ignore_index_4d_log_prob_expanded", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_mean_no_weight_ignore_index_expanded", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_mean_no_weight_ignore_index_log_prob", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_mean_no_weight_ignore_index_log_prob_expanded", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_mean_weight", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_mean_weight_expanded", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_mean_weight_ignore_index", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_mean_weight_ignore_index_3d", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_mean_weight_ignore_index_3d_expanded", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_mean_weight_ignore_index_3d_log_prob", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_mean_weight_ignore_index_3d_log_prob_expanded", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_mean_weight_ignore_index_4d", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_mean_weight_ignore_index_4d_expanded", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_mean_weight_ignore_index_4d_log_prob", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_mean_weight_ignore_index_4d_log_prob_expanded", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_mean_weight_ignore_index_expanded", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_mean_weight_ignore_index_log_prob", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_mean_weight_ignore_index_log_prob_expanded", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_mean_weight_log_prob", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_mean_weight_log_prob_expanded", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_none", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_none_expanded", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_none_log_prob", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_none_log_prob_expanded", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_none_weights", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_none_weights_expanded", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_none_weights_log_prob", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_none_weights_log_prob_expanded", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_sum", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_sum_expanded", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_sum_log_prob", "Shape mismatch"});
|
||||
broken_tests.insert({"softmax_cross_entropy_sum_log_prob_expanded", "Shape mismatch"});
|
||||
}
|
||||
|
||||
if (provider_name == "dml") {
|
||||
broken_tests.insert({"tinyyolov3", "The parameter is incorrect"});
|
||||
broken_tests.insert({"PixelShuffle", "Test requires 6D Reshape, which isn't supported by DirectML"});
|
||||
broken_tests.insert({"operator_permute2", "Test requires 6D Transpose, which isn't supported by DirectML"});
|
||||
broken_tests.insert({"resize_downsample_linear",
|
||||
"ORT 0.4 uses asymmetric but will conform to half_pixel in the next ONNX version."});
|
||||
broken_tests.insert(
|
||||
{"resize_upsample_linear", "ORT 0.4 uses asymmetric but will conform to half_pixel in the next ONNX version."});
|
||||
broken_tests.insert(
|
||||
{"resize_upsample_linear", "ORT 0.4 uses asymmetric but will conform to half_pixel in the next ONNX version."});
|
||||
|
||||
// These tests are temporarily disabled pending investigation
|
||||
broken_tests.insert({"dynamicquantizelinear_expanded", "Temporarily disabled pending investigation"});
|
||||
broken_tests.insert({"dynamicquantizelinear_max_adjusted_expanded", "Temporarily disabled pending investigation"});
|
||||
broken_tests.insert({"dynamicquantizelinear_min_adjusted_expanded", "Temporarily disabled pending investigation"});
|
||||
broken_tests.insert({"mxnet_arcface", "Temporarily disabled pending investigation"});
|
||||
broken_tests.insert({"yolov3", "Temporarily disabled pending investigation"});
|
||||
broken_tests.insert({"tf_inception_v2", "Temporarily disabled pending investigation"});
|
||||
broken_tests.insert({"fp16_inception_v1", "Temporarily disabled pending investigation"});
|
||||
broken_tests.insert({"candy", "Temporarily disabled pending investigation"});
|
||||
broken_tests.insert({"BERT_Squad", "Temporarily disabled pending investigation"});
|
||||
broken_tests.insert({"LSTM_Seq_lens_unpacked", "The parameter is incorrect"});
|
||||
|
||||
broken_tests.insert({"resize_downsample_scales_linear",
|
||||
"DML uses half_pixel and this test assumed \"asymmetric\" but does not include \"mode\""});
|
||||
broken_tests.insert({"resize_downsample_sizes_linear_pytorch_half_pixel",
|
||||
"DML does not support downsampling by such a large factor - skips input pixels"});
|
||||
broken_tests.insert({"resize_downsample_sizes_nearest",
|
||||
"DML uses pixel centers for nearest, rounding 1 value off for the middle column"});
|
||||
broken_tests.insert({"resize_upsample_sizes_nearest",
|
||||
"DML uses pixel centers for nearest, which makes more sense (the 3rd row mismatches)"});
|
||||
broken_tests.insert({"unsqueeze_three_axes", "DML does not support 6D tensors"});
|
||||
broken_tests.insert({"unsqueeze_unsorted_axes", "DMLdoes not support 6D tensors"});
|
||||
|
||||
broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index",
|
||||
"DML does not support 5D+ tensors"});
|
||||
broken_tests.insert(
|
||||
{"negative_log_likelihood_loss_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index_expanded",
|
||||
"DML does not support 5D+ tensors"});
|
||||
broken_tests.insert(
|
||||
{"negative_log_likelihood_loss_input_shape_is_NCd1d2d3d4d5_mean_weight", "DML does not support 5D+ tensors"});
|
||||
broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1d2d3d4d5_mean_weight_expanded",
|
||||
"DML does not support 5D+ tensors"});
|
||||
broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1d2d3d4d5_none_no_weight",
|
||||
"DML does not support 5D+ tensors"});
|
||||
broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1d2d3d4d5_none_no_weight_expanded",
|
||||
"DML does not support 5D+ tensors"});
|
||||
broken_tests.insert({"softmax_cross_entropy_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index",
|
||||
"DML does not support 5D+ tensors"});
|
||||
broken_tests.insert({"softmax_cross_entropy_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index_expanded",
|
||||
"DML does not support 5D+ tensors"});
|
||||
broken_tests.insert({"softmax_cross_entropy_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index_log_prob",
|
||||
"DML does not support 5D+ tensors"});
|
||||
broken_tests.insert(
|
||||
{"softmax_cross_entropy_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index_log_prob_expanded",
|
||||
"DML does not support 5D+ tensors"});
|
||||
broken_tests.insert(
|
||||
{"softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_mean_weight", "DML does not support 5D+ tensors"});
|
||||
broken_tests.insert(
|
||||
{"softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_mean_weight_expanded", "DML does not support 5D+ tensors"});
|
||||
broken_tests.insert(
|
||||
{"softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_mean_weight_log_prob", "DML does not support 5D+ tensors"});
|
||||
broken_tests.insert({"softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_mean_weight_log_prob_expanded",
|
||||
"DML does not support 5D+ tensors"});
|
||||
broken_tests.insert(
|
||||
{"softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_none_no_weight", "DML does not support 5D+ tensors"});
|
||||
broken_tests.insert({"softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_none_no_weight_expanded",
|
||||
"DML does not support 5D+ tensors"});
|
||||
broken_tests.insert({"softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_none_no_weight_log_prob",
|
||||
"DML does not support 5D+ tensors"});
|
||||
broken_tests.insert({"softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_none_no_weight_log_prob_expanded",
|
||||
"DML does not support 5D+ tensors"});
|
||||
}
|
||||
|
||||
#ifdef DISABLE_CONTRIB_OPS
|
||||
broken_tests.insert({"coreml_SqueezeNet_ImageNet", "This model uses contrib ops."});
|
||||
broken_tests.insert({"keras2coreml_Permute_ImageNet", "This model uses contrib ops."});
|
||||
broken_tests.insert({"keras2coreml_ReLU_ImageNet", "This model uses contrib ops."});
|
||||
broken_tests.insert({"keras2coreml_Padding-Upsampling-Normalizer_ImageNet", "This model uses contrib ops."});
|
||||
broken_tests.insert({"tiny_yolov2", "This model uses contrib ops."});
|
||||
broken_tests.insert({"fp16_tiny_yolov2", "This model uses contrib ops."});
|
||||
broken_tests.insert({"keras2coreml_Pooling_ImageNet", "This model uses contrib ops."});
|
||||
broken_tests.insert({"keras2coreml_Padding_ImageNet", "This model uses contrib ops."});
|
||||
broken_tests.insert({"keras2coreml_Normalizer_ImageNet", "This model uses contrib ops."});
|
||||
broken_tests.insert({"keras2coreml_linear_sklearn_load_breast_cancer", "This model uses contrib ops."});
|
||||
broken_tests.insert({"keras2coreml_linear_ImageNet_small", "This model uses contrib ops."});
|
||||
broken_tests.insert({"keras2coreml_linear_ImageNet_large", "This model uses contrib ops."});
|
||||
broken_tests.insert({"keras2coreml_linear_ImageNet", "This model uses contrib ops."});
|
||||
broken_tests.insert({"keras2coreml_leakyrelu_ImageNet", "This model uses contrib ops."});
|
||||
broken_tests.insert({"keras2coreml_hard_sigmoid_ImageNet", "This model uses contrib ops."});
|
||||
broken_tests.insert({"keras2coreml_elu_ImageNet", "This model uses contrib ops."});
|
||||
broken_tests.insert({"keras2coreml_Dense_ImageNet", "This model uses contrib ops."});
|
||||
broken_tests.insert({"keras2coreml_Conv2D_ImageNet", "This model uses contrib ops."});
|
||||
broken_tests.insert({"coreml_VGG16_ImageNet", "This model uses contrib ops."});
|
||||
broken_tests.insert({"coreml_Resnet50_ImageNet", "This model uses contrib ops."});
|
||||
broken_tests.insert({"coreml_Inceptionv3_ImageNet", "This model uses contrib ops."});
|
||||
broken_tests.insert({"coreml_FNS-Candy_ImageNet", "This model uses contrib ops."});
|
||||
broken_tests.insert({"coreml_AgeNet_ImageNet", "This model uses contrib ops."});
|
||||
broken_tests.insert({"keras2coreml_thresholdedrelu_ImageNet_large", "This model uses contrib ops."});
|
||||
broken_tests.insert({"keras2coreml_thresholdedrelu_ImageNet_small", "This model uses contrib ops."});
|
||||
broken_tests.insert({"keras2coreml_thresholdedrelu_sklearn_load_breast_cancer", "This model uses contrib ops."});
|
||||
broken_tests.insert({"thresholdedrelu", "This model uses contrib ops."});
|
||||
broken_tests.insert({"thresholdedrelu_default", "This model uses contrib ops."});
|
||||
broken_tests.insert({"dynamic_slice_default_axes", "This model uses contrib ops."});
|
||||
broken_tests.insert({"thresholdedrelu_example", "This model uses contrib ops."});
|
||||
broken_tests.insert({"dynamic_slice_neg failed", "This model uses contrib ops."});
|
||||
broken_tests.insert({"dynamic_slice_start_out_of_bounds", "This model uses contrib ops."});
|
||||
broken_tests.insert({"dynamic_slice", "This model uses contrib ops."});
|
||||
broken_tests.insert({"dynamic_slice_end_out_of_bounds", "This model uses contrib ops."});
|
||||
broken_tests.insert({"dynamic_slice_neg", "This model uses contrib ops."});
|
||||
broken_tests.insert({"mvn", "This model uses contrib ops.", {"onnx130"}});
|
||||
broken_tests.insert({"cdist_float32_euclidean_1000_2000_1", "This model uses contrib ops."});
|
||||
broken_tests.insert({"cdist_float32_euclidean_1000_2000_500", "This model uses contrib ops."});
|
||||
broken_tests.insert({"cdist_float32_euclidean_1_1_1", "This model uses contrib ops."});
|
||||
broken_tests.insert({"cdist_float32_sqeuclidean_1000_2000_1", "This model uses contrib ops."});
|
||||
broken_tests.insert({"cdist_float32_sqeuclidean_1000_2000_500", "This model uses contrib ops."});
|
||||
broken_tests.insert({"cdist_float32_sqeuclidean_1_1_1", "This model uses contrib ops."});
|
||||
broken_tests.insert({"cdist_float64_euclidean_1000_2000_1", "This model uses contrib ops."});
|
||||
broken_tests.insert({"cdist_float64_euclidean_1000_2000_500", "This model uses contrib ops."});
|
||||
broken_tests.insert({"cdist_float64_euclidean_1_1_1", "This model uses contrib ops."});
|
||||
broken_tests.insert({"cdist_float64_sqeuclidean_1000_2000_1", "This model uses contrib ops."});
|
||||
broken_tests.insert({"cdist_float64_sqeuclidean_1000_2000_500", "This model uses contrib ops."});
|
||||
broken_tests.insert({"cdist_float64_sqeuclidean_1_1_1", "This model uses contrib ops."});
|
||||
#endif
|
||||
|
||||
std::basic_string<ORTCHAR_T> model_dir;
|
||||
(void)GetDirNameFromFilePath(model_path, model_dir);
|
||||
std::basic_string<PATH_CHAR_TYPE> test_case_name = GetLastComponent(model_dir);
|
||||
if (test_case_name.compare(0, 5, ORT_TSTR("test_")) == 0)
|
||||
test_case_name = test_case_name.substr(5);
|
||||
{
|
||||
BrokenTest t = {ToMBString(test_case_name), ""};
|
||||
auto iter = broken_tests.find(t);
|
||||
auto model_version = model_info->GetModelVersion();
|
||||
if (iter != broken_tests.end() &&
|
||||
(model_version == TestModelInfo::unknown_version || iter->broken_versions_.empty() ||
|
||||
iter->broken_versions_.find(model_version) != iter->broken_versions_.end())) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
bool is_single_node = !model_info->GetNodeName().empty();
|
||||
std::vector<ExecutionMode> execution_modes = {ExecutionMode::ORT_SEQUENTIAL};
|
||||
if (provider_name == "cpu" && !is_single_node)
|
||||
execution_modes.push_back(ExecutionMode::ORT_PARALLEL);
|
||||
|
||||
std::vector<bool> use_single_thread{false};
|
||||
// Test the model with intra op threadpool disabled
|
||||
if (provider_name == "cpu" && !is_single_node)
|
||||
use_single_thread.push_back(true);
|
||||
|
||||
std::unique_ptr<ITestCase> l = CreateOnnxTestCase(ToMBString(test_case_name), std::move(model_info),
|
||||
per_sample_tolerance, relative_per_sample_tolerance);
|
||||
for (bool is_single_thread : use_single_thread) {
|
||||
for (ExecutionMode execution_mode : execution_modes) {
|
||||
SessionOptions so;
|
||||
if (!is_single_thread)
|
||||
so.use_per_session_threads = false;
|
||||
else
|
||||
so.intra_op_param.thread_pool_size = 1; // Disable intra op thread pool
|
||||
so.execution_mode = execution_mode;
|
||||
so.session_logid = ToMBString(test_case_name);
|
||||
so.session_log_severity_level = (int)logging::Severity::kERROR;
|
||||
InferenceSession session_object(so, (**ort_env).GetEnvironment());
|
||||
if (provider_name == "cuda") {
|
||||
ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(DefaultCudaExecutionProvider()));
|
||||
} else if (provider_name == "dnnl") {
|
||||
ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(DefaultDnnlExecutionProvider()));
|
||||
} else if (provider_name == "ngraph") {
|
||||
ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(DefaultNGraphExecutionProvider()));
|
||||
} else if (provider_name == "nuphar") {
|
||||
ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(DefaultNupharExecutionProvider()));
|
||||
} else if (provider_name == "tensorrt") {
|
||||
ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(DefaultTensorrtExecutionProvider()));
|
||||
} else if (provider_name == "migraphx") {
|
||||
ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(DefaultMIGraphXExecutionProvider()));
|
||||
} else if (provider_name == "openvino") {
|
||||
ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(DefaultOpenVINOExecutionProvider()));
|
||||
} else if (provider_name == "nnapi") {
|
||||
ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(DefaultNnapiExecutionProvider()));
|
||||
} else if (provider_name == "rknpu") {
|
||||
ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(DefaultRknpuExecutionProvider()));
|
||||
} else if (provider_name == "acl") {
|
||||
ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(DefaultAclExecutionProvider()));
|
||||
}
|
||||
if (provider_name == "armnn") {
|
||||
ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(DefaultArmNNExecutionProvider()));
|
||||
}
|
||||
|
||||
ASSERT_STATUS_OK(session_object.Load(model_path));
|
||||
auto st = session_object.Initialize();
|
||||
if (st.Code() == NOT_IMPLEMENTED)
|
||||
return;
|
||||
ASSERT_TRUE(st.IsOK()) << st.ErrorMessage();
|
||||
const size_t data_count = l->GetDataCount();
|
||||
for (size_t task_id = 0; task_id != data_count; ++task_id) {
|
||||
onnxruntime::test::HeapBuffer holder;
|
||||
std::unordered_map<std::string, OrtValue*> feeds;
|
||||
l->LoadTestData(task_id, holder, feeds, true);
|
||||
|
||||
std::pair<common::Status, const OutputDefList*> output_meta_data = session_object.GetModelOutputs();
|
||||
ASSERT_STATUS_OK(output_meta_data.first);
|
||||
// Create output feed
|
||||
size_t output_count = output_meta_data.second->size();
|
||||
std::vector<std::string> output_names(output_count);
|
||||
for (size_t i = 0; i != output_count; ++i) {
|
||||
output_names[i] = (*output_meta_data.second)[i]->Name();
|
||||
}
|
||||
|
||||
std::vector<OrtValue> output_values(output_count);
|
||||
{
|
||||
std::unordered_map<std::string, OrtValue> input;
|
||||
for (auto& p : feeds) {
|
||||
input[p.first] = *p.second;
|
||||
delete p.second;
|
||||
}
|
||||
ASSERT_STATUS_OK(session_object.Run(input, output_names, &output_values));
|
||||
}
|
||||
|
||||
bool post_procesing = false;
|
||||
Status status;
|
||||
ASSERT_STATUS_OK(l->GetPerSampleTolerance(&per_sample_tolerance));
|
||||
ASSERT_STATUS_OK(l->GetRelativePerSampleTolerance(&relative_per_sample_tolerance));
|
||||
ASSERT_STATUS_OK(l->GetPostProcessing(&post_procesing));
|
||||
|
||||
// TODO: if there are no output value files, just skip the validation
|
||||
std::unordered_map<std::string, OrtValue*> expected_output_values;
|
||||
l->LoadTestData(task_id, holder, expected_output_values, false);
|
||||
|
||||
std::unordered_map<std::string, OrtValue*> name_fetch_output_map;
|
||||
std::unordered_map<std::string, const ONNX_NAMESPACE::ValueInfoProto*> name_output_value_info_proto;
|
||||
size_t i = 0;
|
||||
for (auto& output_name : output_names) {
|
||||
// p_fetches is filled in the order of output_names.
|
||||
name_fetch_output_map[output_name] = &output_values[i];
|
||||
const ONNX_NAMESPACE::ValueInfoProto* infoProto = l->GetOutputInfoFromModel(i);
|
||||
if (infoProto != nullptr)
|
||||
name_output_value_info_proto.insert(std::make_pair(infoProto->name(), infoProto));
|
||||
i++;
|
||||
}
|
||||
|
||||
for (auto& output : expected_output_values) {
|
||||
OrtValue* expected_output_value = output.second;
|
||||
const std::string& output_name = output.first;
|
||||
auto iter = name_fetch_output_map.find(output_name);
|
||||
ASSERT_NE(iter, name_fetch_output_map.end());
|
||||
|
||||
OrtValue* actual_output_value = iter->second;
|
||||
std::pair<COMPARE_RESULT, std::string> ret =
|
||||
CompareOrtValue(*actual_output_value, *expected_output_value, per_sample_tolerance,
|
||||
relative_per_sample_tolerance, post_procesing);
|
||||
COMPARE_RESULT compare_result = ret.first;
|
||||
ASSERT_EQ(COMPARE_RESULT::SUCCESS, ret.first) << ret.second;
|
||||
|
||||
const ONNX_NAMESPACE::ValueInfoProto* v = name_output_value_info_proto[output_name];
|
||||
if (v == nullptr)
|
||||
continue;
|
||||
ret = VerifyValueInfo(*v, Ort::Unowned<Ort::Value>{actual_output_value});
|
||||
compare_result = ret.first;
|
||||
ASSERT_EQ(COMPARE_RESULT::SUCCESS, ret.first) << ret.second;
|
||||
|
||||
if (compare_result != COMPARE_RESULT::SUCCESS) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
for (auto& kvp : expected_output_values) {
|
||||
delete kvp.second;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: all providers
|
||||
::std::vector<::std::basic_string<ORTCHAR_T>> GetParameterStrings() {
|
||||
std::vector<const ORTCHAR_T*> provider_names;
|
||||
provider_names.push_back(ORT_TSTR("cpu"));
|
||||
#ifdef USE_TENSORRT
|
||||
provider_names.push_back(ORT_TSTR("tensorrt"));
|
||||
#endif
|
||||
#ifdef USE_MIGRAPHX
|
||||
provider_names.push_back(ORT_TSTR("migraphx"));
|
||||
#endif
|
||||
#ifdef USE_OPENVINO
|
||||
provider_names.push_back(ORT_TSTR("openvino"));
|
||||
#endif
|
||||
#ifdef USE_CUDA
|
||||
provider_names.push_back(ORT_TSTR("cuda"));
|
||||
#endif
|
||||
#ifdef USE_DNNL
|
||||
provider_names.push_back(ORT_TSTR("dnnl"));
|
||||
#endif
|
||||
#ifdef USE_NGRAPH
|
||||
provider_names.push_back(ORT_TSTR("ngraph"));
|
||||
#endif
|
||||
#ifdef USE_NUPHAR
|
||||
provider_names.push_back(ORT_TSTR("nuphar"));
|
||||
#endif
|
||||
#ifdef USE_NNAPI
|
||||
provider_names.push_back(ORT_TSTR("nnapi"));
|
||||
#endif
|
||||
#ifdef USE_RKNPU
|
||||
provider_names.push_back(ORT_TSTR("rknpu"));
|
||||
#endif
|
||||
#ifdef USE_ACL
|
||||
provider_names.push_back(ORT_TSTR("acl"));
|
||||
#endif
|
||||
#ifdef USE_ARMNN
|
||||
provider_names.push_back(ORT_TSTR("armnn");
|
||||
#endif
|
||||
std::vector<std::basic_string<ORTCHAR_T>> v;
|
||||
// Permanently exclude following tests because ORT support only opset starting from 7,
|
||||
// Please make no more changes to the list
|
||||
static const ORTCHAR_T* immutable_broken_tests[] = {
|
||||
ORT_TSTR("AvgPool1d"),
|
||||
ORT_TSTR("AvgPool1d_stride"),
|
||||
ORT_TSTR("AvgPool2d"),
|
||||
ORT_TSTR("AvgPool2d_stride"),
|
||||
ORT_TSTR("AvgPool3d"),
|
||||
ORT_TSTR("AvgPool3d_stride"),
|
||||
ORT_TSTR("AvgPool3d_stride1_pad0_gpu_input"),
|
||||
ORT_TSTR("BatchNorm1d_3d_input_eval"),
|
||||
ORT_TSTR("BatchNorm2d_eval"),
|
||||
ORT_TSTR("BatchNorm2d_momentum_eval"),
|
||||
ORT_TSTR("BatchNorm3d_eval"),
|
||||
ORT_TSTR("BatchNorm3d_momentum_eval"),
|
||||
ORT_TSTR("GLU"),
|
||||
ORT_TSTR("GLU_dim"),
|
||||
ORT_TSTR("Linear"),
|
||||
ORT_TSTR("PReLU_1d"),
|
||||
ORT_TSTR("PReLU_1d_multiparam"),
|
||||
ORT_TSTR("PReLU_2d"),
|
||||
ORT_TSTR("PReLU_2d_multiparam"),
|
||||
ORT_TSTR("PReLU_3d"),
|
||||
ORT_TSTR("PReLU_3d_multiparam"),
|
||||
ORT_TSTR("PoissonNLLLLoss_no_reduce"),
|
||||
ORT_TSTR("Softsign"),
|
||||
ORT_TSTR("operator_add_broadcast"),
|
||||
ORT_TSTR("operator_add_size1_broadcast"),
|
||||
ORT_TSTR("operator_add_size1_right_broadcast"),
|
||||
ORT_TSTR("operator_add_size1_singleton_broadcast"),
|
||||
ORT_TSTR("operator_addconstant"),
|
||||
ORT_TSTR("operator_addmm"),
|
||||
ORT_TSTR("operator_basic"),
|
||||
ORT_TSTR("operator_mm"),
|
||||
ORT_TSTR("operator_non_float_params"),
|
||||
ORT_TSTR("operator_params"),
|
||||
ORT_TSTR("operator_pow"),
|
||||
};
|
||||
|
||||
static const ORTCHAR_T* cuda_flaky_tests[] = {
|
||||
ORT_TSTR("fp16_inception_v1"),
|
||||
ORT_TSTR("fp16_shufflenet"), ORT_TSTR("fp16_tiny_yolov2"),ORT_TSTR("candy"),
|
||||
ORT_TSTR("tinyyolov3"),
|
||||
ORT_TSTR("mlperf_ssd_mobilenet_300"),
|
||||
ORT_TSTR("mlperf_ssd_resnet34_1200"),
|
||||
ORT_TSTR("tf_inception_v1"),
|
||||
ORT_TSTR("faster_rcnn"),
|
||||
ORT_TSTR("split_zero_size_splits"),
|
||||
ORT_TSTR("convtranspose_3d")};
|
||||
static const ORTCHAR_T* openvino_disabled_tests[] = {ORT_TSTR("operator_permute2"),
|
||||
ORT_TSTR("operator_repeat"),
|
||||
ORT_TSTR("operator_repeat_dim_overflow"),
|
||||
ORT_TSTR("mlperf_ssd_resnet34_1200"),
|
||||
ORT_TSTR("candy"),
|
||||
ORT_TSTR("cntk_simple_seg"),
|
||||
ORT_TSTR("negative_log_likelihood_loss_input_shape_is_NCd1d2d3d4d5_mean_weight"),
|
||||
ORT_TSTR("negative_log_likelihood_loss_input_shape_is_NCd1d2d3d4d5_mean_weight_expanded"),
|
||||
ORT_TSTR("negative_log_likelihood_loss_input_shape_is_NCd1d2d3d4d5_none_no_weight"),
|
||||
ORT_TSTR("negative_log_likelihood_loss_input_shape_is_NCd1d2d3d4d5_none_no_weight_expanded"),
|
||||
ORT_TSTR("softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_mean_weight"),
|
||||
ORT_TSTR("softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_mean_weight_expanded"),
|
||||
ORT_TSTR("softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_mean_weight_log_prob"),
|
||||
ORT_TSTR("softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_mean_weight_log_prob_expanded"),
|
||||
ORT_TSTR("softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_none_no_weight"),
|
||||
ORT_TSTR("softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_none_no_weight_expanded"),
|
||||
ORT_TSTR("softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_none_no_weight_log_prob"),
|
||||
ORT_TSTR("softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_none_no_weight_log_prob_expanded")};
|
||||
static const ORTCHAR_T* dml_disabled_tests[] = {ORT_TSTR("mlperf_ssd_resnet34_1200"),
|
||||
ORT_TSTR("mlperf_ssd_mobilenet_300"), ORT_TSTR("mask_rcnn"),
|
||||
ORT_TSTR("faster_rcnn"), ORT_TSTR("tf_pnasnet_large"),
|
||||
ORT_TSTR("zfnet512"),ORT_TSTR("keras2coreml_Dense_ImageNet") };
|
||||
static const ORTCHAR_T* dnnl_disabled_tests[] = {ORT_TSTR("test_densenet121"), ORT_TSTR("test_resnet18v2"),
|
||||
ORT_TSTR("test_resnet34v2"), ORT_TSTR("test_resnet50v2"),
|
||||
ORT_TSTR("test_resnet101v2"),
|
||||
ORT_TSTR("test_resnet101v2"), ORT_TSTR("test_vgg19"),
|
||||
ORT_TSTR("tf_inception_resnet_v2"), ORT_TSTR("tf_inception_v1"),
|
||||
ORT_TSTR("tf_inception_v3"), ORT_TSTR("tf_inception_v4"),
|
||||
ORT_TSTR("tf_mobilenet_v1_1.0_224"),
|
||||
ORT_TSTR("tf_mobilenet_v2_1.0_224"),
|
||||
ORT_TSTR("tf_mobilenet_v2_1.4_224"), ORT_TSTR("tf_nasnet_large"),
|
||||
ORT_TSTR("tf_pnasnet_large"), ORT_TSTR("tf_resnet_v1_50"),
|
||||
ORT_TSTR("tf_resnet_v1_101"), ORT_TSTR("tf_resnet_v1_101"),
|
||||
ORT_TSTR("tf_resnet_v2_101"), ORT_TSTR("tf_resnet_v2_152"),
|
||||
ORT_TSTR("batchnorm_example_training_mode"),
|
||||
ORT_TSTR("batchnorm_epsilon_training_mode"),
|
||||
ORT_TSTR("tf_mobilenet_v2_1.0_224"),
|
||||
ORT_TSTR("tf_mobilenet_v2_1.4_224"),
|
||||
ORT_TSTR("tf_mobilenet_v1_1.0_224"),
|
||||
ORT_TSTR("mobilenetv2-1.0"),
|
||||
ORT_TSTR("candy"),
|
||||
ORT_TSTR("range_float_type_positive_delta_expanded"),
|
||||
ORT_TSTR("range_int32_type_negative_delta_expanded"),
|
||||
ORT_TSTR("averagepool_2d_ceil"),
|
||||
ORT_TSTR("maxpool_2d_ceil"),
|
||||
ORT_TSTR("maxpool_2d_dilations"),
|
||||
ORT_TSTR("mlperf_ssd_resnet34_1200"),
|
||||
ORT_TSTR("convtranspose_1d"),
|
||||
ORT_TSTR("convtranspose_3d"),
|
||||
ORT_TSTR("maxpool_2d_uint8")};
|
||||
static const ORTCHAR_T* tensorrt_disabled_tests[] = {ORT_TSTR("udnie"), ORT_TSTR("rain_princess"),
|
||||
ORT_TSTR("pointilism"), ORT_TSTR("mosaic"),
|
||||
ORT_TSTR("LSTM_Seq_lens_unpacked"),
|
||||
ORT_TSTR("cgan"), ORT_TSTR("candy"),
|
||||
ORT_TSTR("tinyyolov3"), ORT_TSTR("yolov3"),
|
||||
ORT_TSTR("mlperf_ssd_resnet34_1200"), ORT_TSTR("mlperf_ssd_mobilenet_300"),
|
||||
ORT_TSTR("mask_rcnn"),
|
||||
ORT_TSTR("faster_rcnn"),
|
||||
ORT_TSTR("fp16_shufflenet"),
|
||||
ORT_TSTR("fp16_inception_v1"),
|
||||
ORT_TSTR("fp16_tiny_yolov2"),
|
||||
ORT_TSTR("tf_inception_v3"),
|
||||
ORT_TSTR("tf_mobilenet_v1_1.0_224"),
|
||||
ORT_TSTR("tf_mobilenet_v2_1.0_224"),
|
||||
ORT_TSTR("tf_mobilenet_v2_1.4_224"),
|
||||
ORT_TSTR("tf_resnet_v1_101"),
|
||||
ORT_TSTR("tf_resnet_v1_152"),
|
||||
ORT_TSTR("tf_resnet_v1_50"),
|
||||
ORT_TSTR("tf_resnet_v2_101"),
|
||||
ORT_TSTR("tf_resnet_v2_152"),
|
||||
ORT_TSTR("tf_resnet_v2_50"),
|
||||
ORT_TSTR("convtranspose_1d"),
|
||||
ORT_TSTR("convtranspose_3d"),
|
||||
ORT_TSTR("conv_with_strides_and_asymmetric_padding"),
|
||||
ORT_TSTR("conv_with_strides_padding"),
|
||||
ORT_TSTR("size") //INVALID_ARGUMENT: Cannot find binding of given name: x
|
||||
};
|
||||
for (const ORTCHAR_T* provider_name : provider_names) {
|
||||
std::unordered_set<std::basic_string<ORTCHAR_T>> all_disabled_tests(std::begin(immutable_broken_tests),
|
||||
std::end(immutable_broken_tests));
|
||||
if (CompareCString(provider_name, ORT_TSTR("cuda")) == 0) {
|
||||
all_disabled_tests.insert(std::begin(cuda_flaky_tests), std::end(cuda_flaky_tests));
|
||||
} else if (CompareCString(provider_name, ORT_TSTR("dml")) == 0) {
|
||||
all_disabled_tests.insert(std::begin(dml_disabled_tests), std::end(dml_disabled_tests));
|
||||
} else if (CompareCString(provider_name, ORT_TSTR("dnnl")) == 0) {
|
||||
// these models run but disabled tests to keep memory utilization low
|
||||
// This will be removed after LRU implementation
|
||||
all_disabled_tests.insert(std::begin(dnnl_disabled_tests), std::end(dnnl_disabled_tests));
|
||||
} else if (CompareCString(provider_name, ORT_TSTR("tensorrt")) == 0) {
|
||||
// these models run but disabled tests to keep memory utilization low
|
||||
// This will be removed after LRU implementation
|
||||
all_disabled_tests.insert(std::begin(tensorrt_disabled_tests), std::end(tensorrt_disabled_tests));
|
||||
} else if (CompareCString(provider_name, ORT_TSTR("openvino")) == 0) {
|
||||
// these models run but disabled tests to keep memory utilization low
|
||||
// This will be removed after LRU implementation
|
||||
all_disabled_tests.insert(std::begin(openvino_disabled_tests), std::end(openvino_disabled_tests));
|
||||
}
|
||||
|
||||
#if !defined(__amd64__) && !defined(_M_AMD64)
|
||||
// out of memory
|
||||
static const ORTCHAR_T* x86_disabled_tests[] = {ORT_TSTR("mlperf_ssd_resnet34_1200"),
|
||||
ORT_TSTR("mask_rcnn_keras"),
|
||||
ORT_TSTR("mask_rcnn"),
|
||||
ORT_TSTR("faster_rcnn"),
|
||||
ORT_TSTR("vgg19"),
|
||||
ORT_TSTR("zfnet512"),
|
||||
ORT_TSTR("coreml_VGG16_ImageNet")};
|
||||
all_disabled_tests.insert(std::begin(x86_disabled_tests), std::end(x86_disabled_tests));
|
||||
#endif
|
||||
|
||||
std::vector<std::basic_string<ORTCHAR_T>> paths;
|
||||
#ifdef NDEBUG
|
||||
#ifdef _WIN32
|
||||
paths.push_back(ORT_TSTR("..\\models"));
|
||||
#else
|
||||
paths.push_back(ORT_TSTR("../models"));
|
||||
#endif
|
||||
#endif
|
||||
|
||||
// TENSORRT has too many test failures in the single node tests
|
||||
#if !defined(_WIN32) && !defined(USE_TENSORRT)
|
||||
paths.push_back("/data/onnx");
|
||||
#endif
|
||||
while (!paths.empty()) {
|
||||
std::basic_string<ORTCHAR_T> node_data_root_path = paths.back();
|
||||
paths.pop_back();
|
||||
std::basic_string<ORTCHAR_T> my_dir_name = GetLastComponent(node_data_root_path);
|
||||
try {
|
||||
LoopDir(node_data_root_path, [&](const ORTCHAR_T* filename, OrtFileType f_type) -> bool {
|
||||
if (filename[0] == ORT_TSTR('.'))
|
||||
return true;
|
||||
if (f_type == OrtFileType::TYPE_DIR) {
|
||||
std::basic_string<PATH_CHAR_TYPE> p = ConcatPathComponent<PATH_CHAR_TYPE>(node_data_root_path, filename);
|
||||
paths.push_back(p);
|
||||
return true;
|
||||
}
|
||||
std::basic_string<PATH_CHAR_TYPE> filename_str = filename;
|
||||
if (!HasExtensionOf(filename_str, ORT_TSTR("onnx")))
|
||||
return true;
|
||||
|
||||
std::basic_string<PATH_CHAR_TYPE> test_case_name = my_dir_name;
|
||||
if (test_case_name.compare(0, 5, ORT_TSTR("test_")) == 0)
|
||||
test_case_name = test_case_name.substr(5);
|
||||
if (all_disabled_tests.find(test_case_name) != all_disabled_tests.end())
|
||||
return true;
|
||||
|
||||
#ifdef DISABLE_ML_OPS
|
||||
auto starts_with = [](const std::basic_string<PATH_CHAR_TYPE>& find_in,
|
||||
const std::basic_string<PATH_CHAR_TYPE>& find_what) {
|
||||
return find_in.compare(0, find_what.size(), find_what) == 0;
|
||||
};
|
||||
if (starts_with(test_case_name, ORT_TSTR("XGBoost_")) || starts_with(test_case_name, ORT_TSTR("coreml_")) ||
|
||||
starts_with(test_case_name, ORT_TSTR("scikit_")) || starts_with(test_case_name, ORT_TSTR("libsvm_"))) {
|
||||
return true;
|
||||
}
|
||||
#endif
|
||||
std::basic_string<PATH_CHAR_TYPE> p = ConcatPathComponent<PATH_CHAR_TYPE>(node_data_root_path, filename_str);
|
||||
std::basic_string<PATH_CHAR_TYPE> r = provider_name;
|
||||
r.append(ORT_TSTR("_")).append(p);
|
||||
v.emplace_back(r);
|
||||
return true;
|
||||
});
|
||||
} catch (std::exception&) {
|
||||
} // ignore non-exist dir
|
||||
}
|
||||
}
|
||||
return v;
|
||||
}
|
||||
|
||||
INSTANTIATE_TEST_SUITE_P(ModelTests, ModelTest, testing::ValuesIn(GetParameterStrings()));
|
||||
|
||||
} // namespace test
|
||||
} // namespace onnxruntime
|
||||
|
|
@ -643,6 +643,7 @@ void OpTester::Run(
|
|||
const CustomOutputVerifierFn& custom_output_verifier,
|
||||
const Graph::ResolveOptions& options) {
|
||||
SessionOptions so;
|
||||
so.use_per_session_threads = false;
|
||||
so.session_logid = op_;
|
||||
so.session_log_verbosity_level = 1;
|
||||
so.execution_mode = execution_mode;
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@
|
|||
#endif
|
||||
|
||||
#include "core/session/onnxruntime_cxx_api.h"
|
||||
#include "core/util/thread_utils.h"
|
||||
#include "gtest/gtest.h"
|
||||
#include "test/test_environment.h"
|
||||
|
||||
|
|
@ -46,7 +47,8 @@ int main(int argc, char** argv) {
|
|||
int status = 0;
|
||||
try {
|
||||
::testing::InitGoogleTest(&argc, argv);
|
||||
ort_env.reset(new Ort::Env(ORT_LOGGING_LEVEL_WARNING, "Default"));
|
||||
OrtThreadingOptions tpo;
|
||||
ort_env.reset(new Ort::Env(&tpo, ORT_LOGGING_LEVEL_WARNING, "Default"));
|
||||
status = RUN_ALL_TESTS();
|
||||
} catch (const std::exception& ex) {
|
||||
std::cerr << ex.what();
|
||||
|
|
|
|||
|
|
@ -319,10 +319,11 @@ TEST(CApiTest, RegisterCustomOpForCPUAndCUDA) {
|
|||
}
|
||||
#endif
|
||||
|
||||
#ifndef __ANDROID__
|
||||
TEST(CApiTest, test_custom_op_library) {
|
||||
#else
|
||||
//It has memory leak. The OrtCustomOpDomain created in custom_op_library.cc:RegisterCustomOps function was not freed
|
||||
#if defined(__ANDROID__) || defined(ONNXRUNTIME_ENABLE_MEMLEAK_CHECK)
|
||||
TEST(CApiTest, DISABLED_test_custom_op_library) {
|
||||
#else
|
||||
TEST(CApiTest, test_custom_op_library) {
|
||||
#endif
|
||||
std::cout << "Running inference using custom op shared library" << std::endl;
|
||||
|
||||
|
|
|
|||
|
|
@ -1510,7 +1510,7 @@ TEST(GradientGraphBuilderTest, TrainingSession_WithPipeline) {
|
|||
|
||||
std::vector<PathString> sub_model_files(num_subs);
|
||||
for (size_t sub_id = 0; sub_id < num_subs; ++sub_id) {
|
||||
sub_model_files[sub_id] = GenerateFileNameWithIndex(ORT_TSTR("sub_"), sub_id, ORT_TSTR(".onnx"));
|
||||
sub_model_files[sub_id] = GenerateFileNameWithIndex(ORT_TSTR("sub_"), static_cast<int>(sub_id), ORT_TSTR(".onnx"));
|
||||
}
|
||||
|
||||
PipelineSplitter splitter;
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ import glob
|
|||
import logging
|
||||
import multiprocessing
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
|
|
@ -1313,39 +1312,6 @@ def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs):
|
|||
cwd=cwd, dll_path=dll_path)
|
||||
|
||||
|
||||
def run_onnx_tests(build_dir, configs, onnx_test_data_dir, provider,
|
||||
enable_multi_device_test, enable_parallel_executor_test,
|
||||
num_parallel_models, num_parallel_tests=0):
|
||||
for config in configs:
|
||||
cwd = get_config_build_dir(build_dir, config)
|
||||
if is_windows():
|
||||
exe = os.path.join(cwd, config, 'onnx_test_runner')
|
||||
model_dir = os.path.join(cwd, "models")
|
||||
else:
|
||||
exe = os.path.join(cwd, 'onnx_test_runner')
|
||||
model_dir = os.path.join(build_dir, "models")
|
||||
|
||||
cmd = []
|
||||
if provider:
|
||||
cmd += ["-e", provider]
|
||||
if num_parallel_tests != 0:
|
||||
cmd += ['-c', str(num_parallel_tests)]
|
||||
if num_parallel_models > 0:
|
||||
cmd += ["-j", str(num_parallel_models)]
|
||||
if enable_multi_device_test:
|
||||
cmd += ['-d', '1']
|
||||
# Even in release mode nuphar needs 40 minutes to run all the models tests
|
||||
if config != 'Debug' and os.path.exists(model_dir) and provider != 'nuphar':
|
||||
cmd.append(model_dir)
|
||||
if os.path.exists(onnx_test_data_dir):
|
||||
cmd.append(onnx_test_data_dir)
|
||||
if config == 'Debug' and provider == 'nuphar':
|
||||
return
|
||||
run_subprocess([exe] + cmd, cwd=cwd)
|
||||
if enable_parallel_executor_test:
|
||||
run_subprocess([exe, '-x'] + cmd, cwd=cwd)
|
||||
|
||||
|
||||
def tensorrt_run_onnx_tests(args, build_dir, configs, onnx_test_data_dir,
|
||||
provider, num_parallel_models,
|
||||
num_parallel_tests=0):
|
||||
|
|
@ -1845,52 +1811,16 @@ def main():
|
|||
args, build_dir, configs, trt_onnx_test_data_dir,
|
||||
"tensorrt", 1)
|
||||
|
||||
if args.use_cuda and not args.use_tensorrt:
|
||||
run_onnx_tests(
|
||||
build_dir, configs, onnx_test_data_dir, 'cuda',
|
||||
args.enable_multi_device_test, False, 2)
|
||||
|
||||
# ngraph doesn't support opset12 yet.
|
||||
# if args.use_ngraph:
|
||||
# run_onnx_tests(
|
||||
# build_dir, configs, onnx_test_data_dir, 'ngraph',
|
||||
# args.enable_multi_device_test, True, 1)
|
||||
|
||||
if args.use_openvino:
|
||||
openvino_run_onnx_tests(
|
||||
build_dir, configs, onnx_test_data_dir, 'openvino',
|
||||
1, 1)
|
||||
# TODO: parallel executor test fails on MacOS
|
||||
if args.use_nuphar:
|
||||
run_onnx_tests(
|
||||
build_dir, configs, onnx_test_data_dir, 'nuphar',
|
||||
args.enable_multi_device_test, False, 1, 1)
|
||||
|
||||
if args.use_dml:
|
||||
run_onnx_tests(
|
||||
build_dir, configs, onnx_test_data_dir, 'dml',
|
||||
args.enable_multi_device_test, False, 1)
|
||||
|
||||
if args.use_acl:
|
||||
run_onnx_tests(
|
||||
build_dir, configs, onnx_test_data_dir, 'acl',
|
||||
args.enable_multi_device_test, False, 1, 1)
|
||||
|
||||
# Run some models are disabled to keep memory utilization
|
||||
# under control.
|
||||
if args.use_dnnl:
|
||||
dnnl_run_onnx_tests(build_dir, configs, onnx_test_data_dir)
|
||||
|
||||
if args.use_tensorrt:
|
||||
tensorrt_run_onnx_tests(
|
||||
args, build_dir, configs, onnx_test_data_dir, None, 1)
|
||||
else:
|
||||
run_onnx_tests(
|
||||
build_dir, configs, onnx_test_data_dir, None,
|
||||
args.enable_multi_device_test, False,
|
||||
1 if args.x86 or platform.system() == 'Darwin' else 0,
|
||||
1 if args.x86 or platform.system() == 'Darwin' else 0)
|
||||
|
||||
# run nuphar python tests last, as it installs ONNX 1.5.0
|
||||
if args.enable_pybind and not args.skip_onnx_tests and args.use_nuphar:
|
||||
nuphar_run_python_tests(build_dir, configs)
|
||||
|
|
|
|||
|
|
@ -134,6 +134,13 @@ jobs:
|
|||
workingDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)'
|
||||
displayName: 'Run tests'
|
||||
|
||||
- task: PublishTestResults@2
|
||||
displayName: 'Publish unit test results'
|
||||
inputs:
|
||||
testResultsFiles: '**/*.results.xml'
|
||||
searchFolder: '$(Build.BinariesDirectory)'
|
||||
testRunTitle: 'Unit Test Run'
|
||||
condition: succeededOrFailed()
|
||||
|
||||
- template: templates/component-governance-component-detection-steps.yml
|
||||
parameters :
|
||||
|
|
@ -263,6 +270,14 @@ jobs:
|
|||
workingDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)'
|
||||
displayName: 'Run tests'
|
||||
|
||||
- task: PublishTestResults@2
|
||||
displayName: 'Publish unit test results'
|
||||
inputs:
|
||||
testResultsFiles: '**/*.results.xml'
|
||||
searchFolder: '$(Build.BinariesDirectory)'
|
||||
testRunTitle: 'Unit Test Run'
|
||||
condition: succeededOrFailed()
|
||||
|
||||
- template: templates/component-governance-component-detection-steps.yml
|
||||
parameters :
|
||||
condition : 'succeeded'
|
||||
|
|
@ -380,6 +395,14 @@ jobs:
|
|||
workingDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)'
|
||||
displayName: 'Run tests'
|
||||
|
||||
- task: PublishTestResults@2
|
||||
displayName: 'Publish unit test results'
|
||||
inputs:
|
||||
testResultsFiles: '**/*.results.xml'
|
||||
searchFolder: '$(Build.BinariesDirectory)'
|
||||
testRunTitle: 'Unit Test Run'
|
||||
condition: succeededOrFailed()
|
||||
|
||||
- template: templates/component-governance-component-detection-steps.yml
|
||||
parameters :
|
||||
condition : 'succeeded'
|
||||
|
|
@ -499,6 +522,13 @@ jobs:
|
|||
workingDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)'
|
||||
displayName: 'Run tests'
|
||||
|
||||
- task: PublishTestResults@2
|
||||
displayName: 'Publish unit test results'
|
||||
inputs:
|
||||
testResultsFiles: '**/*.results.xml'
|
||||
searchFolder: '$(Build.BinariesDirectory)'
|
||||
testRunTitle: 'Unit Test Run'
|
||||
condition: succeededOrFailed()
|
||||
|
||||
- template: templates/component-governance-component-detection-steps.yml
|
||||
parameters :
|
||||
|
|
|
|||
Loading…
Reference in a new issue