diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index ec0903a90a..3467db2895 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -439,6 +439,58 @@ target_include_directories(onnxruntime_test_utils PUBLIC "${TEST_SRC_DIR}/util/i ${eigen_INCLUDE_DIRS} ${ONNXRUNTIME_ROOT}) set_target_properties(onnxruntime_test_utils PROPERTIES FOLDER "ONNXRuntimeTest") +set(onnx_test_runner_src_dir ${TEST_SRC_DIR}/onnx) +set(onnx_test_runner_common_srcs + ${onnx_test_runner_src_dir}/TestResultStat.cc + ${onnx_test_runner_src_dir}/TestResultStat.h + ${onnx_test_runner_src_dir}/testenv.h + ${onnx_test_runner_src_dir}/FixedCountFinishCallback.h + ${onnx_test_runner_src_dir}/TestCaseResult.cc + ${onnx_test_runner_src_dir}/TestCaseResult.h + ${onnx_test_runner_src_dir}/testenv.cc + ${onnx_test_runner_src_dir}/heap_buffer.h + ${onnx_test_runner_src_dir}/heap_buffer.cc + ${onnx_test_runner_src_dir}/OrtValueList.h + ${onnx_test_runner_src_dir}/runner.h + ${onnx_test_runner_src_dir}/runner.cc + ${onnx_test_runner_src_dir}/TestCase.cc + ${onnx_test_runner_src_dir}/TestCase.h + ${onnx_test_runner_src_dir}/onnxruntime_event.h + ${onnx_test_runner_src_dir}/sync_api.h + ${onnx_test_runner_src_dir}/sync_api.cc + ${onnx_test_runner_src_dir}/callback.h + ${onnx_test_runner_src_dir}/callback.cc + ${onnx_test_runner_src_dir}/pb_helper.h + ${onnx_test_runner_src_dir}/pb_helper.cc + ${onnx_test_runner_src_dir}/mem_buffer.h + ${onnx_test_runner_src_dir}/tensorprotoutils.h + ${onnx_test_runner_src_dir}/tensorprotoutils.cc + ${onnx_test_runner_src_dir}/onnx_model_info.h + ${onnx_test_runner_src_dir}/onnx_model_info.cc) + + +add_library(onnx_test_runner_common ${onnx_test_runner_common_srcs}) +if(MSVC) + target_compile_options(onnx_test_runner_common PRIVATE "$<$:SHELL:--compiler-options /utf-8>" + "$<$>:/utf-8>") +else() + target_compile_definitions(onnx_test_runner_common PUBLIC -DNSYNC_ATOMIC_CPP11) + target_include_directories(onnx_test_runner_common PRIVATE ${CMAKE_CURRENT_BINARY_DIR} ${ONNXRUNTIME_ROOT} + "${CMAKE_CURRENT_SOURCE_DIR}/external/nsync/public") +endif() +if (MSVC AND NOT CMAKE_SIZEOF_VOID_P EQUAL 8) + #TODO: fix the warnings, they are dangerous + target_compile_options(onnx_test_runner_common PRIVATE "/wd4244") +endif() +onnxruntime_add_include_to_target(onnx_test_runner_common onnxruntime_common onnxruntime_framework + onnxruntime_test_utils onnx onnx_proto re2::re2) + +add_dependencies(onnx_test_runner_common onnx_test_data_proto ${onnxruntime_EXTERNAL_DEPENDENCIES}) +target_include_directories(onnx_test_runner_common PRIVATE ${eigen_INCLUDE_DIRS} ${RE2_INCLUDE_DIR} + ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}/onnx ${ONNXRUNTIME_ROOT}) + +set_target_properties(onnx_test_runner_common PROPERTIES FOLDER "ONNXRuntimeTest") + set(all_tests ${onnxruntime_test_common_src} ${onnxruntime_test_ir_src} ${onnxruntime_test_optimizer_src} ${onnxruntime_test_framework_src} ${onnxruntime_test_providers_src}) if(NOT TARGET onnxruntime) @@ -474,7 +526,7 @@ set(all_dependencies ${onnxruntime_test_providers_dependencies} ) AddTest( TARGET onnxruntime_test_all SOURCES ${all_tests} - LIBS ${onnxruntime_test_providers_libs} ${onnxruntime_test_common_libs} + LIBS onnx_test_runner_common ${onnxruntime_test_providers_libs} ${onnxruntime_test_common_libs} re2::re2 onnx_test_data_proto DEPENDS ${all_dependencies} ) @@ -484,7 +536,6 @@ set(all_dependencies ${onnxruntime_test_providers_dependencies} ) if (onnxruntime_USE_FEATURIZERS) target_include_directories(onnxruntime_test_all PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/external/FeaturizersLibrary/src) endif() - if (onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS) target_link_libraries(onnxruntime_test_all PRIVATE onnxruntime_language_interop onnxruntime_pyop) endif() @@ -559,32 +610,7 @@ target_include_directories(onnx_test_data_proto PRIVATE ${CMAKE_CURRENT_BINARY_D set_target_properties(onnx_test_data_proto PROPERTIES FOLDER "ONNXRuntimeTest") onnxruntime_protobuf_generate(APPEND_PATH IMPORT_DIRS ${ONNXRUNTIME_ROOT}/core/protobuf TARGET onnx_test_data_proto) -set(onnx_test_runner_src_dir ${TEST_SRC_DIR}/onnx) -set(onnx_test_runner_common_srcs - ${onnx_test_runner_src_dir}/TestResultStat.cc - ${onnx_test_runner_src_dir}/TestResultStat.h - ${onnx_test_runner_src_dir}/testenv.h - ${onnx_test_runner_src_dir}/FixedCountFinishCallback.h - ${onnx_test_runner_src_dir}/TestCaseResult.cc - ${onnx_test_runner_src_dir}/TestCaseResult.h - ${onnx_test_runner_src_dir}/testenv.cc - ${onnx_test_runner_src_dir}/heap_buffer.h - ${onnx_test_runner_src_dir}/heap_buffer.cc - ${onnx_test_runner_src_dir}/OrtValueList.h - ${onnx_test_runner_src_dir}/runner.h - ${onnx_test_runner_src_dir}/runner.cc - ${onnx_test_runner_src_dir}/TestCase.cc - ${onnx_test_runner_src_dir}/TestCase.h - ${onnx_test_runner_src_dir}/onnxruntime_event.h - ${onnx_test_runner_src_dir}/sync_api.h - ${onnx_test_runner_src_dir}/sync_api.cc - ${onnx_test_runner_src_dir}/callback.h - ${onnx_test_runner_src_dir}/callback.cc - ${onnx_test_runner_src_dir}/pb_helper.h - ${onnx_test_runner_src_dir}/pb_helper.cc - ${onnx_test_runner_src_dir}/mem_buffer.h - ${onnx_test_runner_src_dir}/tensorprotoutils.h - ${onnx_test_runner_src_dir}/tensorprotoutils.cc) + if(WIN32) set(wide_get_opt_src_dir ${TEST_SRC_DIR}/win_getopt/wide) @@ -595,27 +621,6 @@ if(WIN32) set(GETOPT_LIB_WIDE win_getopt_wide) endif() -add_library(onnx_test_runner_common ${onnx_test_runner_common_srcs}) -if(MSVC) - target_compile_options(onnx_test_runner_common PRIVATE "$<$:SHELL:--compiler-options /utf-8>" - "$<$>:/utf-8>") -else() - target_compile_definitions(onnx_test_runner_common PUBLIC -DNSYNC_ATOMIC_CPP11) - target_include_directories(onnx_test_runner_common PRIVATE ${CMAKE_CURRENT_BINARY_DIR} ${ONNXRUNTIME_ROOT} - "${CMAKE_CURRENT_SOURCE_DIR}/external/nsync/public") -endif() -if (MSVC AND NOT CMAKE_SIZEOF_VOID_P EQUAL 8) - #TODO: fix the warnings, they are dangerous - target_compile_options(onnx_test_runner_common PRIVATE "/wd4244") -endif() -onnxruntime_add_include_to_target(onnx_test_runner_common onnxruntime_common onnxruntime_framework - onnxruntime_test_utils onnx onnx_proto re2::re2) - -add_dependencies(onnx_test_runner_common onnx_test_data_proto ${onnxruntime_EXTERNAL_DEPENDENCIES}) -target_include_directories(onnx_test_runner_common PRIVATE ${eigen_INCLUDE_DIRS} ${RE2_INCLUDE_DIR} - ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}/onnx ${ONNXRUNTIME_ROOT}) - -set_target_properties(onnx_test_runner_common PROPERTIES FOLDER "ONNXRuntimeTest") set(onnx_test_libs onnxruntime_test_utils @@ -723,7 +728,7 @@ if (WIN32) endif() if (onnxruntime_BUILD_SHARED_LIB) - set(onnxruntime_perf_test_libs onnxruntime_test_utils onnx_test_runner_common onnxruntime_common re2::re2 + set(onnxruntime_perf_test_libs onnx_test_runner_common onnxruntime_test_utils onnxruntime_common re2::re2 onnx_test_data_proto onnx_proto ${PROTOBUF_LIB} ${GETOPT_LIB_WIDE} onnxruntime ${SYS_PATH_LIB} ${CMAKE_DL_LIBS}) if(NOT WIN32) diff --git a/onnxruntime/test/onnx/TestCase.cc b/onnxruntime/test/onnx/TestCase.cc index 86958af1ea..f5df1440bf 100644 --- a/onnxruntime/test/onnx/TestCase.cc +++ b/onnxruntime/test/onnx/TestCase.cc @@ -20,6 +20,7 @@ #include #include #include "OrtValueList.h" +#include "onnx_model_info.h" #include "pb_helper.h" @@ -152,85 +153,7 @@ static int ExtractFileNo(const std::basic_string& name) { } using PATH_STRING_TYPE = std::basic_string; -class OnnxModelInfo : public TestModelInfo { - private: - std::string node_name_; - std::string onnx_commit_tag_; - std::vector input_value_info_; - std::vector output_value_info_; - template - static void RepeatedPtrFieldToVector(const ::google::protobuf::RepeatedPtrField& input_value_info, - std::vector& out) { - for (int i = 0; i != input_value_info.size(); ++i) { - out.push_back(input_value_info[i]); - } - } - const std::basic_string model_url_; - - public: - OnnxModelInfo(_In_ const PATH_CHAR_TYPE* model_url) : model_url_(model_url) { - // parse model - int model_fd; - auto st = Env::Default().FileOpenRd(model_url, model_fd); - if (!st.IsOK()) { - ORT_THROW(st.ErrorMessage()); - } - - ONNX_NAMESPACE::ModelProto model_pb; - ::google::protobuf::io::FileInputStream input(model_fd, protobuf_block_size_in_bytes); - const bool parse_result = model_pb.ParseFromZeroCopyStream(&input) && input.GetErrno() == 0; - if (!parse_result) { - (void)Env::Default().FileClose(model_fd); - ORT_THROW("Failed to load model because protobuf parsing failed."); - } - (void)Env::Default().FileClose(model_fd); - { - const RE2::Anchor re2_anchor = RE2::UNANCHORED; - const std::string model_url_string = ToMBString(model_url); - re2::StringPiece text(model_url_string); - re2::StringPiece submatch; - re2::RE2 regex("onnx[0-9a-z]{3}", re2::RE2::Options()); //e.g. onnx141, onnx150, onnxtip - if (!regex.ok()) { - ORT_THROW("Failed to parse regex: onnx[0-9a-z]{3}"); - } - bool match = regex.Match(text, 0, text.length(), re2_anchor, &submatch, 1); - if (match) { - onnx_commit_tag_.assign(submatch.data(), submatch.length()); - } else { - onnx_commit_tag_ = TestModelInfo::unknown_version; - } - } - const ONNX_NAMESPACE::GraphProto& graph = model_pb.graph(); - if (graph.node().size() == 1) { - node_name_ = graph.node()[0].op_type(); - } - std::unordered_set initializer_names; - for (const auto& init : graph.initializer()) { - if (!init.has_name()) continue; - initializer_names.insert(init.name()); - } - //Ignore the inputs that are already in initializers - for (const auto& p : graph.input()) { - if (!p.has_name()) ORT_THROW("input without name??"); - if (initializer_names.find(p.name()) == initializer_names.end()) input_value_info_.push_back(p); - } - RepeatedPtrFieldToVector(graph.output(), output_value_info_); - } - - const PATH_CHAR_TYPE* GetModelUrl() const override { return model_url_.c_str(); } - std::string GetModelVersion() const override { return onnx_commit_tag_; } - - const std::string& GetNodeName() const override { return node_name_; } - const ONNX_NAMESPACE::ValueInfoProto* GetOutputInfoFromModel(size_t i) const override { - return &output_value_info_[i]; - } - int GetInputCount() const override { return static_cast(input_value_info_.size()); } - int GetOutputCount() const override { return static_cast(output_value_info_.size()); } - const std::string& GetInputName(size_t i) const override { return input_value_info_[i].name(); } - - const std::string& GetOutputName(size_t i) const override { return output_value_info_[i].name(); } -}; static void SortTensorFileNames(std::vector>& input_pb_files) { if (input_pb_files.size() <= 1) return; diff --git a/onnxruntime/test/onnx/onnx_model_info.cc b/onnxruntime/test/onnx/onnx_model_info.cc new file mode 100644 index 0000000000..2318d402a3 --- /dev/null +++ b/onnxruntime/test/onnx/onnx_model_info.cc @@ -0,0 +1,72 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "onnx_model_info.h" +#include "core/platform/env.h" +#include "re2/re2.h" +#include "pb_helper.h" + +using namespace onnxruntime; +static constexpr int protobuf_block_size_in_bytes = 4 * 1024 * 1024; + +template +static void RepeatedPtrFieldToVector(const ::google::protobuf::RepeatedPtrField& input_value_info, + std::vector& out) { + for (int i = 0; i != input_value_info.size(); ++i) { + out.push_back(input_value_info[i]); + } +} + +OnnxModelInfo::OnnxModelInfo(_In_ const PATH_CHAR_TYPE* model_url) : model_url_(model_url) { + // parse model + int model_fd; + auto st = Env::Default().FileOpenRd(model_url, model_fd); + if (!st.IsOK()) { + ORT_THROW(st.ErrorMessage()); + } + + ONNX_NAMESPACE::ModelProto model_pb; + ::google::protobuf::io::FileInputStream input(model_fd, protobuf_block_size_in_bytes); + const bool parse_result = model_pb.ParseFromZeroCopyStream(&input) && input.GetErrno() == 0; + if (!parse_result) { + (void)Env::Default().FileClose(model_fd); + ORT_THROW("Failed to load model because protobuf parsing failed."); + } + (void)Env::Default().FileClose(model_fd); + { + const RE2::Anchor re2_anchor = RE2::UNANCHORED; + const std::string model_url_string = ToMBString(model_url); + re2::StringPiece text(model_url_string); + re2::StringPiece submatch; + re2::RE2 regex("onnx[0-9a-z]{3}", re2::RE2::Options()); //e.g. onnx141, onnx150, onnxtip + if (!regex.ok()) { + ORT_THROW("Failed to parse regex: onnx[0-9a-z]{3}"); + } + bool match = regex.Match(text, 0, text.length(), re2_anchor, &submatch, 1); + if (match) { + onnx_commit_tag_.assign(submatch.data(), submatch.length()); + } else { + onnx_commit_tag_ = TestModelInfo::unknown_version; + } + } + for (const auto& opset : model_pb.opset_import()) { + std::string s = opset.domain(); + if (s == "ai.onnx") s = ""; + domain_to_version_[s] = opset.version(); + } + const ONNX_NAMESPACE::GraphProto& graph = model_pb.graph(); + if (graph.node().size() == 1) { + node_name_ = graph.node()[0].op_type(); + } + std::unordered_set initializer_names; + for (const auto& init : graph.initializer()) { + if (!init.has_name()) continue; + initializer_names.insert(init.name()); + } + //Ignore the inputs that are already in initializers + for (const auto& p : graph.input()) { + if (!p.has_name()) ORT_THROW("input without name??"); + if (initializer_names.find(p.name()) == initializer_names.end()) input_value_info_.push_back(p); + } + RepeatedPtrFieldToVector(graph.output(), output_value_info_); +} diff --git a/onnxruntime/test/onnx/onnx_model_info.h b/onnxruntime/test/onnx/onnx_model_info.h new file mode 100644 index 0000000000..5c66daad76 --- /dev/null +++ b/onnxruntime/test/onnx/onnx_model_info.h @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include "core/graph/onnx_protobuf.h" +#include "TestCase.h" + +class OnnxModelInfo : public TestModelInfo { + private: + std::string node_name_; + std::string onnx_commit_tag_; + std::vector input_value_info_; + std::vector output_value_info_; + std::unordered_map domain_to_version_; + const std::basic_string model_url_; + + public: + OnnxModelInfo(_In_ const PATH_CHAR_TYPE* model_url); + + bool HasDomain(const std::string& name) const { + return domain_to_version_.find(name) != domain_to_version_.end(); + } + + int64_t GetONNXOpSetVersion() const { + auto iter = domain_to_version_.find(""); + return iter == domain_to_version_.end() ? -1 : iter->second; + } + + const PATH_CHAR_TYPE* GetModelUrl() const override { return model_url_.c_str(); } + std::string GetModelVersion() const override { return onnx_commit_tag_; } + + const std::string& GetNodeName() const override { return node_name_; } + const ONNX_NAMESPACE::ValueInfoProto* GetOutputInfoFromModel(size_t i) const override { + return &output_value_info_[i]; + } + int GetInputCount() const override { return static_cast(input_value_info_.size()); } + int GetOutputCount() const override { return static_cast(output_value_info_.size()); } + const std::string& GetInputName(size_t i) const override { return input_value_info_[i].name(); } + + const std::string& GetOutputName(size_t i) const override { return output_value_info_[i].name(); } +}; \ No newline at end of file diff --git a/onnxruntime/test/providers/cpu/model_tests.cc b/onnxruntime/test/providers/cpu/model_tests.cc new file mode 100644 index 0000000000..2ae2b08a2f --- /dev/null +++ b/onnxruntime/test/providers/cpu/model_tests.cc @@ -0,0 +1,831 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/session/onnxruntime_c_api.h" +#include "core/session/onnxruntime_cxx_api.h" +#include "core/session/inference_session.h" +#include "core/session/ort_env.h" +#include "asserts.h" +#include +#include "gtest/gtest.h" +#include +#include "test/onnx/TestCase.h" +#include "test/onnx/runner.h" +#include "test/compare_ortvalue.h" +#include "default_providers.h" +#include "test/onnx/onnx_model_info.h" + +extern std::unique_ptr ort_env; + +namespace onnxruntime { +namespace test { +// parameter is provider_name + "_" + model_path +class ModelTest : public testing::TestWithParam> {}; +namespace { +struct BrokenTest { + std::string test_name_; + std::string reason_; + std::set broken_versions_ = {}; // apply to all versions if empty + BrokenTest(std::string name, std::string reason) : test_name_(std::move(name)), reason_(std::move(reason)) { + } + + BrokenTest(std::string name, std::string reason, const std::initializer_list& versions) + : test_name_(std::move(name)), reason_(std::move(reason)), broken_versions_(versions) { + } + + bool operator<(const struct BrokenTest& test) const { + return strcmp(test_name_.c_str(), test.test_name_.c_str()) < 0; + } +}; +} // namespace +TEST_P(ModelTest, Run) { + std::basic_string param = GetParam(); + size_t pos = param.find(ORT_TSTR("_")); + ASSERT_NE(pos, std::string::npos); + std::string provider_name = ToMBString(param.substr(0, pos)); + std::basic_string model_path = param.substr(pos + 1); + double per_sample_tolerance = 1e-3; + // when cuda is enabled, set it to a larger value for resolving random MNIST test failure + // when openvino is enabled, set it to a larger value for resolving MNIST accuracy mismatch + double relative_per_sample_tolerance = 1e-3; + if (provider_name == "openvino") { + relative_per_sample_tolerance = 0.009; + } + + std::unique_ptr model_info = onnxruntime::make_unique(model_path.c_str()); + if (model_info->GetONNXOpSetVersion() != 8 && provider_name == "tensorrt") { + // TensorRT can run most of the model tests, but only part of + // them is enabled here to save CI build time. + return; + } +#ifndef ENABLE_TRAINING + if (model_info->HasDomain(ONNX_NAMESPACE::AI_ONNX_TRAINING_DOMAIN) || + model_info->HasDomain(ONNX_NAMESPACE::AI_ONNX_PREVIEW_TRAINING_DOMAIN)) { + return; + } +#endif + // TODO: filter model based on opset + std::set broken_tests = { + {"mnist", "Input data isn't in valid range"}, + {"BERT_Squad", "test data bug"}, + {"constantofshape_float_ones", "test data bug", {"onnx141", "onnx150"}}, + {"constantofshape_int_zeros", "test data bug", {"onnx141", "onnx150"}}, + {"cast_STRING_to_FLOAT", "Linux CI has old ONNX python package with bad test data", {"onnx141"}}, + // Numpy float to string has unexpected rounding for some results given numpy default precision is meant to be 8. + // "e.g. 0.296140194 -> '0.2961402' not '0.29614019'. ORT produces the latter with precision set to 8, + // which doesn't match the expected output that was generated with numpy. + {"cast_FLOAT_to_STRING", "Numpy float to string has unexpected rounding for some results."}, + {"tf_nasnet_large", "disable temporarily"}, + {"tf_nasnet_mobile", "disable temporarily"}, + {"tf_pnasnet_large", "disable temporarily"}, + {"shrink", "test case is wrong", {"onnx141"}}, + {"maxpool_with_argmax_2d_precomputed_strides", "ShapeInferenceError"}, + {"tf_inception_v2", "result mismatch"}, + {"tf_resnet_v1_50", "result mismatch when Conv BN Fusion is applied"}, + {"tf_resnet_v1_101", "result mismatch when Conv BN Fusion is applied"}, + {"tf_resnet_v1_152", "result mismatch when Conv BN Fusion is applied"}, + {"mxnet_arcface", "Model is an invalid ONNX model"}, + {"unique_not_sorted_without_axis", "Expected data for 'Y' is incorrect and in sorted order."}, + {"cumsum_1d_reverse_exclusive", "only failing linux GPU CI. Likely build error."}, + {"resize_downsample_scales_cubic_align_corners", "results mismatch with onnx tests"}, + {"resize_downsample_scales_linear_align_corners", "results mismatch with onnx tests"}, + {"resize_tf_crop_and_resize", "Bad onnx test output. Needs test fix."}, + {"resize_upsample_sizes_nearest_ceil_half_pixel", "Bad onnx test output. Needs test fix."}, + {"resize_upsample_sizes_nearest_floor_align_corners", "Bad onnx test output. Needs test fix."}, + {"resize_upsample_sizes_nearest_round_prefer_ceil_asymmetric", "Bad onnx test output. Needs test fix."}, + {"bitshift_right_uint16", "BitShift(11) uint16 support not enabled currently"}, + {"bitshift_left_uint16", "BitShift(11) uint16 support not enabled currently"}, + {"maxunpool_export_with_output_shape", + "Invalid output in ONNX test. See https://github.com/onnx/onnx/issues/2398"}, + {"training_dropout", "result differs", {}}, // Temporary, subsequent PR will remove this. + {"training_dropout_default", "result differs", {}}, // Temporary, subsequent PR will remove this. + {"training_dropout_default_mask", "result differs", {}}, // Temporary, subsequent PR will remove this. + {"training_dropout_mask", "result differs", {}}, // Temporary, subsequent PR will remove this. +#ifdef ENABLE_TRAINING + {"adagrad", "not a registered function/op", {}}, // Op not registered. + {"adagrad_multiple", "not a registered function/op", {}}, // Op not registered. + {"adam", "not a registered function/op", {}}, // Op not registered. + {"adam_multiple", "not a registered function/op", {}}, // Op not registered. + {"gradient_of_add", "not a registered function/op", {}}, // Op not registered. + {"gradient_of_add_and_mul", "not a registered function/op", {}}, // Op not registered. + {"momentum", "not a registered function/op", {}}, // Op not registered. + {"momentum_multiple", "not a registered function/op", {}}, // Op not registered. + {"nesterov_momentum", "not a registered function/op", {}}, // Op not registered. +#endif + {"mask_rcnn_keras", "this model currently has an invalid contrib op version set to 10", {}}}; + + if (provider_name == "ngraph") { + broken_tests.insert({"qlinearconv", "ambiguity in scalar dimensions [] vs [1]"}); + broken_tests.insert({"clip_splitbounds", "not implemented yet for opset 11"}); + broken_tests.insert({"clip_outbounds", "not implemented yet for opset 11"}); + broken_tests.insert({"clip_example", "not implemented yet for opset 11"}); + broken_tests.insert({"clip_default_min", "not implemented yet for opset 11"}); + broken_tests.insert({"clip_default_max", "not implemented yet for opset 11"}); + broken_tests.insert({"clip", "not implemented yet for opset 11"}); + broken_tests.insert({"depthtospace_crd_mode_example", "NGraph does not support CRD mode"}); + broken_tests.insert({"depthtospace_crd_mode", "NGraph does not support CRD mode"}); + broken_tests.insert({"gemm_default_no_bias", "not implemented yet for opset 11"}); + broken_tests.insert({"quantizelinear", "ambiguity in scalar dimensions [] vs [1]", {"onnx150"}}); + broken_tests.insert({"dequantizelinear", "ambiguity in scalar dimensions [] vs [1]", {"onnx150"}}); + broken_tests.insert({"mlperf_ssd_resnet34_1200", "Results mismatch"}); + broken_tests.insert({"BERT_Squad", "Invalid Feed Input Name:input4"}); + broken_tests.insert({"candy", "Results mismatch: 2 of 150528"}); + broken_tests.insert({"tf_mobilenet_v1_1.0_224", "Results mismatch"}); + broken_tests.insert({"tf_mobilenet_v2_1.0_224", "Results mismatch"}); + broken_tests.insert({"tf_mobilenet_v2_1.4_224", "Results mismatch"}); + broken_tests.insert({"convtranspose_1d", "1d convtranspose not supported yet"}); + broken_tests.insert({"convtranspose_3d", "3d convtranspose not supported yet"}); + } + + if (provider_name == "nnapi") { + broken_tests.insert({"scan9_sum", "Error with the extra graph"}); + broken_tests.insert({"scan_sum", "Error with the extra graph"}); + broken_tests.insert({"mvn_expanded", "Failed to find kernel for MemcpyFromHost(1) (node Memcpy_1)"}); + broken_tests.insert({"dynamicquantizelinear_expanded", "Temporarily disabled pending investigation"}); + broken_tests.insert({"dynamicquantizelinear_max_adjusted_expanded", "Temporarily disabled pending investigation"}); + broken_tests.insert({"dynamicquantizelinear_min_adjusted_expanded", "Temporarily disabled pending investigation"}); + broken_tests.insert({"gemm_transposeB", "Temporarily disabled pending investigation"}); + broken_tests.insert({"range_float_type_positive_delta_expanded", "Temporarily disabled pending investigation"}); + broken_tests.insert({"range_int32_type_negative_delta_expanded", "Temporarily disabled pending investigation"}); + broken_tests.insert({"convtranspose_1d", "1d convtranspose not supported yet"}); + broken_tests.insert({"convtranspose_3d", "3d convtranspose not supported yet"}); + broken_tests.insert({"maxpool_2d_uint8", "result mismatch"}); + broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NC_expanded", "shape mismatch"}); + broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1d2_expanded", "shape mismatch"}); + broken_tests.insert( + {"negative_log_likelihood_loss_input_shape_is_NCd1d2_reduction_mean_expanded", "shape mismatch"}); + broken_tests.insert( + {"negative_log_likelihood_loss_input_shape_is_NCd1d2_reduction_sum_expanded", "shape mismatch"}); + broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_expanded", "shape mismatch"}); + broken_tests.insert( + {"negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_reduction_mean_expanded", "shape mismatch"}); + broken_tests.insert( + {"negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_reduction_sum_expanded", "shape mismatch"}); + // Disable based on George Wu's recommendation. + broken_tests.insert( + {"negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_reduction_sum_ignore_index_expanded", + "shape mismatch"}); + broken_tests.insert({"negative_log_likelihood_loss_iinput_shape_is_NCd1_weight_ignore_index", "Shape mismatch"}); + broken_tests.insert( + {"negative_log_likelihood_loss_iinput_shape_is_NCd1_weight_ignore_index_expanded", "Shape mismatch"}); + broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NC", "Shape mismatch"}); + broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1", "Shape mismatch"}); + broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1_expanded", "Shape mismatch"}); + broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1_ignore_index", "Shape mismatch"}); + broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1_ignore_index_expanded", "Shape mismatch"}); + broken_tests.insert( + {"negative_log_likelihood_loss_input_shape_is_NCd1_mean_weight_negative_ignore_index", "Shape mismatch"}); + broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1_mean_weight_negative_ignore_index_expanded", + "Shape mismatch"}); + broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1_weight", "Shape mismatch"}); + broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1_weight_expanded", "Shape mismatch"}); + broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1d2", "Shape mismatch"}); + broken_tests.insert( + {"negative_log_likelihood_loss_input_shape_is_NCd1d2_no_weight_reduction_mean_ignore_index", "Shape mismatch"}); + broken_tests.insert( + {"negative_log_likelihood_loss_input_shape_is_NCd1d2_no_weight_reduction_mean_ignore_index_expanded", + "Shape mismatch"}); + broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1d2_reduction_mean", "Shape mismatch"}); + broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1d2_reduction_sum", "Shape mismatch"}); + broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight", "Shape mismatch"}); + broken_tests.insert( + {"negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_reduction_mean", "Shape mismatch"}); + broken_tests.insert( + {"negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_reduction_sum", "Shape mismatch"}); + broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_reduction_sum_ignore_index", + "Shape mismatch"}); + broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index", + "Shape mismatch"}); + broken_tests.insert( + {"negative_log_likelihood_loss_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index_expanded", + "Shape mismatch"}); + broken_tests.insert( + {"negative_log_likelihood_loss_input_shape_is_NCd1d2d3_sum_weight_high_ignore_index", "Shape mismatch"}); + broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1d2d3_sum_weight_high_ignore_index_expanded", + "Shape mismatch"}); + broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1d2d3d4d5_mean_weight", "Shape mismatch"}); + broken_tests.insert( + {"negative_log_likelihood_loss_input_shape_is_NCd1d2d3d4d5_mean_weight_expanded", "Shape mismatch"}); + broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1d2d3d4d5_none_no_weight", "Shape mismatch"}); + broken_tests.insert( + {"negative_log_likelihood_loss_input_shape_is_NCd1d2d3d4d5_none_no_weight_expanded", "Shape mismatch"}); + broken_tests.insert( + {"softmax_cross_entropy_input_shape_is_NCd1_mean_weight_negative_ignore_index", "Shape mismatch"}); + broken_tests.insert( + {"softmax_cross_entropy_input_shape_is_NCd1_mean_weight_negative_ignore_index_expanded", "Shape mismatch"}); + broken_tests.insert( + {"softmax_cross_entropy_input_shape_is_NCd1_mean_weight_negative_ignore_index_log_prob", "Shape mismatch"}); + broken_tests.insert( + {"softmax_cross_entropy_input_shape_is_NCd1_mean_weight_negative_ignore_index_log_prob_expanded", + "Shape mismatch"}); + broken_tests.insert( + {"softmax_cross_entropy_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index_expanded", + "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index_log_prob", + "Shape mismatch"}); + broken_tests.insert( + {"softmax_cross_entropy_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index_log_prob_expanded", + "Shape mismatch"}); + broken_tests.insert( + {"softmax_cross_entropy_input_shape_is_NCd1d2d3_sum_weight_high_ignore_index", "Shape mismatch"}); + broken_tests.insert( + {"softmax_cross_entropy_input_shape_is_NCd1d2d3_sum_weight_high_ignore_index_expanded", "Shape mismatch"}); + broken_tests.insert( + {"softmax_cross_entropy_input_shape_is_NCd1d2d3_sum_weight_high_ignore_index_log_prob", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_input_shape_is_NCd1d2d3_sum_weight_high_ignore_index_log_prob_expanded", + "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_mean_weight", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_mean_weight_expanded", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_mean_weight_log_prob", "Shape mismatch"}); + broken_tests.insert( + {"softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_mean_weight_log_prob_expanded", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_none_no_weight", "Shape mismatch"}); + broken_tests.insert( + {"softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_none_no_weight_expanded", "Shape mismatch"}); + broken_tests.insert( + {"softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_none_no_weight_log_prob", "Shape mismatch"}); + broken_tests.insert( + {"softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_none_no_weight_log_prob_expanded", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_mean", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_mean_3d", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_mean_3d_expanded", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_mean_3d_log_prob", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_mean_3d_log_prob_expanded", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_mean_expanded", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_mean_log_prob", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_mean_log_prob_expanded", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_mean_no_weight_ignore_index", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_mean_no_weight_ignore_index_3d", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_mean_no_weight_ignore_index_3d_expanded", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_mean_no_weight_ignore_index_3d_log_prob", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_mean_no_weight_ignore_index_3d_log_prob_expanded", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_mean_no_weight_ignore_index_4d", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_mean_no_weight_ignore_index_4d_expanded", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_mean_no_weight_ignore_index_4d_log_prob", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_mean_no_weight_ignore_index_4d_log_prob_expanded", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_mean_no_weight_ignore_index_expanded", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_mean_no_weight_ignore_index_log_prob", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_mean_no_weight_ignore_index_log_prob_expanded", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_mean_weight", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_mean_weight_expanded", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_mean_weight_ignore_index", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_mean_weight_ignore_index_3d", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_mean_weight_ignore_index_3d_expanded", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_mean_weight_ignore_index_3d_log_prob", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_mean_weight_ignore_index_3d_log_prob_expanded", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_mean_weight_ignore_index_4d", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_mean_weight_ignore_index_4d_expanded", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_mean_weight_ignore_index_4d_log_prob", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_mean_weight_ignore_index_4d_log_prob_expanded", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_mean_weight_ignore_index_expanded", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_mean_weight_ignore_index_log_prob", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_mean_weight_ignore_index_log_prob_expanded", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_mean_weight_log_prob", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_mean_weight_log_prob_expanded", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_none", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_none_expanded", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_none_log_prob", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_none_log_prob_expanded", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_none_weights", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_none_weights_expanded", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_none_weights_log_prob", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_none_weights_log_prob_expanded", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_sum", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_sum_expanded", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_sum_log_prob", "Shape mismatch"}); + broken_tests.insert({"softmax_cross_entropy_sum_log_prob_expanded", "Shape mismatch"}); + } + + if (provider_name == "dml") { + broken_tests.insert({"tinyyolov3", "The parameter is incorrect"}); + broken_tests.insert({"PixelShuffle", "Test requires 6D Reshape, which isn't supported by DirectML"}); + broken_tests.insert({"operator_permute2", "Test requires 6D Transpose, which isn't supported by DirectML"}); + broken_tests.insert({"resize_downsample_linear", + "ORT 0.4 uses asymmetric but will conform to half_pixel in the next ONNX version."}); + broken_tests.insert( + {"resize_upsample_linear", "ORT 0.4 uses asymmetric but will conform to half_pixel in the next ONNX version."}); + broken_tests.insert( + {"resize_upsample_linear", "ORT 0.4 uses asymmetric but will conform to half_pixel in the next ONNX version."}); + + // These tests are temporarily disabled pending investigation + broken_tests.insert({"dynamicquantizelinear_expanded", "Temporarily disabled pending investigation"}); + broken_tests.insert({"dynamicquantizelinear_max_adjusted_expanded", "Temporarily disabled pending investigation"}); + broken_tests.insert({"dynamicquantizelinear_min_adjusted_expanded", "Temporarily disabled pending investigation"}); + broken_tests.insert({"mxnet_arcface", "Temporarily disabled pending investigation"}); + broken_tests.insert({"yolov3", "Temporarily disabled pending investigation"}); + broken_tests.insert({"tf_inception_v2", "Temporarily disabled pending investigation"}); + broken_tests.insert({"fp16_inception_v1", "Temporarily disabled pending investigation"}); + broken_tests.insert({"candy", "Temporarily disabled pending investigation"}); + broken_tests.insert({"BERT_Squad", "Temporarily disabled pending investigation"}); + broken_tests.insert({"LSTM_Seq_lens_unpacked", "The parameter is incorrect"}); + + broken_tests.insert({"resize_downsample_scales_linear", + "DML uses half_pixel and this test assumed \"asymmetric\" but does not include \"mode\""}); + broken_tests.insert({"resize_downsample_sizes_linear_pytorch_half_pixel", + "DML does not support downsampling by such a large factor - skips input pixels"}); + broken_tests.insert({"resize_downsample_sizes_nearest", + "DML uses pixel centers for nearest, rounding 1 value off for the middle column"}); + broken_tests.insert({"resize_upsample_sizes_nearest", + "DML uses pixel centers for nearest, which makes more sense (the 3rd row mismatches)"}); + broken_tests.insert({"unsqueeze_three_axes", "DML does not support 6D tensors"}); + broken_tests.insert({"unsqueeze_unsorted_axes", "DMLdoes not support 6D tensors"}); + + broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index", + "DML does not support 5D+ tensors"}); + broken_tests.insert( + {"negative_log_likelihood_loss_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index_expanded", + "DML does not support 5D+ tensors"}); + broken_tests.insert( + {"negative_log_likelihood_loss_input_shape_is_NCd1d2d3d4d5_mean_weight", "DML does not support 5D+ tensors"}); + broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1d2d3d4d5_mean_weight_expanded", + "DML does not support 5D+ tensors"}); + broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1d2d3d4d5_none_no_weight", + "DML does not support 5D+ tensors"}); + broken_tests.insert({"negative_log_likelihood_loss_input_shape_is_NCd1d2d3d4d5_none_no_weight_expanded", + "DML does not support 5D+ tensors"}); + broken_tests.insert({"softmax_cross_entropy_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index", + "DML does not support 5D+ tensors"}); + broken_tests.insert({"softmax_cross_entropy_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index_expanded", + "DML does not support 5D+ tensors"}); + broken_tests.insert({"softmax_cross_entropy_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index_log_prob", + "DML does not support 5D+ tensors"}); + broken_tests.insert( + {"softmax_cross_entropy_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index_log_prob_expanded", + "DML does not support 5D+ tensors"}); + broken_tests.insert( + {"softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_mean_weight", "DML does not support 5D+ tensors"}); + broken_tests.insert( + {"softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_mean_weight_expanded", "DML does not support 5D+ tensors"}); + broken_tests.insert( + {"softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_mean_weight_log_prob", "DML does not support 5D+ tensors"}); + broken_tests.insert({"softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_mean_weight_log_prob_expanded", + "DML does not support 5D+ tensors"}); + broken_tests.insert( + {"softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_none_no_weight", "DML does not support 5D+ tensors"}); + broken_tests.insert({"softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_none_no_weight_expanded", + "DML does not support 5D+ tensors"}); + broken_tests.insert({"softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_none_no_weight_log_prob", + "DML does not support 5D+ tensors"}); + broken_tests.insert({"softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_none_no_weight_log_prob_expanded", + "DML does not support 5D+ tensors"}); + } + +#ifdef DISABLE_CONTRIB_OPS + broken_tests.insert({"coreml_SqueezeNet_ImageNet", "This model uses contrib ops."}); + broken_tests.insert({"keras2coreml_Permute_ImageNet", "This model uses contrib ops."}); + broken_tests.insert({"keras2coreml_ReLU_ImageNet", "This model uses contrib ops."}); + broken_tests.insert({"keras2coreml_Padding-Upsampling-Normalizer_ImageNet", "This model uses contrib ops."}); + broken_tests.insert({"tiny_yolov2", "This model uses contrib ops."}); + broken_tests.insert({"fp16_tiny_yolov2", "This model uses contrib ops."}); + broken_tests.insert({"keras2coreml_Pooling_ImageNet", "This model uses contrib ops."}); + broken_tests.insert({"keras2coreml_Padding_ImageNet", "This model uses contrib ops."}); + broken_tests.insert({"keras2coreml_Normalizer_ImageNet", "This model uses contrib ops."}); + broken_tests.insert({"keras2coreml_linear_sklearn_load_breast_cancer", "This model uses contrib ops."}); + broken_tests.insert({"keras2coreml_linear_ImageNet_small", "This model uses contrib ops."}); + broken_tests.insert({"keras2coreml_linear_ImageNet_large", "This model uses contrib ops."}); + broken_tests.insert({"keras2coreml_linear_ImageNet", "This model uses contrib ops."}); + broken_tests.insert({"keras2coreml_leakyrelu_ImageNet", "This model uses contrib ops."}); + broken_tests.insert({"keras2coreml_hard_sigmoid_ImageNet", "This model uses contrib ops."}); + broken_tests.insert({"keras2coreml_elu_ImageNet", "This model uses contrib ops."}); + broken_tests.insert({"keras2coreml_Dense_ImageNet", "This model uses contrib ops."}); + broken_tests.insert({"keras2coreml_Conv2D_ImageNet", "This model uses contrib ops."}); + broken_tests.insert({"coreml_VGG16_ImageNet", "This model uses contrib ops."}); + broken_tests.insert({"coreml_Resnet50_ImageNet", "This model uses contrib ops."}); + broken_tests.insert({"coreml_Inceptionv3_ImageNet", "This model uses contrib ops."}); + broken_tests.insert({"coreml_FNS-Candy_ImageNet", "This model uses contrib ops."}); + broken_tests.insert({"coreml_AgeNet_ImageNet", "This model uses contrib ops."}); + broken_tests.insert({"keras2coreml_thresholdedrelu_ImageNet_large", "This model uses contrib ops."}); + broken_tests.insert({"keras2coreml_thresholdedrelu_ImageNet_small", "This model uses contrib ops."}); + broken_tests.insert({"keras2coreml_thresholdedrelu_sklearn_load_breast_cancer", "This model uses contrib ops."}); + broken_tests.insert({"thresholdedrelu", "This model uses contrib ops."}); + broken_tests.insert({"thresholdedrelu_default", "This model uses contrib ops."}); + broken_tests.insert({"dynamic_slice_default_axes", "This model uses contrib ops."}); + broken_tests.insert({"thresholdedrelu_example", "This model uses contrib ops."}); + broken_tests.insert({"dynamic_slice_neg failed", "This model uses contrib ops."}); + broken_tests.insert({"dynamic_slice_start_out_of_bounds", "This model uses contrib ops."}); + broken_tests.insert({"dynamic_slice", "This model uses contrib ops."}); + broken_tests.insert({"dynamic_slice_end_out_of_bounds", "This model uses contrib ops."}); + broken_tests.insert({"dynamic_slice_neg", "This model uses contrib ops."}); + broken_tests.insert({"mvn", "This model uses contrib ops.", {"onnx130"}}); + broken_tests.insert({"cdist_float32_euclidean_1000_2000_1", "This model uses contrib ops."}); + broken_tests.insert({"cdist_float32_euclidean_1000_2000_500", "This model uses contrib ops."}); + broken_tests.insert({"cdist_float32_euclidean_1_1_1", "This model uses contrib ops."}); + broken_tests.insert({"cdist_float32_sqeuclidean_1000_2000_1", "This model uses contrib ops."}); + broken_tests.insert({"cdist_float32_sqeuclidean_1000_2000_500", "This model uses contrib ops."}); + broken_tests.insert({"cdist_float32_sqeuclidean_1_1_1", "This model uses contrib ops."}); + broken_tests.insert({"cdist_float64_euclidean_1000_2000_1", "This model uses contrib ops."}); + broken_tests.insert({"cdist_float64_euclidean_1000_2000_500", "This model uses contrib ops."}); + broken_tests.insert({"cdist_float64_euclidean_1_1_1", "This model uses contrib ops."}); + broken_tests.insert({"cdist_float64_sqeuclidean_1000_2000_1", "This model uses contrib ops."}); + broken_tests.insert({"cdist_float64_sqeuclidean_1000_2000_500", "This model uses contrib ops."}); + broken_tests.insert({"cdist_float64_sqeuclidean_1_1_1", "This model uses contrib ops."}); +#endif + + std::basic_string model_dir; + (void)GetDirNameFromFilePath(model_path, model_dir); + std::basic_string test_case_name = GetLastComponent(model_dir); + if (test_case_name.compare(0, 5, ORT_TSTR("test_")) == 0) + test_case_name = test_case_name.substr(5); + { + BrokenTest t = {ToMBString(test_case_name), ""}; + auto iter = broken_tests.find(t); + auto model_version = model_info->GetModelVersion(); + if (iter != broken_tests.end() && + (model_version == TestModelInfo::unknown_version || iter->broken_versions_.empty() || + iter->broken_versions_.find(model_version) != iter->broken_versions_.end())) { + return; + } + } + bool is_single_node = !model_info->GetNodeName().empty(); + std::vector execution_modes = {ExecutionMode::ORT_SEQUENTIAL}; + if (provider_name == "cpu" && !is_single_node) + execution_modes.push_back(ExecutionMode::ORT_PARALLEL); + + std::vector use_single_thread{false}; + // Test the model with intra op threadpool disabled + if (provider_name == "cpu" && !is_single_node) + use_single_thread.push_back(true); + + std::unique_ptr l = CreateOnnxTestCase(ToMBString(test_case_name), std::move(model_info), + per_sample_tolerance, relative_per_sample_tolerance); + for (bool is_single_thread : use_single_thread) { + for (ExecutionMode execution_mode : execution_modes) { + SessionOptions so; + if (!is_single_thread) + so.use_per_session_threads = false; + else + so.intra_op_param.thread_pool_size = 1; // Disable intra op thread pool + so.execution_mode = execution_mode; + so.session_logid = ToMBString(test_case_name); + so.session_log_severity_level = (int)logging::Severity::kERROR; + InferenceSession session_object(so, (**ort_env).GetEnvironment()); + if (provider_name == "cuda") { + ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(DefaultCudaExecutionProvider())); + } else if (provider_name == "dnnl") { + ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(DefaultDnnlExecutionProvider())); + } else if (provider_name == "ngraph") { + ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(DefaultNGraphExecutionProvider())); + } else if (provider_name == "nuphar") { + ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(DefaultNupharExecutionProvider())); + } else if (provider_name == "tensorrt") { + ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(DefaultTensorrtExecutionProvider())); + } else if (provider_name == "migraphx") { + ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(DefaultMIGraphXExecutionProvider())); + } else if (provider_name == "openvino") { + ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(DefaultOpenVINOExecutionProvider())); + } else if (provider_name == "nnapi") { + ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(DefaultNnapiExecutionProvider())); + } else if (provider_name == "rknpu") { + ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(DefaultRknpuExecutionProvider())); + } else if (provider_name == "acl") { + ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(DefaultAclExecutionProvider())); + } + if (provider_name == "armnn") { + ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(DefaultArmNNExecutionProvider())); + } + + ASSERT_STATUS_OK(session_object.Load(model_path)); + auto st = session_object.Initialize(); + if (st.Code() == NOT_IMPLEMENTED) + return; + ASSERT_TRUE(st.IsOK()) << st.ErrorMessage(); + const size_t data_count = l->GetDataCount(); + for (size_t task_id = 0; task_id != data_count; ++task_id) { + onnxruntime::test::HeapBuffer holder; + std::unordered_map feeds; + l->LoadTestData(task_id, holder, feeds, true); + + std::pair output_meta_data = session_object.GetModelOutputs(); + ASSERT_STATUS_OK(output_meta_data.first); + // Create output feed + size_t output_count = output_meta_data.second->size(); + std::vector output_names(output_count); + for (size_t i = 0; i != output_count; ++i) { + output_names[i] = (*output_meta_data.second)[i]->Name(); + } + + std::vector output_values(output_count); + { + std::unordered_map input; + for (auto& p : feeds) { + input[p.first] = *p.second; + delete p.second; + } + ASSERT_STATUS_OK(session_object.Run(input, output_names, &output_values)); + } + + bool post_procesing = false; + Status status; + ASSERT_STATUS_OK(l->GetPerSampleTolerance(&per_sample_tolerance)); + ASSERT_STATUS_OK(l->GetRelativePerSampleTolerance(&relative_per_sample_tolerance)); + ASSERT_STATUS_OK(l->GetPostProcessing(&post_procesing)); + + // TODO: if there are no output value files, just skip the validation + std::unordered_map expected_output_values; + l->LoadTestData(task_id, holder, expected_output_values, false); + + std::unordered_map name_fetch_output_map; + std::unordered_map name_output_value_info_proto; + size_t i = 0; + for (auto& output_name : output_names) { + // p_fetches is filled in the order of output_names. + name_fetch_output_map[output_name] = &output_values[i]; + const ONNX_NAMESPACE::ValueInfoProto* infoProto = l->GetOutputInfoFromModel(i); + if (infoProto != nullptr) + name_output_value_info_proto.insert(std::make_pair(infoProto->name(), infoProto)); + i++; + } + + for (auto& output : expected_output_values) { + OrtValue* expected_output_value = output.second; + const std::string& output_name = output.first; + auto iter = name_fetch_output_map.find(output_name); + ASSERT_NE(iter, name_fetch_output_map.end()); + + OrtValue* actual_output_value = iter->second; + std::pair ret = + CompareOrtValue(*actual_output_value, *expected_output_value, per_sample_tolerance, + relative_per_sample_tolerance, post_procesing); + COMPARE_RESULT compare_result = ret.first; + ASSERT_EQ(COMPARE_RESULT::SUCCESS, ret.first) << ret.second; + + const ONNX_NAMESPACE::ValueInfoProto* v = name_output_value_info_proto[output_name]; + if (v == nullptr) + continue; + ret = VerifyValueInfo(*v, Ort::Unowned{actual_output_value}); + compare_result = ret.first; + ASSERT_EQ(COMPARE_RESULT::SUCCESS, ret.first) << ret.second; + + if (compare_result != COMPARE_RESULT::SUCCESS) { + break; + } + } + for (auto& kvp : expected_output_values) { + delete kvp.second; + } + } + } + } +} + +// TODO: all providers +::std::vector<::std::basic_string> GetParameterStrings() { + std::vector provider_names; + provider_names.push_back(ORT_TSTR("cpu")); +#ifdef USE_TENSORRT + provider_names.push_back(ORT_TSTR("tensorrt")); +#endif +#ifdef USE_MIGRAPHX + provider_names.push_back(ORT_TSTR("migraphx")); +#endif +#ifdef USE_OPENVINO + provider_names.push_back(ORT_TSTR("openvino")); +#endif +#ifdef USE_CUDA + provider_names.push_back(ORT_TSTR("cuda")); +#endif +#ifdef USE_DNNL + provider_names.push_back(ORT_TSTR("dnnl")); +#endif +#ifdef USE_NGRAPH + provider_names.push_back(ORT_TSTR("ngraph")); +#endif +#ifdef USE_NUPHAR + provider_names.push_back(ORT_TSTR("nuphar")); +#endif +#ifdef USE_NNAPI + provider_names.push_back(ORT_TSTR("nnapi")); +#endif +#ifdef USE_RKNPU + provider_names.push_back(ORT_TSTR("rknpu")); +#endif +#ifdef USE_ACL + provider_names.push_back(ORT_TSTR("acl")); +#endif +#ifdef USE_ARMNN + provider_names.push_back(ORT_TSTR("armnn"); +#endif + std::vector> v; + // Permanently exclude following tests because ORT support only opset starting from 7, + // Please make no more changes to the list + static const ORTCHAR_T* immutable_broken_tests[] = { + ORT_TSTR("AvgPool1d"), + ORT_TSTR("AvgPool1d_stride"), + ORT_TSTR("AvgPool2d"), + ORT_TSTR("AvgPool2d_stride"), + ORT_TSTR("AvgPool3d"), + ORT_TSTR("AvgPool3d_stride"), + ORT_TSTR("AvgPool3d_stride1_pad0_gpu_input"), + ORT_TSTR("BatchNorm1d_3d_input_eval"), + ORT_TSTR("BatchNorm2d_eval"), + ORT_TSTR("BatchNorm2d_momentum_eval"), + ORT_TSTR("BatchNorm3d_eval"), + ORT_TSTR("BatchNorm3d_momentum_eval"), + ORT_TSTR("GLU"), + ORT_TSTR("GLU_dim"), + ORT_TSTR("Linear"), + ORT_TSTR("PReLU_1d"), + ORT_TSTR("PReLU_1d_multiparam"), + ORT_TSTR("PReLU_2d"), + ORT_TSTR("PReLU_2d_multiparam"), + ORT_TSTR("PReLU_3d"), + ORT_TSTR("PReLU_3d_multiparam"), + ORT_TSTR("PoissonNLLLLoss_no_reduce"), + ORT_TSTR("Softsign"), + ORT_TSTR("operator_add_broadcast"), + ORT_TSTR("operator_add_size1_broadcast"), + ORT_TSTR("operator_add_size1_right_broadcast"), + ORT_TSTR("operator_add_size1_singleton_broadcast"), + ORT_TSTR("operator_addconstant"), + ORT_TSTR("operator_addmm"), + ORT_TSTR("operator_basic"), + ORT_TSTR("operator_mm"), + ORT_TSTR("operator_non_float_params"), + ORT_TSTR("operator_params"), + ORT_TSTR("operator_pow"), + }; + + static const ORTCHAR_T* cuda_flaky_tests[] = { + ORT_TSTR("fp16_inception_v1"), + ORT_TSTR("fp16_shufflenet"), ORT_TSTR("fp16_tiny_yolov2"),ORT_TSTR("candy"), + ORT_TSTR("tinyyolov3"), + ORT_TSTR("mlperf_ssd_mobilenet_300"), + ORT_TSTR("mlperf_ssd_resnet34_1200"), + ORT_TSTR("tf_inception_v1"), + ORT_TSTR("faster_rcnn"), + ORT_TSTR("split_zero_size_splits"), + ORT_TSTR("convtranspose_3d")}; + static const ORTCHAR_T* openvino_disabled_tests[] = {ORT_TSTR("operator_permute2"), + ORT_TSTR("operator_repeat"), + ORT_TSTR("operator_repeat_dim_overflow"), + ORT_TSTR("mlperf_ssd_resnet34_1200"), + ORT_TSTR("candy"), + ORT_TSTR("cntk_simple_seg"), + ORT_TSTR("negative_log_likelihood_loss_input_shape_is_NCd1d2d3d4d5_mean_weight"), + ORT_TSTR("negative_log_likelihood_loss_input_shape_is_NCd1d2d3d4d5_mean_weight_expanded"), + ORT_TSTR("negative_log_likelihood_loss_input_shape_is_NCd1d2d3d4d5_none_no_weight"), + ORT_TSTR("negative_log_likelihood_loss_input_shape_is_NCd1d2d3d4d5_none_no_weight_expanded"), + ORT_TSTR("softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_mean_weight"), + ORT_TSTR("softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_mean_weight_expanded"), + ORT_TSTR("softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_mean_weight_log_prob"), + ORT_TSTR("softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_mean_weight_log_prob_expanded"), + ORT_TSTR("softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_none_no_weight"), + ORT_TSTR("softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_none_no_weight_expanded"), + ORT_TSTR("softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_none_no_weight_log_prob"), + ORT_TSTR("softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_none_no_weight_log_prob_expanded")}; + static const ORTCHAR_T* dml_disabled_tests[] = {ORT_TSTR("mlperf_ssd_resnet34_1200"), + ORT_TSTR("mlperf_ssd_mobilenet_300"), ORT_TSTR("mask_rcnn"), + ORT_TSTR("faster_rcnn"), ORT_TSTR("tf_pnasnet_large"), + ORT_TSTR("zfnet512"),ORT_TSTR("keras2coreml_Dense_ImageNet") }; + static const ORTCHAR_T* dnnl_disabled_tests[] = {ORT_TSTR("test_densenet121"), ORT_TSTR("test_resnet18v2"), + ORT_TSTR("test_resnet34v2"), ORT_TSTR("test_resnet50v2"), + ORT_TSTR("test_resnet101v2"), + ORT_TSTR("test_resnet101v2"), ORT_TSTR("test_vgg19"), + ORT_TSTR("tf_inception_resnet_v2"), ORT_TSTR("tf_inception_v1"), + ORT_TSTR("tf_inception_v3"), ORT_TSTR("tf_inception_v4"), + ORT_TSTR("tf_mobilenet_v1_1.0_224"), + ORT_TSTR("tf_mobilenet_v2_1.0_224"), + ORT_TSTR("tf_mobilenet_v2_1.4_224"), ORT_TSTR("tf_nasnet_large"), + ORT_TSTR("tf_pnasnet_large"), ORT_TSTR("tf_resnet_v1_50"), + ORT_TSTR("tf_resnet_v1_101"), ORT_TSTR("tf_resnet_v1_101"), + ORT_TSTR("tf_resnet_v2_101"), ORT_TSTR("tf_resnet_v2_152"), + ORT_TSTR("batchnorm_example_training_mode"), + ORT_TSTR("batchnorm_epsilon_training_mode"), + ORT_TSTR("tf_mobilenet_v2_1.0_224"), + ORT_TSTR("tf_mobilenet_v2_1.4_224"), + ORT_TSTR("tf_mobilenet_v1_1.0_224"), + ORT_TSTR("mobilenetv2-1.0"), + ORT_TSTR("candy"), + ORT_TSTR("range_float_type_positive_delta_expanded"), + ORT_TSTR("range_int32_type_negative_delta_expanded"), + ORT_TSTR("averagepool_2d_ceil"), + ORT_TSTR("maxpool_2d_ceil"), + ORT_TSTR("maxpool_2d_dilations"), + ORT_TSTR("mlperf_ssd_resnet34_1200"), + ORT_TSTR("convtranspose_1d"), + ORT_TSTR("convtranspose_3d"), + ORT_TSTR("maxpool_2d_uint8")}; + static const ORTCHAR_T* tensorrt_disabled_tests[] = {ORT_TSTR("udnie"), ORT_TSTR("rain_princess"), + ORT_TSTR("pointilism"), ORT_TSTR("mosaic"), + ORT_TSTR("LSTM_Seq_lens_unpacked"), + ORT_TSTR("cgan"), ORT_TSTR("candy"), + ORT_TSTR("tinyyolov3"), ORT_TSTR("yolov3"), + ORT_TSTR("mlperf_ssd_resnet34_1200"), ORT_TSTR("mlperf_ssd_mobilenet_300"), + ORT_TSTR("mask_rcnn"), + ORT_TSTR("faster_rcnn"), + ORT_TSTR("fp16_shufflenet"), + ORT_TSTR("fp16_inception_v1"), + ORT_TSTR("fp16_tiny_yolov2"), + ORT_TSTR("tf_inception_v3"), + ORT_TSTR("tf_mobilenet_v1_1.0_224"), + ORT_TSTR("tf_mobilenet_v2_1.0_224"), + ORT_TSTR("tf_mobilenet_v2_1.4_224"), + ORT_TSTR("tf_resnet_v1_101"), + ORT_TSTR("tf_resnet_v1_152"), + ORT_TSTR("tf_resnet_v1_50"), + ORT_TSTR("tf_resnet_v2_101"), + ORT_TSTR("tf_resnet_v2_152"), + ORT_TSTR("tf_resnet_v2_50"), + ORT_TSTR("convtranspose_1d"), + ORT_TSTR("convtranspose_3d"), + ORT_TSTR("conv_with_strides_and_asymmetric_padding"), + ORT_TSTR("conv_with_strides_padding"), + ORT_TSTR("size") //INVALID_ARGUMENT: Cannot find binding of given name: x + }; + for (const ORTCHAR_T* provider_name : provider_names) { + std::unordered_set> all_disabled_tests(std::begin(immutable_broken_tests), + std::end(immutable_broken_tests)); + if (CompareCString(provider_name, ORT_TSTR("cuda")) == 0) { + all_disabled_tests.insert(std::begin(cuda_flaky_tests), std::end(cuda_flaky_tests)); + } else if (CompareCString(provider_name, ORT_TSTR("dml")) == 0) { + all_disabled_tests.insert(std::begin(dml_disabled_tests), std::end(dml_disabled_tests)); + } else if (CompareCString(provider_name, ORT_TSTR("dnnl")) == 0) { + // these models run but disabled tests to keep memory utilization low + // This will be removed after LRU implementation + all_disabled_tests.insert(std::begin(dnnl_disabled_tests), std::end(dnnl_disabled_tests)); + } else if (CompareCString(provider_name, ORT_TSTR("tensorrt")) == 0) { + // these models run but disabled tests to keep memory utilization low + // This will be removed after LRU implementation + all_disabled_tests.insert(std::begin(tensorrt_disabled_tests), std::end(tensorrt_disabled_tests)); + } else if (CompareCString(provider_name, ORT_TSTR("openvino")) == 0) { + // these models run but disabled tests to keep memory utilization low + // This will be removed after LRU implementation + all_disabled_tests.insert(std::begin(openvino_disabled_tests), std::end(openvino_disabled_tests)); + } + +#if !defined(__amd64__) && !defined(_M_AMD64) + // out of memory + static const ORTCHAR_T* x86_disabled_tests[] = {ORT_TSTR("mlperf_ssd_resnet34_1200"), + ORT_TSTR("mask_rcnn_keras"), + ORT_TSTR("mask_rcnn"), + ORT_TSTR("faster_rcnn"), + ORT_TSTR("vgg19"), + ORT_TSTR("zfnet512"), + ORT_TSTR("coreml_VGG16_ImageNet")}; + all_disabled_tests.insert(std::begin(x86_disabled_tests), std::end(x86_disabled_tests)); +#endif + + std::vector> paths; +#ifdef NDEBUG +#ifdef _WIN32 + paths.push_back(ORT_TSTR("..\\models")); +#else + paths.push_back(ORT_TSTR("../models")); +#endif +#endif + +// TENSORRT has too many test failures in the single node tests +#if !defined(_WIN32) && !defined(USE_TENSORRT) + paths.push_back("/data/onnx"); +#endif + while (!paths.empty()) { + std::basic_string node_data_root_path = paths.back(); + paths.pop_back(); + std::basic_string my_dir_name = GetLastComponent(node_data_root_path); + try { + LoopDir(node_data_root_path, [&](const ORTCHAR_T* filename, OrtFileType f_type) -> bool { + if (filename[0] == ORT_TSTR('.')) + return true; + if (f_type == OrtFileType::TYPE_DIR) { + std::basic_string p = ConcatPathComponent(node_data_root_path, filename); + paths.push_back(p); + return true; + } + std::basic_string filename_str = filename; + if (!HasExtensionOf(filename_str, ORT_TSTR("onnx"))) + return true; + + std::basic_string test_case_name = my_dir_name; + if (test_case_name.compare(0, 5, ORT_TSTR("test_")) == 0) + test_case_name = test_case_name.substr(5); + if (all_disabled_tests.find(test_case_name) != all_disabled_tests.end()) + return true; + +#ifdef DISABLE_ML_OPS + auto starts_with = [](const std::basic_string& find_in, + const std::basic_string& find_what) { + return find_in.compare(0, find_what.size(), find_what) == 0; + }; + if (starts_with(test_case_name, ORT_TSTR("XGBoost_")) || starts_with(test_case_name, ORT_TSTR("coreml_")) || + starts_with(test_case_name, ORT_TSTR("scikit_")) || starts_with(test_case_name, ORT_TSTR("libsvm_"))) { + return true; + } +#endif + std::basic_string p = ConcatPathComponent(node_data_root_path, filename_str); + std::basic_string r = provider_name; + r.append(ORT_TSTR("_")).append(p); + v.emplace_back(r); + return true; + }); + } catch (std::exception&) { + } // ignore non-exist dir + } + } + return v; +} + +INSTANTIATE_TEST_SUITE_P(ModelTests, ModelTest, testing::ValuesIn(GetParameterStrings())); + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/provider_test_utils.cc b/onnxruntime/test/providers/provider_test_utils.cc index 4b9cfcfee1..8aa90982bc 100644 --- a/onnxruntime/test/providers/provider_test_utils.cc +++ b/onnxruntime/test/providers/provider_test_utils.cc @@ -643,6 +643,7 @@ void OpTester::Run( const CustomOutputVerifierFn& custom_output_verifier, const Graph::ResolveOptions& options) { SessionOptions so; + so.use_per_session_threads = false; so.session_logid = op_; so.session_log_verbosity_level = 1; so.execution_mode = execution_mode; diff --git a/onnxruntime/test/providers/test_main.cc b/onnxruntime/test/providers/test_main.cc index dae5e1d16c..e1d013e17d 100644 --- a/onnxruntime/test/providers/test_main.cc +++ b/onnxruntime/test/providers/test_main.cc @@ -37,6 +37,7 @@ #endif #include "core/session/onnxruntime_cxx_api.h" +#include "core/util/thread_utils.h" #include "gtest/gtest.h" #include "test/test_environment.h" @@ -46,7 +47,8 @@ int main(int argc, char** argv) { int status = 0; try { ::testing::InitGoogleTest(&argc, argv); - ort_env.reset(new Ort::Env(ORT_LOGGING_LEVEL_WARNING, "Default")); + OrtThreadingOptions tpo; + ort_env.reset(new Ort::Env(&tpo, ORT_LOGGING_LEVEL_WARNING, "Default")); status = RUN_ALL_TESTS(); } catch (const std::exception& ex) { std::cerr << ex.what(); diff --git a/onnxruntime/test/shared_lib/test_inference.cc b/onnxruntime/test/shared_lib/test_inference.cc index 2a4a23391e..589f46f33e 100644 --- a/onnxruntime/test/shared_lib/test_inference.cc +++ b/onnxruntime/test/shared_lib/test_inference.cc @@ -319,10 +319,11 @@ TEST(CApiTest, RegisterCustomOpForCPUAndCUDA) { } #endif -#ifndef __ANDROID__ -TEST(CApiTest, test_custom_op_library) { -#else +//It has memory leak. The OrtCustomOpDomain created in custom_op_library.cc:RegisterCustomOps function was not freed +#if defined(__ANDROID__) || defined(ONNXRUNTIME_ENABLE_MEMLEAK_CHECK) TEST(CApiTest, DISABLED_test_custom_op_library) { +#else +TEST(CApiTest, test_custom_op_library) { #endif std::cout << "Running inference using custom op shared library" << std::endl; diff --git a/orttraining/orttraining/test/graph/gradient_graph_builder_test.cc b/orttraining/orttraining/test/graph/gradient_graph_builder_test.cc index e5ff643cec..d8810ab73a 100644 --- a/orttraining/orttraining/test/graph/gradient_graph_builder_test.cc +++ b/orttraining/orttraining/test/graph/gradient_graph_builder_test.cc @@ -1510,7 +1510,7 @@ TEST(GradientGraphBuilderTest, TrainingSession_WithPipeline) { std::vector sub_model_files(num_subs); for (size_t sub_id = 0; sub_id < num_subs; ++sub_id) { - sub_model_files[sub_id] = GenerateFileNameWithIndex(ORT_TSTR("sub_"), sub_id, ORT_TSTR(".onnx")); + sub_model_files[sub_id] = GenerateFileNameWithIndex(ORT_TSTR("sub_"), static_cast(sub_id), ORT_TSTR(".onnx")); } PipelineSplitter splitter; diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 9462795096..7e4d5b0a37 100755 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -7,7 +7,6 @@ import glob import logging import multiprocessing import os -import platform import re import shutil import subprocess @@ -1313,39 +1312,6 @@ def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs): cwd=cwd, dll_path=dll_path) -def run_onnx_tests(build_dir, configs, onnx_test_data_dir, provider, - enable_multi_device_test, enable_parallel_executor_test, - num_parallel_models, num_parallel_tests=0): - for config in configs: - cwd = get_config_build_dir(build_dir, config) - if is_windows(): - exe = os.path.join(cwd, config, 'onnx_test_runner') - model_dir = os.path.join(cwd, "models") - else: - exe = os.path.join(cwd, 'onnx_test_runner') - model_dir = os.path.join(build_dir, "models") - - cmd = [] - if provider: - cmd += ["-e", provider] - if num_parallel_tests != 0: - cmd += ['-c', str(num_parallel_tests)] - if num_parallel_models > 0: - cmd += ["-j", str(num_parallel_models)] - if enable_multi_device_test: - cmd += ['-d', '1'] - # Even in release mode nuphar needs 40 minutes to run all the models tests - if config != 'Debug' and os.path.exists(model_dir) and provider != 'nuphar': - cmd.append(model_dir) - if os.path.exists(onnx_test_data_dir): - cmd.append(onnx_test_data_dir) - if config == 'Debug' and provider == 'nuphar': - return - run_subprocess([exe] + cmd, cwd=cwd) - if enable_parallel_executor_test: - run_subprocess([exe, '-x'] + cmd, cwd=cwd) - - def tensorrt_run_onnx_tests(args, build_dir, configs, onnx_test_data_dir, provider, num_parallel_models, num_parallel_tests=0): @@ -1845,52 +1811,16 @@ def main(): args, build_dir, configs, trt_onnx_test_data_dir, "tensorrt", 1) - if args.use_cuda and not args.use_tensorrt: - run_onnx_tests( - build_dir, configs, onnx_test_data_dir, 'cuda', - args.enable_multi_device_test, False, 2) - - # ngraph doesn't support opset12 yet. - # if args.use_ngraph: - # run_onnx_tests( - # build_dir, configs, onnx_test_data_dir, 'ngraph', - # args.enable_multi_device_test, True, 1) - if args.use_openvino: openvino_run_onnx_tests( build_dir, configs, onnx_test_data_dir, 'openvino', 1, 1) - # TODO: parallel executor test fails on MacOS - if args.use_nuphar: - run_onnx_tests( - build_dir, configs, onnx_test_data_dir, 'nuphar', - args.enable_multi_device_test, False, 1, 1) - - if args.use_dml: - run_onnx_tests( - build_dir, configs, onnx_test_data_dir, 'dml', - args.enable_multi_device_test, False, 1) - - if args.use_acl: - run_onnx_tests( - build_dir, configs, onnx_test_data_dir, 'acl', - args.enable_multi_device_test, False, 1, 1) # Run some models are disabled to keep memory utilization # under control. if args.use_dnnl: dnnl_run_onnx_tests(build_dir, configs, onnx_test_data_dir) - if args.use_tensorrt: - tensorrt_run_onnx_tests( - args, build_dir, configs, onnx_test_data_dir, None, 1) - else: - run_onnx_tests( - build_dir, configs, onnx_test_data_dir, None, - args.enable_multi_device_test, False, - 1 if args.x86 or platform.system() == 'Darwin' else 0, - 1 if args.x86 or platform.system() == 'Darwin' else 0) - # run nuphar python tests last, as it installs ONNX 1.5.0 if args.enable_pybind and not args.skip_onnx_tests and args.use_nuphar: nuphar_run_python_tests(build_dir, configs) diff --git a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml index c9b8b6dab4..d5106a25f8 100644 --- a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml @@ -134,6 +134,13 @@ jobs: workingDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)' displayName: 'Run tests' + - task: PublishTestResults@2 + displayName: 'Publish unit test results' + inputs: + testResultsFiles: '**/*.results.xml' + searchFolder: '$(Build.BinariesDirectory)' + testRunTitle: 'Unit Test Run' + condition: succeededOrFailed() - template: templates/component-governance-component-detection-steps.yml parameters : @@ -263,6 +270,14 @@ jobs: workingDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)' displayName: 'Run tests' + - task: PublishTestResults@2 + displayName: 'Publish unit test results' + inputs: + testResultsFiles: '**/*.results.xml' + searchFolder: '$(Build.BinariesDirectory)' + testRunTitle: 'Unit Test Run' + condition: succeededOrFailed() + - template: templates/component-governance-component-detection-steps.yml parameters : condition : 'succeeded' @@ -380,6 +395,14 @@ jobs: workingDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)' displayName: 'Run tests' + - task: PublishTestResults@2 + displayName: 'Publish unit test results' + inputs: + testResultsFiles: '**/*.results.xml' + searchFolder: '$(Build.BinariesDirectory)' + testRunTitle: 'Unit Test Run' + condition: succeededOrFailed() + - template: templates/component-governance-component-detection-steps.yml parameters : condition : 'succeeded' @@ -499,6 +522,13 @@ jobs: workingDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)' displayName: 'Run tests' + - task: PublishTestResults@2 + displayName: 'Publish unit test results' + inputs: + testResultsFiles: '**/*.results.xml' + searchFolder: '$(Build.BinariesDirectory)' + testRunTitle: 'Unit Test Run' + condition: succeededOrFailed() - template: templates/component-governance-component-detection-steps.yml parameters :