diff --git a/cmake/winml_unittests.cmake b/cmake/winml_unittests.cmake index f16f0ac0e1..76f5aab37f 100644 --- a/cmake/winml_unittests.cmake +++ b/cmake/winml_unittests.cmake @@ -148,6 +148,18 @@ function(get_winml_test_image_src set(${output_winml_test_image_src} ${winml_test_image_src} PARENT_SCOPE) endfunction() +function (get_winml_test_model_src + winml_test_src_path + output_winml_test_model_src + winml_test_model_libs) + file(GLOB winml_test_model_src CONFIGURE_DEPENDS + "${winml_test_src_path}/model/*.h" + "${winml_test_src_path}/model/*.cpp") + set(${output_winml_test_model_src} ${winml_test_model_src} PARENT_SCOPE) + set(${winml_test_model_libs} onnx_test_data_proto onnx_test_runner_common onnxruntime_common onnxruntime_mlas + onnxruntime_graph onnxruntime_test_utils onnxruntime_framework onnxruntime_flatbuffers PARENT_SCOPE) +endfunction() + file(GLOB winml_test_common_src CONFIGURE_DEPENDS "${WINML_TEST_SRC_DIR}/common/*.h" "${WINML_TEST_SRC_DIR}/common/*.cpp") @@ -253,6 +265,17 @@ add_dependencies(winml_test_adapter ${onnxruntime_EXTERNAL_DEPENDENCIES}) target_include_directories(winml_test_adapter PRIVATE ${winml_adapter_dir}) target_include_directories(winml_test_adapter PRIVATE ${winml_lib_common_dir}/inc) +# Onnxruntime memory leak checker doesn't work well with GTest static mutexes that create critical sections that cannot be freed prematurely. +if(NOT onnxruntime_ENABLE_MEMLEAK_CHECKER) + get_winml_test_model_src(${WINML_TEST_SRC_DIR} winml_test_model_src winml_test_model_libs) + add_winml_test( + TARGET winml_test_model + SOURCES ${winml_test_model_src} + LIBS winml_test_common ${winml_test_model_libs} + ) + target_precompiled_header(winml_test_model testPch.h) +endif() + # During build time, copy any modified collaterals. # configure_file(source destination COPYONLY), which configures CMake to copy the file whenever source is modified, # can't be used here because we don't know the destination during configure time (in multi-configuration generators, diff --git a/winml/lib/Common/inc/common.h b/winml/lib/Common/inc/common.h index ccfa418b49..bc6df85882 100644 --- a/winml/lib/Common/inc/common.h +++ b/winml/lib/Common/inc/common.h @@ -22,6 +22,7 @@ #include #include #include +#include // WIL #include diff --git a/winml/test/common/std.h b/winml/test/common/std.h index 1a8f132bd7..a232e3600a 100644 --- a/winml/test/common/std.h +++ b/winml/test/common/std.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -18,5 +19,4 @@ #include #include #include - #include "test.h" diff --git a/winml/test/model/model_tests.cpp b/winml/test/model/model_tests.cpp new file mode 100644 index 0000000000..8f2863a41d --- /dev/null +++ b/winml/test/model/model_tests.cpp @@ -0,0 +1,212 @@ +#include "testPch.h" +#include "test/onnx/TestCase.h" +#include "test/onnx/heap_buffer.h" +#include "test/util/include/test/compare_ortvalue.h" +#include "ort_value_helper.h" +#include "onnxruntime_cxx_api.h" +#include "StringHelpers.h" +#include "skip_model_tests.h" + +#ifndef BUILD_GOOGLE_TEST +#error Must use googletest for value-parameterized tests +#endif + +using namespace onnxruntime::test; +using namespace winml; +using namespace onnxruntime; + +namespace WinML { +// Global needed to keep the actual ITestCase alive while the tests are going on. Only ITestCase* are used as test parameters. +std::vector> ownedTests; + +class ModelTest : public testing::TestWithParam> { + protected: + void SetUp() override { + std::tie(m_testCase, m_deviceKind) = GetParam(); + WINML_EXPECT_NO_THROW(m_testCase->GetPerSampleTolerance(&m_perSampleTolerance)); + WINML_EXPECT_NO_THROW(m_testCase->GetRelativePerSampleTolerance(&m_relativePerSampleTolerance)); + WINML_EXPECT_NO_THROW(m_testCase->GetPostProcessing(&m_postProcessing)); + } + // Called after the last test in this test suite. + static void TearDownTestSuite() { + ownedTests.clear(); // clear the global vector + } + winml::LearningModelDeviceKind m_deviceKind; + ITestCase* m_testCase; + double m_perSampleTolerance = 1e-3; + double m_relativePerSampleTolerance = 1e-3; + bool m_postProcessing = false; + + void BindInputsFromFeed(LearningModelBinding& binding, std::unordered_map& feed) { + for (auto& [name, value] : feed) { + ITensor bindingValue; + WINML_EXPECT_NO_THROW(bindingValue = OrtValueHelpers::LoadTensorFromOrtValue(value)); + WINML_EXPECT_NO_THROW(binding.Bind(_winml::Strings::WStringFromString(name), bindingValue)); + } + } + + void CompareEvaluationResults(LearningModelEvaluationResult& results, + std::unordered_map& expectedOutputFeeds) { + for (const auto& [name, value] : expectedOutputFeeds) { + // Extract the output buffer from the evaluation output + std::wstring outputName = _winml::Strings::WStringFromString(name); + auto actualOutputTensorValue = results.Outputs().Lookup(outputName).as(); + BYTE* actualData; + uint32_t actualSizeInBytes; + WINML_EXPECT_HRESULT_SUCCEEDED(actualOutputTensorValue->GetBuffer(&actualData, &actualSizeInBytes)); + + // Create a copy of Ort::Value from evaluation output + auto expectedShapeAndTensorType = Ort::TensorTypeAndShapeInfo{nullptr}; + auto memoryInfo = Ort::MemoryInfo{nullptr}; + WINML_EXPECT_NO_THROW(expectedShapeAndTensorType = value.GetTensorTypeAndShapeInfo()); + WINML_EXPECT_NO_THROW(memoryInfo = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault)); + Ort::Value actualOutput = Ort::Value{nullptr}; + WINML_EXPECT_NO_THROW( + actualOutput = Ort::Value::CreateTensor( + memoryInfo, + actualData, + actualSizeInBytes, + expectedShapeAndTensorType.GetShape().data(), + expectedShapeAndTensorType.GetShape().size(), + expectedShapeAndTensorType.GetElementType())); + + // Use the expected and actual OrtValues to compare + std::pair ret = CompareOrtValue(*actualOutput, *value, m_perSampleTolerance, m_relativePerSampleTolerance, m_postProcessing); + WINML_EXPECT_EQUAL(COMPARE_RESULT::SUCCESS, ret.first) << ret.second; + } + } +}; + +TEST_P(ModelTest, Run) { + LearningModel model = nullptr; + LearningModelDevice device = nullptr; + LearningModelSession session = nullptr; + LearningModelBinding binding = nullptr; + WINML_EXPECT_NO_THROW(model = LearningModel::LoadFromFilePath(m_testCase->GetModelUrl())); + WINML_EXPECT_NO_THROW(device = LearningModelDevice(m_deviceKind)); + WINML_EXPECT_NO_THROW(session = LearningModelSession(model, device)); + WINML_EXPECT_NO_THROW(binding = LearningModelBinding(session)); + for (size_t i = 0; i < m_testCase->GetDataCount(); i++) { + // Load and bind inputs + onnxruntime::test::HeapBuffer inputHolder; + std::unordered_map inputFeeds; + WINML_EXPECT_NO_THROW(m_testCase->LoadTestData(i, inputHolder, inputFeeds, true)); + WINML_EXPECT_NO_THROW(BindInputsFromFeed(binding, inputFeeds)); + + // evaluate + LearningModelEvaluationResult results = nullptr; + WINML_EXPECT_NO_THROW(results = session.Evaluate(binding, L"Testing")); + + // Load expected outputs + onnxruntime::test::HeapBuffer outputHolder; + std::unordered_map outputFeeds; + WINML_EXPECT_NO_THROW(m_testCase->LoadTestData(i, outputHolder, outputFeeds, false)); + + // compare results + CompareEvaluationResults(results, outputFeeds); + } +} + +// Get the path of the model test collateral. Will return empty string if it doesn't exist. +std::string GetTestDataPath() { + std::string testDataPath(MAX_PATH, '\0'); + auto environmentVariableFetchSuceeded = GetEnvironmentVariableA("WINML_TEST_DATA_PATH", testDataPath.data(), MAX_PATH); + if (environmentVariableFetchSuceeded == 0 && GetLastError() == ERROR_ENVVAR_NOT_FOUND || environmentVariableFetchSuceeded > MAX_PATH) { + // if the WINML_TEST_DATA_PATH environment variable cannot be found, attempt to find the hardcoded models folder + std::wstring modulePath = FileHelpers::GetModulePath(); + std::filesystem::path currPath = modulePath.substr(0,modulePath.find_last_of(L"\\")); + std::filesystem::path parentPath = currPath.parent_path(); + auto hardcodedModelPath = parentPath.string() + "\\models"; + if (std::filesystem::exists(hardcodedModelPath) && hardcodedModelPath.length() <= MAX_PATH) { + return hardcodedModelPath; + } + } + return testDataPath; +} + +// This function returns the list of all test cases inside model test collateral +static std::vector GetAllTestCases() { + std::vector tests; + std::vector> whitelistedTestCases; + double perSampleTolerance = 1e-3; + double relativePerSampleTolerance = 1e-3; + std::unordered_set> allDisabledTests; + std::vector> dataDirs; + auto testDataPath = GetTestDataPath(); + if (testDataPath == "") return tests; + + for (auto& p : std::filesystem::directory_iterator(testDataPath.c_str())) { + if (p.is_directory()) { + dataDirs.push_back(std::move(p.path())); + } + } + + WINML_EXPECT_NO_THROW(LoadTests(dataDirs, whitelistedTestCases, perSampleTolerance, relativePerSampleTolerance, + allDisabledTests, + [&tests](std::unique_ptr l) { + tests.push_back(l.get()); + ownedTests.push_back(std::move(l)); + })); + return tests; +} + +// determine if test should be disabled +void DetermineIfDisableTest(std::string& testName, winml::LearningModelDeviceKind deviceKind) { + bool shouldSkip = false; + std::string reason = "Reason not found."; + if (disabledTests.find(testName) != disabledTests.end()) { + reason = disabledTests.at(testName); + shouldSkip = true; + } else if (deviceKind == LearningModelDeviceKind::DirectX) { + if (SkipGpuTests()) { + reason = "GPU tests are not enabled for this build."; + shouldSkip = true; + } else if (disabledGpuTests.find(testName) != disabledGpuTests.end()) { + reason = disabledGpuTests.at(testName); + shouldSkip = true; + } + } else if (disabledx86Tests.find(testName) != disabledx86Tests.end()) { +#if !defined(__amd64__) && !defined(_M_AMD64) + reason = disabledx86Tests.at(testName); + shouldSkip = true; +#endif + } + if (shouldSkip) { + printf("Disabling %s test because : %s\n", testName.c_str(), reason.c_str()); + testName = "DISABLED_" + testName; + } +} + +// This function gets the name of the test +static std::string GetNameOfTest(const testing::TestParamInfo& info) { + std::string name = ""; + auto modelPath = std::wstring(std::get<0>(info.param)->GetModelUrl()); + auto modelPathStr = _winml::Strings::UTF8FromUnicode(modelPath.c_str(), modelPath.length()); + std::vector tokenizedModelPath; + std::istringstream ss(modelPathStr); + std::string token; + while (std::getline(ss, token, '\\')) { + tokenizedModelPath.push_back(std::move(token)); + } + // The model path is structured like this "//model.onnx + // The desired naming of the test is like this __ + name += tokenizedModelPath[tokenizedModelPath.size() - 2] += "_"; // model name + name += tokenizedModelPath[tokenizedModelPath.size() - 3]; // opset version + + std::replace_if(name.begin(), name.end(), [](char c) { return !google::protobuf::ascii_isalnum(c); }, '_'); + + auto deviceKind = std::get<1>(info.param); + // Determine if test should be skipped + DetermineIfDisableTest(name, deviceKind); + if (deviceKind == winml::LearningModelDeviceKind::Cpu) { + name += "_CPU"; + } else { + name += "_GPU"; + } + + return name; +} + +INSTANTIATE_TEST_SUITE_P(ModelTests, ModelTest, testing::Combine(testing::ValuesIn(GetAllTestCases()), testing::Values(winml::LearningModelDeviceKind::Cpu, winml::LearningModelDeviceKind::DirectX)), + GetNameOfTest); +} // namespace WinML \ No newline at end of file diff --git a/winml/test/model/ort_value_helper.cpp b/winml/test/model/ort_value_helper.cpp new file mode 100644 index 0000000000..d3db477f41 --- /dev/null +++ b/winml/test/model/ort_value_helper.cpp @@ -0,0 +1,85 @@ +#include "testPch.h" +#include "ort_value_helper.h" +using namespace winml; + +namespace OrtValueHelpers { + +template +winml::ITensor CreateTensorFromShape(std::vector& shape) +{ + using WinMLTensorKind = typename ONNXTensorElementDataTypeToWinMLTensorKind::Type; + ITensor tensor = nullptr; + WINML_EXPECT_NO_THROW(tensor = WinMLTensorKind::Create(shape)); + return tensor; +} + +// This function takes in an Ort::Value and returns a copy of winml::ITensor +// TODO: String types still need to be implemented. +winml::ITensor LoadTensorFromOrtValue(Ort::Value& val) { + ITensor tensor = nullptr; + auto tensorTypeAndShape = val.GetTensorTypeAndShapeInfo(); + auto shape = tensorTypeAndShape.GetShape(); + switch (tensorTypeAndShape.GetElementType()) { + case (ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT): { + tensor = CreateTensorFromShape(shape); + break; + } + case (ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8): { + tensor = CreateTensorFromShape(shape); + break; + } + case (ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8): { + tensor = CreateTensorFromShape(shape); + } + case (ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16): { + tensor = CreateTensorFromShape(shape); + break; + } + case (ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16): { + tensor = CreateTensorFromShape(shape); + break; + } + case (ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32): { + tensor = CreateTensorFromShape(shape); + break; + } + case (ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64): { + tensor = CreateTensorFromShape(shape); + break; + } + case (ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL): { + tensor = CreateTensorFromShape(shape); + break; + } + case (ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16): { + tensor = CreateTensorFromShape(shape); + break; + } + case (ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE): { + tensor = CreateTensorFromShape(shape); + break; + } + case (ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32): { + tensor = CreateTensorFromShape(shape); + break; + } + case (ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64): { + tensor = CreateTensorFromShape(shape); + break; + } + case (ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16): { + tensor = CreateTensorFromShape(shape); + break; + } + default: + throw winrt::hresult_invalid_argument(L"TensorType not implemented yet."); + } + BYTE* actualData = nullptr; + uint32_t actualSizeInBytes = 0; + WINML_EXPECT_NO_THROW(tensor.as()->GetBuffer(&actualData, &actualSizeInBytes)); + void* ortValueTensorData = nullptr; + WINML_EXPECT_NO_THROW(Ort::GetApi().GetTensorMutableData(val, &ortValueTensorData)); + WINML_EXPECT_NO_THROW(memcpy(actualData, ortValueTensorData, actualSizeInBytes * sizeof(char))); + return tensor; +} +} // namespace OrtValueHelpers \ No newline at end of file diff --git a/winml/test/model/ort_value_helper.h b/winml/test/model/ort_value_helper.h new file mode 100644 index 0000000000..a0d46f429b --- /dev/null +++ b/winml/test/model/ort_value_helper.h @@ -0,0 +1,82 @@ +#include "testPch.h" +#include "onnxruntime_cxx_api.h" + +namespace OrtValueHelpers { +winml::ITensor LoadTensorFromOrtValue(Ort::Value& val); +} + +template +struct ONNXTensorElementDataTypeToWinMLTensorKind { + // Invalid ONNXTensorElementDataType to TensorKind + static_assert(sizeof(T) == -1, "No WinML TensorKind mapped for given ONNX Tensor Element type!"); +}; + +template <> +struct ONNXTensorElementDataTypeToWinMLTensorKind { + typedef winml::TensorFloat Type; +}; + +template <> +struct ONNXTensorElementDataTypeToWinMLTensorKind { + typedef winml::TensorUInt8Bit Type; +}; + +template <> +struct ONNXTensorElementDataTypeToWinMLTensorKind { + typedef winml::TensorInt8Bit Type; +}; + +template <> +struct ONNXTensorElementDataTypeToWinMLTensorKind { + typedef winml::TensorUInt16Bit Type; +}; + +template <> +struct ONNXTensorElementDataTypeToWinMLTensorKind { + typedef winml::TensorInt16Bit Type; +}; + +template <> +struct ONNXTensorElementDataTypeToWinMLTensorKind { + typedef winml::TensorInt32Bit Type; +}; + +template <> +struct ONNXTensorElementDataTypeToWinMLTensorKind { + typedef winml::TensorInt64Bit Type; +}; + +template <> +struct ONNXTensorElementDataTypeToWinMLTensorKind { + typedef winml::TensorString Type; +}; + +template <> +struct ONNXTensorElementDataTypeToWinMLTensorKind { + typedef winml::TensorBoolean Type; +}; + +template <> +struct ONNXTensorElementDataTypeToWinMLTensorKind { + typedef winml::TensorFloat16Bit Type; +}; + +template <> +struct ONNXTensorElementDataTypeToWinMLTensorKind { + typedef winml::TensorDouble Type; +}; + +template <> +struct ONNXTensorElementDataTypeToWinMLTensorKind { + typedef winml::TensorUInt32Bit Type; +}; + +template <> +struct ONNXTensorElementDataTypeToWinMLTensorKind { + typedef winml::TensorUInt64Bit Type; +}; + +template <> +struct ONNXTensorElementDataTypeToWinMLTensorKind { + typedef winml::TensorFloat16Bit Type; +}; \ No newline at end of file diff --git a/winml/test/model/skip_model_tests.h b/winml/test/model/skip_model_tests.h new file mode 100644 index 0000000000..070cea2321 --- /dev/null +++ b/winml/test/model/skip_model_tests.h @@ -0,0 +1,132 @@ +#pragma once +#include "common.h" + +//Need to file bugs for failing tests and add to reason. Before that happens, default reasons will be used. +static const std::string disabledTestDefaultReason = "Model not working on CPU and GPU. Please file bug and replace this reason message."; +static const std::string disabledGpuTestDefaultReason = "Model not working on GPU. Please file bug and replace this reason message."; +static const std::string disabledx86TestDefaultReason = "Model not working on x86. Please file bug and replace this reason message."; + +// {"model test name", "reason for why it is happening and bug filed for it."} +std::unordered_map disabledTests( + {// Tier 3 models + {"mxnet_arcface_opset8", disabledTestDefaultReason}, + {"XGBoost_XGClassifier_sklearn_load_wine_opset7", disabledTestDefaultReason}, + {"XGBoost_XGClassifier_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason}, + {"XGBoost_XGClassifier_OpenML_312_scene_opset7", disabledTestDefaultReason}, + {"XGBoost_XGClassifier_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason}, + {"scikit_SVC_sklearn_load_wine_opset7", disabledTestDefaultReason}, + {"scikit_SVC_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason}, + {"scikit_SVC_OpenML_312_scene_opset7", disabledTestDefaultReason}, + {"scikit_SVC_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason}, + {"scikit_Scaler_LogisticRegression_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason}, + {"scikit_Scaler_LogisticRegression_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason}, + {"scikit_RandomForestClassifier_sklearn_load_wine_opset7", disabledTestDefaultReason}, + {"scikit_RandomForestClassifier_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason}, + {"scikit_RandomForestClassifier_OpenML_312_scene_opset7", disabledTestDefaultReason}, + {"scikit_Nu_SVC_sklearn_load_wine_opset7", disabledTestDefaultReason}, + {"scikit_Nu_SVC_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason}, + {"scikit_Nu_SVC_OpenML_312_scene_opset7", disabledTestDefaultReason}, + {"scikit_Nu_SVC_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason}, + {"scikit_Normalizer_RandomForestClassifier_sklearn_load_wine_opset7", disabledTestDefaultReason}, + {"scikit_Normalizer_LinearSVC_sklearn_load_wine_opset7", disabledTestDefaultReason}, + {"scikit_LogisticRegression_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason}, + {"scikit_LogisticRegression_OpenML_31_credit_opset7", disabledTestDefaultReason}, + {"scikit_LogisticRegression_OpenML_312_scene_opset7", disabledTestDefaultReason}, + {"scikit_LogisticRegression_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason}, + {"scikit_LinearSVC_sklearn_load_wine_opset7", disabledTestDefaultReason}, + {"scikit_LabelEncoder_OpenML_3_chess_opset7", disabledTestDefaultReason}, + {"scikit_LabelEncoder_BikeSharing_opset7", disabledTestDefaultReason}, + {"scikit_Imputer_LogisticRegression_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason}, + {"scikit_Imputer_LogisticRegression_OpenML_1464_blood_transfusion_missing_opset7", disabledTestDefaultReason}, + {"scikit_Imputer_GradientBoostingClassifier_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason}, + {"scikit_Imputer_GradientBoostingClassifier_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason}, + {"scikit_GradientBoostingClassifier_sklearn_load_wine_opset7", disabledTestDefaultReason}, + {"scikit_GradientBoostingClassifier_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason}, + {"scikit_GradientBoostingClassifier_OpenML_312_scene_opset7", disabledTestDefaultReason}, + {"scikit_GradientBoostingClassifier_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason}, + {"scikit_DictVectorizer_sklearn_load_Iris_missing_opset7", disabledTestDefaultReason}, + {"scikit_DictVectorizer_sklearn_load_digits_opset7", disabledTestDefaultReason}, + {"scikit_DictVectorizer_sklearn_load_diabetes_missing_opset7", disabledTestDefaultReason}, + {"scikit_DictVectorizer_OpenML_31_credit_opset7", disabledTestDefaultReason}, + {"scikit_DictVectorizer_RandomForestRegressor_sklearn_load_diabetes_opset7", disabledTestDefaultReason}, + {"scikit_DictVectorizer_RandomForestClassifier_sklearn_load_wine_opset7", disabledTestDefaultReason}, + {"scikit_DictVectorizer_LinearSVC_sklearn_load_wine_opset7", disabledTestDefaultReason}, + {"scikit_DictVectorizer_LinearRegression_sklearn_load_diabetes_opset7", disabledTestDefaultReason}, + {"scikit_DictVectorizer_GradientBoostingRegressor_sklearn_load_boston_opset7", disabledTestDefaultReason}, + {"scikit_DecisionTreeClassifier_sklearn_load_wine_opset7", disabledTestDefaultReason}, + {"scikit_DecisionTreeClassifier_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason}, + {"scikit_DecisionTreeClassifier_OpenML_312_scene_opset7", disabledTestDefaultReason}, + {"scikit_DecisionTreeClassifier_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason}, + {"scikit_Binarization_DecisionTreeClassifier_OpenML_1492_plants_opset7", disabledTestDefaultReason}, + {"scikit_Binarization_DecisionTreeClassifier_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason}, + {"libsvm_Nu_SVC_sklearn_load_wine_opset7", disabledTestDefaultReason}, + {"libsvm_Nu_SVC_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason}, + {"libsvm_Nu_SVC_OpenML_312_scene_opset7", disabledTestDefaultReason}, + {"libsvm_Nu_SVC_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason}, + {"coreml_VGG16_ImageNet_opset7", disabledTestDefaultReason}, + {"coreml_SVC_sklearn_load_wine_opset7", disabledTestDefaultReason}, + {"coreml_SVC_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason}, + {"coreml_SVC_OpenML_312_scene_opset7", disabledTestDefaultReason}, + {"coreml_SVC_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason}, + {"coreml_SqueezeNet_ImageNet_opset7", disabledTestDefaultReason}, + {"coreml_Scaler_LogisticRegression_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason}, + {"coreml_Scaler_LogisticRegression_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason}, + {"coreml_Resnet50_ImageNet_opset7", disabledTestDefaultReason}, + {"coreml_RandomForestClassifier_sklearn_load_wine_opset7", disabledTestDefaultReason}, + {"coreml_RandomForestClassifier_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason}, + {"coreml_RandomForestClassifier_OpenML_312_scene_opset7", disabledTestDefaultReason}, + {"coreml_RandomForestClassifier_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason}, + {"coreml_Normalizer_RandomForestClassifier_sklearn_load_wine_opset7", disabledTestDefaultReason}, + {"coreml_Normalizer_LinearSVC_sklearn_load_wine_opset7", disabledTestDefaultReason}, + {"coreml_LogisticRegression_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason}, + {"coreml_LogisticRegression_OpenML_31_credit_opset7", disabledTestDefaultReason}, + {"coreml_LogisticRegression_OpenML_312_scene_opset7", disabledTestDefaultReason}, + {"coreml_LogisticRegression_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason}, + {"coreml_LinearSVC_sklearn_load_wine_opset7", disabledTestDefaultReason}, + {"coreml_LinearSVC_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason}, + {"coreml_LinearSVC_OpenML_312_scene_opset7", disabledTestDefaultReason}, + {"coreml_LinearSVC_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason}, + {"coreml_Inceptionv3_ImageNet_opset7", disabledTestDefaultReason}, + {"coreml_Imputer_LogisticRegression_OpenML_1464_blood_transfusion_missing_opset7", disabledTestDefaultReason}, + {"coreml_Imputer_GradientBoostingClassifier_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason}, + {"coreml_Imputer_GradientBoostingClassifier_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason}, + {"coreml_GradientBoostingClassifier_sklearn_load_wine_opset7", disabledTestDefaultReason}, + {"coreml_GradientBoostingClassifier_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason}, + {"coreml_GradientBoostingClassifier_OpenML_312_scene_opset7", disabledTestDefaultReason}, + {"coreml_GradientBoostingClassifier_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason}, + {"coreml_GradientBoostingClassifier_Criteo_opset7", disabledTestDefaultReason}, + {"coreml_GradientBoostingClassifier_BingClick_opset7", disabledTestDefaultReason}, + {"coreml_DictVectorizer_sklearn_load_Iris_missing_opset7", disabledTestDefaultReason}, + {"coreml_DictVectorizer_sklearn_load_digits_opset7", disabledTestDefaultReason}, + {"coreml_DictVectorizer_sklearn_load_diabetes_missing_opset7", disabledTestDefaultReason}, + {"coreml_DictVectorizer_OpenML_31_credit_opset7", disabledTestDefaultReason}, + {"coreml_DictVectorizer_RandomForestRegressor_sklearn_load_diabetes_opset7", disabledTestDefaultReason}, + {"coreml_DictVectorizer_RandomForestClassifier_sklearn_load_wine_opset7", disabledTestDefaultReason}, + {"coreml_DictVectorizer_LinearSVC_sklearn_load_wine_opset7", disabledTestDefaultReason}, + {"coreml_DictVectorizer_LinearRegression_sklearn_load_diabetes_opset7", disabledTestDefaultReason}, + {"coreml_DictVectorizer_GradientBoostingRegressor_sklearn_load_boston_opset7", disabledTestDefaultReason}, + {"coreml_DecisionTreeClassifier_sklearn_load_wine_opset7", disabledTestDefaultReason}, + {"coreml_DecisionTreeClassifier_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason}, + {"coreml_DecisionTreeClassifier_OpenML_312_scene_opset7", disabledTestDefaultReason}, + {"coreml_DecisionTreeClassifier_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason}, + {"coreml_AgeNet_ImageNet_opset7", disabledTestDefaultReason}}); + +std::unordered_map disabledGpuTests( + {{"LSTM_Seq_lens_unpacked_opset9", disabledGpuTestDefaultReason}, + {"fp16_inception_v1_opset8", disabledGpuTestDefaultReason}, + {"fp16_inception_v1_opset7", disabledGpuTestDefaultReason}, + {"mlperf_ssd_mobilenet_300_opset10", disabledGpuTestDefaultReason}, + {"mask_rcnn_opset10", disabledGpuTestDefaultReason}, + {"faster_rcnn_opset10", disabledGpuTestDefaultReason}, + {"BERT_Squad_opset10", disabledGpuTestDefaultReason}}); + +std::unordered_map disabledx86Tests( + {{"mlperf_ssd_resnet34_1200_opset10", disabledx86TestDefaultReason}, + {"mask_rcnn_opset10", disabledx86TestDefaultReason}, + {"faster_rcnn_opset10", disabledx86TestDefaultReason}, + {"test_vgg19_opset7", disabledx86TestDefaultReason}, + {"test_vgg19_opset8", disabledx86TestDefaultReason}, + {"coreml_VGG16_ImageNet_opset7", disabledx86TestDefaultReason}, + {"GPT2_LM_HEAD_opset10", disabledx86TestDefaultReason}, + {"GPT2_opset10", disabledx86TestDefaultReason}, + {"BERT_Squad_opset10", disabledx86TestDefaultReason}}); \ No newline at end of file