mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-10 17:37:14 +00:00
Add WinML Model testing (#5417)
* Model test start with float * Clean up code and add environment variable detection * Move into namespace * PR comments * Fix linker errors in latest merge to master and also fix warning * add skipping model test mechanism * Return std::string instead of writing to buffer * Address case where env variable is larger than max_path * use const static string for test reason * Disable x86 tests and don't build if ort memory checker is enabled * Add comment * Add additional failing x86 tests and ifdef for checking fo rx86 build * PR comments
This commit is contained in:
Ryan Lai
GitHub
parent
b991ee4c69
commit
f207f0bf5e
7 changed files with 536 additions and 1 deletions
|
|
@ -148,6 +148,18 @@ function(get_winml_test_image_src
|
|||
set(${output_winml_test_image_src} ${winml_test_image_src} PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
function (get_winml_test_model_src
|
||||
winml_test_src_path
|
||||
output_winml_test_model_src
|
||||
winml_test_model_libs)
|
||||
file(GLOB winml_test_model_src CONFIGURE_DEPENDS
|
||||
"${winml_test_src_path}/model/*.h"
|
||||
"${winml_test_src_path}/model/*.cpp")
|
||||
set(${output_winml_test_model_src} ${winml_test_model_src} PARENT_SCOPE)
|
||||
set(${winml_test_model_libs} onnx_test_data_proto onnx_test_runner_common onnxruntime_common onnxruntime_mlas
|
||||
onnxruntime_graph onnxruntime_test_utils onnxruntime_framework onnxruntime_flatbuffers PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
file(GLOB winml_test_common_src CONFIGURE_DEPENDS
|
||||
"${WINML_TEST_SRC_DIR}/common/*.h"
|
||||
"${WINML_TEST_SRC_DIR}/common/*.cpp")
|
||||
|
|
@ -253,6 +265,17 @@ add_dependencies(winml_test_adapter ${onnxruntime_EXTERNAL_DEPENDENCIES})
|
|||
target_include_directories(winml_test_adapter PRIVATE ${winml_adapter_dir})
|
||||
target_include_directories(winml_test_adapter PRIVATE ${winml_lib_common_dir}/inc)
|
||||
|
||||
# Onnxruntime memory leak checker doesn't work well with GTest static mutexes that create critical sections that cannot be freed prematurely.
|
||||
if(NOT onnxruntime_ENABLE_MEMLEAK_CHECKER)
|
||||
get_winml_test_model_src(${WINML_TEST_SRC_DIR} winml_test_model_src winml_test_model_libs)
|
||||
add_winml_test(
|
||||
TARGET winml_test_model
|
||||
SOURCES ${winml_test_model_src}
|
||||
LIBS winml_test_common ${winml_test_model_libs}
|
||||
)
|
||||
target_precompiled_header(winml_test_model testPch.h)
|
||||
endif()
|
||||
|
||||
# During build time, copy any modified collaterals.
|
||||
# configure_file(source destination COPYONLY), which configures CMake to copy the file whenever source is modified,
|
||||
# can't be used here because we don't know the destination during configure time (in multi-configuration generators,
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@
|
|||
#include <variant>
|
||||
#include <vector>
|
||||
#include <thread>
|
||||
#include <tuple>
|
||||
|
||||
// WIL
|
||||
#include <wil/cppwinrt.h>
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@
|
|||
#include <algorithm>
|
||||
#include <codecvt>
|
||||
#include <fcntl.h>
|
||||
#include <filesystem>
|
||||
#include <future>
|
||||
#include <io.h>
|
||||
#include <locale>
|
||||
|
|
@ -18,5 +19,4 @@
|
|||
#include <utility>
|
||||
#include <vector>
|
||||
#include <sstream>
|
||||
|
||||
#include "test.h"
|
||||
|
|
|
|||
212
winml/test/model/model_tests.cpp
Normal file
212
winml/test/model/model_tests.cpp
Normal file
|
|
@ -0,0 +1,212 @@
|
|||
#include "testPch.h"
|
||||
#include "test/onnx/TestCase.h"
|
||||
#include "test/onnx/heap_buffer.h"
|
||||
#include "test/util/include/test/compare_ortvalue.h"
|
||||
#include "ort_value_helper.h"
|
||||
#include "onnxruntime_cxx_api.h"
|
||||
#include "StringHelpers.h"
|
||||
#include "skip_model_tests.h"
|
||||
|
||||
#ifndef BUILD_GOOGLE_TEST
|
||||
#error Must use googletest for value-parameterized tests
|
||||
#endif
|
||||
|
||||
using namespace onnxruntime::test;
|
||||
using namespace winml;
|
||||
using namespace onnxruntime;
|
||||
|
||||
namespace WinML {
|
||||
// Global needed to keep the actual ITestCase alive while the tests are going on. Only ITestCase* are used as test parameters.
|
||||
std::vector<std::unique_ptr<ITestCase>> ownedTests;
|
||||
|
||||
class ModelTest : public testing::TestWithParam<std::tuple<ITestCase*, winml::LearningModelDeviceKind>> {
|
||||
protected:
|
||||
void SetUp() override {
|
||||
std::tie(m_testCase, m_deviceKind) = GetParam();
|
||||
WINML_EXPECT_NO_THROW(m_testCase->GetPerSampleTolerance(&m_perSampleTolerance));
|
||||
WINML_EXPECT_NO_THROW(m_testCase->GetRelativePerSampleTolerance(&m_relativePerSampleTolerance));
|
||||
WINML_EXPECT_NO_THROW(m_testCase->GetPostProcessing(&m_postProcessing));
|
||||
}
|
||||
// Called after the last test in this test suite.
|
||||
static void TearDownTestSuite() {
|
||||
ownedTests.clear(); // clear the global vector
|
||||
}
|
||||
winml::LearningModelDeviceKind m_deviceKind;
|
||||
ITestCase* m_testCase;
|
||||
double m_perSampleTolerance = 1e-3;
|
||||
double m_relativePerSampleTolerance = 1e-3;
|
||||
bool m_postProcessing = false;
|
||||
|
||||
void BindInputsFromFeed(LearningModelBinding& binding, std::unordered_map<std::string, Ort::Value>& feed) {
|
||||
for (auto& [name, value] : feed) {
|
||||
ITensor bindingValue;
|
||||
WINML_EXPECT_NO_THROW(bindingValue = OrtValueHelpers::LoadTensorFromOrtValue(value));
|
||||
WINML_EXPECT_NO_THROW(binding.Bind(_winml::Strings::WStringFromString(name), bindingValue));
|
||||
}
|
||||
}
|
||||
|
||||
void CompareEvaluationResults(LearningModelEvaluationResult& results,
|
||||
std::unordered_map<std::string, Ort::Value>& expectedOutputFeeds) {
|
||||
for (const auto& [name, value] : expectedOutputFeeds) {
|
||||
// Extract the output buffer from the evaluation output
|
||||
std::wstring outputName = _winml::Strings::WStringFromString(name);
|
||||
auto actualOutputTensorValue = results.Outputs().Lookup(outputName).as<ITensorNative>();
|
||||
BYTE* actualData;
|
||||
uint32_t actualSizeInBytes;
|
||||
WINML_EXPECT_HRESULT_SUCCEEDED(actualOutputTensorValue->GetBuffer(&actualData, &actualSizeInBytes));
|
||||
|
||||
// Create a copy of Ort::Value from evaluation output
|
||||
auto expectedShapeAndTensorType = Ort::TensorTypeAndShapeInfo{nullptr};
|
||||
auto memoryInfo = Ort::MemoryInfo{nullptr};
|
||||
WINML_EXPECT_NO_THROW(expectedShapeAndTensorType = value.GetTensorTypeAndShapeInfo());
|
||||
WINML_EXPECT_NO_THROW(memoryInfo = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault));
|
||||
Ort::Value actualOutput = Ort::Value{nullptr};
|
||||
WINML_EXPECT_NO_THROW(
|
||||
actualOutput = Ort::Value::CreateTensor(
|
||||
memoryInfo,
|
||||
actualData,
|
||||
actualSizeInBytes,
|
||||
expectedShapeAndTensorType.GetShape().data(),
|
||||
expectedShapeAndTensorType.GetShape().size(),
|
||||
expectedShapeAndTensorType.GetElementType()));
|
||||
|
||||
// Use the expected and actual OrtValues to compare
|
||||
std::pair<COMPARE_RESULT, std::string> ret = CompareOrtValue(*actualOutput, *value, m_perSampleTolerance, m_relativePerSampleTolerance, m_postProcessing);
|
||||
WINML_EXPECT_EQUAL(COMPARE_RESULT::SUCCESS, ret.first) << ret.second;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
TEST_P(ModelTest, Run) {
|
||||
LearningModel model = nullptr;
|
||||
LearningModelDevice device = nullptr;
|
||||
LearningModelSession session = nullptr;
|
||||
LearningModelBinding binding = nullptr;
|
||||
WINML_EXPECT_NO_THROW(model = LearningModel::LoadFromFilePath(m_testCase->GetModelUrl()));
|
||||
WINML_EXPECT_NO_THROW(device = LearningModelDevice(m_deviceKind));
|
||||
WINML_EXPECT_NO_THROW(session = LearningModelSession(model, device));
|
||||
WINML_EXPECT_NO_THROW(binding = LearningModelBinding(session));
|
||||
for (size_t i = 0; i < m_testCase->GetDataCount(); i++) {
|
||||
// Load and bind inputs
|
||||
onnxruntime::test::HeapBuffer inputHolder;
|
||||
std::unordered_map<std::string, Ort::Value> inputFeeds;
|
||||
WINML_EXPECT_NO_THROW(m_testCase->LoadTestData(i, inputHolder, inputFeeds, true));
|
||||
WINML_EXPECT_NO_THROW(BindInputsFromFeed(binding, inputFeeds));
|
||||
|
||||
// evaluate
|
||||
LearningModelEvaluationResult results = nullptr;
|
||||
WINML_EXPECT_NO_THROW(results = session.Evaluate(binding, L"Testing"));
|
||||
|
||||
// Load expected outputs
|
||||
onnxruntime::test::HeapBuffer outputHolder;
|
||||
std::unordered_map<std::string, Ort::Value> outputFeeds;
|
||||
WINML_EXPECT_NO_THROW(m_testCase->LoadTestData(i, outputHolder, outputFeeds, false));
|
||||
|
||||
// compare results
|
||||
CompareEvaluationResults(results, outputFeeds);
|
||||
}
|
||||
}
|
||||
|
||||
// Get the path of the model test collateral. Will return empty string if it doesn't exist.
|
||||
std::string GetTestDataPath() {
|
||||
std::string testDataPath(MAX_PATH, '\0');
|
||||
auto environmentVariableFetchSuceeded = GetEnvironmentVariableA("WINML_TEST_DATA_PATH", testDataPath.data(), MAX_PATH);
|
||||
if (environmentVariableFetchSuceeded == 0 && GetLastError() == ERROR_ENVVAR_NOT_FOUND || environmentVariableFetchSuceeded > MAX_PATH) {
|
||||
// if the WINML_TEST_DATA_PATH environment variable cannot be found, attempt to find the hardcoded models folder
|
||||
std::wstring modulePath = FileHelpers::GetModulePath();
|
||||
std::filesystem::path currPath = modulePath.substr(0,modulePath.find_last_of(L"\\"));
|
||||
std::filesystem::path parentPath = currPath.parent_path();
|
||||
auto hardcodedModelPath = parentPath.string() + "\\models";
|
||||
if (std::filesystem::exists(hardcodedModelPath) && hardcodedModelPath.length() <= MAX_PATH) {
|
||||
return hardcodedModelPath;
|
||||
}
|
||||
}
|
||||
return testDataPath;
|
||||
}
|
||||
|
||||
// This function returns the list of all test cases inside model test collateral
|
||||
static std::vector<ITestCase*> GetAllTestCases() {
|
||||
std::vector<ITestCase*> tests;
|
||||
std::vector<std::basic_string<PATH_CHAR_TYPE>> whitelistedTestCases;
|
||||
double perSampleTolerance = 1e-3;
|
||||
double relativePerSampleTolerance = 1e-3;
|
||||
std::unordered_set<std::basic_string<ORTCHAR_T>> allDisabledTests;
|
||||
std::vector<std::basic_string<PATH_CHAR_TYPE>> dataDirs;
|
||||
auto testDataPath = GetTestDataPath();
|
||||
if (testDataPath == "") return tests;
|
||||
|
||||
for (auto& p : std::filesystem::directory_iterator(testDataPath.c_str())) {
|
||||
if (p.is_directory()) {
|
||||
dataDirs.push_back(std::move(p.path()));
|
||||
}
|
||||
}
|
||||
|
||||
WINML_EXPECT_NO_THROW(LoadTests(dataDirs, whitelistedTestCases, perSampleTolerance, relativePerSampleTolerance,
|
||||
allDisabledTests,
|
||||
[&tests](std::unique_ptr<ITestCase> l) {
|
||||
tests.push_back(l.get());
|
||||
ownedTests.push_back(std::move(l));
|
||||
}));
|
||||
return tests;
|
||||
}
|
||||
|
||||
// determine if test should be disabled
|
||||
void DetermineIfDisableTest(std::string& testName, winml::LearningModelDeviceKind deviceKind) {
|
||||
bool shouldSkip = false;
|
||||
std::string reason = "Reason not found.";
|
||||
if (disabledTests.find(testName) != disabledTests.end()) {
|
||||
reason = disabledTests.at(testName);
|
||||
shouldSkip = true;
|
||||
} else if (deviceKind == LearningModelDeviceKind::DirectX) {
|
||||
if (SkipGpuTests()) {
|
||||
reason = "GPU tests are not enabled for this build.";
|
||||
shouldSkip = true;
|
||||
} else if (disabledGpuTests.find(testName) != disabledGpuTests.end()) {
|
||||
reason = disabledGpuTests.at(testName);
|
||||
shouldSkip = true;
|
||||
}
|
||||
} else if (disabledx86Tests.find(testName) != disabledx86Tests.end()) {
|
||||
#if !defined(__amd64__) && !defined(_M_AMD64)
|
||||
reason = disabledx86Tests.at(testName);
|
||||
shouldSkip = true;
|
||||
#endif
|
||||
}
|
||||
if (shouldSkip) {
|
||||
printf("Disabling %s test because : %s\n", testName.c_str(), reason.c_str());
|
||||
testName = "DISABLED_" + testName;
|
||||
}
|
||||
}
|
||||
|
||||
// This function gets the name of the test
|
||||
static std::string GetNameOfTest(const testing::TestParamInfo<ModelTest::ParamType>& info) {
|
||||
std::string name = "";
|
||||
auto modelPath = std::wstring(std::get<0>(info.param)->GetModelUrl());
|
||||
auto modelPathStr = _winml::Strings::UTF8FromUnicode(modelPath.c_str(), modelPath.length());
|
||||
std::vector<std::string> tokenizedModelPath;
|
||||
std::istringstream ss(modelPathStr);
|
||||
std::string token;
|
||||
while (std::getline(ss, token, '\\')) {
|
||||
tokenizedModelPath.push_back(std::move(token));
|
||||
}
|
||||
// The model path is structured like this "<opset>/<model_name>/model.onnx
|
||||
// The desired naming of the test is like this <model_name>_<opset>_<CPU/GPU>
|
||||
name += tokenizedModelPath[tokenizedModelPath.size() - 2] += "_"; // model name
|
||||
name += tokenizedModelPath[tokenizedModelPath.size() - 3]; // opset version
|
||||
|
||||
std::replace_if(name.begin(), name.end(), [](char c) { return !google::protobuf::ascii_isalnum(c); }, '_');
|
||||
|
||||
auto deviceKind = std::get<1>(info.param);
|
||||
// Determine if test should be skipped
|
||||
DetermineIfDisableTest(name, deviceKind);
|
||||
if (deviceKind == winml::LearningModelDeviceKind::Cpu) {
|
||||
name += "_CPU";
|
||||
} else {
|
||||
name += "_GPU";
|
||||
}
|
||||
|
||||
return name;
|
||||
}
|
||||
|
||||
INSTANTIATE_TEST_SUITE_P(ModelTests, ModelTest, testing::Combine(testing::ValuesIn(GetAllTestCases()), testing::Values(winml::LearningModelDeviceKind::Cpu, winml::LearningModelDeviceKind::DirectX)),
|
||||
GetNameOfTest);
|
||||
} // namespace WinML
|
||||
85
winml/test/model/ort_value_helper.cpp
Normal file
85
winml/test/model/ort_value_helper.cpp
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
#include "testPch.h"
|
||||
#include "ort_value_helper.h"
|
||||
using namespace winml;
|
||||
|
||||
namespace OrtValueHelpers {
|
||||
|
||||
template <ONNXTensorElementDataType T>
|
||||
winml::ITensor CreateTensorFromShape(std::vector<int64_t>& shape)
|
||||
{
|
||||
using WinMLTensorKind = typename ONNXTensorElementDataTypeToWinMLTensorKind<T>::Type;
|
||||
ITensor tensor = nullptr;
|
||||
WINML_EXPECT_NO_THROW(tensor = WinMLTensorKind::Create(shape));
|
||||
return tensor;
|
||||
}
|
||||
|
||||
// This function takes in an Ort::Value and returns a copy of winml::ITensor
|
||||
// TODO: String types still need to be implemented.
|
||||
winml::ITensor LoadTensorFromOrtValue(Ort::Value& val) {
|
||||
ITensor tensor = nullptr;
|
||||
auto tensorTypeAndShape = val.GetTensorTypeAndShapeInfo();
|
||||
auto shape = tensorTypeAndShape.GetShape();
|
||||
switch (tensorTypeAndShape.GetElementType()) {
|
||||
case (ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT): {
|
||||
tensor = CreateTensorFromShape<ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT>(shape);
|
||||
break;
|
||||
}
|
||||
case (ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8): {
|
||||
tensor = CreateTensorFromShape<ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8>(shape);
|
||||
break;
|
||||
}
|
||||
case (ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8): {
|
||||
tensor = CreateTensorFromShape<ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8>(shape);
|
||||
}
|
||||
case (ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16): {
|
||||
tensor = CreateTensorFromShape<ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16>(shape);
|
||||
break;
|
||||
}
|
||||
case (ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16): {
|
||||
tensor = CreateTensorFromShape<ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16>(shape);
|
||||
break;
|
||||
}
|
||||
case (ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32): {
|
||||
tensor = CreateTensorFromShape<ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32>(shape);
|
||||
break;
|
||||
}
|
||||
case (ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64): {
|
||||
tensor = CreateTensorFromShape<ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64>(shape);
|
||||
break;
|
||||
}
|
||||
case (ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL): {
|
||||
tensor = CreateTensorFromShape<ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL>(shape);
|
||||
break;
|
||||
}
|
||||
case (ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16): {
|
||||
tensor = CreateTensorFromShape<ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16>(shape);
|
||||
break;
|
||||
}
|
||||
case (ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE): {
|
||||
tensor = CreateTensorFromShape<ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE>(shape);
|
||||
break;
|
||||
}
|
||||
case (ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32): {
|
||||
tensor = CreateTensorFromShape<ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32>(shape);
|
||||
break;
|
||||
}
|
||||
case (ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64): {
|
||||
tensor = CreateTensorFromShape<ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64>(shape);
|
||||
break;
|
||||
}
|
||||
case (ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16): {
|
||||
tensor = CreateTensorFromShape<ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16>(shape);
|
||||
break;
|
||||
}
|
||||
default:
|
||||
throw winrt::hresult_invalid_argument(L"TensorType not implemented yet.");
|
||||
}
|
||||
BYTE* actualData = nullptr;
|
||||
uint32_t actualSizeInBytes = 0;
|
||||
WINML_EXPECT_NO_THROW(tensor.as<ITensorNative>()->GetBuffer(&actualData, &actualSizeInBytes));
|
||||
void* ortValueTensorData = nullptr;
|
||||
WINML_EXPECT_NO_THROW(Ort::GetApi().GetTensorMutableData(val, &ortValueTensorData));
|
||||
WINML_EXPECT_NO_THROW(memcpy(actualData, ortValueTensorData, actualSizeInBytes * sizeof(char)));
|
||||
return tensor;
|
||||
}
|
||||
} // namespace OrtValueHelpers
|
||||
82
winml/test/model/ort_value_helper.h
Normal file
82
winml/test/model/ort_value_helper.h
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
#include "testPch.h"
|
||||
#include "onnxruntime_cxx_api.h"
|
||||
|
||||
namespace OrtValueHelpers {
|
||||
winml::ITensor LoadTensorFromOrtValue(Ort::Value& val);
|
||||
}
|
||||
|
||||
template <ONNXTensorElementDataType T>
|
||||
struct ONNXTensorElementDataTypeToWinMLTensorKind {
|
||||
// Invalid ONNXTensorElementDataType to TensorKind
|
||||
static_assert(sizeof(T) == -1, "No WinML TensorKind mapped for given ONNX Tensor Element type!");
|
||||
};
|
||||
|
||||
template <>
|
||||
struct ONNXTensorElementDataTypeToWinMLTensorKind<ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT> {
|
||||
typedef winml::TensorFloat Type;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct ONNXTensorElementDataTypeToWinMLTensorKind<ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8> {
|
||||
typedef winml::TensorUInt8Bit Type;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct ONNXTensorElementDataTypeToWinMLTensorKind<ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8> {
|
||||
typedef winml::TensorInt8Bit Type;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct ONNXTensorElementDataTypeToWinMLTensorKind<ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16> {
|
||||
typedef winml::TensorUInt16Bit Type;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct ONNXTensorElementDataTypeToWinMLTensorKind<ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16> {
|
||||
typedef winml::TensorInt16Bit Type;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct ONNXTensorElementDataTypeToWinMLTensorKind<ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32> {
|
||||
typedef winml::TensorInt32Bit Type;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct ONNXTensorElementDataTypeToWinMLTensorKind<ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64> {
|
||||
typedef winml::TensorInt64Bit Type;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct ONNXTensorElementDataTypeToWinMLTensorKind<ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING> {
|
||||
typedef winml::TensorString Type;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct ONNXTensorElementDataTypeToWinMLTensorKind<ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL> {
|
||||
typedef winml::TensorBoolean Type;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct ONNXTensorElementDataTypeToWinMLTensorKind<ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16> {
|
||||
typedef winml::TensorFloat16Bit Type;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct ONNXTensorElementDataTypeToWinMLTensorKind<ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE> {
|
||||
typedef winml::TensorDouble Type;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct ONNXTensorElementDataTypeToWinMLTensorKind<ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32> {
|
||||
typedef winml::TensorUInt32Bit Type;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct ONNXTensorElementDataTypeToWinMLTensorKind<ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64> {
|
||||
typedef winml::TensorUInt64Bit Type;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct ONNXTensorElementDataTypeToWinMLTensorKind<ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16> {
|
||||
typedef winml::TensorFloat16Bit Type;
|
||||
};
|
||||
132
winml/test/model/skip_model_tests.h
Normal file
132
winml/test/model/skip_model_tests.h
Normal file
|
|
@ -0,0 +1,132 @@
|
|||
#pragma once
|
||||
#include "common.h"
|
||||
|
||||
//Need to file bugs for failing tests and add to reason. Before that happens, default reasons will be used.
|
||||
static const std::string disabledTestDefaultReason = "Model not working on CPU and GPU. Please file bug and replace this reason message.";
|
||||
static const std::string disabledGpuTestDefaultReason = "Model not working on GPU. Please file bug and replace this reason message.";
|
||||
static const std::string disabledx86TestDefaultReason = "Model not working on x86. Please file bug and replace this reason message.";
|
||||
|
||||
// {"model test name", "reason for why it is happening and bug filed for it."}
|
||||
std::unordered_map<std::string, std::string> disabledTests(
|
||||
{// Tier 3 models
|
||||
{"mxnet_arcface_opset8", disabledTestDefaultReason},
|
||||
{"XGBoost_XGClassifier_sklearn_load_wine_opset7", disabledTestDefaultReason},
|
||||
{"XGBoost_XGClassifier_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason},
|
||||
{"XGBoost_XGClassifier_OpenML_312_scene_opset7", disabledTestDefaultReason},
|
||||
{"XGBoost_XGClassifier_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason},
|
||||
{"scikit_SVC_sklearn_load_wine_opset7", disabledTestDefaultReason},
|
||||
{"scikit_SVC_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason},
|
||||
{"scikit_SVC_OpenML_312_scene_opset7", disabledTestDefaultReason},
|
||||
{"scikit_SVC_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason},
|
||||
{"scikit_Scaler_LogisticRegression_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason},
|
||||
{"scikit_Scaler_LogisticRegression_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason},
|
||||
{"scikit_RandomForestClassifier_sklearn_load_wine_opset7", disabledTestDefaultReason},
|
||||
{"scikit_RandomForestClassifier_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason},
|
||||
{"scikit_RandomForestClassifier_OpenML_312_scene_opset7", disabledTestDefaultReason},
|
||||
{"scikit_Nu_SVC_sklearn_load_wine_opset7", disabledTestDefaultReason},
|
||||
{"scikit_Nu_SVC_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason},
|
||||
{"scikit_Nu_SVC_OpenML_312_scene_opset7", disabledTestDefaultReason},
|
||||
{"scikit_Nu_SVC_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason},
|
||||
{"scikit_Normalizer_RandomForestClassifier_sklearn_load_wine_opset7", disabledTestDefaultReason},
|
||||
{"scikit_Normalizer_LinearSVC_sklearn_load_wine_opset7", disabledTestDefaultReason},
|
||||
{"scikit_LogisticRegression_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason},
|
||||
{"scikit_LogisticRegression_OpenML_31_credit_opset7", disabledTestDefaultReason},
|
||||
{"scikit_LogisticRegression_OpenML_312_scene_opset7", disabledTestDefaultReason},
|
||||
{"scikit_LogisticRegression_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason},
|
||||
{"scikit_LinearSVC_sklearn_load_wine_opset7", disabledTestDefaultReason},
|
||||
{"scikit_LabelEncoder_OpenML_3_chess_opset7", disabledTestDefaultReason},
|
||||
{"scikit_LabelEncoder_BikeSharing_opset7", disabledTestDefaultReason},
|
||||
{"scikit_Imputer_LogisticRegression_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason},
|
||||
{"scikit_Imputer_LogisticRegression_OpenML_1464_blood_transfusion_missing_opset7", disabledTestDefaultReason},
|
||||
{"scikit_Imputer_GradientBoostingClassifier_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason},
|
||||
{"scikit_Imputer_GradientBoostingClassifier_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason},
|
||||
{"scikit_GradientBoostingClassifier_sklearn_load_wine_opset7", disabledTestDefaultReason},
|
||||
{"scikit_GradientBoostingClassifier_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason},
|
||||
{"scikit_GradientBoostingClassifier_OpenML_312_scene_opset7", disabledTestDefaultReason},
|
||||
{"scikit_GradientBoostingClassifier_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason},
|
||||
{"scikit_DictVectorizer_sklearn_load_Iris_missing_opset7", disabledTestDefaultReason},
|
||||
{"scikit_DictVectorizer_sklearn_load_digits_opset7", disabledTestDefaultReason},
|
||||
{"scikit_DictVectorizer_sklearn_load_diabetes_missing_opset7", disabledTestDefaultReason},
|
||||
{"scikit_DictVectorizer_OpenML_31_credit_opset7", disabledTestDefaultReason},
|
||||
{"scikit_DictVectorizer_RandomForestRegressor_sklearn_load_diabetes_opset7", disabledTestDefaultReason},
|
||||
{"scikit_DictVectorizer_RandomForestClassifier_sklearn_load_wine_opset7", disabledTestDefaultReason},
|
||||
{"scikit_DictVectorizer_LinearSVC_sklearn_load_wine_opset7", disabledTestDefaultReason},
|
||||
{"scikit_DictVectorizer_LinearRegression_sklearn_load_diabetes_opset7", disabledTestDefaultReason},
|
||||
{"scikit_DictVectorizer_GradientBoostingRegressor_sklearn_load_boston_opset7", disabledTestDefaultReason},
|
||||
{"scikit_DecisionTreeClassifier_sklearn_load_wine_opset7", disabledTestDefaultReason},
|
||||
{"scikit_DecisionTreeClassifier_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason},
|
||||
{"scikit_DecisionTreeClassifier_OpenML_312_scene_opset7", disabledTestDefaultReason},
|
||||
{"scikit_DecisionTreeClassifier_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason},
|
||||
{"scikit_Binarization_DecisionTreeClassifier_OpenML_1492_plants_opset7", disabledTestDefaultReason},
|
||||
{"scikit_Binarization_DecisionTreeClassifier_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason},
|
||||
{"libsvm_Nu_SVC_sklearn_load_wine_opset7", disabledTestDefaultReason},
|
||||
{"libsvm_Nu_SVC_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason},
|
||||
{"libsvm_Nu_SVC_OpenML_312_scene_opset7", disabledTestDefaultReason},
|
||||
{"libsvm_Nu_SVC_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason},
|
||||
{"coreml_VGG16_ImageNet_opset7", disabledTestDefaultReason},
|
||||
{"coreml_SVC_sklearn_load_wine_opset7", disabledTestDefaultReason},
|
||||
{"coreml_SVC_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason},
|
||||
{"coreml_SVC_OpenML_312_scene_opset7", disabledTestDefaultReason},
|
||||
{"coreml_SVC_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason},
|
||||
{"coreml_SqueezeNet_ImageNet_opset7", disabledTestDefaultReason},
|
||||
{"coreml_Scaler_LogisticRegression_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason},
|
||||
{"coreml_Scaler_LogisticRegression_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason},
|
||||
{"coreml_Resnet50_ImageNet_opset7", disabledTestDefaultReason},
|
||||
{"coreml_RandomForestClassifier_sklearn_load_wine_opset7", disabledTestDefaultReason},
|
||||
{"coreml_RandomForestClassifier_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason},
|
||||
{"coreml_RandomForestClassifier_OpenML_312_scene_opset7", disabledTestDefaultReason},
|
||||
{"coreml_RandomForestClassifier_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason},
|
||||
{"coreml_Normalizer_RandomForestClassifier_sklearn_load_wine_opset7", disabledTestDefaultReason},
|
||||
{"coreml_Normalizer_LinearSVC_sklearn_load_wine_opset7", disabledTestDefaultReason},
|
||||
{"coreml_LogisticRegression_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason},
|
||||
{"coreml_LogisticRegression_OpenML_31_credit_opset7", disabledTestDefaultReason},
|
||||
{"coreml_LogisticRegression_OpenML_312_scene_opset7", disabledTestDefaultReason},
|
||||
{"coreml_LogisticRegression_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason},
|
||||
{"coreml_LinearSVC_sklearn_load_wine_opset7", disabledTestDefaultReason},
|
||||
{"coreml_LinearSVC_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason},
|
||||
{"coreml_LinearSVC_OpenML_312_scene_opset7", disabledTestDefaultReason},
|
||||
{"coreml_LinearSVC_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason},
|
||||
{"coreml_Inceptionv3_ImageNet_opset7", disabledTestDefaultReason},
|
||||
{"coreml_Imputer_LogisticRegression_OpenML_1464_blood_transfusion_missing_opset7", disabledTestDefaultReason},
|
||||
{"coreml_Imputer_GradientBoostingClassifier_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason},
|
||||
{"coreml_Imputer_GradientBoostingClassifier_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason},
|
||||
{"coreml_GradientBoostingClassifier_sklearn_load_wine_opset7", disabledTestDefaultReason},
|
||||
{"coreml_GradientBoostingClassifier_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason},
|
||||
{"coreml_GradientBoostingClassifier_OpenML_312_scene_opset7", disabledTestDefaultReason},
|
||||
{"coreml_GradientBoostingClassifier_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason},
|
||||
{"coreml_GradientBoostingClassifier_Criteo_opset7", disabledTestDefaultReason},
|
||||
{"coreml_GradientBoostingClassifier_BingClick_opset7", disabledTestDefaultReason},
|
||||
{"coreml_DictVectorizer_sklearn_load_Iris_missing_opset7", disabledTestDefaultReason},
|
||||
{"coreml_DictVectorizer_sklearn_load_digits_opset7", disabledTestDefaultReason},
|
||||
{"coreml_DictVectorizer_sklearn_load_diabetes_missing_opset7", disabledTestDefaultReason},
|
||||
{"coreml_DictVectorizer_OpenML_31_credit_opset7", disabledTestDefaultReason},
|
||||
{"coreml_DictVectorizer_RandomForestRegressor_sklearn_load_diabetes_opset7", disabledTestDefaultReason},
|
||||
{"coreml_DictVectorizer_RandomForestClassifier_sklearn_load_wine_opset7", disabledTestDefaultReason},
|
||||
{"coreml_DictVectorizer_LinearSVC_sklearn_load_wine_opset7", disabledTestDefaultReason},
|
||||
{"coreml_DictVectorizer_LinearRegression_sklearn_load_diabetes_opset7", disabledTestDefaultReason},
|
||||
{"coreml_DictVectorizer_GradientBoostingRegressor_sklearn_load_boston_opset7", disabledTestDefaultReason},
|
||||
{"coreml_DecisionTreeClassifier_sklearn_load_wine_opset7", disabledTestDefaultReason},
|
||||
{"coreml_DecisionTreeClassifier_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason},
|
||||
{"coreml_DecisionTreeClassifier_OpenML_312_scene_opset7", disabledTestDefaultReason},
|
||||
{"coreml_DecisionTreeClassifier_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason},
|
||||
{"coreml_AgeNet_ImageNet_opset7", disabledTestDefaultReason}});
|
||||
|
||||
std::unordered_map<std::string, std::string> disabledGpuTests(
|
||||
{{"LSTM_Seq_lens_unpacked_opset9", disabledGpuTestDefaultReason},
|
||||
{"fp16_inception_v1_opset8", disabledGpuTestDefaultReason},
|
||||
{"fp16_inception_v1_opset7", disabledGpuTestDefaultReason},
|
||||
{"mlperf_ssd_mobilenet_300_opset10", disabledGpuTestDefaultReason},
|
||||
{"mask_rcnn_opset10", disabledGpuTestDefaultReason},
|
||||
{"faster_rcnn_opset10", disabledGpuTestDefaultReason},
|
||||
{"BERT_Squad_opset10", disabledGpuTestDefaultReason}});
|
||||
|
||||
std::unordered_map<std::string, std::string> disabledx86Tests(
|
||||
{{"mlperf_ssd_resnet34_1200_opset10", disabledx86TestDefaultReason},
|
||||
{"mask_rcnn_opset10", disabledx86TestDefaultReason},
|
||||
{"faster_rcnn_opset10", disabledx86TestDefaultReason},
|
||||
{"test_vgg19_opset7", disabledx86TestDefaultReason},
|
||||
{"test_vgg19_opset8", disabledx86TestDefaultReason},
|
||||
{"coreml_VGG16_ImageNet_opset7", disabledx86TestDefaultReason},
|
||||
{"GPT2_LM_HEAD_opset10", disabledx86TestDefaultReason},
|
||||
{"GPT2_opset10", disabledx86TestDefaultReason},
|
||||
{"BERT_Squad_opset10", disabledx86TestDefaultReason}});
|
||||
Loading…
Reference in a new issue