From 0494a0f95fae2bd381cc4ef2aaef5b49f18d9577 Mon Sep 17 00:00:00 2001 From: Ryan Lai Date: Tue, 22 Dec 2020 15:20:23 -0800 Subject: [PATCH] Add ability to skip GPU tests based on GPU adapter name (#6198) * Implement conversion from ortvalue to Itensor for string tensors and comparing sequence of maps of strings to floats * PR comments * Add ability to skip gpu tests according to adapter description * spacing * spacing * spacing --- winml/test/model/model_tests.cpp | 93 ++++++++++++++++++++++++++- winml/test/model/ort_value_helper.cpp | 2 +- winml/test/model/skip_model_tests.h | 9 +++ 3 files changed, 101 insertions(+), 3 deletions(-) diff --git a/winml/test/model/model_tests.cpp b/winml/test/model/model_tests.cpp index c474eb1c98..0576c328be 100644 --- a/winml/test/model/model_tests.cpp +++ b/winml/test/model/model_tests.cpp @@ -7,6 +7,8 @@ #include "StringHelpers.h" #include "skip_model_tests.h" #include "compare_feature_value.h" +#include +#include "CommonDeviceHelpers.h" #ifndef BUILD_GOOGLE_TEST #error Must use googletest for value-parameterized tests @@ -164,13 +166,13 @@ static std::vector GetAllTestCases() { std::vector> dataDirs; auto testDataPath = GetTestDataPath(); if (testDataPath == "") return tests; - + for (auto& p : std::filesystem::directory_iterator(testDataPath.c_str())) { if (p.is_directory()) { dataDirs.push_back(std::move(p.path())); } } - + #if !defined(__amd64__) && !defined(_M_AMD64) // Should match "x86_disabled_tests" in onnxruntime/test/providers/cpu/model_tests.cc // However there are more tests skipped. TODO: bugs must be filed for difference in models. @@ -225,6 +227,90 @@ static std::vector GetAllTestCases() { return tests; } +bool ShouldSkipTestOnGpuAdapterDxgi(std::string& testName) { + winrt::com_ptr spFactory; + winrt::com_ptr spAdapter; + UINT i = 0; + WINML_EXPECT_HRESULT_SUCCEEDED(CreateDXGIFactory1(IID_PPV_ARGS(spFactory.put()))); + while (spFactory->EnumAdapters1(i, spAdapter.put()) != DXGI_ERROR_NOT_FOUND) { + DXGI_ADAPTER_DESC1 pDesc; + WINML_EXPECT_HRESULT_SUCCEEDED(spAdapter->GetDesc1(&pDesc)); + + // Check if WARP adapter + // see here for documentation on filtering WARP adapter: + // https://docs.microsoft.com/en-us/windows/desktop/direct3ddxgi/d3d10-graphics-programming-guide-dxgi#new-info-about-enumerating-adapters-for-windows-8 + auto isBasicRenderDriverVendorId = pDesc.VendorId == 0x1414; + auto isBasicRenderDriverDeviceId = pDesc.DeviceId == 0x8c; + auto isSoftwareAdapter = pDesc.Flags == DXGI_ADAPTER_FLAG_SOFTWARE; + bool isWarpAdapter = isSoftwareAdapter || (isBasicRenderDriverVendorId && isBasicRenderDriverDeviceId); + + if (!isWarpAdapter) { + // Found an adapter that is not WARP. This is the adapter that will be used by WinML. + std::string regex = disabledGpuAdapterTests[testName].first; + std::wstring adapterDescription = pDesc.Description; + return std::regex_search( + _winml::Strings::UTF8FromUnicode(adapterDescription.c_str(), adapterDescription.length()), + std::regex(regex, std::regex_constants::icase | std::regex_constants::nosubs)); + } + spAdapter = nullptr; + i++; + } + // If no adapters can be enumerated or none of them are hardware, might as well skip this test + return true; +} +#ifdef ENABLE_DXCORE +bool ShouldSkipTestOnGpuAdapterDxcore(std::string& testName) { + winrt::com_ptr spFactory; + WINML_EXPECT_HRESULT_SUCCEEDED(DXCoreCreateAdapterFactory(IID_PPV_ARGS(spFactory.put()))); + + winrt::com_ptr spAdapterList; + const GUID gpuFilter[] = {DXCORE_ADAPTER_ATTRIBUTE_D3D12_GRAPHICS}; + WINML_EXPECT_HRESULT_SUCCEEDED(spFactory->CreateAdapterList(1, gpuFilter, IID_PPV_ARGS(spAdapterList.put()))); + + winrt::com_ptr firstHardwareAdapter; + + // select first hardware adapter + for (uint32_t i = 0; i < spAdapterList->GetAdapterCount(); i++) { + winrt::com_ptr spCurrAdapter; + WINML_EXPECT_HRESULT_SUCCEEDED(spAdapterList->GetAdapter(i, IID_PPV_ARGS(spCurrAdapter.put()))); + + bool isHardware = false; + WINML_EXPECT_HRESULT_SUCCEEDED(spCurrAdapter->GetProperty(DXCoreAdapterProperty::IsHardware, &isHardware)); + + if (isHardware) { + // Found an adapter that is not WARP. This is the adapter that will be used by WinML. + std::string regex = disabledGpuAdapterTests[testName].first; + std::string adapterDescription; + WINML_EXPECT_HRESULT_SUCCEEDED(spCurrAdapter->GetProperty(DXCoreAdapterProperty::DriverDescription, &adapterDescription)); + return std::regex_search( + adapterDescription, + std::regex(regex, std::regex_constants::icase | std::regex_constants::nosubs)); + } + } + // If no adapters can be enumerated or none of them are hardware, might as well skip this test + return true; +} +#endif + +bool ShouldSkipTestOnGpuAdapter(std::string& testName) { + CommonDeviceHelpers::AdapterEnumerationSupport support; + if (FAILED(CommonDeviceHelpers::GetAdapterEnumerationSupport(&support))) { + WINML_LOG_ERROR("Unable to load DXGI or DXCore"); + // If cannot load DXGI or DXCore, then don't run the GPU test + return true; + } + if (support.has_dxgi) { + return ShouldSkipTestOnGpuAdapterDxgi(testName); + } +#ifdef ENABLE_DXCORE + if (support.has_dxcore) { + return ShouldSkipTestOnGpuAdapterDxcore(testName); + } +#endif + // don't skip by default (shouldn't really hit this case) + return false; +} + // determine if test should be disabled void DetermineIfDisableTest(std::string& testName, winml::LearningModelDeviceKind deviceKind) { bool shouldSkip = false; @@ -239,6 +325,9 @@ void DetermineIfDisableTest(std::string& testName, winml::LearningModelDeviceKin } else if (disabledGpuTests.find(testName) != disabledGpuTests.end()) { reason = disabledGpuTests.at(testName); shouldSkip = true; + } else if (disabledGpuAdapterTests.find(testName) != disabledGpuAdapterTests.end() && ShouldSkipTestOnGpuAdapter(testName)) { + reason = disabledGpuAdapterTests[testName].second; + shouldSkip = true; } } if (shouldSkip) { diff --git a/winml/test/model/ort_value_helper.cpp b/winml/test/model/ort_value_helper.cpp index 6f615daad3..45749a1642 100644 --- a/winml/test/model/ort_value_helper.cpp +++ b/winml/test/model/ort_value_helper.cpp @@ -206,7 +206,7 @@ Ort::Value CreateOrtValueFromITensor(winml::ITensor winmlTensor) { std::vector utf8Strs; auto strValues = winmlTensor.as().GetAsVectorView(); for (winrt::hstring str : strValues) { - utf8Strs.push_back(std::move(_winml::Strings::UTF8FromHString(str))); + utf8Strs.push_back(_winml::Strings::UTF8FromHString(str)); strData.push_back(utf8Strs.back().c_str()); } WINML_EXPECT_NO_THROW(ortValueCreated.FillStringTensor(strData.data(), strData.size())); diff --git a/winml/test/model/skip_model_tests.h b/winml/test/model/skip_model_tests.h index 6fc96d69f6..604f9b9d9b 100644 --- a/winml/test/model/skip_model_tests.h +++ b/winml/test/model/skip_model_tests.h @@ -122,4 +122,13 @@ std::unordered_map disabledGpuTests( {"fp16_test_tiny_yolov2_opset7", "Bug 31005780: Result of fp16_test_tiny_yolov2_opset7 and fp16_coreml_FNS_Candy_opset7 models on DirectML aren't as accurate as on CPU https://microsoft.visualstudio.com/OS/_workitems/edit/31005780"}, {"fp16_coreml_FNS_Candy_opset7", "Bug 31005780: Result of fp16_test_tiny_yolov2_opset7 and fp16_coreml_FNS_Candy_opset7 models on DirectML aren't as accurate as on CPU https://microsoft.visualstudio.com/OS/_workitems/edit/31005780"}, {"mlperf_ssd_mobilenet_300_opset10", "Bug 31005624: mlperf_ssd_mobilenet_300 opset 10 model fails to evaluate in DirectML https://microsoft.visualstudio.com/OS/_workitems/edit/31005624"} + }); + +/* + model name -> (adapter name regex, skipped test reason) +*/ +std::unordered_map> disabledGpuAdapterTests( + { + {"fp16_inception_v1_opset7", std::make_pair("AMD Radeon VII|Intel\\(R\\) (UHD )?Graphics|TITAN X", "Bug 31144419: Results of fp16_inception_v1 opset7 and opset8 aren't accurate enough on AMD Radeon VII & Intel(R) UHD Graphics 630 & NVIDIA TITAN X (Pascal) https://microsoft.visualstudio.com/OS/_workitems/edit/31144419")}, + {"fp16_inception_v1_opset8", std::make_pair("AMD Radeon VII|Intel\\(R\\) (UHD )?Graphics|TITAN X", "Bug 31144419: Results of fp16_inception_v1 opset7 and opset8 aren't accurate enough on AMD Radeon VII & Intel(R) UHD Graphics 630 & NVIDIA TITAN X (Pascal) https://microsoft.visualstudio.com/OS/_workitems/edit/31144419")}, }); \ No newline at end of file