diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index c8eb667871..b58bc52833 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -83,7 +83,9 @@ option(tensorflow_C_PACKAGE_PATH "Path to tensorflow C package installation dir" option(onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS "Enable operator implemented in language other than cpp" OFF) option(onnxruntime_DEBUG_NODE_INPUTS_OUTPUTS "Dump node input shapes and output data to standard output when executing the model." OFF) option(onnxruntime_USE_DML "Build with DirectML support" OFF) +option(onnxruntime_USE_WINML "Build with WinML support" OFF) option(onnxruntime_USE_ACL "Build with ACL support" OFF) +option(onnxruntime_USE_TELEMETRY "Build with Telemetry" OFF) set(protobuf_BUILD_TESTS OFF CACHE BOOL "Build protobuf tests" FORCE) #nsync tests failed on Mac Build @@ -747,4 +749,4 @@ if (onnxruntime_BUILD_CSHARP) message(STATUS "CSharp Build is enabled") # set_property(GLOBAL PROPERTY VS_DOTNET_TARGET_FRAMEWORK_VERSION "netstandard2.0") include(onnxruntime_csharp.cmake) -endif() \ No newline at end of file +endif() diff --git a/cmake/onnxruntime_common.cmake b/cmake/onnxruntime_common.cmake index 12180fd25a..d1c983b87f 100644 --- a/cmake/onnxruntime_common.cmake +++ b/cmake/onnxruntime_common.cmake @@ -52,6 +52,10 @@ source_group(TREE ${REPO_ROOT} FILES ${onnxruntime_common_src}) add_library(onnxruntime_common ${onnxruntime_common_src}) +if (onnxruntime_USE_TELEMETRY) + set_target_properties(onnxruntime_common PROPERTIES COMPILE_FLAGS "/FI${ONNXRUNTIME_INCLUDE_DIR}/core/platform/windows/TraceLoggingConfigPrivate.h") +endif() + if (onnxruntime_USE_MIMALLOC) if(onnxruntime_USE_CUDA OR onnxruntime_USE_OPENVINO) message(WARNING "Ignoring directive to use mimalloc on unimplemented targets") diff --git a/cmake/winml.cmake b/cmake/winml.cmake index 44c2a01943..cd6fd862ca 100644 --- a/cmake/winml.cmake +++ b/cmake/winml.cmake @@ -31,7 +31,7 @@ convert_forward_slashes_to_back(${exclusions} CPPWINRT_COMPONENT_EXCLUSION_LIST) # For winrt idl files: # 1) the file name must match the casing of the file on disk. # 2) for winrt idls the casing must match the namespaces within exactly (Window.AI.MachineLearning). -# target_cppwinrt will attempt to create a winmd with the name and same casing as the supplied +# target_cppwinrt will attempt to create a winmd with the name and same casing as the supplied # idl file. If the name of the winmd file does not match the contained namespaces, cppwinrt.exe # will generate component template files with fully qualified names, which will not match the existing # generated component files. @@ -90,6 +90,9 @@ add_library(winml_lib_telemetry STATIC # Compiler options target_compile_features(winml_lib_telemetry PRIVATE cxx_std_17) target_compile_options(winml_lib_telemetry PRIVATE /GR- /await /wd4238) +if (onnxruntime_USE_TELEMETRY) + set_target_properties(winml_lib_telemetry PROPERTIES COMPILE_FLAGS "/FI${ONNXRUNTIME_INCLUDE_DIR}/core/platform/windows/TraceLoggingConfigPrivate.h") +endif() # Compiler flags target_compile_definitions(winml_lib_telemetry PRIVATE PLATFORM_WINDOWS) @@ -236,7 +239,7 @@ add_library(winml_lib_image STATIC ) # Compiler options -target_compile_features(winml_lib_image PRIVATE cxx_std_17) +target_compile_features(winml_lib_image PRIVATE cxx_std_17) target_compile_options(winml_lib_image PRIVATE /GR- /await /wd4238) # Compiler flags @@ -323,7 +326,7 @@ add_library(winml_lib_api STATIC ) # Compiler options -target_compile_features(winml_lib_api PRIVATE cxx_std_17) +target_compile_features(winml_lib_api PRIVATE cxx_std_17) target_compile_options(winml_lib_api PRIVATE /GR- /await /bigobj /wd4238) # Compiler flags @@ -400,7 +403,7 @@ add_library(winml_dll SHARED ) # Compiler options -target_compile_features(winml_dll PRIVATE cxx_std_17) +target_compile_features(winml_dll PRIVATE cxx_std_17) target_compile_options(winml_dll PRIVATE /GR- /await /bigobj /wd4238) # Compiler definitions @@ -493,4 +496,8 @@ target_link_libraries(winml_dll PRIVATE ${DBGHELP}) # rather than waiting for it to fail and retry and resolve incorrectly. if("${CMAKE_BUILD_TYPE}" STREQUAL "Release") set_target_properties(winml_dll PROPERTIES VS_GLOBAL_PreferredToolArchitecture "x64") -endif("${CMAKE_BUILD_TYPE}" STREQUAL "Release") \ No newline at end of file +endif("${CMAKE_BUILD_TYPE}" STREQUAL "Release") + +if (onnxruntime_BUILD_WINML_TESTS) + include(winml_unittests.cmake) +endif() diff --git a/cmake/winml_unittests.cmake b/cmake/winml_unittests.cmake new file mode 100644 index 0000000000..13ff2464f6 --- /dev/null +++ b/cmake/winml_unittests.cmake @@ -0,0 +1,90 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +set(WINML_TEST_SRC_DIR ${REPO_ROOT}/winml/test) +set(WINML_TEST_INC_DIR + ${REPO_ROOT}/winml/test/common + ${REPO_ROOT}/winml/lib/Api.Image/inc + ${REPO_ROOT}/winml/lib/Common/inc + ${REPO_ROOT}/onnxruntime + ${REPO_ROOT}/onnxruntime/core/providers/dml/DmlExecutionProvider/src/External/D3DX12 + ${REPO_ROOT}/cmake/external/googletest/googletest/include + ${REPO_ROOT}/cmake/external/protobuf/src + ${REPO_ROOT}/cmake/external/wil/include + ${CMAKE_CURRENT_BINARY_DIR} + ${CMAKE_CURRENT_BINARY_DIR}/winml_api + ${CMAKE_CURRENT_BINARY_DIR}/winml_api/comp_generated + ${CMAKE_CURRENT_BINARY_DIR}/winml/sdk/cppwinrt/include) + +function(set_winml_target_properties target) + set_target_properties(${target} PROPERTIES + FOLDER "WinMLTest" + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED YES + CXX_EXTENSIONS NO + ) + target_include_directories(${target} PRIVATE ${WINML_TEST_INC_DIR}) +endfunction() + +function(add_winml_test) + # Add a test target and make it discoverable by CTest by calling add_test + cmake_parse_arguments(_UT "DYN" "TARGET" "LIBS;SOURCES;DEPENDS" ${ARGN}) + if(_UT_LIBS) + list(REMOVE_DUPLICATES _UT_LIBS) + endif() + list(REMOVE_DUPLICATES _UT_SOURCES) + if (_UT_DEPENDS) + list(REMOVE_DUPLICATES _UT_DEPENDS) + endif() + + add_executable(${_UT_TARGET} ${_UT_SOURCES}) + source_group(TREE ${WINML_TEST_SRC_DIR} FILES ${_UT_SOURCES}) + set_winml_target_properties(${_UT_TARGET}) + + if (_UT_DEPENDS) + add_dependencies(${_UT_TARGET} ${_UT_DEPENDS}) + endif() + target_link_libraries(${_UT_TARGET} PRIVATE ${_UT_LIBS} gtest_main windowsapp winml_lib_image ${onnxruntime_EXTERNAL_LIBRARIES}) + + add_test(NAME ${_UT_TARGET} + COMMAND ${_UT_TARGET} + WORKING_DIRECTORY $ + ) +endfunction() + + +file(GLOB winml_test_common_src CONFIGURE_DEPENDS "${WINML_TEST_SRC_DIR}/common/*.cpp") +add_library(winml_test_common STATIC ${winml_test_common_src}) +set_winml_target_properties(winml_test_common) + +file(GLOB winml_test_api_src CONFIGURE_DEPENDS "${WINML_TEST_SRC_DIR}/api/*.cpp") +add_winml_test( + TARGET winml_test_api + SOURCES ${winml_test_api_src} + LIBS winml_test_common +) +target_precompiled_header(winml_test_api testPch.h) + +# During build time, copy any modified collaterals. +# configure_file(source destination COPYONLY), which configures CMake to copy the file whenever source is modified, +# can't be used here because we don't know the destination during configure time (in multi-configuration generators, +# such as VS, one can switch between Debug/Release builds in the same build tree, and the destination depends on the +# build mode). +function(add_winml_collateral source) + get_filename_component(source_directory ${source} DIRECTORY) + file(GLOB_RECURSE collaterals RELATIVE ${source_directory} ${source}) + foreach(collateral ${collaterals}) + set(collateral_path ${source_directory}/${collateral}) + if(NOT IS_DIRECTORY ${collateral_path}) + add_custom_command(TARGET winml_test_common + POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different ${collateral_path} "$/${collateral}") + endif() + endforeach() +endfunction() + +add_winml_collateral("${WINML_TEST_SRC_DIR}/api/models/*.onnx") +add_winml_collateral("${WINML_TEST_SRC_DIR}/collateral/images/*.png") +add_winml_collateral("${WINML_TEST_SRC_DIR}/collateral/models/*.onnx") +add_winml_collateral("${WINML_TEST_SRC_DIR}/common/testdata/squeezenet/*") +add_winml_collateral("${WINML_TEST_SRC_DIR}/scenario/cppwinrt/*.onnx") diff --git a/include/onnxruntime/core/platform/windows/TraceLoggingConfig.h b/include/onnxruntime/core/platform/windows/TraceLoggingConfig.h index 02f55f954a..c217054506 100644 --- a/include/onnxruntime/core/platform/windows/TraceLoggingConfig.h +++ b/include/onnxruntime/core/platform/windows/TraceLoggingConfig.h @@ -31,8 +31,11 @@ Environment: // Configuration macro for use in TRACELOGGING_DEFINE_PROVIDER. The definition // in this file configures the provider as a normal (non-telemetry) provider. +#ifndef TraceLoggingOptionMicrosoftTelemetry #define TraceLoggingOptionMicrosoftTelemetry() \ + TraceLoggingOptionGroup(0000000000, 00000, 00000, 0000, 0000, 0000, 0000, 0000, 000, 0000, 0000) // Empty definition for TraceLoggingOptionMicrosoftTelemetry +#endif // Configuration macro for use in TRACELOGGING_DEFINE_PROVIDER. The definition // in this file configures the provider as a normal (non-telemetry) provider. diff --git a/onnxruntime/core/graph/model.cc b/onnxruntime/core/graph/model.cc index 601375f96f..01eb169672 100644 --- a/onnxruntime/core/graph/model.cc +++ b/onnxruntime/core/graph/model.cc @@ -91,6 +91,10 @@ Model::Model(std::unique_ptr model_proto, const IOnnxRuntimeOpSchema " specifies which version of the ONNX OperatorSet is being imported."); } + if (!model_proto->has_ir_version() || model_proto->ir_version() > ONNX_NAMESPACE::Version::IR_VERSION) { + throw std::invalid_argument("Unknown model file format version."); + } + model_proto_ = std::move(model_proto); for (auto& prop : model_proto_->metadata_props()) { model_metadata_[prop.key()] = prop.value(); diff --git a/onnxruntime/core/graph/model.h b/onnxruntime/core/graph/model.h index a70521ef29..4ceb91ec22 100644 --- a/onnxruntime/core/graph/model.h +++ b/onnxruntime/core/graph/model.h @@ -21,7 +21,7 @@ using IOnnxRuntimeOpSchemaRegistryList = std::list it->second.second) { + // The domain is beyond what is registered. + return true; + } + + // Either all ONNX domains were within range, or the domains were not ONNX. + return false; +} + // Return the schema with biggest version, which is not greater than specified // in specified domain. The value of earliest_opset_where_unchanged // is also set to the earliest version preceding op_set_version where the operator @@ -238,10 +255,14 @@ void SchemaRegistryManager::GetSchemaAndHistory( checked_registry_indices.push_back(index); } - // if not found in registered custom schema registry, search in ONNX schema registry - *latest_schema = ONNX_NAMESPACE::OpSchemaRegistry::Schema(key, version, domain); - if (*latest_schema != nullptr) { - *earliest_opset_where_unchanged = (*latest_schema)->SinceVersion(); + // Reject versions greater than what is actually supported. + *latest_schema = nullptr; + if (!IsDomainVersionBeyondSupportedRange(domain, version)) { + // if not found in registered custom schema registry, search in ONNX schema registry + *latest_schema = ONNX_NAMESPACE::OpSchemaRegistry::Schema(key, version, domain); + if (*latest_schema != nullptr) { + *earliest_opset_where_unchanged = (*latest_schema)->SinceVersion(); + } } } diff --git a/onnxruntime/test/ir/onnx_model_test.cc b/onnxruntime/test/ir/onnx_model_test.cc index 5c4dce0452..48d30c1a77 100644 --- a/onnxruntime/test/ir/onnx_model_test.cc +++ b/onnxruntime/test/ir/onnx_model_test.cc @@ -72,6 +72,14 @@ TEST(ONNXModelsTest, non_existing_model) { #endif } +TEST(ONNXModelsTest, future_opset) { + // NOTE: this requires the current directory to be where onnxruntime_ir_UT.exe is located + std::shared_ptr model; + common::Status st = Model::Load("./testdata/add_opset_314159.onnx", model); + ASSERT_FALSE(st.IsOK()); + ASSERT_EQ(st.Code(), common::INVALID_GRAPH); +} + #ifdef ORT_RUN_EXTERNAL_ONNX_TESTS TEST(ONNXModelsTest1, bvlc_alexnet_1) { using ::google::protobuf::io::CodedInputStream; diff --git a/onnxruntime/test/testdata/add_opset_314159.onnx b/onnxruntime/test/testdata/add_opset_314159.onnx new file mode 100644 index 0000000000..296b5719cb Binary files /dev/null and b/onnxruntime/test/testdata/add_opset_314159.onnx differ diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 9c3fc913b2..29912bdae6 100755 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -154,6 +154,7 @@ Use the individual flags to only run the specified stages. parser.add_argument("--use_full_protobuf", action='store_true', help="Use the full protobuf library") parser.add_argument("--disable_contrib_ops", action='store_true', help="Disable contrib ops (reduces binary size)") parser.add_argument("--skip_onnx_tests", action='store_true', help="Explicitly disable all onnx related tests") + parser.add_argument("--skip_winml_tests", action='store_true', help="Explicitly disable all WinML related tests") parser.add_argument("--enable_msvc_static_runtime", action='store_true', help="Enable static linking of MSVC runtimes.") parser.add_argument("--enable_language_interop_ops", action='store_true', help="Enable operator implemented in language other than cpp") parser.add_argument("--cmake_generator", choices=['Visual Studio 15 2017', 'Visual Studio 16 2019'], @@ -161,6 +162,7 @@ Use the individual flags to only run the specified stages. parser.add_argument("--enable_multi_device_test", action='store_true', help="Test with multi-device. Mostly used for multi-device GPU") parser.add_argument("--use_dml", action='store_true', help="Build with DirectML.") parser.add_argument("--use_winml", action='store_true', help="Build with WinML.") + parser.add_argument("--use_telemetry", action='store_true', help="Only official builds can set this flag to enable telemetry.") return parser.parse_args() def resolve_executable_path(command_or_path): @@ -326,8 +328,9 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home # for now, disable jemalloc if pybind is also enabled. cmake_args = [cmake_path, cmake_dir, "-Donnxruntime_RUN_ONNX_TESTS=" + ("ON" if args.enable_onnx_tests else "OFF"), + "-Donnxruntime_BUILD_WINML_TESTS=" + ("OFF" if args.skip_winml_tests else "ON"), "-Donnxruntime_GENERATE_TEST_REPORTS=ON", - "-Donnxruntime_DEV_MODE=" + ("OFF" if args.android else "ON"), + "-Donnxruntime_DEV_MODE=" + ("OFF" if args.android or args.use_winml and not args.skip_winml_tests else "ON"), "-DPYTHON_EXECUTABLE=" + sys.executable, "-Donnxruntime_USE_CUDA=" + ("ON" if args.use_cuda else "OFF"), "-Donnxruntime_USE_NSYNC=" + ("OFF" if is_windows() or not args.use_nsync else "ON"), @@ -374,6 +377,7 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home "-Donnxruntime_ENABLE_LANGUAGE_INTEROP_OPS=" + ("ON" if args.enable_language_interop_ops or (args.config != 'Debug' and bool(os.getenv('NIGHTLY_BUILD') == '1')) else "OFF"), "-Donnxruntime_USE_DML=" + ("ON" if args.use_dml else "OFF"), "-Donnxruntime_USE_WINML=" + ("ON" if args.use_winml else "OFF"), + "-Donnxruntime_USE_TELEMETRY=" + ("ON" if args.use_telemetry else "OFF"), ] if args.use_brainslice: bs_pkg_name = args.brain_slice_package_name.split('.', 1) diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml index 417f4bb406..874ad996a3 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml @@ -32,6 +32,16 @@ jobs: CUDA_VERSION: ${{ parameters.CudaVersion }} steps: + - powershell: | + if($env:WINMLTELEMETRYGUID) + { + $length = $env:WINMLTELEMETRYGUID.length + $fileContent = "#define TraceLoggingOptionMicrosoftTelemetry() \ + TraceLoggingOptionGroup("+$env:WINMLTELEMETRYGUID.substring(1, $length-2)+")" + New-Item -Path "$(Build.SourcesDirectory)\include\onnxruntime\core\platform\windows\TraceLoggingConfigPrivate.h" -ItemType "file" -Value "$fileContent" -Force + } + displayName: 'Create TraceLoggingConfigPrivate.h For WinML Telemetry' + - template: set-test-data-variables-step.yml - template: windows-build-tools-setup-steps.yml parameters: diff --git a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml index 8269507da1..e65c0a821a 100644 --- a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml @@ -4,7 +4,7 @@ jobs: AgentPool : 'Win-CPU' DoDebugBuild: 'true' DoCompliance: 'false' - BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_tvm --use_automl --enable_pybind --use_mkldnn --use_openmp --use_dml --build_shared_lib --build_csharp --enable_onnx_tests' + BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_tvm --use_automl --enable_pybind --use_mkldnn --use_openmp --use_dml --use_winml --build_shared_lib --build_csharp --enable_onnx_tests' JobName: 'Windows_CI_Dev' DoNugetPack: 'false' NuPackScript : '' diff --git a/winml/test/.gitignore b/winml/test/.gitignore new file mode 100644 index 0000000000..dbc16e37ec --- /dev/null +++ b/winml/test/.gitignore @@ -0,0 +1 @@ +!*.onnx diff --git a/winml/test/api/APITest.h b/winml/test/api/APITest.h new file mode 100644 index 0000000000..68e68f724d --- /dev/null +++ b/winml/test/api/APITest.h @@ -0,0 +1,41 @@ +//----------------------------------------------------------------------------- +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// +//----------------------------------------------------------------------------- + +#pragma once + +#include + +class APITest : public ::testing::Test +{ +protected: + void LoadModel(const std::wstring& modelPath) + { + std::wstring fullPath = FileHelpers::GetModulePath() + modelPath; + m_model = winrt::Windows::AI::MachineLearning::LearningModel::LoadFromFilePath(fullPath); + } + + winrt::Windows::AI::MachineLearning::LearningModel m_model = nullptr; + winrt::Windows::AI::MachineLearning::LearningModelDevice m_device = nullptr; + winrt::Windows::AI::MachineLearning::LearningModelSession m_session = nullptr; + + uint64_t GetAdapterIdQuadPart() + { + LARGE_INTEGER id; + id.LowPart = m_device.AdapterId().LowPart; + id.HighPart = m_device.AdapterId().HighPart; + return id.QuadPart; + }; + + _LUID GetAdapterIdAsLUID() + { + _LUID id; + id.LowPart = m_device.AdapterId().LowPart; + id.HighPart = m_device.AdapterId().HighPart; + return id; + } + + bool m_runGPUTests = true; +}; diff --git a/winml/test/api/LearningModelAPITest.cpp b/winml/test/api/LearningModelAPITest.cpp new file mode 100644 index 0000000000..a629beaeb2 --- /dev/null +++ b/winml/test/api/LearningModelAPITest.cpp @@ -0,0 +1,261 @@ +#include "testPch.h" +#include "APITest.h" + +#include +#include +#include +#include + +using namespace winrt; +using namespace winrt::Windows::AI::MachineLearning; +using namespace winrt::Windows::Foundation::Collections; +using namespace winrt::Windows::Graphics::Imaging; +using namespace winrt::Windows::Media; +using namespace winrt::Windows::Storage; +using namespace winrt::Windows::Storage::Streams; + +class LearningModelAPITest : public APITest +{ +protected: + LearningModelAPITest() { + init_apartment(); + m_model = nullptr; + m_device = nullptr; + m_session = nullptr; + } +}; + +class LearningModelAPITestGpu : public LearningModelAPITest +{}; + +TEST_F(LearningModelAPITest, CreateModelFromFilePath) +{ + EXPECT_NO_THROW(LoadModel(L"squeezenet_modifiedforruntimestests.onnx")); +} + +TEST_F(LearningModelAPITest, CreateModelFromIStorage) +{ + std::wstring path = FileHelpers::GetModulePath() + L"squeezenet_modifiedforruntimestests.onnx"; + auto storageFile = winrt::Windows::Storage::StorageFile::GetFileFromPathAsync(path).get(); + EXPECT_NO_THROW(m_model = LearningModel::LoadFromStorageFileAsync(storageFile).get()); + EXPECT_TRUE(m_model != nullptr); + + // check the author so we know the model was populated correctly. + std::wstring author(m_model.Author()); + EXPECT_EQ(L"onnx-caffe2", author); +} + +TEST_F(LearningModelAPITest, CreateModelFromIStorageOutsideCwd) +{ + std::wstring path = FileHelpers::GetModulePath() + L"ModelSubdirectory\\ModelInSubdirectory.onnx"; + auto storageFile = winrt::Windows::Storage::StorageFile::GetFileFromPathAsync(path).get(); + EXPECT_NO_THROW(m_model = LearningModel::LoadFromStorageFileAsync(storageFile).get()); + EXPECT_TRUE(m_model != nullptr); + + // check the author so we know the model was populated correctly. + std::wstring author(m_model.Author()); + EXPECT_EQ(L"onnx-caffe2", author); +} + +TEST_F(LearningModelAPITest, CreateModelFromIStream) +{ + std::wstring path = FileHelpers::GetModulePath() + L"squeezenet_modifiedforruntimestests.onnx"; + auto storageFile = winrt::Windows::Storage::StorageFile::GetFileFromPathAsync(path).get(); + winrt::Windows::Storage::Streams::IRandomAccessStreamReference streamref; + storageFile.as(streamref); + + EXPECT_NO_THROW(m_model = LearningModel::LoadFromStreamAsync(streamref).get()); + EXPECT_TRUE(m_model != nullptr); + + // check the author so we know the model was populated correctly. + std::wstring author(m_model.Author()); + EXPECT_EQ(L"onnx-caffe2", author); +} + +TEST_F(LearningModelAPITest, GetAuthor) +{ + EXPECT_NO_THROW(LoadModel(L"squeezenet_modifiedforruntimestests.onnx")); + std::wstring author(m_model.Author()); + EXPECT_EQ(L"onnx-caffe2", author); +} + +TEST_F(LearningModelAPITest, GetName) +{ + EXPECT_NO_THROW(LoadModel(L"squeezenet_modifiedforruntimestests.onnx")); + std::wstring name(m_model.Name()); + EXPECT_EQ(L"squeezenet_old", name); +} + +TEST_F(LearningModelAPITest, GetDomain) +{ + EXPECT_NO_THROW(LoadModel(L"squeezenet_modifiedforruntimestests.onnx")); + std::wstring domain(m_model.Domain()); + EXPECT_EQ(L"test-domain", domain); +} + +TEST_F(LearningModelAPITest, GetDescription) +{ + EXPECT_NO_THROW(LoadModel(L"squeezenet_modifiedforruntimestests.onnx")); + std::wstring description(m_model.Description()); + EXPECT_EQ(L"test-doc_string", description); +} + +TEST_F(LearningModelAPITest, GetVersion) +{ + EXPECT_NO_THROW(LoadModel(L"squeezenet_modifiedforruntimestests.onnx")); + int64_t version(m_model.Version()); + (void)(version); +} + +typedef std::vector> Metadata; + +class MetadataTest : public LearningModelAPITest, public testing::WithParamInterface> +{}; + +TEST_P(MetadataTest, GetMetaData) +{ + std::wstring fileName; + std::vector> keyValuePairs; + + tie(fileName, keyValuePairs) = GetParam(); + EXPECT_NO_THROW(LoadModel(fileName.c_str())); + EXPECT_TRUE(m_model.Metadata() != nullptr); + EXPECT_EQ(keyValuePairs.size(), m_model.Metadata().Size()); + + auto iter = m_model.Metadata().First(); + for (auto& keyValue : keyValuePairs) + { + EXPECT_TRUE(iter.HasCurrent()); + EXPECT_EQ(keyValue.first, std::wstring(iter.Current().Key())); + EXPECT_EQ(keyValue.second, std::wstring(iter.Current().Value())); + iter.MoveNext(); + } +} + +INSTANTIATE_TEST_SUITE_P( + ModelMetadata, + MetadataTest, + ::testing::Values( + std::pair(L"squeezenet_modifiedforruntimestests.onnx", Metadata{}), + std::pair(L"modelWithMetaData.onnx", Metadata{{L"thisisalongkey", L"thisisalongvalue"}}), + std::pair(L"modelWith2MetaData.onnx", Metadata{{L"thisisalongkey", L"thisisalongvalue"}, {L"key2", L"val2"}}) +)); + +TEST_F(LearningModelAPITest, EnumerateInputs) +{ + EXPECT_NO_THROW(LoadModel(L"squeezenet_modifiedforruntimestests.onnx")); + + // purposely don't cache "InputFeatures" in order to exercise calling it multiple times + EXPECT_TRUE(m_model.InputFeatures().First().HasCurrent()); + + std::wstring name(m_model.InputFeatures().First().Current().Name()); + EXPECT_EQ(L"data_0", name); + + // make sure it's either tensor or image + TensorFeatureDescriptor tensorDescriptor = nullptr; + m_model.InputFeatures().First().Current().try_as(tensorDescriptor); + if (tensorDescriptor == nullptr) + { + ImageFeatureDescriptor imageDescriptor = nullptr; + EXPECT_NO_THROW(m_model.InputFeatures().First().Current().as(imageDescriptor)); + } + + auto modelDataKind = tensorDescriptor.TensorKind(); + EXPECT_EQ(TensorKind::Float, modelDataKind); + + EXPECT_TRUE(tensorDescriptor.IsRequired()); + + std::vector expectedShapes = { 1,3,224,224 }; + EXPECT_EQ(expectedShapes.size(), tensorDescriptor.Shape().Size()); + for (uint32_t j = 0; j < tensorDescriptor.Shape().Size(); j++) + { + EXPECT_EQ(expectedShapes.at(j), tensorDescriptor.Shape().GetAt(j)); + } + + auto first = m_model.InputFeatures().First(); + first.MoveNext(); + EXPECT_FALSE(first.HasCurrent()); +} + +TEST_F(LearningModelAPITest, EnumerateOutputs) +{ + EXPECT_NO_THROW(LoadModel(L"squeezenet_modifiedforruntimestests.onnx")); + + // purposely don't cache "OutputFeatures" in order to exercise calling it multiple times + std::wstring name(m_model.OutputFeatures().First().Current().Name()); + EXPECT_EQ(L"softmaxout_1", name); + + TensorFeatureDescriptor tensorDescriptor = nullptr; + EXPECT_NO_THROW(m_model.OutputFeatures().First().Current().as(tensorDescriptor)); + EXPECT_TRUE(tensorDescriptor != nullptr); + + auto tensorName = tensorDescriptor.Name(); + EXPECT_EQ(L"softmaxout_1", tensorName); + + auto modelDataKind = tensorDescriptor.TensorKind(); + EXPECT_EQ(TensorKind::Float, modelDataKind); + + EXPECT_TRUE(tensorDescriptor.IsRequired()); + + std::vector expectedShapes = { 1, 1000, 1, 1 }; + EXPECT_EQ(expectedShapes.size(), tensorDescriptor.Shape().Size()); + for (uint32_t j = 0; j < tensorDescriptor.Shape().Size(); j++) + { + EXPECT_EQ(expectedShapes.at(j), tensorDescriptor.Shape().GetAt(j)); + } + + auto first = m_model.OutputFeatures().First(); + first.MoveNext(); + EXPECT_FALSE(first.HasCurrent()); +} + +TEST_F(LearningModelAPITest, CloseModelCheckMetadata) +{ + EXPECT_NO_THROW(LoadModel(L"squeezenet_modifiedforruntimestests.onnx")); + EXPECT_NO_THROW(m_model.Close()); + std::wstring author(m_model.Author()); + EXPECT_EQ(L"onnx-caffe2", author); + std::wstring name(m_model.Name()); + EXPECT_EQ(L"squeezenet_old", name); + std::wstring domain(m_model.Domain()); + EXPECT_EQ(L"test-domain", domain); + std::wstring description(m_model.Description()); + EXPECT_EQ(L"test-doc_string", description); + int64_t version(m_model.Version()); + EXPECT_EQ(123456, version); +} + +TEST_F(LearningModelAPITestGpu, CloseModelCheckEval) +{ + EXPECT_NO_THROW(LoadModel(L"model.onnx")); + LearningModelSession session = nullptr; + EXPECT_NO_THROW(session = LearningModelSession(m_model)); + EXPECT_NO_THROW(m_model.Close()); + + std::wstring fullImagePath = FileHelpers::GetModulePath() + L"kitten_224.png"; + StorageFile imagefile = StorageFile::GetFileFromPathAsync(fullImagePath).get(); + IRandomAccessStream stream = imagefile.OpenAsync(FileAccessMode::Read).get(); + SoftwareBitmap softwareBitmap = (BitmapDecoder::CreateAsync(stream).get()).GetSoftwareBitmapAsync().get(); + VideoFrame frame = VideoFrame::CreateWithSoftwareBitmap(softwareBitmap); + + LearningModelBinding binding = nullptr; + EXPECT_NO_THROW(binding = LearningModelBinding(session)); + EXPECT_NO_THROW(binding.Bind(m_model.InputFeatures().First().Current().Name(), frame)); + + EXPECT_NO_THROW(session.Evaluate(binding, L"")); +} + +TEST_F(LearningModelAPITest, CloseModelNoNewSessions) +{ + EXPECT_NO_THROW(LoadModel(L"model.onnx")); + EXPECT_NO_THROW(m_model.Close()); + LearningModelSession session = nullptr; + EXPECT_THROW( + try { + session = LearningModelSession(m_model); + } catch (const winrt::hresult_error& e) { + EXPECT_EQ(E_INVALIDARG, e.code()); + throw; + } + , winrt::hresult_error); +} diff --git a/winml/test/api/LearningModelBindingAPITest.cpp b/winml/test/api/LearningModelBindingAPITest.cpp new file mode 100644 index 0000000000..ad372a52b4 --- /dev/null +++ b/winml/test/api/LearningModelBindingAPITest.cpp @@ -0,0 +1,615 @@ +#include "testPch.h" +#include "APITest.h" +#include "SqueezeNetValidator.h" + +#include +#include +#include "winrt/Windows.Storage.h" +#include "DeviceHelpers.h" + +using namespace winrt; +using namespace winrt::Windows::AI::MachineLearning; +using namespace winrt::Windows::Foundation::Collections; +using namespace winrt::Windows::Graphics::Imaging; +using namespace winrt::Windows::Media; +using namespace winrt::Windows::Storage; + +class LearningModelBindingAPITest : public APITest +{}; + +class LearningModelBindingAPITestGpu : public LearningModelBindingAPITest +{}; + +TEST_F(LearningModelBindingAPITest, CpuSqueezeNet) +{ + std::string cpuInstance("CPU"); + WinML::Engine::Test::ModelValidator::SqueezeNet(cpuInstance, LearningModelDeviceKind::Cpu, /*dataTolerance*/ 0.00001f, false); +} + +TEST_F(LearningModelBindingAPITest, CpuSqueezeNetEmptyOutputs) +{ + std::string cpuInstance("CPU"); + WinML::Engine::Test::ModelValidator::SqueezeNet( + cpuInstance, + LearningModelDeviceKind::Cpu, + /*dataTolerance*/ 0.00001f, + false, + OutputBindingStrategy::Empty); +} + +TEST_F(LearningModelBindingAPITest, CpuSqueezeNetUnboundOutputs) +{ + std::string cpuInstance("CPU"); + WinML::Engine::Test::ModelValidator::SqueezeNet( + cpuInstance, + LearningModelDeviceKind::Cpu, + /*dataTolerance*/ 0.00001f, + false, + OutputBindingStrategy::Unbound); +} + +TEST_F(LearningModelBindingAPITest, CpuSqueezeNetBindInputTensorAsInspectable) +{ + std::string cpuInstance("CPU"); + WinML::Engine::Test::ModelValidator::SqueezeNet( + cpuInstance, + LearningModelDeviceKind::Cpu, + /*dataTolerance*/ 0.00001f, + false, + OutputBindingStrategy::Bound /* empty outputs */, + true /* bind inputs as inspectables */); +} + +TEST_F(LearningModelBindingAPITest, CastMapInt64) +{ + EXPECT_NO_THROW(LoadModel(L"castmap-int64.onnx")); + // TODO: Check Descriptor +} + +TEST_F(LearningModelBindingAPITest, DictionaryVectorizerMapInt64) +{ + EXPECT_NO_THROW(LoadModel(L"dictvectorizer-int64.onnx")); + + auto inputDescriptor = m_model.InputFeatures().First().Current(); + EXPECT_TRUE(inputDescriptor.Kind() == LearningModelFeatureKind::Map); + auto mapDescriptor = inputDescriptor.as(); + EXPECT_TRUE(mapDescriptor.KeyKind() == TensorKind::Int64); + EXPECT_TRUE(mapDescriptor.ValueDescriptor().Kind() == LearningModelFeatureKind::Tensor); + auto tensorDescriptor = mapDescriptor.ValueDescriptor().as(); + // empty size means tensor of scalar value + EXPECT_TRUE(tensorDescriptor.Shape().Size() == 0); + EXPECT_TRUE(tensorDescriptor.TensorKind() == TensorKind::Float); + + LearningModelSession modelSession(m_model); + LearningModelBinding binding(modelSession); + std::unordered_map map; + map[1] = 1.f; + map[10] = 10.f; + map[3] = 3.f; + + + auto mapInputName = inputDescriptor.Name(); + + // Bind as IMap + auto abiMap = winrt::single_threaded_map(std::move(map)); + binding.Bind(mapInputName, abiMap); + auto mapInputInspectable = abiMap.as(); + auto first = binding.First(); + EXPECT_TRUE(first.Current().Key() == mapInputName); + EXPECT_TRUE(first.Current().Value() == mapInputInspectable); + EXPECT_TRUE(binding.Lookup(mapInputName) == mapInputInspectable); + + // Bind as IMapView + auto mapView = abiMap.GetView(); + binding.Bind(mapInputName, mapView); + mapInputInspectable = mapView.as(); + first = binding.First(); + EXPECT_TRUE(first.Current().Key() == mapInputName); + EXPECT_TRUE(first.Current().Value() == mapView); + EXPECT_TRUE(binding.Lookup(mapInputName) == mapView); + +} + +TEST_F(LearningModelBindingAPITest, DictionaryVectorizerMapString) +{ + EXPECT_NO_THROW(LoadModel(L"dictvectorizer-string.onnx")); + + auto inputDescriptor = m_model.InputFeatures().First().Current(); + EXPECT_TRUE(inputDescriptor.Kind() == LearningModelFeatureKind::Map); + + auto mapDescriptor = inputDescriptor.as(); + EXPECT_TRUE(mapDescriptor.KeyKind() == TensorKind::String); + EXPECT_TRUE(mapDescriptor.ValueDescriptor().Kind() == LearningModelFeatureKind::Tensor); + + auto tensorDescriptor = mapDescriptor.ValueDescriptor().as(); + // empty size means tensor of scalar value + EXPECT_TRUE(tensorDescriptor.Shape().Size() == 0); + EXPECT_TRUE(tensorDescriptor.TensorKind() == TensorKind::Float); + + LearningModelSession modelSession(m_model); + LearningModelBinding binding(modelSession); + std::unordered_map map; + map[L"1"] = 1.f; + map[L"10"] = 10.f; + map[L"2"] = 2.f; + + auto mapInputName = inputDescriptor.Name(); + auto abiMap = winrt::single_threaded_map(std::move(map)); + binding.Bind(mapInputName, abiMap); + + auto mapInputInspectable = abiMap.as(); + auto first = binding.First(); + EXPECT_TRUE(first.Current().Key() == mapInputName); + EXPECT_TRUE(first.Current().Value() == mapInputInspectable); + EXPECT_TRUE(binding.Lookup(mapInputName) == mapInputInspectable); +} + +static void RunZipMapInt64( + winrt::Windows::AI::MachineLearning::LearningModel model, + OutputBindingStrategy bindingStrategy) +{ + auto outputFeatures = model.OutputFeatures(); + auto outputDescriptor = outputFeatures.First().Current(); + EXPECT_TRUE(outputDescriptor.Kind() == LearningModelFeatureKind::Sequence); + + auto seqDescriptor = outputDescriptor.as(); + auto mapDescriptor = seqDescriptor.ElementDescriptor().as(); + EXPECT_TRUE(mapDescriptor.KeyKind() == TensorKind::Int64); + + EXPECT_TRUE(mapDescriptor.ValueDescriptor().Kind() == LearningModelFeatureKind::Tensor); + auto tensorDescriptor = mapDescriptor.ValueDescriptor().as(); + EXPECT_TRUE(tensorDescriptor.TensorKind() == TensorKind::Float); + + LearningModelSession session(model); + LearningModelBinding binding(session); + + std::vector inputs = { 0.5f, 0.25f, 0.125f }; + std::vector shape = { 1, 3 }; + + // Bind inputs + auto inputTensor = + TensorFloat::CreateFromArray( + shape, + winrt::array_view(std::move(inputs))); + binding.Bind(winrt::hstring(L"X"), inputTensor); + + typedef IMap ABIMap; + typedef IVector ABISequeneceOfMap; + + ABISequeneceOfMap abiOutput = nullptr; + // Bind outputs + if (bindingStrategy == OutputBindingStrategy::Bound) + { + abiOutput = winrt::single_threaded_vector(); + binding.Bind(winrt::hstring(L"Y"), abiOutput); + } + + // Evaluate + auto result = session.Evaluate(binding, L"0").Outputs(); + + if (bindingStrategy == OutputBindingStrategy::Bound) + { + // from output binding + const auto &out1 = abiOutput.GetAt(0); + const auto &out2 = result.Lookup(L"Y").as>().GetAt(0); + SCOPED_TRACE((std::ostringstream() << "size: " << out1.Size()).str()); + // check outputs + auto iter1 = out1.First(); + auto iter2 = out2.First(); + for (uint32_t i = 0, size = (uint32_t)inputs.size(); i < size; ++i) + { + EXPECT_TRUE(iter1.HasCurrent()); + EXPECT_TRUE(iter2.HasCurrent()); + const auto &pair1 = iter1.Current(); + const auto &pair2 = iter2.Current(); + SCOPED_TRACE((std::ostringstream() << "key: " << pair1.Key() << ", value: " << pair2.Value()).str()); + EXPECT_TRUE(pair1.Key() == i && pair2.Key() == i); + EXPECT_TRUE(pair1.Value() == inputs[i] && pair2.Value() == inputs[i]); + iter1.MoveNext(); + iter2.MoveNext(); + } + EXPECT_TRUE(!iter1.HasCurrent()); + EXPECT_TRUE(!iter2.HasCurrent()); + } + else + { + abiOutput = result.Lookup(L"Y").as(); + EXPECT_TRUE(abiOutput.Size() == 1); + ABIMap map = abiOutput.GetAt(0); + EXPECT_TRUE(map.Size() == 3); + EXPECT_TRUE(map.Lookup(0) == 0.5); + EXPECT_TRUE(map.Lookup(1) == .25); + EXPECT_TRUE(map.Lookup(2) == .125); + } +} + +TEST_F(LearningModelBindingAPITest, ZipMapInt64) +{ + EXPECT_NO_THROW(LoadModel(L"zipmap-int64.onnx")); + RunZipMapInt64(m_model, OutputBindingStrategy::Bound); +} + +TEST_F(LearningModelBindingAPITest, ZipMapInt64Unbound) +{ + EXPECT_NO_THROW(LoadModel(L"zipmap-int64.onnx")); + RunZipMapInt64(m_model, OutputBindingStrategy::Unbound); +} + +TEST_F(LearningModelBindingAPITest, ZipMapString) +{ + // output constraint: "seq(map(string, float))" or "seq(map(int64, float))" + EXPECT_NO_THROW(LoadModel(L"zipmap-string.onnx")); + auto outputs = m_model.OutputFeatures(); + auto outputDescriptor = outputs.First().Current(); + EXPECT_TRUE(outputDescriptor.Kind() == LearningModelFeatureKind::Sequence); + auto mapDescriptor = outputDescriptor.as().ElementDescriptor().as(); + EXPECT_TRUE(mapDescriptor.KeyKind() == TensorKind::String); + EXPECT_TRUE(mapDescriptor.ValueDescriptor().Kind() == LearningModelFeatureKind::Tensor); + auto tensorDescriptor = mapDescriptor.ValueDescriptor().as(); + EXPECT_TRUE(tensorDescriptor.TensorKind() == TensorKind::Float); + + LearningModelSession session(m_model); + LearningModelBinding binding(session); + + std::vector inputs = { 0.5f, 0.25f, 0.125f }; + std::vector shape = { 1, 3 }; + std::vector labels = { L"cat", L"dog", L"lion" }; + std::map mapData = { { L"cat", 0.0f }, { L"dog", 0.0f }, { L"lion", 0.0f } }; + typedef IMap ABIMap; + ABIMap abiMap = winrt::single_threaded_map(std::move(mapData)); + std::vector seqOutput = { abiMap }; + IVector ABIOutput = winrt::single_threaded_vector(std::move(seqOutput)); + + TensorFloat inputTensor = TensorFloat::CreateFromArray(shape, winrt::array_view(std::move(inputs))); + binding.Bind(winrt::hstring(L"X"), inputTensor); + binding.Bind(winrt::hstring(L"Y"), ABIOutput); + auto result = session.Evaluate(binding, L"0").Outputs(); + // from output binding + const auto &out1 = ABIOutput.GetAt(0); + const auto &out2 = result.Lookup(L"Y").as>().GetAt(0); + SCOPED_TRACE((std::ostringstream() << "size: " << out1.Size()).str()); + // single key,value pair for each map + auto iter1 = out1.First(); + auto iter2 = out2.First(); + for (uint32_t i = 0, size = (uint32_t)inputs.size(); i < size; ++i) + { + EXPECT_TRUE(iter2.HasCurrent()); + const auto &pair1 = iter1.Current(); + const auto &pair2 = iter2.Current(); + SCOPED_TRACE((std::ostringstream() << "key: " << pair1.Key().c_str() << ", value " << pair2.Value()).str()); + EXPECT_TRUE(std::wstring(pair1.Key().c_str()).compare(labels[i]) == 0); + EXPECT_TRUE(std::wstring(pair2.Key().c_str()).compare(labels[i]) == 0); + EXPECT_TRUE(pair1.Value() == inputs[i] && pair2.Value() == inputs[i]); + iter1.MoveNext(); + iter2.MoveNext(); + } + EXPECT_TRUE(!iter1.HasCurrent()); + EXPECT_TRUE(!iter2.HasCurrent()); +} + +TEST_F(LearningModelBindingAPITestGpu, GpuSqueezeNet) +{ + + std::string gpuInstance("GPU"); + WinML::Engine::Test::ModelValidator::SqueezeNet( + gpuInstance, + LearningModelDeviceKind::DirectX, + /*dataTolerance*/ 0.00001f); +} + +TEST_F(LearningModelBindingAPITestGpu, GpuSqueezeNetEmptyOutputs) +{ + + std::string gpuInstance("GPU"); + WinML::Engine::Test::ModelValidator::SqueezeNet( + gpuInstance, + LearningModelDeviceKind::DirectX, + /*dataTolerance*/ 0.00001f, + false, + OutputBindingStrategy::Empty); +} + +TEST_F(LearningModelBindingAPITestGpu, GpuSqueezeNetUnboundOutputs) +{ + + std::string gpuInstance("GPU"); + WinML::Engine::Test::ModelValidator::SqueezeNet( + gpuInstance, + LearningModelDeviceKind::DirectX, + /*dataTolerance*/ 0.00001f, + false, + OutputBindingStrategy::Unbound); +} + +// Validates that when the input image is the same as the model expects, the binding step is executed correctly. +TEST_F(LearningModelBindingAPITestGpu, ImageBindingDimensions) +{ + LearningModelBinding m_binding = nullptr; + std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; + // load a model with expected input size: 224 x 224 + EXPECT_NO_THROW(m_device = LearningModelDevice(LearningModelDeviceKind::Default)); + EXPECT_NO_THROW(m_model = LearningModel::LoadFromFilePath(filePath)); + EXPECT_TRUE(m_model != nullptr); + EXPECT_NO_THROW(m_session = LearningModelSession(m_model, m_device)); + EXPECT_NO_THROW(m_binding = LearningModelBinding(m_session)); + + // Create input images and execute bind + // Test Case 1: both width and height are larger than model expects + VideoFrame inputImage1(BitmapPixelFormat::Rgba8, 1000, 1000); + ImageFeatureValue inputTensor = ImageFeatureValue::CreateFromVideoFrame(inputImage1); + EXPECT_NO_THROW(m_binding.Bind(L"data_0", inputTensor)); + + // Test Case 2: only height is larger, while width is smaller + VideoFrame inputImage2(BitmapPixelFormat::Rgba8, 20, 1000); + inputTensor = ImageFeatureValue::CreateFromVideoFrame(inputImage2); + EXPECT_NO_THROW(m_binding.Bind(L"data_0", inputTensor)); + + // Test Case 3: only width is larger, while height is smaller + VideoFrame inputImage3(BitmapPixelFormat::Rgba8, 1000, 20); + inputTensor = ImageFeatureValue::CreateFromVideoFrame(inputImage3); + EXPECT_NO_THROW(m_binding.Bind(L"data_0", inputTensor)); + + // Test Case 4: both width and height are smaller than model expects + VideoFrame inputImage4(BitmapPixelFormat::Rgba8, 20, 20); + inputTensor = ImageFeatureValue::CreateFromVideoFrame(inputImage4); + EXPECT_NO_THROW(m_binding.Bind(L"data_0", inputTensor)); +} + +TEST_F(LearningModelBindingAPITestGpu, VerifyInvalidBindExceptions) +{ + + EXPECT_NO_THROW(LoadModel(L"zipmap-int64.onnx")); + + LearningModelSession session(m_model); + LearningModelBinding binding(session); + + std::vector inputs = { 0.5f, 0.25f, 0.125f }; + std::vector shape = { 1, 3 }; + + auto matchException = + [](const winrt::hresult_error& e, HRESULT hr) -> bool + { + return e.code() == hr; + }; + + auto ensureWinmlSizeMismatch = std::bind(matchException, std::placeholders::_1, WINML_ERR_SIZE_MISMATCH); + auto ensureWinmlInvalidBinding = std::bind(matchException, std::placeholders::_1, WINML_ERR_INVALID_BINDING); + + /* + Verify tensor bindings throw correct bind exceptions + */ + + // Bind invalid image as tensorfloat input + auto image = FileHelpers::LoadImageFeatureValue(L"227x227.png"); + EXPECT_THROW_SPECIFIC(binding.Bind(L"X", image), winrt::hresult_error, ensureWinmlSizeMismatch); + + // Bind invalid map as tensorfloat input + std::unordered_map map; + auto abiMap = winrt::single_threaded_map(std::move(map)); + EXPECT_THROW_SPECIFIC(binding.Bind(L"X", abiMap), winrt::hresult_error, ensureWinmlInvalidBinding); + + // Bind invalid sequence as tensorfloat input + std::vector sequence; + auto abiSequence = winrt::single_threaded_vector(std::move(sequence)); + EXPECT_THROW_SPECIFIC(binding.Bind(L"X", abiSequence), winrt::hresult_error, ensureWinmlInvalidBinding); + + // Bind invalid tensor size as tensorfloat input + auto tensorBoolean = TensorBoolean::Create(); + EXPECT_THROW_SPECIFIC(binding.Bind(L"X", tensorBoolean), winrt::hresult_error, ensureWinmlInvalidBinding); + + // Bind invalid tensor shape as tensorfloat input + auto tensorInvalidShape = TensorFloat::Create(std::vector { 2, 3, 4 }); + EXPECT_THROW_SPECIFIC(binding.Bind(L"X", tensorInvalidShape), winrt::hresult_error, ensureWinmlInvalidBinding); + + /* + Verify sequence bindings throw correct bind exceptions + */ + + // Bind invalid image as sequence output + EXPECT_THROW_SPECIFIC(binding.Bind(L"Y", image), winrt::hresult_error, ensureWinmlInvalidBinding); + + // Bind invalid map as sequence output + EXPECT_THROW_SPECIFIC(binding.Bind(L"Y", abiMap), winrt::hresult_error, ensureWinmlInvalidBinding); + + // Bind invalid sequence as sequence output + EXPECT_THROW_SPECIFIC(binding.Bind(L"Y", abiSequence), winrt::hresult_error, ensureWinmlInvalidBinding); + + // Bind invalid tensor as sequence output + EXPECT_THROW_SPECIFIC(binding.Bind(L"Y", tensorBoolean), winrt::hresult_error, ensureWinmlInvalidBinding); + + /* + Verify image bindings throw correct bind exceptions + */ + + // EXPECT_NO_THROW(LoadModel(L"fns-candy.onnx")); + + // LearningModelSession imageSession(m_model); + // LearningModelBinding imageBinding(imageSession); + + // auto inputName = m_model.InputFeatures().First().Current().Name(); + + // // Bind invalid map as image input + // EXPECT_THROW_SPECIFIC(imageBinding.Bind(inputName, abiMap), winrt::hresult_error, ensureWinmlInvalidBinding); + + // // Bind invalid sequence as image input + // EXPECT_THROW_SPECIFIC(imageBinding.Bind(inputName, abiSequence), winrt::hresult_error, ensureWinmlInvalidBinding); + + // // Bind invalid tensor type as image input + // EXPECT_THROW_SPECIFIC(imageBinding.Bind(inputName, tensorBoolean), winrt::hresult_error, ensureWinmlInvalidBinding); + + // // Bind invalid tensor size as image input + // auto tensorFloat = TensorFloat::Create(std::vector { 1, 1, 100, 100 }); + // EXPECT_THROW_SPECIFIC(imageBinding.Bind(inputName, tensorFloat), winrt::hresult_error, ensureWinmlInvalidBinding); + + // // Bind invalid tensor shape as image input + // EXPECT_THROW_SPECIFIC(imageBinding.Bind(inputName, tensorInvalidShape), winrt::hresult_error, ensureWinmlInvalidBinding); + + /* + Verify map bindings throw correct bind exceptions + */ + EXPECT_NO_THROW(LoadModel(L"dictvectorizer-int64.onnx")); + + LearningModelSession mapSession(m_model); + LearningModelBinding mapBinding(mapSession); + + auto inputName = m_model.InputFeatures().First().Current().Name(); + + // Bind invalid image as image input + auto smallImage = FileHelpers::LoadImageFeatureValue(L"100x100.png"); + EXPECT_THROW_SPECIFIC(mapBinding.Bind(inputName, smallImage), winrt::hresult_error, ensureWinmlInvalidBinding); + + // Bind invalid map as image input + EXPECT_THROW_SPECIFIC(mapBinding.Bind(inputName, abiMap), winrt::hresult_error, ensureWinmlInvalidBinding); + + // Bind invalid sequence as image input + EXPECT_THROW_SPECIFIC(mapBinding.Bind(inputName, abiSequence), winrt::hresult_error, ensureWinmlInvalidBinding); + + // Bind invalid tensor type as image input + EXPECT_THROW_SPECIFIC(mapBinding.Bind(inputName, tensorBoolean), winrt::hresult_error, ensureWinmlInvalidBinding); +} + +// Verify that it throws an error when binding an invalid name. +TEST_F(LearningModelBindingAPITestGpu, BindInvalidInputName) +{ + LearningModelBinding m_binding = nullptr; + std::wstring modelPath = FileHelpers::GetModulePath() + L"Add_ImageNet1920.onnx"; + EXPECT_NO_THROW(m_model = LearningModel::LoadFromFilePath(modelPath)); + EXPECT_TRUE(m_model != nullptr); + EXPECT_NO_THROW(m_device = LearningModelDevice(LearningModelDeviceKind::Default)); + EXPECT_NO_THROW(m_session = LearningModelSession(m_model, m_device)); + EXPECT_NO_THROW(m_binding = LearningModelBinding(m_session)); + + VideoFrame iuputImage(BitmapPixelFormat::Rgba8, 1920, 1080); + ImageFeatureValue inputTensor = ImageFeatureValue::CreateFromVideoFrame(iuputImage); + + auto first = m_model.InputFeatures().First(); + std::wstring testInvalidName = L"0"; + + // Verify that testInvalidName is not in model's InputFeatures + while (first.HasCurrent()) + { + EXPECT_NE(testInvalidName, first.Current().Name()); + first.MoveNext(); + } + + // Bind inputTensor to a valid input name + EXPECT_NO_THROW(m_binding.Bind(L"input_39:0", inputTensor)); + + // Bind inputTensor to an invalid input name + EXPECT_THROW_SPECIFIC(m_binding.Bind(testInvalidName, inputTensor), + winrt::hresult_error, + [](const winrt::hresult_error& e) -> bool + { + return e.code() == WINML_ERR_INVALID_BINDING; + }); +} + +TEST_F(LearningModelBindingAPITest, VerifyOutputAfterEvaluateAsyncCalledTwice) +{ + LearningModelBinding m_binding = nullptr; + std::wstring filePath = FileHelpers::GetModulePath() + L"relu.onnx"; + EXPECT_NO_THROW(m_device = LearningModelDevice(LearningModelDeviceKind::Default)); + EXPECT_NO_THROW(m_model = LearningModel::LoadFromFilePath(filePath)); + EXPECT_TRUE(m_model != nullptr); + EXPECT_NO_THROW(m_session = LearningModelSession(m_model, m_device)); + EXPECT_NO_THROW(m_binding = LearningModelBinding(m_session)); + + auto inputShape = std::vector{ 5 }; + auto inputData1 = std::vector{ -50.f, -25.f, 0.f, 25.f, 50.f }; + auto inputValue1 = + TensorFloat::CreateFromIterable( + inputShape, + single_threaded_vector(std::move(inputData1)).GetView()); + + auto inputData2 = std::vector{ 50.f, 25.f, 0.f, -25.f, -50.f }; + auto inputValue2 = + TensorFloat::CreateFromIterable( + inputShape, + single_threaded_vector(std::move(inputData2)).GetView()); + + EXPECT_NO_THROW(m_binding.Bind(L"X", inputValue1)); + + auto outputValue = TensorFloat::Create(); + EXPECT_NO_THROW(m_binding.Bind(L"Y", outputValue)); + + EXPECT_NO_THROW(m_session.Evaluate(m_binding, L"")); + + auto buffer1 = outputValue.GetAsVectorView(); + EXPECT_TRUE(buffer1 != nullptr); + + // The second evaluation + // If we don't bind output again, the output value will not change + EXPECT_NO_THROW(m_binding.Bind(L"X", inputValue2)); + EXPECT_NO_THROW(m_session.Evaluate(m_binding, L"")); + auto buffer2 = outputValue.GetAsVectorView(); + EXPECT_EQ(buffer1.Size(), buffer2.Size()); + bool isSame = true; + for (uint32_t i = 0; i < buffer1.Size(); ++i) + { + if (buffer1.GetAt(i) != buffer2.GetAt(i)) + { + isSame = false; + break; + } + } + EXPECT_FALSE(isSame); +} + +static VideoFrame CreateVideoFrame(const wchar_t* path) +{ + auto imagefile = StorageFile::GetFileFromPathAsync(path).get(); + auto stream = imagefile.OpenAsync(FileAccessMode::Read).get(); + auto decoder = BitmapDecoder::CreateAsync(stream).get(); + auto softwareBitmap = decoder.GetSoftwareBitmapAsync().get(); + return VideoFrame::CreateWithSoftwareBitmap(softwareBitmap); +} + +TEST_F(LearningModelBindingAPITest, VerifyOutputAfterImageBindCalledTwice) +{ + std::wstring fullModelPath = FileHelpers::GetModulePath() + L"model.onnx"; + std::wstring fullImagePath1 = FileHelpers::GetModulePath() + L"kitten_224.png"; + std::wstring fullImagePath2 = FileHelpers::GetModulePath() + L"fish.png"; + + // winml model creation + LearningModel model = nullptr; + EXPECT_NO_THROW(model = LearningModel::LoadFromFilePath(fullModelPath)); + LearningModelSession modelSession = nullptr; + EXPECT_NO_THROW(modelSession = LearningModelSession(model, LearningModelDevice(LearningModelDeviceKind::Default))); + LearningModelBinding modelBinding(modelSession); + + // create the tensor for the actual output + auto output = TensorFloat::Create(); + modelBinding.Bind(L"softmaxout_1", output); + + // Bind image 1 and evaluate + auto frame = CreateVideoFrame(fullImagePath1.c_str()); + auto imageTensor = ImageFeatureValue::CreateFromVideoFrame(frame); + EXPECT_NO_THROW(modelBinding.Bind(L"data_0", imageTensor)); + EXPECT_NO_THROW(modelSession.Evaluate(modelBinding, L"")); + + // Store 1st result + auto outputVectorView1 = output.GetAsVectorView(); + + // Bind image 2 and evaluate + // In this scenario, the backing videoframe is updated, and the imagefeaturevalue is rebound. + // The expected result is that the videoframe will be re-tensorized at bind + auto frame2 = CreateVideoFrame(fullImagePath2.c_str()); + frame2.CopyToAsync(frame).get(); + EXPECT_NO_THROW(modelBinding.Bind(L"data_0", imageTensor)); + EXPECT_NO_THROW(modelSession.Evaluate(modelBinding, L"")); + + // Store 2nd result + auto outputVectorView2 = output.GetAsVectorView(); + + EXPECT_EQ(outputVectorView1.Size(), outputVectorView2.Size()); + bool isSame = true; + for (uint32_t i = 0; i < outputVectorView1.Size(); ++i) + { + if (outputVectorView1.GetAt(i) != outputVectorView2.GetAt(i)) + { + isSame = false; + break; + } + } + EXPECT_FALSE(isSame); +} diff --git a/winml/test/api/LearningModelSessionAPITest.cpp b/winml/test/api/LearningModelSessionAPITest.cpp new file mode 100644 index 0000000000..1269a8c2a9 --- /dev/null +++ b/winml/test/api/LearningModelSessionAPITest.cpp @@ -0,0 +1,373 @@ +#include "testPch.h" +#include "APITest.h" + +#include "winrt/Windows.Storage.h" + +#include "DeviceHelpers.h" +#include "protobufHelpers.h" + +#include +#include +#include "Psapi.h" + +using namespace winrt; +using namespace winrt::Windows::AI::MachineLearning; +using namespace winrt::Windows::Foundation::Collections; + +using winrt::Windows::Foundation::IPropertyValue; + +class LearningModelSessionAPITests : public APITest +{}; + +class LearningModelSessionAPITestsGpu : public APITest +{}; // TODO create a constructor that calls GTEST_SKIP when GPU tests are disabled + +class LearningModelSessionAPITestsSkipEdgeCore : public LearningModelSessionAPITestsGpu +{}; // TODO create a constructor that calls GTEST_SKIP when on EdgeCore + +TEST_F(LearningModelSessionAPITests, CreateSessionDeviceDefault) +{ + EXPECT_NO_THROW(LoadModel(L"model.onnx")); + + EXPECT_NO_THROW(m_device = LearningModelDevice(LearningModelDeviceKind::Default)); + EXPECT_NO_THROW(m_session = LearningModelSession(m_model, m_device)); +} + +TEST_F(LearningModelSessionAPITests, CreateSessionDeviceCpu) +{ + EXPECT_NO_THROW(LoadModel(L"model.onnx")); + + EXPECT_NO_THROW(m_device = LearningModelDevice(LearningModelDeviceKind::Cpu)); + EXPECT_NO_THROW(m_session = LearningModelSession(m_model, m_device)); + // for the CPU device, make sure that we get back NULL and 0 for any device properties + EXPECT_FALSE(m_device.Direct3D11Device()); + LARGE_INTEGER id; + id.QuadPart = GetAdapterIdQuadPart(); + EXPECT_EQ(id.LowPart, static_cast(0)); + EXPECT_EQ(id.HighPart, 0); +} + +TEST_F(LearningModelSessionAPITests, CreateSessionWithModelLoadedFromStream) +{ + std::wstring path = FileHelpers::GetModulePath() + L"model.onnx"; + auto storageFile = winrt::Windows::Storage::StorageFile::GetFileFromPathAsync(path).get(); + + EXPECT_NO_THROW(m_model = LearningModel::LoadFromStream(storageFile)); + + EXPECT_NO_THROW(m_device = LearningModelDevice(LearningModelDeviceKind::Default)); + EXPECT_NO_THROW(m_session = LearningModelSession(m_model, m_device)); +} + +TEST_F(LearningModelSessionAPITestsGpu, CreateSessionDeviceDirectX) +{ + + EXPECT_NO_THROW(LoadModel(L"model.onnx")); + + EXPECT_NO_THROW(m_device = LearningModelDevice(LearningModelDeviceKind::DirectX)); + EXPECT_NO_THROW(m_session = LearningModelSession(m_model, m_device)); +} + +TEST_F(LearningModelSessionAPITestsGpu, CreateSessionDeviceDirectXHighPerformance) +{ + + EXPECT_NO_THROW(LoadModel(L"model.onnx")); + + EXPECT_NO_THROW(m_device = LearningModelDevice(LearningModelDeviceKind::DirectXHighPerformance)); + EXPECT_NO_THROW(m_session = LearningModelSession(m_model, m_device)); +} + +TEST_F(LearningModelSessionAPITestsGpu, CreateSessionDeviceDirectXMinimumPower) +{ + + EXPECT_NO_THROW(LoadModel(L"model.onnx")); + + EXPECT_NO_THROW(m_device = LearningModelDevice(LearningModelDeviceKind::DirectXMinPower)); + EXPECT_NO_THROW(m_session = LearningModelSession(m_model, m_device)); +} + +TEST_F(LearningModelSessionAPITestsSkipEdgeCore, AdapterIdAndDevice) +{ + EXPECT_NO_THROW(LoadModel(L"model.onnx")); + + com_ptr factory; + EXPECT_HRESULT_SUCCEEDED(CreateDXGIFactory1(__uuidof(IDXGIFactory6), factory.put_void())); + com_ptr adapter; + + m_device = LearningModelDevice(LearningModelDeviceKind::DirectX); + EXPECT_HRESULT_SUCCEEDED(factory->EnumAdapters(0, adapter.put())); + DXGI_ADAPTER_DESC desc; + EXPECT_HRESULT_SUCCEEDED(adapter->GetDesc(&desc)); + LARGE_INTEGER id; + id.QuadPart = GetAdapterIdQuadPart(); + EXPECT_EQ(desc.AdapterLuid.LowPart, id.LowPart); + EXPECT_EQ(desc.AdapterLuid.HighPart, id.HighPart); + EXPECT_TRUE(m_device.Direct3D11Device() != nullptr); + + m_device = LearningModelDevice(LearningModelDeviceKind::DirectXHighPerformance); + adapter = nullptr; + EXPECT_HRESULT_SUCCEEDED(factory->EnumAdapterByGpuPreference(0, DXGI_GPU_PREFERENCE_HIGH_PERFORMANCE, __uuidof(IDXGIAdapter), adapter.put_void())); + EXPECT_HRESULT_SUCCEEDED(adapter->GetDesc(&desc)); + id.QuadPart = GetAdapterIdQuadPart(); + EXPECT_EQ(desc.AdapterLuid.LowPart, id.LowPart); + EXPECT_EQ(desc.AdapterLuid.HighPart, id.HighPart); + EXPECT_TRUE(m_device.Direct3D11Device() != nullptr); + + adapter = nullptr; + m_device = LearningModelDevice(LearningModelDeviceKind::DirectXMinPower); + EXPECT_HRESULT_SUCCEEDED(factory->EnumAdapterByGpuPreference(0, DXGI_GPU_PREFERENCE_MINIMUM_POWER, __uuidof(IDXGIAdapter), adapter.put_void())); + EXPECT_HRESULT_SUCCEEDED(adapter->GetDesc(&desc)); + id.QuadPart = GetAdapterIdQuadPart(); + EXPECT_EQ(desc.AdapterLuid.LowPart, id.LowPart); + EXPECT_EQ(desc.AdapterLuid.HighPart, id.HighPart); + EXPECT_TRUE(m_device.Direct3D11Device() != nullptr); + + EXPECT_NO_THROW(m_session = LearningModelSession(m_model, m_device)); + EXPECT_EQ(m_session.Device().AdapterId(), m_device.AdapterId()); +} + +TEST_F(LearningModelSessionAPITests, EvaluateFeatures) +{ + std::vector shape = { 4 }; + std::vector data = { L"one", L"two", L"three", L"four" }; + + // create from buffer + auto tensor = TensorString::CreateFromArray(shape, data); + EXPECT_EQ(tensor.GetAsVectorView().Size(), data.size()); + EXPECT_TRUE(std::equal(data.cbegin(), data.cend(), begin(tensor.GetAsVectorView()))); + + // create from vector view + auto dataCopy = data; + tensor = TensorString::CreateFromIterable( + shape, winrt::single_threaded_vector(std::move(dataCopy)).GetView()); + EXPECT_EQ(tensor.GetAsVectorView().Size(), data.size()); + EXPECT_TRUE(std::equal(data.cbegin(), data.cend(), begin(tensor.GetAsVectorView()))); + + EXPECT_NO_THROW(LoadModel(L"id-tensor-string.onnx")); + LearningModelSession session(m_model); + + auto outputTensor = TensorString::Create(); + + std::map featuresstandardmap; + featuresstandardmap[L"X"] = tensor; + featuresstandardmap[L"Y"] = outputTensor; + auto featureswinrtmap = winrt::single_threaded_map(std::move(featuresstandardmap)); + session.EvaluateFeatures(featureswinrtmap, L"0"); + + // verify identity model round-trip works + EXPECT_EQ(outputTensor.GetAsVectorView().Size(), data.size()); + EXPECT_TRUE(std::equal(data.cbegin(), data.cend(), begin(outputTensor.GetAsVectorView()))); +} + +TEST_F(LearningModelSessionAPITests, EvaluateFeaturesAsync) +{ + std::vector shape = { 4 }; + std::vector data = { L"one", L"two", L"three", L"four" }; + + // create from buffer + auto tensor = TensorString::CreateFromArray(shape, data); + EXPECT_EQ(tensor.GetAsVectorView().Size(), data.size()); + EXPECT_TRUE(std::equal(data.cbegin(), data.cend(), begin(tensor.GetAsVectorView()))); + + // create from vector view + auto dataCopy = data; + tensor = TensorString::CreateFromIterable( + shape, winrt::single_threaded_vector(std::move(dataCopy)).GetView()); + EXPECT_EQ(tensor.GetAsVectorView().Size(), data.size()); + EXPECT_TRUE(std::equal(data.cbegin(), data.cend(), begin(tensor.GetAsVectorView()))); + + EXPECT_NO_THROW(LoadModel(L"id-tensor-string.onnx")); + LearningModelSession session(m_model); + + auto outputTensor = TensorString::Create(shape); + + std::map featuresstandardmap; + featuresstandardmap[L"X"] = tensor; + featuresstandardmap[L"Y"] = outputTensor; + auto featureswinrtmap = winrt::single_threaded_map(std::move(featuresstandardmap)); + session.EvaluateFeaturesAsync(featureswinrtmap, L"0").get(); + + // verify identity model round-trip works + EXPECT_EQ(outputTensor.GetAsVectorView().Size(), data.size()); + EXPECT_TRUE(std::equal(data.cbegin(), data.cend(), begin(outputTensor.GetAsVectorView()))); +} + +TEST_F(LearningModelSessionAPITests, EvaluationProperties) +{ + // load a model + EXPECT_NO_THROW(LoadModel(L"model.onnx")); + // create a session + m_session = LearningModelSession(m_model); + // set a property + auto value = winrt::Windows::Foundation::PropertyValue::CreateBoolean(true); + m_session.EvaluationProperties().Insert(L"propName1", value); + // get the property and make sure it's there with the right value + auto value2 = m_session.EvaluationProperties().Lookup(L"propName1"); + EXPECT_EQ(value2.as().GetBoolean(), true); +} + +static LearningModelSession CreateSession(LearningModel model) +{ + LearningModelDevice device(nullptr); + EXPECT_NO_THROW(device = LearningModelDevice(LearningModelDeviceKind::DirectX)); + + LearningModelSession session(nullptr); + if (DeviceHelpers::IsFloat16Supported(device)) + { + EXPECT_NO_THROW(session = LearningModelSession(model, device)); + } + else + { + EXPECT_THROW_SPECIFIC( + session = LearningModelSession(model, device), + winrt::hresult_error, + [](const winrt::hresult_error& e) -> bool + { + return e.code() == DXGI_ERROR_UNSUPPORTED; + }); + } + + return session; +} + +TEST_F(LearningModelSessionAPITests, CreateSessionWithCastToFloat16InModel) +{ + // load a model + EXPECT_NO_THROW(LoadModel(L"fp16-truncate-with-cast.onnx")); + + CreateSession(m_model); +} + +TEST_F(LearningModelSessionAPITests, DISABLED_CreateSessionWithFloat16InitializersInModel) +{ + // Disabled due to https://microsoft.visualstudio.com/DefaultCollection/OS/_workitems/edit/21624720: + // Model fails to resolve due to ORT using incorrect IR version within partition + + // load a model + EXPECT_NO_THROW(LoadModel(L"fp16-initializer.onnx")); + + CreateSession(m_model); +} + +static void EvaluateSessionAndCloseModel( + LearningModelDeviceKind kind, + bool close_model_on_session_creation) +{ + auto shape = std::vector{ 1, 1000 }; + + auto model = ProtobufHelpers::CreateModel(TensorKind::Float, shape, 1000); + + auto device = LearningModelDevice(kind); + auto options = LearningModelSessionOptions(); + + // close the model on session creation + options.CloseModelOnSessionCreation(close_model_on_session_creation); + + // ensure you can create a session from the model + LearningModelSession session(nullptr); + + EXPECT_NO_THROW(session = LearningModelSession(model, device, options)); + + std::vector input(1000); + std::iota(std::begin(input), std::end(input), 0.0f); + auto tensor_input = TensorFloat::CreateFromShapeArrayAndDataArray(shape, input); + auto binding = LearningModelBinding(session); + binding.Bind(L"input", tensor_input); + + LearningModelEvaluationResult result(nullptr); + EXPECT_NO_THROW(result = session.Evaluate(binding, L"")); + + if (close_model_on_session_creation) + { + // ensure that the model has been closed + EXPECT_THROW_SPECIFIC( + LearningModelSession(model, device, options), + winrt::hresult_error, + [](const winrt::hresult_error& e) -> bool + { + return e.code() == E_INVALIDARG; + }); + } + else + { + EXPECT_NO_THROW(LearningModelSession(model, device, options)); + } +} + +TEST_F(LearningModelSessionAPITests, EvaluateSessionAndCloseModel) +{ + EXPECT_NO_THROW(::EvaluateSessionAndCloseModel(LearningModelDeviceKind::Cpu, true)); + EXPECT_NO_THROW(::EvaluateSessionAndCloseModel(LearningModelDeviceKind::Cpu, false)); +} + +TEST_F(LearningModelSessionAPITests, CloseSession) +{ + EXPECT_NO_THROW(LoadModel(L"model.onnx")); + LearningModelSession session = nullptr; + + /* + HANDLE currentProcessHandle = NULL; + try + { + currentProcessHandle = GetCurrentProcess(); + } + catch (...) + { + VERIFY_FAIL(L"Failed to get current process handle."); + } + PROCESS_MEMORY_COUNTERS pmc = { 0 }; + SIZE_T beforeSessionCloseWorkingSetSize = 0; + SIZE_T afterSessionCloseWorkingSetSize = 0; + bool getProcessMemoryInfoSuccess = false; + */ + EXPECT_NO_THROW(session = LearningModelSession(m_model)); + + /* + // Get the current process memory info after session creation. + getProcessMemoryInfoSuccess = GetProcessMemoryInfo(currentProcessHandle, &pmc, sizeof(pmc)); + if (!getProcessMemoryInfoSuccess) + { + VERIFY_FAIL(L"Failed to get current process memory info."); + } + beforeSessionCloseWorkingSetSize = pmc.WorkingSetSize; + pmc = { 0 }; + */ + EXPECT_NO_THROW(session.Close()); + + /* + Bug 23659026: Working set difference tolerance is too tight for LearningModelSessionAPITests::CloseSession + https://microsoft.visualstudio.com/OS/_workitems/edit/23659026 + + // Check that working set size has dropped after session close + getProcessMemoryInfoSuccess = GetProcessMemoryInfo(currentProcessHandle, &pmc, sizeof(pmc)); + if (!getProcessMemoryInfoSuccess) + { + VERIFY_FAIL(L"Failed to get current process memory info."); + } + afterSessionCloseWorkingSetSize = pmc.WorkingSetSize; + pmc = { 0 }; + + // expected working set difference of session close. It is approximately 2x the size of the weights of model.onnx + // there needs to be a tolerance because the working set difference varies from run to run. + + // Bug 23739697: Closing Session API in LearningModelSessionAPITests::CloseSession doesn't always result in ~2x working set memory reduction. + // https://microsoft.visualstudio.com/OS/_workitems/edit/23739697 + float tolerance = 0.4f; + int64_t expectedWorkingSetDifference = 9662464; + VERIFY_IS_LESS_THAN(expectedWorkingSetDifference - (beforeSessionCloseWorkingSetSize - afterSessionCloseWorkingSetSize), expectedWorkingSetDifference * tolerance); + */ + + // verify that model still has metadata info after session close + std::wstring author(m_model.Author()); + EXPECT_EQ(author, L"onnx-caffe2"); + + // verify that session throws RO_E_CLOSED error + std::vector input(1 * 3 * 224 * 224, 0); + std::vector shape = { 1, 3, 224, 224 }; + auto tensor_input = TensorFloat::CreateFromShapeArrayAndDataArray(shape, input); + EXPECT_THROW_SPECIFIC(LearningModelBinding binding(session), + winrt::hresult_error, + [](const winrt::hresult_error &e) -> bool + { + return e.code() == RO_E_CLOSED; + }); + } diff --git a/winml/test/api/models/fp16-initializer.onnx b/winml/test/api/models/fp16-initializer.onnx new file mode 100644 index 0000000000..36e7e5e8e0 --- /dev/null +++ b/winml/test/api/models/fp16-initializer.onnx @@ -0,0 +1,12 @@ +sheilk:Š +6 +fp16_initializercast_to_float_output"Cast* +to +! +X +cast_to_float_outputY"Add* +*Bfp16_initializerZ +X +b +Y +B \ No newline at end of file diff --git a/winml/test/api/models/fp16-truncate-with-cast.onnx b/winml/test/api/models/fp16-truncate-with-cast.onnx new file mode 100644 index 0000000000..754dd91b0e --- /dev/null +++ b/winml/test/api/models/fp16-truncate-with-cast.onnx @@ -0,0 +1,12 @@ +sheilk:r +, +Xcast_to_float16_output"Cast* +to +  +, +cast_to_float16_outputY"Cast* +to  Z +X +b +Y + B \ No newline at end of file diff --git a/winml/test/api/models/id-tensor-string.onnx b/winml/test/api/models/id-tensor-string.onnx new file mode 100644 index 0000000000..aa9f059376 --- /dev/null +++ b/winml/test/api/models/id-tensor-string.onnx @@ -0,0 +1,11 @@ +dwayner:> + +XYIdentity"IdentityZ +X + + +b +Y + + +B \ No newline at end of file diff --git a/winml/test/collateral/images/100x100.png b/winml/test/collateral/images/100x100.png new file mode 100644 index 0000000000..ef222ed264 Binary files /dev/null and b/winml/test/collateral/images/100x100.png differ diff --git a/winml/test/collateral/images/227x227.png b/winml/test/collateral/images/227x227.png new file mode 100644 index 0000000000..facc9cbd6e Binary files /dev/null and b/winml/test/collateral/images/227x227.png differ diff --git a/winml/test/collateral/images/LICENSE.md b/winml/test/collateral/images/LICENSE.md new file mode 100644 index 0000000000..dd8020a1e4 --- /dev/null +++ b/winml/test/collateral/images/LICENSE.md @@ -0,0 +1,6 @@ +# Licenses + +| Image | Source | License | +| ----------- | ---------- | ----------- | +| [fish_720](fish_720.png), [fish](fish.png) | https://commons.wikimedia.org/wiki/File:Tinca_tinca.jpeg | CC Attribution 3.0 Unported | +| [kitten_224](kitten_224.png) | https://clipart.info/42-cat-png-image-download-picture-kitten-8177 | CC BY 4.0 | diff --git a/winml/test/collateral/images/fish.png b/winml/test/collateral/images/fish.png new file mode 100644 index 0000000000..5c083f805f Binary files /dev/null and b/winml/test/collateral/images/fish.png differ diff --git a/winml/test/collateral/images/fish_720.png b/winml/test/collateral/images/fish_720.png new file mode 100644 index 0000000000..3d413cfc2a Binary files /dev/null and b/winml/test/collateral/images/fish_720.png differ diff --git a/winml/test/collateral/images/kitten_224.png b/winml/test/collateral/images/kitten_224.png new file mode 100644 index 0000000000..e47ca94863 Binary files /dev/null and b/winml/test/collateral/images/kitten_224.png differ diff --git a/winml/test/collateral/models/Add_ImageNet1920.onnx b/winml/test/collateral/models/Add_ImageNet1920.onnx new file mode 100644 index 0000000000..694f3e5c65 --- /dev/null +++ b/winml/test/collateral/models/Add_ImageNet1920.onnx @@ -0,0 +1,27 @@ + OnnxMLTools +0.1.0.0000"onnxml:Ä +/ + +input_39:0 + +input_40:0 add_3/add:0Add"Addkeras_Add_ImageNet_smallZ& + +input_39:0 + + + +¸ +€Z& + +input_40:0 + + + +¸ +€b' + add_3/add:0 + + + +¸ +€B \ No newline at end of file diff --git a/winml/test/collateral/models/Add_ImageNet1920WithImageMetadataBgr8_LINEAR_0_255.onnx b/winml/test/collateral/models/Add_ImageNet1920WithImageMetadataBgr8_LINEAR_0_255.onnx new file mode 100644 index 0000000000..3943966a5b --- /dev/null +++ b/winml/test/collateral/models/Add_ImageNet1920WithImageMetadataBgr8_LINEAR_0_255.onnx @@ -0,0 +1,27 @@ + OnnxMLTools +0.1.0.0000"onnxml:í +% +input_39 +input_40add_3Add"Addkeras_Add_ImageNet_smallZ$ +input_39 + + + +¸ +€Z$ +input_40 + + + +¸ +€b^ +add_3U +LH + +DATA_BATCH + DATA_CHANNEL +¸ DATA_FEATURE +€ DATA_FEATURE2IMAGEBr +Image.BitmapPixelFormatBgr8r +Image.ColorSpaceGammaLinearr- +Image.NominalPixelRangeNominalRange_0_255 \ No newline at end of file diff --git a/winml/test/collateral/models/Add_ImageNet1920WithImageMetadataBgr8_SRGB_0_1.onnx b/winml/test/collateral/models/Add_ImageNet1920WithImageMetadataBgr8_SRGB_0_1.onnx new file mode 100644 index 0000000000..88fb241f32 --- /dev/null +++ b/winml/test/collateral/models/Add_ImageNet1920WithImageMetadataBgr8_SRGB_0_1.onnx @@ -0,0 +1,27 @@ + OnnxMLTools +0.1.0.0000"onnxml:í +% +input_39 +input_40add_3Add"Addkeras_Add_ImageNet_smallZ$ +input_39 + + + +¸ +€Z$ +input_40 + + + +¸ +€b^ +add_3U +LH + +DATA_BATCH + DATA_CHANNEL +¸ DATA_FEATURE +€ DATA_FEATURE2IMAGEBr +Image.BitmapPixelFormatBgr8r +Image.ColorSpaceGammaSRGBr) +Image.NominalPixelRangeNormalized_0_1 \ No newline at end of file diff --git a/winml/test/collateral/models/Add_ImageNet1920WithImageMetadataBgr8_SRGB_0_255.onnx b/winml/test/collateral/models/Add_ImageNet1920WithImageMetadataBgr8_SRGB_0_255.onnx new file mode 100644 index 0000000000..11f1bf6178 --- /dev/null +++ b/winml/test/collateral/models/Add_ImageNet1920WithImageMetadataBgr8_SRGB_0_255.onnx @@ -0,0 +1,26 @@ + OnnxMLTools +0.1.0.0000"onnxml:· +% +input_39 +input_40add_3Add"Addkeras_Add_ImageNet_smallZ$ +input_39 + + + +¸ +€Z$ +input_40 + + + +¸ +€b( +add_3 + + + +¸ +€2IMAGEBr +Image.BitmapPixelFormatBgr8r +Image.ColorSpaceGammaSRGBr- +Image.NominalPixelRangeNominalRange_0_255 \ No newline at end of file diff --git a/winml/test/collateral/models/Add_ImageNet1920WithImageMetadataBgr8_SRGB_16_235.onnx b/winml/test/collateral/models/Add_ImageNet1920WithImageMetadataBgr8_SRGB_16_235.onnx new file mode 100644 index 0000000000..6a0b5692ce --- /dev/null +++ b/winml/test/collateral/models/Add_ImageNet1920WithImageMetadataBgr8_SRGB_16_235.onnx @@ -0,0 +1,27 @@ + OnnxMLTools +0.1.0.0000"onnxml:í +% +input_39 +input_40add_3Add"Addkeras_Add_ImageNet_smallZ$ +input_39 + + + +¸ +€Z$ +input_40 + + + +¸ +€b^ +add_3U +LH + +DATA_BATCH + DATA_CHANNEL +¸ DATA_FEATURE +€ DATA_FEATURE2IMAGEBr +Image.BitmapPixelFormatBgr8r +Image.ColorSpaceGammaSRGBr. +Image.NominalPixelRangeNominalRange_16_235 \ No newline at end of file diff --git a/winml/test/collateral/models/Add_ImageNet1920WithImageMetadataBgr8_SRGB_1_1.onnx b/winml/test/collateral/models/Add_ImageNet1920WithImageMetadataBgr8_SRGB_1_1.onnx new file mode 100644 index 0000000000..c3faef843f --- /dev/null +++ b/winml/test/collateral/models/Add_ImageNet1920WithImageMetadataBgr8_SRGB_1_1.onnx @@ -0,0 +1,27 @@ + OnnxMLTools +0.1.0.0000"onnxml:í +% +input_39 +input_40add_3Add"Addkeras_Add_ImageNet_smallZ$ +input_39 + + + +¸ +€Z$ +input_40 + + + +¸ +€b^ +add_3U +LH + +DATA_BATCH + DATA_CHANNEL +¸ DATA_FEATURE +€ DATA_FEATURE2IMAGEBr +Image.BitmapPixelFormatBgr8r +Image.ColorSpaceGammaSRGBr) +Image.NominalPixelRangeNormalized_1_1 \ No newline at end of file diff --git a/winml/test/collateral/models/Add_ImageNet1920WithImageMetadataBgra8_SRGB_0_255.onnx b/winml/test/collateral/models/Add_ImageNet1920WithImageMetadataBgra8_SRGB_0_255.onnx new file mode 100644 index 0000000000..dfb06e76ce --- /dev/null +++ b/winml/test/collateral/models/Add_ImageNet1920WithImageMetadataBgra8_SRGB_0_255.onnx @@ -0,0 +1,26 @@ + OnnxMLTools +0.1.0.0000"onnxml:· +% +input_39 +input_40add_3Add"Addkeras_Add_ImageNet_smallZ$ +input_39 + + + +¸ +€Z$ +input_40 + + + +¸ +€b( +add_3 + + + +¸ +€2IMAGEBr +Image.BitmapPixelFormatBgra8r +Image.ColorSpaceGammaSRGBr- +Image.NominalPixelRangeNominalRange_0_255 \ No newline at end of file diff --git a/winml/test/collateral/models/Add_ImageNet1920WithImageMetadataRgb8_SRGB_0_255.onnx b/winml/test/collateral/models/Add_ImageNet1920WithImageMetadataRgb8_SRGB_0_255.onnx new file mode 100644 index 0000000000..7478cb96e5 --- /dev/null +++ b/winml/test/collateral/models/Add_ImageNet1920WithImageMetadataRgb8_SRGB_0_255.onnx @@ -0,0 +1,26 @@ + OnnxMLTools +0.1.0.0000"onnxml:· +% +input_39 +input_40add_3Add"Addkeras_Add_ImageNet_smallZ$ +input_39 + + + +¸ +€Z$ +input_40 + + + +¸ +€b( +add_3 + + + +¸ +€2IMAGEBr +Image.BitmapPixelFormatRgb8r +Image.ColorSpaceGammaSRGBr- +Image.NominalPixelRangeNominalRange_0_255 \ No newline at end of file diff --git a/winml/test/collateral/models/Add_ImageNet1920WithImageMetadataRgba8_SRGB_0_255.onnx b/winml/test/collateral/models/Add_ImageNet1920WithImageMetadataRgba8_SRGB_0_255.onnx new file mode 100644 index 0000000000..cd325f227f --- /dev/null +++ b/winml/test/collateral/models/Add_ImageNet1920WithImageMetadataRgba8_SRGB_0_255.onnx @@ -0,0 +1,26 @@ + OnnxMLTools +0.1.0.0000"onnxml:· +% +input_39 +input_40add_3Add"Addkeras_Add_ImageNet_smallZ$ +input_39 + + + +¸ +€Z$ +input_40 + + + +¸ +€b( +add_3 + + + +¸ +€2IMAGEBr +Image.BitmapPixelFormatRgba8r +Image.ColorSpaceGammaSRGBr- +Image.NominalPixelRangeNominalRange_0_255 \ No newline at end of file diff --git a/winml/test/collateral/models/LICENSE.md b/winml/test/collateral/models/LICENSE.md new file mode 100644 index 0000000000..f6b9ebbecc --- /dev/null +++ b/winml/test/collateral/models/LICENSE.md @@ -0,0 +1,65 @@ +# Licenses + +| Model | Source | License | +| ----------- | ---------- | ----------- | +| SqueezeNet | https://github.com/DeepScale/SqueezeNet | BSD 2-Clause | +| Starry Night | https://github.com/pytorch/examples/tree/master/fast_neural_style | BSD 3-Clause | + +SqueezeNet license: + +``` +BSD LICENSE. + +Redistribution and use in source and binary forms, with or without modification, are permitted +provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions +and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions +and the following disclaimer in the documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER +IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +``` + +Starry Night license: + +``` +BSD 3-Clause License + +Copyright (c) 2017, +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +``` diff --git a/winml/test/collateral/models/ModelSubdirectory/ModelInSubdirectory.onnx b/winml/test/collateral/models/ModelSubdirectory/ModelInSubdirectory.onnx new file mode 100644 index 0000000000..64e447620c Binary files /dev/null and b/winml/test/collateral/models/ModelSubdirectory/ModelInSubdirectory.onnx differ diff --git a/winml/test/collateral/models/bad_names.onnx b/winml/test/collateral/models/bad_names.onnx new file mode 100644 index 0000000000..5a0645c0d1 --- /dev/null +++ b/winml/test/collateral/models/bad_names.onnx @@ -0,0 +1,13 @@ +kiyoung:^ +* + +input/nameoutput:0Identity"IdentityZ + +input/name + + +b +output:0 + + + \ No newline at end of file diff --git a/winml/test/collateral/models/castmap-int64.onnx b/winml/test/collateral/models/castmap-int64.onnx new file mode 100644 index 0000000000..e80ecfa15e --- /dev/null +++ b/winml/test/collateral/models/castmap-int64.onnx @@ -0,0 +1,12 @@ +dwayner:N +$ +XYCastMap"CastMap: +ai.onnx.mlZ +X* + + +b +Y + + +B \ No newline at end of file diff --git a/winml/test/collateral/models/conv-float.onnx b/winml/test/collateral/models/conv-float.onnx new file mode 100644 index 0000000000..76206b93c1 Binary files /dev/null and b/winml/test/collateral/models/conv-float.onnx differ diff --git a/winml/test/collateral/models/dictvectorizer-int64.onnx b/winml/test/collateral/models/dictvectorizer-int64.onnx new file mode 100644 index 0000000000..063580f747 Binary files /dev/null and b/winml/test/collateral/models/dictvectorizer-int64.onnx differ diff --git a/winml/test/collateral/models/dictvectorizer-string.onnx b/winml/test/collateral/models/dictvectorizer-string.onnx new file mode 100644 index 0000000000..9146e3172d Binary files /dev/null and b/winml/test/collateral/models/dictvectorizer-string.onnx differ diff --git a/winml/test/collateral/models/foo.onnx b/winml/test/collateral/models/foo.onnx new file mode 100644 index 0000000000..51558c9981 Binary files /dev/null and b/winml/test/collateral/models/foo.onnx differ diff --git a/winml/test/collateral/models/foo_truncated.onnx b/winml/test/collateral/models/foo_truncated.onnx new file mode 100644 index 0000000000..e034badc0a Binary files /dev/null and b/winml/test/collateral/models/foo_truncated.onnx differ diff --git a/winml/test/collateral/models/free_dimensional_imageDes.onnx b/winml/test/collateral/models/free_dimensional_imageDes.onnx new file mode 100644 index 0000000000..87687688f9 --- /dev/null +++ b/winml/test/collateral/models/free_dimensional_imageDes.onnx @@ -0,0 +1,33 @@ + OnnxMLTools +0.1.0.0000"onnxml:¬ +/ + +input_39:0 + +input_40:0 add_3/add:0Add"Addkeras_Add_ImageNet_smallZs + +input_39:0e +\X + +DATA_BATCH + DATA_CHANNEL +ÿÿÿÿÿÿÿÿÿ DATA_FEATURE +ÿÿÿÿÿÿÿÿÿ DATA_FEATURE2IMAGEZs + +input_40:0e +\X + +DATA_BATCH + DATA_CHANNEL +ÿÿÿÿÿÿÿÿÿ DATA_FEATURE +ÿÿÿÿÿÿÿÿÿ DATA_FEATURE2IMAGEbu + add_3/add:0f +\X + +DATA_BATCH + DATA_CHANNEL +ÿÿÿÿÿÿÿÿÿ DATA_FEATURE +ÿÿÿÿÿÿÿÿÿ DATA_FEATURE2TENSORBr +Image.BitmapPixelFormatBgr8r +Image.ColorSpaceGammaSRGBr- +Image.NominalPixelRangeNominalRange_0_255 \ No newline at end of file diff --git a/winml/test/collateral/models/free_dimensional_image_input.onnx b/winml/test/collateral/models/free_dimensional_image_input.onnx new file mode 100644 index 0000000000..6750f70262 --- /dev/null +++ b/winml/test/collateral/models/free_dimensional_image_input.onnx @@ -0,0 +1,27 @@ + OnnxMLTools +0.1.0.0000"onnxml:ô +/ + +input_39:0 + +input_40:0 add_3/add:0Add"Addkeras_Add_ImageNet_smallZ6 + +input_39:0( +&" + + + ÿÿÿÿÿÿÿÿÿ + ÿÿÿÿÿÿÿÿÿZ6 + +input_40:0( +&" + + + ÿÿÿÿÿÿÿÿÿ + ÿÿÿÿÿÿÿÿÿb7 + add_3/add:0( +&" + + + ÿÿÿÿÿÿÿÿÿ + ÿÿÿÿÿÿÿÿÿB \ No newline at end of file diff --git a/winml/test/collateral/models/mnist.onnx b/winml/test/collateral/models/mnist.onnx new file mode 100644 index 0000000000..bb189a52ea Binary files /dev/null and b/winml/test/collateral/models/mnist.onnx differ diff --git a/winml/test/collateral/models/modelWith2MetaData.onnx b/winml/test/collateral/models/modelWith2MetaData.onnx new file mode 100644 index 0000000000..bd6dfdc0c3 Binary files /dev/null and b/winml/test/collateral/models/modelWith2MetaData.onnx differ diff --git a/winml/test/collateral/models/modelWithMetaData.onnx b/winml/test/collateral/models/modelWithMetaData.onnx new file mode 100644 index 0000000000..f8d7e1613d Binary files /dev/null and b/winml/test/collateral/models/modelWithMetaData.onnx differ diff --git a/winml/test/collateral/models/mul.onnx b/winml/test/collateral/models/mul.onnx new file mode 100644 index 0000000000..0ddb8e2c0f Binary files /dev/null and b/winml/test/collateral/models/mul.onnx differ diff --git a/winml/test/collateral/models/relu.onnx b/winml/test/collateral/models/relu.onnx new file mode 100644 index 0000000000..e316dd1eb0 --- /dev/null +++ b/winml/test/collateral/models/relu.onnx @@ -0,0 +1,11 @@ +justoeck:0 + +XY"ReluZ +X + + +b +Y + + +B \ No newline at end of file diff --git a/winml/test/collateral/models/squeezenet_modifiedforruntimestests.onnx b/winml/test/collateral/models/squeezenet_modifiedforruntimestests.onnx new file mode 100644 index 0000000000..64e447620c Binary files /dev/null and b/winml/test/collateral/models/squeezenet_modifiedforruntimestests.onnx differ diff --git a/winml/test/collateral/models/squeezenet_tensor_input.onnx b/winml/test/collateral/models/squeezenet_tensor_input.onnx new file mode 100644 index 0000000000..e14d8502a8 Binary files /dev/null and b/winml/test/collateral/models/squeezenet_tensor_input.onnx differ diff --git a/winml/test/collateral/models/starry-night-fp16.onnx b/winml/test/collateral/models/starry-night-fp16.onnx new file mode 100644 index 0000000000..9a00201905 Binary files /dev/null and b/winml/test/collateral/models/starry-night-fp16.onnx differ diff --git a/winml/test/collateral/models/zipmap-int64.onnx b/winml/test/collateral/models/zipmap-int64.onnx new file mode 100644 index 0000000000..70a0f83227 Binary files /dev/null and b/winml/test/collateral/models/zipmap-int64.onnx differ diff --git a/winml/test/collateral/models/zipmap-string.onnx b/winml/test/collateral/models/zipmap-string.onnx new file mode 100644 index 0000000000..07de232b1b Binary files /dev/null and b/winml/test/collateral/models/zipmap-string.onnx differ diff --git a/winml/test/common/SqueezeNetValidator.cpp b/winml/test/common/SqueezeNetValidator.cpp new file mode 100644 index 0000000000..2b1377868e --- /dev/null +++ b/winml/test/common/SqueezeNetValidator.cpp @@ -0,0 +1,335 @@ +#include "SqueezeNetValidator.h" +#include "protobufHelpers.h" +#include "fileHelpers.h" +#include +#include +#include +#include +#include + +#include "WinMLProfiler.h" + +// using namespace winrt::Windows::Foundation; +using namespace winrt::Windows::AI::MachineLearning; +using namespace winrt::Windows::Foundation::Collections; +using namespace winrt::Windows::Graphics::Imaging; +using namespace winrt::Windows::Media; +using namespace winrt::Windows::Storage; +using namespace winrt::Windows::Storage::Streams; + +namespace WinML::Engine::Test{ + +enum WINML_RUNTIME_TEST_PERF +{ + PREP_TEST = 0, + CREATE_RUNTIME, + LOAD_MODEL, + CREATE_EVAL_CONTEXT, + RUN_TEST, + BIND_VALUE, + EVAL_MODEL, + EVAL_MODEL_FIRST_RUN, + kCount +}; + +static std::vector WINML_RUNTIME_TEST_PERF_NAMES = +{ + "PREP TEST ", + " CREATE RUNTIME ", + " LOAD MODEL ", + " CREATE EVAL CONTEXT", + "RUN TEST ", + " BIND VALUE ", + " EVAL MODEL ", + " EVAL MODEL 1st Run " +}; + +#define MAX_PROFILING_LOOP 100 +Profiler g_RuntimeProfiler; + + +static void BindImage( + LearningModelBinding binding, + const wchar_t* name, + const wchar_t* fullImagePath, + bool bindAsInspectable = false) +{ + auto imagefile = StorageFile::GetFileFromPathAsync(fullImagePath).get(); + auto stream = imagefile.OpenAsync(FileAccessMode::Read).get(); + auto decoder = BitmapDecoder::CreateAsync(stream).get(); + auto softwareBitmap = decoder.GetSoftwareBitmapAsync().get(); + auto frame = VideoFrame::CreateWithSoftwareBitmap(softwareBitmap); + + if (bindAsInspectable) + { + EXPECT_NO_THROW(binding.Bind(name, frame)); + } + else + { + auto imagetensor = ImageFeatureValue::CreateFromVideoFrame(frame); + EXPECT_NO_THROW(binding.Bind(name, imagetensor)); + } +} + +static void BindTensor( + LearningModelBinding binding, + const wchar_t* name, + ITensor inputTensor, + bool bindAsInspectable = false) +{ + EXPECT_TRUE(inputTensor != nullptr); + + if (bindAsInspectable) + { + EXPECT_NO_THROW(binding.Bind(name, inputTensor.as().GetAsVectorView())); + } + else + { + EXPECT_NO_THROW(binding.Bind(name, inputTensor)); + } +} + +template +ITensor BindOutput( + OutputBindingStrategy strategy, + LearningModelBinding binding, + const wchar_t* name, + const IVectorView shape = nullptr +) +{ + ITensor outputTensor = nullptr; + switch (strategy) + { + case OutputBindingStrategy::Bound: + outputTensor = T::Create(shape); + EXPECT_NO_THROW(binding.Bind(name, outputTensor)); + break; + case OutputBindingStrategy::Empty: + outputTensor = T::Create(); + EXPECT_NO_THROW(binding.Bind(name, outputTensor)); + break; + case OutputBindingStrategy::Unbound: + __fallthrough; + default: + break; + } + + return outputTensor; +} + +ImageFeatureValue BindImageOutput( + OutputBindingStrategy strategy, + LearningModelBinding binding, + const wchar_t* name +) +{ + ImageFeatureValue outputTensor = nullptr; + switch (strategy) + { + case OutputBindingStrategy::Bound: + { + SoftwareBitmap bitmap(BitmapPixelFormat::Bgra8, 720, 720); + VideoFrame frame = VideoFrame::CreateWithSoftwareBitmap(bitmap); + outputTensor = ImageFeatureValue::CreateFromVideoFrame(frame); + EXPECT_NO_THROW(binding.Bind(name, outputTensor)); + break; + } + case OutputBindingStrategy::Unbound: + __fallthrough; + } + + return outputTensor; +} + + +void ModelValidator::FnsCandy16( + std::string instance, + LearningModelDeviceKind deviceKind, + OutputBindingStrategy outputBindingStrategy, + bool bindInputsAsIInspectable, + float dataTolerance) +{ + WINML_PROFILING_START(g_RuntimeProfiler, WINML_RUNTIME_TEST_PERF::PREP_TEST); + // file name strings + static wchar_t* modelFileName = L"winmlperf_coreml_FNS-Candy_prerelease_fp16.onnx"; + static wchar_t* inputDataImageFileName = L"fish_720.png"; + static wchar_t* outputDataFileName = L"output.png"; + static wchar_t* inputBindingName = L"inputImage"; + static const wchar_t* outputDataBindingName = L"outputImage"; + + auto modulePath = FileHelpers::GetModulePath(); + auto fullModelPath = modulePath + modelFileName; + auto outputFileName = modulePath + outputDataFileName; + WINML_PROFILING_STOP(g_RuntimeProfiler, WINML_RUNTIME_TEST_PERF::PREP_TEST); + + WINML_PROFILING_START(g_RuntimeProfiler, WINML_RUNTIME_TEST_PERF::LOAD_MODEL); + // WinML model creation + LearningModel model = nullptr; + EXPECT_NO_THROW(model = LearningModel::LoadFromFilePath(fullModelPath)); + WINML_PROFILING_STOP(g_RuntimeProfiler, WINML_RUNTIME_TEST_PERF::LOAD_MODEL); + + WINML_PROFILING_START(g_RuntimeProfiler, WINML_RUNTIME_TEST_PERF::RUN_TEST); + WINML_PROFILING_START(g_RuntimeProfiler, WINML_RUNTIME_TEST_PERF::CREATE_EVAL_CONTEXT); + LearningModelSession modelSession = nullptr; + EXPECT_NO_THROW(modelSession = LearningModelSession(model, LearningModelDevice(deviceKind))); + WINML_PROFILING_STOP(g_RuntimeProfiler, WINML_RUNTIME_TEST_PERF::CREATE_EVAL_CONTEXT); + + WINML_PROFILING_START(g_RuntimeProfiler, WINML_RUNTIME_TEST_PERF::BIND_VALUE); + LearningModelBinding modelBinding(modelSession); + auto fullImagePath = modulePath + inputDataImageFileName; + BindImage(modelBinding, inputBindingName, fullImagePath.c_str(), bindInputsAsIInspectable); + + // create the tensor for the actual output + auto output = model.OutputFeatures().First().Current(); + EXPECT_TRUE(output.Kind() == LearningModelFeatureKind::Tensor); + + auto shape = winrt::single_threaded_vector(std::vector {1, 1}); + auto outputTensor = BindImageOutput(outputBindingStrategy, modelBinding, outputDataBindingName); + WINML_PROFILING_STOP(g_RuntimeProfiler, WINML_RUNTIME_TEST_PERF::BIND_VALUE); + + // Evaluate the model + WINML_PROFILING_START(g_RuntimeProfiler, WINML_RUNTIME_TEST_PERF::EVAL_MODEL_FIRST_RUN); + std::cout << "Calling EvaluateSync on instance" << instance << "\n"; + LearningModelEvaluationResult result = nullptr; + EXPECT_NO_THROW(result = modelSession.Evaluate(modelBinding, {})); + WINML_PROFILING_STOP(g_RuntimeProfiler, WINML_RUNTIME_TEST_PERF::EVAL_MODEL_FIRST_RUN); + + // Get results + if (outputBindingStrategy == OutputBindingStrategy::Unbound) + { + // When output binding strategy is unbound, the output tensor was not set on bind. + // Therefore, we need to retrieve it from the LearnignModelEvaluationResult + // TODO: is this right? outputTensorT is unused... + /*auto outputTensorT = */result.Outputs().Lookup(outputDataBindingName).as(); + } + else + { + EXPECT_EQ(result.Outputs().Lookup(outputDataBindingName), outputTensor); + + auto softwareBitmap = outputTensor.VideoFrame().SoftwareBitmap(); + + auto folder = StorageFolder::GetFolderFromPathAsync(modulePath.c_str()).get(); + auto imagefile = folder.CreateFileAsync(outputDataFileName, CreationCollisionOption::ReplaceExisting).get(); + auto stream = imagefile.OpenAsync(FileAccessMode::ReadWrite).get(); + auto encoder = BitmapEncoder::CreateAsync(BitmapEncoder::JpegEncoderId(), stream).get(); + encoder.SetSoftwareBitmap(softwareBitmap); + encoder.FlushAsync(); + + } + WINML_PROFILING_STOP(g_RuntimeProfiler, WINML_RUNTIME_TEST_PERF::RUN_TEST); +} + +void ModelValidator::SqueezeNet( + std::string instance, + LearningModelDeviceKind deviceKind, + float dataTolerance, + bool bindAsImage, + OutputBindingStrategy outputBindingStrategy, + bool bindInputsAsIInspectable) +{ + g_RuntimeProfiler.Enable(ProfilerType::CPU); + g_RuntimeProfiler.Enable(ProfilerType::GPU); + g_RuntimeProfiler.Reset(ProfilerType::CPU); + g_RuntimeProfiler.Reset(ProfilerType::GPU); + + WINML_PROFILING_START(g_RuntimeProfiler, WINML_RUNTIME_TEST_PERF::PREP_TEST); + // file name strings + static wchar_t* modelFileName = L"model.onnx"; + static wchar_t* inputDataFileName = L"test_data_0_input.pb"; + static wchar_t* outputDataFileName = L"test_data_0_output.pb"; + static wchar_t* inputBindingName = L"data_0"; + static wchar_t* inputDataImageFileName = L"kitten_224.png"; + static const wchar_t* outputDataBindingName = L"softmaxout_1"; + + auto modulePath = FileHelpers::GetModulePath(); + auto fullModelPath = modulePath + modelFileName; + auto outputFileName = modulePath + outputDataFileName; + WINML_PROFILING_STOP(g_RuntimeProfiler, WINML_RUNTIME_TEST_PERF::PREP_TEST); + + WINML_PROFILING_START(g_RuntimeProfiler, WINML_RUNTIME_TEST_PERF::LOAD_MODEL); + // WinML model creation + LearningModel model = nullptr; + EXPECT_NO_THROW(model = LearningModel::LoadFromFilePath(fullModelPath)); + WINML_PROFILING_STOP(g_RuntimeProfiler, WINML_RUNTIME_TEST_PERF::LOAD_MODEL); + + WINML_PROFILING_START(g_RuntimeProfiler, WINML_RUNTIME_TEST_PERF::RUN_TEST); + WINML_PROFILING_START(g_RuntimeProfiler, WINML_RUNTIME_TEST_PERF::CREATE_EVAL_CONTEXT); + LearningModelSession modelSession = nullptr; + EXPECT_NO_THROW(modelSession = LearningModelSession(model, LearningModelDevice(deviceKind))); + WINML_PROFILING_STOP(g_RuntimeProfiler, WINML_RUNTIME_TEST_PERF::CREATE_EVAL_CONTEXT); + + WINML_PROFILING_START(g_RuntimeProfiler, WINML_RUNTIME_TEST_PERF::BIND_VALUE); + LearningModelBinding modelBinding(modelSession); + + if (bindAsImage) + { + std::wstring fullImagePath = modulePath + inputDataImageFileName; + BindImage(modelBinding, inputBindingName, fullImagePath.c_str(), bindInputsAsIInspectable); + } + else + { + auto inputDataPath = modulePath + inputDataFileName; + auto inputTensor = ProtobufHelpers::LoadTensorFromProtobufFile(inputDataPath, false); + BindTensor(modelBinding, inputBindingName, inputTensor, bindInputsAsIInspectable); + } + + // load up the expected output + auto expectedResultsTensor = ProtobufHelpers::LoadTensorFromProtobufFile(outputFileName, false); + EXPECT_TRUE(expectedResultsTensor != nullptr); + + // create the tensor for the actual output + auto output = model.OutputFeatures().First().Current(); + EXPECT_TRUE(output.Kind() == LearningModelFeatureKind::Tensor); + + auto outputTensor = BindOutput( + outputBindingStrategy, modelBinding, outputDataBindingName, expectedResultsTensor.Shape()); + WINML_PROFILING_STOP(g_RuntimeProfiler, WINML_RUNTIME_TEST_PERF::BIND_VALUE); + + // Evaluate the model + WINML_PROFILING_START(g_RuntimeProfiler, WINML_RUNTIME_TEST_PERF::EVAL_MODEL_FIRST_RUN); + std::cout << "Calling EvaluateSync on instance" << instance << "\n"; + LearningModelEvaluationResult result = nullptr; + EXPECT_NO_THROW(result = modelSession.Evaluate(modelBinding, {})); + WINML_PROFILING_STOP(g_RuntimeProfiler, WINML_RUNTIME_TEST_PERF::EVAL_MODEL_FIRST_RUN); + + // Get results + if (outputBindingStrategy == OutputBindingStrategy::Unbound) + { + // When output binding strategy is unbound, the output tensor was not set on bind. + // Therefore, we need to retrieve it from the LearnignModelEvaluationResult + outputTensor = result.Outputs().Lookup(outputDataBindingName).as(); + } + else + { + EXPECT_EQ(result.Outputs().Lookup(outputDataBindingName), outputTensor); + } + + auto outDataExpected = expectedResultsTensor.as().GetAsVectorView(); + auto outDataActual = outputTensor.as().GetAsVectorView(); + + EXPECT_TRUE(outDataActual.Size() == outDataExpected.Size()); + for (uint32_t i = 0; i < outDataActual.Size(); i++) + { + float delta = std::abs(outDataActual.GetAt(i) - outDataExpected.GetAt(i)); + if (delta > dataTolerance) + { + ADD_FAILURE() << "EXPECTED: " << outDataExpected.GetAt(i) << " , ACTUAL: " << outDataActual.GetAt(i) + << "instance " << instance << ", element " << i; + + } + } + WINML_PROFILING_STOP(g_RuntimeProfiler, WINML_RUNTIME_TEST_PERF::RUN_TEST); + + std::cout << "Profiling data:\n"; + for (int i = 0; i < WINML_RUNTIME_TEST_PERF::kCount; ++i) + { + std::cout << WINML_RUNTIME_TEST_PERF_NAMES[i] + << ": Time=" << g_RuntimeProfiler[i].GetAverage(CounterType::TIMER) + << "\tCPUUse(%%)=" << g_RuntimeProfiler[i].GetAverage(CounterType::CPU_USAGE) + << "\tAvgWorkingSetDelta(MB)=" << g_RuntimeProfiler[i].GetAverage(CounterType::WORKING_SET_USAGE) + << "\tMaxWorkingSetDelta(MB)=" << g_RuntimeProfiler[i].GetMax(CounterType::WORKING_SET_USAGE) + << "\tGPUUse(%%)=" << g_RuntimeProfiler[i].GetAverage(CounterType::GPU_USAGE) + << "\tGPUDedicatedMem(MB)=" << g_RuntimeProfiler[i].GetAverage(CounterType::GPU_DEDICATED_MEM_USAGE); + } +} +} diff --git a/winml/test/common/SqueezeNetValidator.h b/winml/test/common/SqueezeNetValidator.h new file mode 100644 index 0000000000..68b44ddb2d --- /dev/null +++ b/winml/test/common/SqueezeNetValidator.h @@ -0,0 +1,30 @@ +//----------------------------------------------------------------------------- +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// +//----------------------------------------------------------------------------- + +#pragma once + +#include + +enum OutputBindingStrategy { Bound, Unbound, Empty }; + +namespace WinML::Engine::Test::ModelValidator +{ + void FnsCandy16( + std::string instance, + winrt::Windows::AI::MachineLearning::LearningModelDeviceKind deviceKind, + OutputBindingStrategy outputBindingStrategy, + bool bindInputsAsIInspectable, + float dataTolerance = false); + + void SqueezeNet( + std::string instance, + winrt::Windows::AI::MachineLearning::LearningModelDeviceKind deviceKind, + float dataTolerance, + bool bindAsImage = false, + OutputBindingStrategy outputBindingStrategy = OutputBindingStrategy::Bound, + bool bindInputsAsIInspectable = false + ); +} diff --git a/winml/test/common/dllload.cpp b/winml/test/common/dllload.cpp new file mode 100644 index 0000000000..19a1efdda4 --- /dev/null +++ b/winml/test/common/dllload.cpp @@ -0,0 +1,75 @@ +#include "Std.h" +#include "fileHelpers.h" +#include + +extern "C" +{ + HRESULT __stdcall OS_RoGetActivationFactory(HSTRING classId, GUID const& iid, void** factory) noexcept; +} + +#ifdef _M_IX86 +#pragma comment(linker, "/alternatename:_OS_RoGetActivationFactory@12=_RoGetActivationFactory@12") +#else +#pragma comment(linker, "/alternatename:OS_RoGetActivationFactory=RoGetActivationFactory") +#endif + +bool starts_with(std::wstring_view value, std::wstring_view match) noexcept +{ + return 0 == value.compare(0, match.size(), match); +} + +HRESULT __stdcall WINRT_RoGetActivationFactory(HSTRING classId_hstring, GUID const& iid, void** factory) noexcept +{ + *factory = nullptr; + std::wstring_view name{ WindowsGetStringRawBuffer(classId_hstring, nullptr), WindowsGetStringLen(classId_hstring) }; + HMODULE library{ nullptr }; + + std::wstring winmlDllPath = FileHelpers::GetWinMLPath() + L"Windows.AI.MachineLearning.dll"; + + if (starts_with(name, L"Windows.AI.MachineLearning.")) + { + const wchar_t* libPath = winmlDllPath.c_str(); + library = LoadLibraryW(libPath); + } + else + { + return OS_RoGetActivationFactory(classId_hstring, iid, factory); + } + + if (!library) + { + return HRESULT_FROM_WIN32(GetLastError()); + } + + using DllGetActivationFactory = HRESULT __stdcall(HSTRING classId, void** factory); + auto call = reinterpret_cast(GetProcAddress(library, "DllGetActivationFactory")); + + if (!call) + { + HRESULT const hr = HRESULT_FROM_WIN32(GetLastError()); + WINRT_VERIFY(FreeLibrary(library)); + return hr; + } + + winrt::com_ptr activation_factory; + HRESULT const hr = call(classId_hstring, activation_factory.put_void()); + + if (FAILED(hr)) + { + WINRT_VERIFY(FreeLibrary(library)); + return hr; + } + + if (winrt::guid(iid) != winrt::guid_of()) + { + return activation_factory->QueryInterface(iid, factory); + } + + *factory = activation_factory.detach(); + return S_OK; +} + +int32_t __stdcall WINRT_RoGetActivationFactory(void* classId, winrt::guid const& iid, void** factory) noexcept +{ + return WINRT_RoGetActivationFactory((HSTRING)classId, (GUID)iid, factory); +} diff --git a/winml/test/common/fileHelpers.cpp b/winml/test/common/fileHelpers.cpp new file mode 100644 index 0000000000..cde8646fde --- /dev/null +++ b/winml/test/common/fileHelpers.cpp @@ -0,0 +1,67 @@ +#include "std.h" +#include "fileHelpers.h" +#include "winrt/Windows.Media.h" +#include "winrt/Windows.Storage.h" + +EXTERN_C IMAGE_DOS_HEADER __ImageBase; + +using namespace winrt; +using namespace winrt::Windows::AI::MachineLearning; +using namespace winrt::Windows::Graphics::Imaging; +using namespace winrt::Windows::Storage; + +namespace FileHelpers +{ + std::wstring GetModulePath() + { + std::wstring val; + wchar_t modulePath[MAX_PATH] = { 0 }; + GetModuleFileNameW((HINSTANCE)&__ImageBase, modulePath, _countof(modulePath)); + wchar_t drive[_MAX_DRIVE]; + wchar_t dir[_MAX_DIR]; + wchar_t filename[_MAX_FNAME]; + wchar_t ext[_MAX_EXT]; + _wsplitpath_s(modulePath, drive, _MAX_DRIVE, dir, _MAX_DIR, filename, _MAX_FNAME, ext, _MAX_EXT); + + val = drive; + val += dir; + + return val; + } + + std::wstring GetWinMLPath() + { + // bool inboxDll = false; + // TODO Add command line parsing + // if (SUCCEEDED(WEX::TestExecution::RuntimeParameters::TryGetValue(L"inbox", inboxDll)) && inboxDll) + // { + // return L""; + // } + return GetModulePath(); + } + + + winrt::Windows::Graphics::Imaging::SoftwareBitmap GetSoftwareBitmapFromFile(const std::wstring& filePath) + { + auto storageFile = StorageFile::GetFileFromPathAsync(filePath).get(); + auto stream = storageFile.OpenAsync(FileAccessMode::Read).get(); + auto decoder = BitmapDecoder::CreateAsync(stream).get(); + IBitmapFrameWithSoftwareBitmap bitmapFrameWithSoftwareBitmap; + decoder.as(bitmapFrameWithSoftwareBitmap); + auto softwareBitmap = bitmapFrameWithSoftwareBitmap.GetSoftwareBitmapAsync( + BitmapPixelFormat::Bgra8, + BitmapAlphaMode::Ignore, + BitmapTransform::BitmapTransform(), + ExifOrientationMode::IgnoreExifOrientation, + ColorManagementMode::DoNotColorManage + ).get(); + return softwareBitmap; + } + + ImageFeatureValue LoadImageFeatureValue(const std::wstring& imagePath) + { + auto softwareBitmap = FileHelpers::GetSoftwareBitmapFromFile(FileHelpers::GetModulePath() + imagePath); + auto videoFrame = winrt::Windows::Media::VideoFrame::CreateWithSoftwareBitmap(softwareBitmap); + return ImageFeatureValue::CreateFromVideoFrame(videoFrame); + } +} diff --git a/winml/test/common/fileHelpers.h b/winml/test/common/fileHelpers.h new file mode 100644 index 0000000000..216368592c --- /dev/null +++ b/winml/test/common/fileHelpers.h @@ -0,0 +1,18 @@ +//----------------------------------------------------------------------------- +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// +//----------------------------------------------------------------------------- + +#pragma once +#include "winrt/Windows.Graphics.Imaging.h" +#include "winrt/Windows.AI.MachineLearning.h" + +namespace FileHelpers +{ + std::wstring GetModulePath(); + std::wstring GetWinMLPath(); + + winrt::Windows::Graphics::Imaging::SoftwareBitmap GetSoftwareBitmapFromFile(const std::wstring& filePath); + winrt::Windows::AI::MachineLearning::ImageFeatureValue LoadImageFeatureValue(const std::wstring& imagePath); +} diff --git a/winml/test/common/protobufHelpers.cpp b/winml/test/common/protobufHelpers.cpp new file mode 100644 index 0000000000..025f8d2e73 --- /dev/null +++ b/winml/test/common/protobufHelpers.cpp @@ -0,0 +1,324 @@ +// LotusRT +#include "core/framework/allocatormgr.h" +// #include "core/session/inference_session.h" +#include "core/common/logging/logging.h" +#include "core/common/logging/sinks/clog_sink.h" + +#include "protobufHelpers.h" +#include "onnx/onnx-ml.pb.h" +#include +#include + +#include "winrt/Windows.Storage.Streams.h" + +#pragma warning(disable : 4244) + +using namespace winrt::Windows::Storage::Streams; +using namespace winrt::Windows::AI::MachineLearning; +using namespace winrt::Windows::Foundation::Collections; + +// Copy and pasted from LOTUS as is. temporary code to load tensors from protobufs +int FdOpen(const std::string& name) +{ + int fd = -1; +#ifdef _WIN32 + _sopen_s(&fd, name.c_str(), _O_RDONLY | _O_SEQUENTIAL | _O_BINARY, _SH_DENYWR, _S_IREAD | _S_IWRITE); +#else + fd = open(name.c_str(), O_RDONLY); +#endif + return fd; +}; + +// Copy and pasted from LOTUS as is. temporary code to load tensors from protobufs +void FdClose(int fd) +{ + if (fd >= 0) + { +#ifdef _WIN32 + _close(fd); +#else + close(fd); +#endif + } +} + +// Copy and pasted from LOTUS as is. temporary code to load tensors from protobufs +bool LoadTensorFromPb(onnx::TensorProto& tensor, std::wstring filePath) +{ + // setup a string converter + using convert_type = std::codecvt_utf8; + std::wstring_convert converter; + + // use converter (.to_bytes: wstr->str, .from_bytes: str->wstr) + std::string file = converter.to_bytes(filePath.c_str()); + + std::ifstream stream(file, std::ios::binary | std::ios::ate); + std::streamsize size = stream.tellg(); + stream.seekg(0, std::ios::beg); + + std::vector buffer(size); + if (stream.read(buffer.data(), size)) + { + return tensor.ParseFromArray(buffer.data(), static_cast(size)); + } + else + { + return false; + } + +} + +template +std::vector GetTensorDataFromTensorProto(onnx::TensorProto tensorProto, int elementCount) +{ + if (tensorProto.has_raw_data()) + { + std::vector tensorData; + auto& values = tensorProto.raw_data(); + EXPECT_EQ(elementCount, values.size() / sizeof(DataType)) << L"TensorProto elementcount should match raw data buffer size in elements."; + + tensorData = std::vector(elementCount); + memcpy(tensorData.data(), values.data(), values.size()); + return tensorData; + } + else + { + return std::vector(std::begin(tensorProto.float_data()), std::end(tensorProto.float_data())); + } +} + +static +std::vector GetTensorStringDataFromTensorProto( + onnx::TensorProto tensorProto, + int elementCount) +{ + EXPECT_EQ(tensorProto.string_data_size(), elementCount); + auto& values = tensorProto.string_data(); + auto returnVector = std::vector(elementCount); + std::transform(std::begin(values), std::end(values), std::begin(returnVector), + [](auto& value) { return winrt::to_hstring(value); }); + return returnVector; +} + +ITensor ProtobufHelpers::LoadTensorFromProtobufFile( + const std::wstring& filePath, + bool isFp16) +{ + // load from the file path into the onnx format + onnx::TensorProto tensorProto; + if (LoadTensorFromPb(tensorProto, filePath)) + { + std::vector tensorShape = std::vector(tensorProto.dims().begin(), tensorProto.dims().end()); + int64_t initialValue = 1; + auto elementCount = std::accumulate(tensorShape.begin(), tensorShape.end(), initialValue, std::multiplies()); + + if (!tensorProto.has_data_type()) + { + std::cerr << "WARNING: Loading unknown TensorProto datatype.\n"; + } + if (isFp16) + { + return TensorFloat16Bit::CreateFromIterable(tensorShape, GetTensorDataFromTensorProto(tensorProto, elementCount)); + } + switch (tensorProto.data_type()) + { + case(onnx::TensorProto::DataType::TensorProto_DataType_FLOAT): + return TensorFloat::CreateFromIterable(tensorShape, GetTensorDataFromTensorProto(tensorProto, elementCount)); + case(onnx::TensorProto::DataType::TensorProto_DataType_INT32): + return TensorInt32Bit::CreateFromIterable(tensorShape, GetTensorDataFromTensorProto(tensorProto, elementCount)); + case(onnx::TensorProto::DataType::TensorProto_DataType_INT64): + return TensorInt64Bit::CreateFromIterable(tensorShape, GetTensorDataFromTensorProto(tensorProto, elementCount)); + case(onnx::TensorProto::DataType::TensorProto_DataType_STRING): + return TensorString::CreateFromIterable(tensorShape, GetTensorStringDataFromTensorProto(tensorProto, elementCount)); + default: + ADD_FAILURE() << L"Tensor type for creating tensor from protobuf file not supported."; + break; + } + } + return nullptr; +} + +TensorFloat16Bit ProtobufHelpers::LoadTensorFloat16FromProtobufFile( + const std::wstring& filePath) +{ + // load from the file path into the onnx format + onnx::TensorProto tensorProto; + if (LoadTensorFromPb(tensorProto, filePath)) + { + if (tensorProto.has_data_type()) + { + EXPECT_EQ(onnx::TensorProto::DataType::TensorProto_DataType_FLOAT16, tensorProto.data_type()); + } + else + { + std::cerr << "Loading unknown TensorProto datatype as TensorFloat16Bit.\n"; + } + + auto shape = winrt::single_threaded_vector(std::vector(tensorProto.dims().begin(), tensorProto.dims().end())); + TensorFloat16Bit singleTensorValue = TensorFloat16Bit::Create(shape.GetView()); + + uint16_t* data; + winrt::com_ptr spTensorValueNative; + singleTensorValue.as(spTensorValueNative); + uint32_t sizeInBytes; + spTensorValueNative->GetBuffer(reinterpret_cast(&data), &sizeInBytes); + + EXPECT_TRUE(tensorProto.has_raw_data()) << L"Float16 tensor proto buffers are expected to contain raw data."; + + auto& raw_data = tensorProto.raw_data(); + auto buff = raw_data.c_str(); + const size_t type_size = sizeof(uint16_t); + + memcpy((void*)data, (void*)buff, raw_data.size() * sizeof(char)); + + return singleTensorValue; + } + return nullptr; +} + +winrt::Windows::AI::MachineLearning::LearningModel ProtobufHelpers::CreateModel( + winrt::Windows::AI::MachineLearning::TensorKind kind, + const std::vector& shape, + uint32_t num_elements) +{ + onnx::ModelProto model; + + // Set opset import + auto opsetimportproto = model.add_opset_import(); + opsetimportproto->set_version(7); + + onnx::GraphProto& graph = *model.mutable_graph(); + + uint32_t begin = 0; + uint32_t end = num_elements - 1; + for (uint32_t i = begin; i <= end; i++) + { + onnx::NodeProto& node = *graph.add_node(); + node.set_op_type("Identity"); + if (i == begin && i == end) + { + node.add_input("input"); + node.add_output("output"); + } + else if (i == begin) + { + node.add_input("input"); + node.add_output("output" + std::to_string(i)); + + } + else if (i == end) + { + node.add_input("output" + std::to_string(i-1)); + node.add_output("output"); + } + else + { + node.add_input("output" + std::to_string(i-1)); + node.add_output("output" + std::to_string(i)); + } + } + + onnx::TensorProto_DataType dataType; + switch (kind) + { + case TensorKind::Float: dataType = onnx::TensorProto_DataType_FLOAT; break; + case TensorKind::UInt8: dataType = onnx::TensorProto_DataType_UINT8; break; + case TensorKind::Int8: dataType = onnx::TensorProto_DataType_INT8; break; + case TensorKind::UInt16: dataType = onnx::TensorProto_DataType_UINT16; break; + case TensorKind::Int16: dataType = onnx::TensorProto_DataType_INT16; break; + case TensorKind::Int32: dataType = onnx::TensorProto_DataType_INT32; break; + case TensorKind::Int64: dataType = onnx::TensorProto_DataType_INT64; break; + case TensorKind::String: dataType = onnx::TensorProto_DataType_STRING; break; + case TensorKind::Boolean: dataType = onnx::TensorProto_DataType_BOOL; break; + case TensorKind::Float16: dataType = onnx::TensorProto_DataType_FLOAT16; break; + case TensorKind::Double: dataType = onnx::TensorProto_DataType_DOUBLE; break; + case TensorKind::UInt32: dataType = onnx::TensorProto_DataType_UINT32; break; + case TensorKind::UInt64: dataType = onnx::TensorProto_DataType_UINT64; break; + default: + return nullptr; + } + + char dim_param = 'a'; + // input + { + onnx::ValueInfoProto& variable = *graph.add_input(); + variable.set_name("input"); + //onnx::TypeProto_Tensor* pTensor = variable.mutable_type()->mutable_tensor_type(); + variable.mutable_type()->mutable_tensor_type()->set_elem_type(dataType); + for (auto dim : shape) + { + if (dim == -1) + { + variable.mutable_type()->mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_param(&dim_param, 1); + dim_param++; + } + else + { + variable.mutable_type()->mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(dim); + } + } + + if (shape.size() > 0) + { + variable.mutable_type()->mutable_tensor_type()->mutable_shape()->mutable_dim(0)->set_denotation("DATA_BATCH"); + } + } + + // output + { + onnx::ValueInfoProto& variable = *graph.add_output(); + variable.set_name("output"); + //onnx::TypeProto_Tensor* pTensor = variable.mutable_type()->mutable_tensor_type(); + variable.mutable_type()->mutable_tensor_type()->set_elem_type(dataType); + for (auto dim : shape) + { + if (dim == -1) + { + variable.mutable_type()->mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_param(&dim_param, 1); + dim_param++; + } + else + { + variable.mutable_type()->mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(dim); + } + } + } + + struct BufferStreamAdapter : public std::streambuf + { + RandomAccessStreamReference BufferAsRandomAccessStreamReference() + { + auto buffer = m_dataWriter.DetachBuffer(); + m_dataWriter = DataWriter(); + + InMemoryRandomAccessStream stream; + stream.WriteAsync(buffer).get(); + return RandomAccessStreamReference::CreateFromStream(stream); + } + + protected: + virtual int_type overflow(int_type c) { + if (c != EOF) { + // convert lowercase to uppercase + auto temp = static_cast(c); + + m_dataWriter.WriteByte(temp); + } + return c; + } + + private: + DataWriter m_dataWriter; + }; + + auto size = model.ByteSize(); + auto raw_array = std::unique_ptr(new char[size]); + model.SerializeToArray(raw_array.get(), size); + + BufferStreamAdapter buffer; + std::ostream os(&buffer); + + os.write(raw_array.get(), size); + + return LearningModel::LoadFromStream(buffer.BufferAsRandomAccessStreamReference()); +} diff --git a/winml/test/common/protobufHelpers.h b/winml/test/common/protobufHelpers.h new file mode 100644 index 0000000000..233d9da055 --- /dev/null +++ b/winml/test/common/protobufHelpers.h @@ -0,0 +1,23 @@ +//----------------------------------------------------------------------------- +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// +//----------------------------------------------------------------------------- + +#pragma once + +#include "std.h" + +namespace ProtobufHelpers +{ + // LoadTensorFromProtobufFile take a path to a FP32 data file and loads it into a 32bit array or + // 16bit array based on isFp16 + winrt::Windows::AI::MachineLearning::ITensor LoadTensorFromProtobufFile(const std::wstring& filePath, bool isFp16); + // LoadTensorFloat16FromProtobufFile takes a path to a FP16 data file and loads it into a 16bit array + winrt::Windows::AI::MachineLearning::TensorFloat16Bit LoadTensorFloat16FromProtobufFile(const std::wstring& filePath); + + winrt::Windows::AI::MachineLearning::LearningModel CreateModel( + winrt::Windows::AI::MachineLearning::TensorKind kind, + const std::vector& shape, + uint32_t num_elements = 1); +} diff --git a/winml/test/common/std.h b/winml/test/common/std.h new file mode 100644 index 0000000000..823869e6f6 --- /dev/null +++ b/winml/test/common/std.h @@ -0,0 +1,51 @@ +//----------------------------------------------------------------------------- +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// +//----------------------------------------------------------------------------- + +#pragma once + +// stl +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +// IUnknown must be declared before winrt/base.h is included to light up support for native COM +// interfaces with C++/WinRT types (e.g. winrt::com_ptr). +#include +#include "winrt/base.h" +#include "winrt/Windows.Foundation.Collections.h" +#include "comp_generated/winrt/windows.ai.machinelearning.h" + +// WinML +#include "Windows.AI.MachineLearning.Native.h" + +#define EXPECT_THROW_SPECIFIC(statement, exception, condition) \ + EXPECT_THROW( \ + try { \ + statement; \ + } catch (const exception& e) { \ + EXPECT_TRUE(condition(e)); \ + throw; \ + } \ + , exception); + +// For old versions of gtest without GTEST_SKIP, stream the message and return success instead +#ifndef GTEST_SKIP +#define GTEST_SKIP(message) return GTEST_MESSAGE_(message, ::testing::TestPartResult::kSuccess) +#endif + +#ifndef INSTANTIATE_TEST_SUITE_P +// Use the old name, removed in newer versions of googletest +#define INSTANTIATE_TEST_SUITE_P INSTANTIATE_TEST_CASE_P +#endif diff --git a/winml/test/common/testPch.h b/winml/test/common/testPch.h new file mode 100644 index 0000000000..952f0bbccf --- /dev/null +++ b/winml/test/common/testPch.h @@ -0,0 +1,12 @@ +//----------------------------------------------------------------------------- +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// +//----------------------------------------------------------------------------- +#define _SILENCE_ALL_CXX17_DEPRECATION_WARNINGS +#include "std.h" + +#include +#include + +#include "fileHelpers.h" diff --git a/winml/test/common/testdata/squeezenet/model.onnx b/winml/test/common/testdata/squeezenet/model.onnx new file mode 100644 index 0000000000..b8e1dfce26 Binary files /dev/null and b/winml/test/common/testdata/squeezenet/model.onnx differ diff --git a/winml/test/common/testdata/squeezenet/test_data_0_input.pb b/winml/test/common/testdata/squeezenet/test_data_0_input.pb new file mode 100644 index 0000000000..f521c230e5 Binary files /dev/null and b/winml/test/common/testdata/squeezenet/test_data_0_input.pb differ diff --git a/winml/test/common/testdata/squeezenet/test_data_0_output.pb b/winml/test/common/testdata/squeezenet/test_data_0_output.pb new file mode 100644 index 0000000000..e731eb0234 Binary files /dev/null and b/winml/test/common/testdata/squeezenet/test_data_0_output.pb differ