diff --git a/cmake/winml_unittests.cmake b/cmake/winml_unittests.cmake index 73d0ff7fde..16c7b5600b 100644 --- a/cmake/winml_unittests.cmake +++ b/cmake/winml_unittests.cmake @@ -59,10 +59,14 @@ function(get_winml_test_scenario_src output_winml_test_scenario_libs ) if (onnxruntime_USE_DML) - file(GLOB winml_test_scenario_src CONFIGURE_DEPENDS "${winml_test_src_path}/scenario/cppwinrt/*.cpp") + file(GLOB winml_test_scenario_src CONFIGURE_DEPENDS + "${winml_test_src_path}/scenario/cppwinrt/*.h" + "${winml_test_src_path}/scenario/cppwinrt/*.cpp") set(${output_winml_test_scenario_libs} "onnxruntime_providers_dml" PARENT_SCOPE) else() - set(winml_test_scenario_src "${winml_test_src_path}/scenario/cppwinrt/scenariotestscppwinrt.cpp") + set(winml_test_scenario_src + "${winml_test_src_path}/scenario/cppwinrt/scenariotestscppwinrt.h" + "${winml_test_src_path}/scenario/cppwinrt/scenariotestscppwinrt.cpp") endif() set(${output_winml_test_scenario_src} ${winml_test_scenario_src} PARENT_SCOPE) endfunction() @@ -71,7 +75,9 @@ function(get_winml_test_api_src winml_test_src_path output_winml_test_api_src ) - file(GLOB winml_test_api_src CONFIGURE_DEPENDS "${winml_test_src_path}/api/*.cpp") + file(GLOB winml_test_api_src CONFIGURE_DEPENDS + "${winml_test_src_path}/api/*.h" + "${winml_test_src_path}/api/*.cpp") set(${output_winml_test_api_src} ${winml_test_api_src} PARENT_SCOPE) endfunction() @@ -79,10 +85,24 @@ function(get_winml_test_concurrency_src winml_test_src_path output_winml_test_concurrency_src ) - file(GLOB winml_test_concurrency_src CONFIGURE_DEPENDS "${winml_test_src_path}/concurrency/*.cpp") + file(GLOB winml_test_concurrency_src CONFIGURE_DEPENDS + "${winml_test_src_path}/concurrency/*.h" + "${winml_test_src_path}/concurrency/*.cpp") set(${output_winml_test_concurrency_src} ${winml_test_concurrency_src} PARENT_SCOPE) endfunction() +function(get_winml_test_adapter_src + winml_test_src_path + output_winml_test_adapter_src + output_winml_test_adapter_libs +) + set(${output_winml_test_adapter_libs} "onnxruntime" PARENT_SCOPE) + file(GLOB winml_test_adapter_src CONFIGURE_DEPENDS + "${winml_test_src_path}/adapter/*.h" + "${winml_test_src_path}/adapter/*.cpp") + set(${output_winml_test_adapter_src} ${winml_test_adapter_src} PARENT_SCOPE) +endfunction() + function(get_winml_test_image_src winml_test_src_path output_winml_test_image_src @@ -90,11 +110,15 @@ function(get_winml_test_image_src if (onnxruntime_USE_DML) set(${output_winml_test_scenario_libs} "onnxruntime_providers_dml" PARENT_SCOPE) endif() - file(GLOB winml_test_image_src CONFIGURE_DEPENDS "${winml_test_src_path}/image/*.cpp") + file(GLOB winml_test_image_src CONFIGURE_DEPENDS + "${winml_test_src_path}/image/*.h" + "${winml_test_src_path}/image/*.cpp") set(${output_winml_test_image_src} ${winml_test_image_src} PARENT_SCOPE) endfunction() -file(GLOB winml_test_common_src CONFIGURE_DEPENDS "${WINML_TEST_SRC_DIR}/common/*.cpp") +file(GLOB winml_test_common_src CONFIGURE_DEPENDS + "${WINML_TEST_SRC_DIR}/common/*.h" + "${WINML_TEST_SRC_DIR}/common/*.cpp") add_library(winml_test_common STATIC ${winml_test_common_src}) add_dependencies(winml_test_common onnx @@ -159,6 +183,14 @@ add_winml_test( ) target_include_directories(winml_test_concurrency PRIVATE ${ONNXRUNTIME_ROOT}/core/graph) +get_winml_test_adapter_src(${WINML_TEST_SRC_DIR} winml_test_adapter_src winml_test_adapter_libs) +add_winml_test( + TARGET winml_test_adapter + SOURCES ${winml_test_adapter_src} + LIBS winml_test_common ${winml_test_adapter_libs} +) +target_include_directories(winml_test_adapter PRIVATE ${REPO_ROOT}/winml/adapter) + # During build time, copy any modified collaterals. # configure_file(source destination COPYONLY), which configures CMake to copy the file whenever source is modified, # can't be used here because we don't know the destination during configure time (in multi-configuration generators, diff --git a/winml/test/adapter/adapter_test.cpp b/winml/test/adapter/adapter_test.cpp new file mode 100644 index 0000000000..9f009a97e7 --- /dev/null +++ b/winml/test/adapter/adapter_test.cpp @@ -0,0 +1,331 @@ +#include "testPch.h" +#include "adapter_test.h" +#include "fileHelpers.h" +#include "winrt/Windows.Storage.h" +#include "winrt/Windows.Storage.Streams.h" + +using namespace winrt::Windows::Storage; +using namespace winrt::Windows::Storage::Streams; + +static void AdapterTestSetup() { + ort_api = OrtGetApiBase()->GetApi(2); + winml_adapter_api = OrtGetWinMLAdapter(ort_api); + + // for model tests + std::wstring module_path = FileHelpers::GetModulePath(); + std::string squeezenet_path = std::wstring_convert>().to_bytes(module_path + L"squeezenet_modifiedforruntimestests.onnx"); + std::string metadata_path = std::wstring_convert>().to_bytes(module_path + L"modelWith2MetaData.onnx"); + std::string float16_path = std::wstring_convert>().to_bytes(module_path + L"starry-night-fp16.onnx"); + winml_adapter_api->CreateModelFromPath(squeezenet_path.c_str(), squeezenet_path.size(), &squeezenet_model); + winml_adapter_api->CreateModelFromPath(metadata_path.c_str(), metadata_path.size(), &metadata_model); + winml_adapter_api->CreateModelFromPath(float16_path.c_str(), float16_path.size(), &float16_Model); +} + +static void AdapterTestTeardown() { + winml_adapter_api->ReleaseModel(squeezenet_model); + winml_adapter_api->ReleaseModel(metadata_model); + winml_adapter_api->ReleaseModel(float16_Model); +} + +static void CreateModelFromPath() { + WINML_EXPECT_TRUE(squeezenet_model != nullptr); + WINML_EXPECT_TRUE(metadata_model != nullptr); + WINML_EXPECT_TRUE(float16_Model != nullptr); +} + +static void CreateModelFromData() { + StorageFolder folder = StorageFolder::GetFolderFromPathAsync(FileHelpers::GetModulePath()).get(); + StorageFile file = folder.GetFileAsync(L"squeezenet_modifiedforruntimestests.onnx").get(); + IRandomAccessStream stream = file.OpenAsync(FileAccessMode::Read).get(); + DataReader data_reader(stream.GetInputStreamAt(0)); + data_reader.LoadAsync(static_cast(stream.Size())).get(); + IBuffer data_buffer = data_reader.DetachBuffer(); + OrtModel* squeezenet_model_from_data = nullptr; + winml_adapter_api->CreateModelFromData(data_buffer.data(), data_buffer.Length(), &squeezenet_model_from_data); + WINML_EXPECT_TRUE(squeezenet_model_from_data != nullptr); + // Verify a function in the model for thoroughness + const char* author; + size_t len; + winml_adapter_api->ModelGetAuthor(squeezenet_model_from_data, &author, &len); + std::string author_str(author); + WINML_EXPECT_EQUAL(author_str, "onnx-caffe2"); + winml_adapter_api->ReleaseModel(squeezenet_model_from_data); +} + +static void CloneModel() { + OrtModel* squeezenet_clone = nullptr; + winml_adapter_api->CloneModel(squeezenet_model, &squeezenet_clone); + WINML_EXPECT_TRUE(squeezenet_clone != nullptr); + // Verify a function in clone + const char* author; + size_t len; + winml_adapter_api->ModelGetAuthor(squeezenet_clone, &author, &len); + std::string author_str(author); + WINML_EXPECT_EQUAL(author_str, "onnx-caffe2"); +} + +static void ModelGetAuthor() { + const char* author; + size_t len; + winml_adapter_api->ModelGetAuthor(squeezenet_model, &author, &len); + std::string author_str(author); + WINML_EXPECT_EQUAL(author_str, "onnx-caffe2"); +} + +static void ModelGetName() { + const char* name; + size_t len; + winml_adapter_api->ModelGetName(squeezenet_model, &name, &len); + std::string name_str(name); + WINML_EXPECT_EQUAL(name_str, "squeezenet_old"); +} + +static void ModelGetDomain() { + const char* domain; + size_t len; + winml_adapter_api->ModelGetDomain(squeezenet_model, &domain, &len); + std::string domain_str(domain); + WINML_EXPECT_EQUAL(domain_str, "test-domain"); +} + +static void ModelGetDescription() { + const char* description; + size_t len; + winml_adapter_api->ModelGetDescription(squeezenet_model, &description, &len); + std::string description_str(description); + WINML_EXPECT_EQUAL(description_str, "test-doc_string"); +} + +static void ModelGetVersion() { + int64_t version; + winml_adapter_api->ModelGetVersion(squeezenet_model, &version); + WINML_EXPECT_EQUAL(version, 123456); +} + +static void ModelGetInputCount() { + size_t input_count; + winml_adapter_api->ModelGetInputCount(squeezenet_model, &input_count); + WINML_EXPECT_EQUAL(input_count, 1); +} + +static void ModelGetOutputCount() { + size_t output_count; + winml_adapter_api->ModelGetOutputCount(squeezenet_model, &output_count); + WINML_EXPECT_EQUAL(output_count, 1); +} + +static void ModelGetInputName() { + const char* input_name; + size_t count; + winml_adapter_api->ModelGetInputName(squeezenet_model, 0, &input_name, &count); + std::string input_name_str(input_name); + WINML_EXPECT_EQUAL(input_name_str, "data_0"); +} + +static void ModelGetOutputName() { + const char* output_name; + size_t count; + winml_adapter_api->ModelGetOutputName(squeezenet_model, 0, &output_name, &count); + std::string output_name_str(output_name); + WINML_EXPECT_EQUAL(output_name_str, "softmaxout_1"); +} + +static void ModelGetInputDescription() { + const char* input_description; + size_t count; + winml_adapter_api->ModelGetInputDescription(metadata_model, 0, &input_description, &count); + std::string input_description_str(input_description); + WINML_EXPECT_EQUAL(input_description_str, "this is a long input description!"); +} + +static void ModelGetOutputDescription() { + const char* output_description; + size_t count; + winml_adapter_api->ModelGetOutputDescription(metadata_model, 0, &output_description, &count); + std::string output_description_str(output_description); + WINML_EXPECT_EQUAL(output_description_str, "this is a long output description!"); +} + +static void ModelGetInputTypeInfo() { + OrtTypeInfo* input_type_info; + winml_adapter_api->ModelGetInputTypeInfo(squeezenet_model, 0, &input_type_info); + + ONNXType input_type; + ort_api->GetOnnxTypeFromTypeInfo(input_type_info, &input_type); + WINML_EXPECT_EQUAL(input_type, ONNX_TYPE_TENSOR); + + const OrtTensorTypeAndShapeInfo* tensor_info; + ort_api->CastTypeInfoToTensorInfo(input_type_info, &tensor_info); + + ONNXTensorElementDataType tensor_type; + ort_api->GetTensorElementType(tensor_info, &tensor_type); + WINML_EXPECT_EQUAL(tensor_type, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT); + + size_t dim_count; + ort_api->GetDimensionsCount(tensor_info, &dim_count); + WINML_EXPECT_EQUAL(dim_count, 4); + + int64_t dim_values[4]; + ort_api->GetDimensions(tensor_info, dim_values, 4); + WINML_EXPECT_EQUAL(dim_values[0], 1); + WINML_EXPECT_EQUAL(dim_values[1], 3); + WINML_EXPECT_EQUAL(dim_values[2], 224); + WINML_EXPECT_EQUAL(dim_values[3], 224); + + ort_api->ReleaseTypeInfo(input_type_info); +} + +static void ModelGetOutputTypeInfo() { + OrtTypeInfo* output_type_info; + winml_adapter_api->ModelGetOutputTypeInfo(squeezenet_model, 0, &output_type_info); + + ONNXType output_type; + ort_api->GetOnnxTypeFromTypeInfo(output_type_info, &output_type); + WINML_EXPECT_EQUAL(output_type, ONNX_TYPE_TENSOR); + + const OrtTensorTypeAndShapeInfo* tensor_info; + ort_api->CastTypeInfoToTensorInfo(output_type_info, &tensor_info); + + ONNXTensorElementDataType tensor_type; + ort_api->GetTensorElementType(tensor_info, &tensor_type); + WINML_EXPECT_EQUAL(tensor_type, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT); + + size_t dim_count; + ort_api->GetDimensionsCount(tensor_info, &dim_count); + WINML_EXPECT_EQUAL(dim_count, 4); + + int64_t dim_values[4]; + ort_api->GetDimensions(tensor_info, dim_values, 4); + WINML_EXPECT_EQUAL(dim_values[0], 1); + WINML_EXPECT_EQUAL(dim_values[1], 1000); + WINML_EXPECT_EQUAL(dim_values[2], 1); + WINML_EXPECT_EQUAL(dim_values[3], 1); + + ort_api->ReleaseTypeInfo(output_type_info); +} + +static void ModelGetMetadataCount() { + size_t metadata_count; + winml_adapter_api->ModelGetMetadataCount(metadata_model, &metadata_count); + WINML_EXPECT_EQUAL(metadata_count, 2); +} + +static void ModelGetMetadata() { + const char* metadata_key; + size_t metadata_key_len; + const char* metadata_value; + size_t metadata_value_len; + + winml_adapter_api->ModelGetMetadata(metadata_model, 0, &metadata_key, &metadata_key_len, &metadata_value, &metadata_value_len); + WINML_EXPECT_EQUAL(std::string(metadata_key), "thisisalongkey"); + WINML_EXPECT_EQUAL(metadata_key_len, 14); + WINML_EXPECT_EQUAL(std::string(metadata_value), "thisisalongvalue"); + WINML_EXPECT_EQUAL(metadata_value_len, 16); + + winml_adapter_api->ModelGetMetadata(metadata_model, 1, &metadata_key, &metadata_key_len, &metadata_value, &metadata_value_len); + WINML_EXPECT_EQUAL(std::string(metadata_key), "key2"); + WINML_EXPECT_EQUAL(metadata_key_len, 4); + WINML_EXPECT_EQUAL(std::string(metadata_value), "val2"); + WINML_EXPECT_EQUAL(metadata_value_len, 4); +} + +static void ModelEnsureNoFloat16() { + OrtStatus* float16_error_status; + + float16_error_status = winml_adapter_api->ModelEnsureNoFloat16(squeezenet_model); + WINML_EXPECT_EQUAL(float16_error_status, nullptr); + + float16_error_status = winml_adapter_api->ModelEnsureNoFloat16(float16_Model); + WINML_EXPECT_NOT_EQUAL(float16_error_status, nullptr); + WINML_EXPECT_EQUAL(ort_api->GetErrorCode(float16_error_status), ORT_INVALID_GRAPH); +} + +static void __stdcall TestLoggingCallback(void* param, OrtLoggingLevel severity, const char* category, + const char* logger_id, const char* code_location, const char* message) noexcept { + UNREFERENCED_PARAMETER(param); + UNREFERENCED_PARAMETER(severity); + UNREFERENCED_PARAMETER(category); + UNREFERENCED_PARAMETER(logger_id); + UNREFERENCED_PARAMETER(code_location); + UNREFERENCED_PARAMETER(message); + logging_function_called = true; +} + +static void __stdcall TestProfileEventCallback(const OrtProfilerEventRecord* profiler_record) noexcept { + UNREFERENCED_PARAMETER(profiler_record); + profiling_function_called = true; +} + +static void EnvConfigureCustomLoggerAndProfiler() { + OrtEnv* ort_env = nullptr; + ort_api->CreateEnv(OrtLoggingLevel::ORT_LOGGING_LEVEL_VERBOSE, "Default", &ort_env); + winml_adapter_api->EnvConfigureCustomLoggerAndProfiler(ort_env, + &TestLoggingCallback, &TestProfileEventCallback, nullptr, + OrtLoggingLevel::ORT_LOGGING_LEVEL_VERBOSE, "Default", &ort_env); + logging_function_called = false; + OrtSession* ort_session = nullptr; + std::wstring squeezenet_path = FileHelpers::GetModulePath() + L"relu.onnx"; + ort_api->CreateSession(ort_env, squeezenet_path.c_str(), nullptr, &ort_session); + WINML_EXPECT_TRUE(logging_function_called); + + size_t input_tensor_size = 5; + int64_t input_dimensions[] = {5}; + + std::vector input_tensor_values(input_tensor_size); + std::vector input_node_names = {"X"}; + std::vector output_node_names = {"Y"}; + + // initialize input data with values in [0.0, 1.0] + for (size_t i = 0; i < input_tensor_size; i++) + input_tensor_values[i] = (float)i / (input_tensor_size + 1); + + // create input tensor object from data values + OrtMemoryInfo* memory_info; + ort_api->CreateCpuMemoryInfo(OrtArenaAllocator, OrtMemTypeDefault, &memory_info); + OrtValue* input_tensor = nullptr; + ort_api->CreateTensorWithDataAsOrtValue(memory_info, input_tensor_values.data(), input_tensor_size * sizeof(float), input_dimensions, 1, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, &input_tensor); + int is_tensor; + ort_api->IsTensor(input_tensor, &is_tensor); + assert(is_tensor); + ort_api->ReleaseMemoryInfo(memory_info); + OrtValue* output_tensor = nullptr; + winml_adapter_api->SessionStartProfiling(ort_env, ort_session); + profiling_function_called = false; + ort_api->Run(ort_session, nullptr, input_node_names.data(), (const OrtValue* const*)&input_tensor, 1, output_node_names.data(), 1, &output_tensor); + WINML_EXPECT_TRUE(profiling_function_called); + winml_adapter_api->SessionEndProfiling(ort_session); + + ort_api->ReleaseValue(output_tensor); + ort_api->ReleaseValue(input_tensor); + ort_api->ReleaseSession(ort_session); + ort_api->ReleaseEnv(ort_env); +} + +const AdapterTestApi& getapi() { + static constexpr AdapterTestApi api = + { + AdapterTestSetup, + AdapterTestTeardown, + CreateModelFromPath, + CreateModelFromData, + CloneModel, + ModelGetAuthor, + ModelGetName, + ModelGetDomain, + ModelGetDescription, + ModelGetVersion, + ModelGetInputCount, + ModelGetOutputCount, + ModelGetInputName, + ModelGetOutputName, + ModelGetInputDescription, + ModelGetOutputDescription, + ModelGetInputTypeInfo, + ModelGetOutputTypeInfo, + ModelGetMetadataCount, + ModelGetMetadata, + ModelEnsureNoFloat16, + EnvConfigureCustomLoggerAndProfiler, + }; + return api; +} \ No newline at end of file diff --git a/winml/test/adapter/adapter_test.h b/winml/test/adapter/adapter_test.h new file mode 100644 index 0000000000..f4814afa0d --- /dev/null +++ b/winml/test/adapter/adapter_test.h @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "test.h" +#include "core/providers/winml/winml_provider_factory.h" +#include "winml_adapter_c_api.h" + +struct AdapterTestApi +{ + SetupClass AdapterTestSetup; + TeardownClass AdapterTestTeardown; + VoidTest CreateModelFromPath; + VoidTest CreateModelFromData; + VoidTest CloneModel; + VoidTest ModelGetAuthor; + VoidTest ModelGetName; + VoidTest ModelGetDomain; + VoidTest ModelGetDescription; + VoidTest ModelGetVersion; + VoidTest ModelGetInputCount; + VoidTest ModelGetOutputCount; + VoidTest ModelGetInputName; + VoidTest ModelGetOutputName; + VoidTest ModelGetInputDescription; + VoidTest ModelGetOutputDescription; + VoidTest ModelGetInputTypeInfo; + VoidTest ModelGetOutputTypeInfo; + VoidTest ModelGetMetadataCount; + VoidTest ModelGetMetadata; + VoidTest ModelEnsureNoFloat16; + VoidTest EnvConfigureCustomLoggerAndProfiler; +}; +const AdapterTestApi& getapi(); +const WinmlAdapterApi* winml_adapter_api; +const OrtApi* ort_api; +OrtModel* squeezenet_model = nullptr; +OrtModel* metadata_model = nullptr; +OrtModel* float16_Model = nullptr; +static bool logging_function_called = false; +static bool profiling_function_called = false; + +WINML_TEST_CLASS_BEGIN(AdapterTest) +WINML_TEST_CLASS_SETUP_CLASS(AdapterTestSetup) +WINML_TEST_CLASS_TEARDOWN_CLASS(AdapterTestTeardown) +WINML_TEST_CLASS_BEGIN_TESTS +WINML_TEST(AdapterTest, CreateModelFromPath) +WINML_TEST(AdapterTest, CreateModelFromData) +WINML_TEST(AdapterTest, CloneModel) +WINML_TEST(AdapterTest, ModelGetAuthor) +WINML_TEST(AdapterTest, ModelGetName) +WINML_TEST(AdapterTest, ModelGetDomain) +WINML_TEST(AdapterTest, ModelGetDescription) +WINML_TEST(AdapterTest, ModelGetVersion) +WINML_TEST(AdapterTest, ModelGetInputCount) +WINML_TEST(AdapterTest, ModelGetOutputCount) +WINML_TEST(AdapterTest, ModelGetInputName) +WINML_TEST(AdapterTest, ModelGetOutputName) +WINML_TEST(AdapterTest, ModelGetInputDescription) +WINML_TEST(AdapterTest, ModelGetOutputDescription) +WINML_TEST(AdapterTest, ModelGetInputTypeInfo) +WINML_TEST(AdapterTest, ModelGetOutputTypeInfo) +WINML_TEST(AdapterTest, ModelGetMetadataCount) +WINML_TEST(AdapterTest, ModelGetMetadata) +WINML_TEST(AdapterTest, ModelEnsureNoFloat16) +WINML_TEST(AdapterTest, EnvConfigureCustomLoggerAndProfiler) +WINML_TEST_CLASS_END() diff --git a/winml/test/api/LearningModelAPITest.cpp b/winml/test/api/LearningModelAPITest.cpp index d07981fe28..6687aa1f69 100644 --- a/winml/test/api/LearningModelAPITest.cpp +++ b/winml/test/api/LearningModelAPITest.cpp @@ -17,13 +17,12 @@ using namespace winrt::Windows::Media; using namespace winrt::Windows::Storage; using namespace winrt::Windows::Storage::Streams; -static void LearningModelAPITestSetup() { +static void LearningModelAPITestsClassSetup() { init_apartment(); } -static void LearningModelAPITestGpuSetup() { +static void LearningModelAPITestsGpuMethodSetup() { GPUTEST; - init_apartment(); } static void CreateModelFromFilePath() { @@ -254,11 +253,11 @@ static void CloseModelNoNewSessions() { }); } -const LearningModelApiTestApi& getapi() { - static constexpr LearningModelApiTestApi api = +const LearningModelApiTestsApi& getapi() { + static constexpr LearningModelApiTestsApi api = { - LearningModelAPITestSetup, - LearningModelAPITestGpuSetup, + LearningModelAPITestsClassSetup, + LearningModelAPITestsGpuMethodSetup, CreateModelFromFilePath, CreateModelFromIStorage, CreateModelFromIStorageOutsideCwd, diff --git a/winml/test/api/LearningModelAPITest.h b/winml/test/api/LearningModelAPITest.h index 46d815fc27..376fb661f7 100644 --- a/winml/test/api/LearningModelAPITest.h +++ b/winml/test/api/LearningModelAPITest.h @@ -2,10 +2,10 @@ // Licensed under the MIT License. #include "test.h" -struct LearningModelApiTestApi +struct LearningModelApiTestsApi { - SetupTest LearningModelAPITestSetup; - SetupTest LearningModelAPITestGpuSetup; + SetupClass LearningModelAPITestsClassSetup; + SetupTest LearningModelAPITestsGpuMethodSetup; VoidTest CreateModelFromFilePath; VoidTest CreateModelFromIStorage; VoidTest CreateModelFromIStorageOutsideCwd; @@ -21,24 +21,29 @@ struct LearningModelApiTestApi VoidTest CloseModelCheckEval; VoidTest CloseModelNoNewSessions; }; -const LearningModelApiTestApi& getapi(); +const LearningModelApiTestsApi& getapi(); -WINML_TEST_CLASS_BEGIN_WITH_SETUP(LearningModelAPITest, LearningModelAPITestSetup) -WINML_TEST(LearningModelAPITest, CreateModelFromFilePath) -WINML_TEST(LearningModelAPITest, CreateModelFromIStorage) -WINML_TEST(LearningModelAPITest, CreateModelFromIStorageOutsideCwd) -WINML_TEST(LearningModelAPITest, CreateModelFromIStream) -WINML_TEST(LearningModelAPITest, ModelGetAuthor) -WINML_TEST(LearningModelAPITest, ModelGetName) -WINML_TEST(LearningModelAPITest, ModelGetDomain) -WINML_TEST(LearningModelAPITest, ModelGetDescription) -WINML_TEST(LearningModelAPITest, ModelGetVersion) -WINML_TEST(LearningModelAPITest, EnumerateInputs) -WINML_TEST(LearningModelAPITest, EnumerateOutputs) -WINML_TEST(LearningModelAPITest, CloseModelCheckMetadata) -WINML_TEST(LearningModelAPITest, CloseModelNoNewSessions) +WINML_TEST_CLASS_BEGIN(LearningModelAPITests) +WINML_TEST_CLASS_SETUP_CLASS(LearningModelAPITestsClassSetup) +WINML_TEST_CLASS_BEGIN_TESTS +WINML_TEST(LearningModelAPITests, CreateModelFromFilePath) +WINML_TEST(LearningModelAPITests, CreateModelFromIStorage) +WINML_TEST(LearningModelAPITests, CreateModelFromIStorageOutsideCwd) +WINML_TEST(LearningModelAPITests, CreateModelFromIStream) +WINML_TEST(LearningModelAPITests, ModelGetAuthor) +WINML_TEST(LearningModelAPITests, ModelGetName) +WINML_TEST(LearningModelAPITests, ModelGetDomain) +WINML_TEST(LearningModelAPITests, ModelGetDescription) +WINML_TEST(LearningModelAPITests, ModelGetVersion) +WINML_TEST(LearningModelAPITests, EnumerateInputs) +WINML_TEST(LearningModelAPITests, EnumerateOutputs) +WINML_TEST(LearningModelAPITests, CloseModelCheckMetadata) +WINML_TEST(LearningModelAPITests, CloseModelNoNewSessions) WINML_TEST_CLASS_END() -WINML_TEST_CLASS_BEGIN_WITH_SETUP(LearningModelAPITestGpu, LearningModelAPITestGpuSetup) -WINML_TEST(LearningModelAPITestGpu, CloseModelCheckEval) +WINML_TEST_CLASS_BEGIN(LearningModelAPITestsGpu) +WINML_TEST_CLASS_SETUP_CLASS(LearningModelAPITestsClassSetup) +WINML_TEST_CLASS_SETUP_METHOD(LearningModelAPITestsGpuMethodSetup) +WINML_TEST_CLASS_BEGIN_TESTS +WINML_TEST(LearningModelAPITestsGpu, CloseModelCheckEval) WINML_TEST_CLASS_END() \ No newline at end of file diff --git a/winml/test/api/LearningModelBindingAPITest.cpp b/winml/test/api/LearningModelBindingAPITest.cpp index b09ed33dba..27deb86be0 100644 --- a/winml/test/api/LearningModelBindingAPITest.cpp +++ b/winml/test/api/LearningModelBindingAPITest.cpp @@ -18,13 +18,12 @@ using namespace winrt::Windows::Graphics::Imaging; using namespace winrt::Windows::Media; using namespace winrt::Windows::Storage; -static void LearningModelBindingAPITestSetup() { +static void LearningModelBindingAPITestsClassSetup() { init_apartment(); } -static void LearningModelBindingAPITestGpuSetup() { +static void LearningModelBindingAPITestsGpuMethodSetup() { GPUTEST; - init_apartment(); } static void CpuSqueezeNet() @@ -714,11 +713,11 @@ static void SequenceConstructTensorString() WINML_EXPECT_EQUAL(3, bound_output_sequence.GetAt(1).Shape().GetAt(1)); } -const LearningModelBindingAPITestApi& getapi() { - static constexpr LearningModelBindingAPITestApi api = +const LearningModelBindingAPITestsApi& getapi() { + static constexpr LearningModelBindingAPITestsApi api = { - LearningModelBindingAPITestSetup, - LearningModelBindingAPITestGpuSetup, + LearningModelBindingAPITestsClassSetup, + LearningModelBindingAPITestsGpuMethodSetup, CpuSqueezeNet, CpuSqueezeNetEmptyOutputs, CpuSqueezeNetUnboundOutputs, diff --git a/winml/test/api/LearningModelBindingAPITest.h b/winml/test/api/LearningModelBindingAPITest.h index eccacda971..b752c37065 100644 --- a/winml/test/api/LearningModelBindingAPITest.h +++ b/winml/test/api/LearningModelBindingAPITest.h @@ -3,9 +3,9 @@ #include "test.h" -struct LearningModelBindingAPITestApi { - SetupTest LearningModelBindingAPITestSetup; - SetupTest LearningModelBindingAPITestGpuSetup; +struct LearningModelBindingAPITestsApi { + SetupClass LearningModelBindingAPITestsClassSetup; + SetupTest LearningModelBindingAPITestsGpuMethodSetup; VoidTest CpuSqueezeNet; VoidTest CpuSqueezeNetEmptyOutputs; VoidTest CpuSqueezeNetUnboundOutputs; @@ -27,30 +27,35 @@ struct LearningModelBindingAPITestApi { VoidTest SequenceLengthTensorFloat; VoidTest SequenceConstructTensorString; }; -const LearningModelBindingAPITestApi& getapi(); +const LearningModelBindingAPITestsApi& getapi(); -WINML_TEST_CLASS_BEGIN_WITH_SETUP(LearningModelBindingAPITest, LearningModelBindingAPITestSetup) -WINML_TEST(LearningModelBindingAPITest, CpuSqueezeNet) -WINML_TEST(LearningModelBindingAPITest, CpuSqueezeNetEmptyOutputs) -WINML_TEST(LearningModelBindingAPITest, CpuSqueezeNetUnboundOutputs) -WINML_TEST(LearningModelBindingAPITest, CpuSqueezeNetBindInputTensorAsInspectable) -WINML_TEST(LearningModelBindingAPITest, CastMapInt64) -WINML_TEST(LearningModelBindingAPITest, DictionaryVectorizerMapInt64) -WINML_TEST(LearningModelBindingAPITest, DictionaryVectorizerMapString) -WINML_TEST(LearningModelBindingAPITest, ZipMapInt64) -WINML_TEST(LearningModelBindingAPITest, ZipMapInt64Unbound) -WINML_TEST(LearningModelBindingAPITest, ZipMapString) -WINML_TEST(LearningModelBindingAPITest, VerifyOutputAfterEvaluateAsyncCalledTwice) -WINML_TEST(LearningModelBindingAPITest, VerifyOutputAfterImageBindCalledTwice) -WINML_TEST(LearningModelBindingAPITest, SequenceLengthTensorFloat) -WINML_TEST(LearningModelBindingAPITest, SequenceConstructTensorString) +WINML_TEST_CLASS_BEGIN(LearningModelBindingAPITests) +WINML_TEST_CLASS_SETUP_CLASS(LearningModelBindingAPITestsClassSetup) +WINML_TEST_CLASS_BEGIN_TESTS +WINML_TEST(LearningModelBindingAPITests, CpuSqueezeNet) +WINML_TEST(LearningModelBindingAPITests, CpuSqueezeNetEmptyOutputs) +WINML_TEST(LearningModelBindingAPITests, CpuSqueezeNetUnboundOutputs) +WINML_TEST(LearningModelBindingAPITests, CpuSqueezeNetBindInputTensorAsInspectable) +WINML_TEST(LearningModelBindingAPITests, CastMapInt64) +WINML_TEST(LearningModelBindingAPITests, DictionaryVectorizerMapInt64) +WINML_TEST(LearningModelBindingAPITests, DictionaryVectorizerMapString) +WINML_TEST(LearningModelBindingAPITests, ZipMapInt64) +WINML_TEST(LearningModelBindingAPITests, ZipMapInt64Unbound) +WINML_TEST(LearningModelBindingAPITests, ZipMapString) +WINML_TEST(LearningModelBindingAPITests, VerifyOutputAfterEvaluateAsyncCalledTwice) +WINML_TEST(LearningModelBindingAPITests, VerifyOutputAfterImageBindCalledTwice) +WINML_TEST(LearningModelBindingAPITests, SequenceLengthTensorFloat) +WINML_TEST(LearningModelBindingAPITests, SequenceConstructTensorString) WINML_TEST_CLASS_END() -WINML_TEST_CLASS_BEGIN_WITH_SETUP(LearningModelBindingAPITestGpu, LearningModelBindingAPITestGpuSetup) -WINML_TEST(LearningModelBindingAPITestGpu, GpuSqueezeNet) -WINML_TEST(LearningModelBindingAPITestGpu, GpuSqueezeNetEmptyOutputs) -WINML_TEST(LearningModelBindingAPITestGpu, GpuSqueezeNetUnboundOutputs) -WINML_TEST(LearningModelBindingAPITestGpu, ImageBindingDimensions) -WINML_TEST(LearningModelBindingAPITestGpu, VerifyInvalidBindExceptions) -WINML_TEST(LearningModelBindingAPITestGpu, BindInvalidInputName) +WINML_TEST_CLASS_BEGIN(LearningModelBindingAPITestsGPU) +WINML_TEST_CLASS_SETUP_CLASS(LearningModelBindingAPITestsClassSetup) +WINML_TEST_CLASS_SETUP_METHOD(LearningModelBindingAPITestsGpuMethodSetup) +WINML_TEST_CLASS_BEGIN_TESTS +WINML_TEST(LearningModelBindingAPITestsGPU, GpuSqueezeNet) +WINML_TEST(LearningModelBindingAPITestsGPU, GpuSqueezeNetEmptyOutputs) +WINML_TEST(LearningModelBindingAPITestsGPU, GpuSqueezeNetUnboundOutputs) +WINML_TEST(LearningModelBindingAPITestsGPU, ImageBindingDimensions) +WINML_TEST(LearningModelBindingAPITestsGPU, VerifyInvalidBindExceptions) +WINML_TEST(LearningModelBindingAPITestsGPU, BindInvalidInputName) WINML_TEST_CLASS_END() \ No newline at end of file diff --git a/winml/test/api/LearningModelSessionAPITest.cpp b/winml/test/api/LearningModelSessionAPITest.cpp index f60ae6503d..481e18b9e6 100644 --- a/winml/test/api/LearningModelSessionAPITest.cpp +++ b/winml/test/api/LearningModelSessionAPITest.cpp @@ -19,17 +19,16 @@ using namespace winrt::Windows::Foundation::Collections; using winrt::Windows::Foundation::IPropertyValue; -static void LearningModelSessionAPITestSetup() { +static void LearningModelSessionAPITestsClassSetup() { init_apartment(); } -static void LearningModelSessionAPITestGpuSetup() { +static void LearningModelSessionAPITestsGpuMethodSetup() { GPUTEST; - init_apartment(); } -static void LearningModelSessionAPITestsSkipEdgeCoreSetup() { - LearningModelSessionAPITestGpuSetup(); +static void LearningModelSessionAPITestsGpuSkipEdgeCoreMethodSetup() { + LearningModelSessionAPITestsGpuMethodSetup(); SKIP_EDGECORE } @@ -398,12 +397,12 @@ static void CloseSession() }); } -const LearningModelSesssionAPITestApi& getapi() { - static constexpr LearningModelSesssionAPITestApi api = +const LearningModelSesssionAPITestsApi& getapi() { + static constexpr LearningModelSesssionAPITestsApi api = { - LearningModelSessionAPITestSetup, - LearningModelSessionAPITestGpuSetup, - LearningModelSessionAPITestsSkipEdgeCoreSetup, + LearningModelSessionAPITestsClassSetup, + LearningModelSessionAPITestsGpuMethodSetup, + LearningModelSessionAPITestsGpuSkipEdgeCoreMethodSetup, CreateSessionDeviceDefault, CreateSessionDeviceCpu, CreateSessionWithModelLoadedFromStream, diff --git a/winml/test/api/LearningModelSessionAPITest.h b/winml/test/api/LearningModelSessionAPITest.h index 618fab0ea7..02b7ad2d05 100644 --- a/winml/test/api/LearningModelSessionAPITest.h +++ b/winml/test/api/LearningModelSessionAPITest.h @@ -3,10 +3,10 @@ #include "test.h" -struct LearningModelSesssionAPITestApi { - SetupTest LearningModelSessionAPITestSetup; - SetupTest LearningModelSessionAPITestGpuSetup; - SetupTest LearningModelSessionAPITestsSkipEdgeCoreSetup; +struct LearningModelSesssionAPITestsApi { + SetupClass LearningModelSessionAPITestsClassSetup; + SetupTest LearningModelSessionAPITestsGpuMethodSetup; + SetupTest LearningModelSessionAPITestsGpuSkipEdgeCoreMethodSetup; VoidTest CreateSessionDeviceDefault; VoidTest CreateSessionDeviceCpu; VoidTest CreateSessionWithModelLoadedFromStream; @@ -22,26 +22,34 @@ struct LearningModelSesssionAPITestApi { VoidTest EvaluateSessionAndCloseModel; VoidTest CloseSession; }; -const LearningModelSesssionAPITestApi& getapi(); +const LearningModelSesssionAPITestsApi& getapi(); -WINML_TEST_CLASS_BEGIN_WITH_SETUP(LearningModelSessionAPITest, LearningModelSessionAPITestSetup) -WINML_TEST(LearningModelSessionAPITest, CreateSessionDeviceDefault) -WINML_TEST(LearningModelSessionAPITest,CreateSessionDeviceCpu) -WINML_TEST(LearningModelSessionAPITest,CreateSessionWithModelLoadedFromStream) -WINML_TEST(LearningModelSessionAPITest,EvaluateFeatures) -WINML_TEST(LearningModelSessionAPITest,EvaluateFeaturesAsync) -WINML_TEST(LearningModelSessionAPITest,EvaluationProperties) -WINML_TEST(LearningModelSessionAPITest,EvaluateSessionAndCloseModel) +WINML_TEST_CLASS_BEGIN(LearningModelSessionAPITests) +WINML_TEST_CLASS_SETUP_CLASS(LearningModelSessionAPITestsClassSetup) +WINML_TEST_CLASS_BEGIN_TESTS +WINML_TEST(LearningModelSessionAPITests, CreateSessionDeviceDefault) +WINML_TEST(LearningModelSessionAPITests,CreateSessionDeviceCpu) +WINML_TEST(LearningModelSessionAPITests,CreateSessionWithModelLoadedFromStream) +WINML_TEST(LearningModelSessionAPITests,EvaluateFeatures) +WINML_TEST(LearningModelSessionAPITests,EvaluateFeaturesAsync) +WINML_TEST(LearningModelSessionAPITests,EvaluationProperties) +WINML_TEST(LearningModelSessionAPITests,EvaluateSessionAndCloseModel) WINML_TEST_CLASS_END() -WINML_TEST_CLASS_BEGIN_WITH_SETUP(LearningModelSessionAPITestGpu, LearningModelSessionAPITestGpuSetup) -WINML_TEST(LearningModelSessionAPITestGpu, CreateSessionDeviceDirectX) -WINML_TEST(LearningModelSessionAPITestGpu, CreateSessionDeviceDirectXHighPerformance) -WINML_TEST(LearningModelSessionAPITestGpu, CreateSessionDeviceDirectXMinimumPower) -WINML_TEST(LearningModelSessionAPITestGpu, CreateSessionWithCastToFloat16InModel) -WINML_TEST(LearningModelSessionAPITestGpu, DISABLED_CreateSessionWithFloat16InitializersInModel) +WINML_TEST_CLASS_BEGIN(LearningModelSessionAPITestsGpu) +WINML_TEST_CLASS_SETUP_CLASS(LearningModelSessionAPITestsClassSetup) +WINML_TEST_CLASS_SETUP_METHOD(LearningModelSessionAPITestsGpuMethodSetup) +WINML_TEST_CLASS_BEGIN_TESTS +WINML_TEST(LearningModelSessionAPITestsGpu, CreateSessionDeviceDirectX) +WINML_TEST(LearningModelSessionAPITestsGpu, CreateSessionDeviceDirectXHighPerformance) +WINML_TEST(LearningModelSessionAPITestsGpu, CreateSessionDeviceDirectXMinimumPower) +WINML_TEST(LearningModelSessionAPITestsGpu, CreateSessionWithCastToFloat16InModel) +WINML_TEST(LearningModelSessionAPITestsGpu, DISABLED_CreateSessionWithFloat16InitializersInModel) WINML_TEST_CLASS_END() -WINML_TEST_CLASS_BEGIN_WITH_SETUP(LearningModelSessionAPITestsSkipEdgeCore, LearningModelSessionAPITestsSkipEdgeCoreSetup) -WINML_TEST(LearningModelSessionAPITestsSkipEdgeCore, AdapterIdAndDevice) +WINML_TEST_CLASS_BEGIN(LearningModelSessionAPITestsGpuSkipEdgeCore) +WINML_TEST_CLASS_SETUP_CLASS(LearningModelSessionAPITestsClassSetup) +WINML_TEST_CLASS_SETUP_METHOD(LearningModelSessionAPITestsGpuSkipEdgeCoreMethodSetup) +WINML_TEST_CLASS_BEGIN_TESTS +WINML_TEST(LearningModelSessionAPITestsGpuSkipEdgeCore, AdapterIdAndDevice) WINML_TEST_CLASS_END() \ No newline at end of file diff --git a/winml/test/common/googleTestMacros.h b/winml/test/common/googleTestMacros.h index fc52d4fa08..19e054b095 100644 --- a/winml/test/common/googleTestMacros.h +++ b/winml/test/common/googleTestMacros.h @@ -12,22 +12,39 @@ getapi().test_name(); \ } -#define WINML_TEST_CLASS_BEGIN_NO_SETUP(test_class_name) \ - namespace { \ - class test_class_name : public ::testing::Test { \ - }; +#define WINML_TEST_CLASS_BEGIN(test_class_name) \ + namespace { \ + class test_class_name : public ::testing::Test { -#define WINML_TEST_CLASS_BEGIN_WITH_SETUP(test_class_name, setup_method) \ - namespace { \ - class test_class_name : public ::testing::Test { \ - protected: \ - void SetUp() override { \ - getapi().setup_method(); \ - } \ - }; +#define WINML_TEST_CLASS_SETUP_CLASS(setup_class) \ + protected: \ + static void SetUpTestSuite() { \ + getapi().setup_class(); \ + } +#define WINML_TEST_CLASS_TEARDOWN_CLASS(teardown_class) \ + protected: \ + static void TearDownTestSuite() { \ + getapi().teardown_class(); \ + } + +#define WINML_TEST_CLASS_SETUP_METHOD(setup_method) \ + protected: \ + void SetUp() override { \ + getapi().setup_method(); \ + } + +#define WINML_TEST_CLASS_TEARDOWN_METHOD(teardown_method) \ + protected: \ + void TearDown() override { \ + getapi().teardown_method(); \ + } + +#define WINML_TEST_CLASS_BEGIN_TESTS }; + #define WINML_TEST_CLASS_END() } + // For old versions of gtest without GTEST_SKIP, stream the message and return success instead #ifndef GTEST_SKIP #define GTEST_SKIP_(message) \ diff --git a/winml/test/common/taefTestMacros.h b/winml/test/common/taefTestMacros.h index a4b3b6beab..269d404c89 100644 --- a/winml/test/common/taefTestMacros.h +++ b/winml/test/common/taefTestMacros.h @@ -11,14 +11,36 @@ using namespace WEX::TestExecution; #define WINML_EXPECT_NO_THROW(statement) VERIFY_NO_THROW(statement) -#define WINML_TEST_CLASS_BEGIN_WITH_SETUP(test_class_name, setup_method) \ - class test_class_name { \ - TEST_CLASS(test_class_name); \ - TEST_CLASS_SETUP(TestClassSetup) { \ - getapi().setup_method(); \ - return true; \ +#define WINML_TEST_CLASS_BEGIN(test_class_name) \ + class test_class_name { \ + TEST_CLASS(test_class_name); + +#define WINML_TEST_CLASS_SETUP_CLASS(setup_class) \ + TEST_CLASS_SETUP(TestMethodSetup) { \ + getapi().setup_class(); \ + return true; \ } +#define WINML_TEST_CLASS_TEARDOWN_CLASS(teardown_class) \ + TEST_CLASS_CLEANUP(TestClassCleanup) { \ + getapi().teardown_class(); \ + return true; \ + } + +#define WINML_TEST_CLASS_SETUP_METHOD(setup_method) \ + TEST_METHOD_SETUP(TestMethodSetup) { \ + getapi().setup_method(); \ + return true; \ + } + +#define WINML_TEST_CLASS_TEARDOWN_METHOD(teardown_method) \ + TEST_METHOD_CLEANUP(TestClassCleanup) { \ + getapi().teardown_method(); \ + return true; \ + } + +#define WINML_TEST_CLASS_BEGIN_TESTS + #define WINML_TEST_CLASS_END() \ } \ ; diff --git a/winml/test/common/test.h b/winml/test/common/test.h index 7c122e992c..e30736b0f7 100644 --- a/winml/test/common/test.h +++ b/winml/test/common/test.h @@ -4,7 +4,10 @@ #pragma once using VoidTest = void (*)(); +using SetupClass = VoidTest; +using TeardownClass = VoidTest; using SetupTest = VoidTest; +using TeardownTest = VoidTest; constexpr bool alwaysTrue() { return true; diff --git a/winml/test/concurrency/ConcurrencyTests.cpp b/winml/test/concurrency/ConcurrencyTests.cpp index e5c1808148..9b622e59ff 100644 --- a/winml/test/concurrency/ConcurrencyTests.cpp +++ b/winml/test/concurrency/ConcurrencyTests.cpp @@ -37,11 +37,15 @@ void LoadBindEvalSqueezenetRealDataWithValidationConcurrently() { } } -void ConcurrencyTestsApiSetup() { +void ConcurrencyTestsClassSetup() { init_apartment(); std::srand(static_cast(std::time(nullptr))); } +void ConcurrencyTestsGpuMethodSetup() { + GPUTEST; +} + struct EvaluationUnit { LearningModel model; LearningModelSession session; @@ -253,7 +257,6 @@ void MultiThreadMultiSession() { } void MultiThreadMultiSessionGpu() { - GPUTEST; MultiThreadMultiSessionOnDevice(LearningModelDeviceKind::DirectX); } @@ -323,14 +326,14 @@ void MultiThreadSingleSession() { } void MultiThreadSingleSessionGpu() { - GPUTEST; MultiThreadSingleSessionOnDevice(LearningModelDeviceKind::DirectX); } } const ConcurrencyTestsApi& getapi() { static constexpr ConcurrencyTestsApi api = { - ConcurrencyTestsApiSetup, + ConcurrencyTestsClassSetup, + ConcurrencyTestsGpuMethodSetup, LoadBindEvalSqueezenetRealDataWithValidationConcurrently, MultiThreadLoadModel, MultiThreadMultiSession, diff --git a/winml/test/concurrency/ConcurrencyTests.h b/winml/test/concurrency/ConcurrencyTests.h index 2bfb8a4896..0fa8e4a45f 100644 --- a/winml/test/concurrency/ConcurrencyTests.h +++ b/winml/test/concurrency/ConcurrencyTests.h @@ -6,7 +6,8 @@ struct ConcurrencyTestsApi { - SetupTest ConcurrencyTestsApiSetup; + SetupClass ConcurrencyTestsClassSetup; + SetupTest ConcurrencyTestsGpuMethodSetup; VoidTest LoadBindEvalSqueezenetRealDataWithValidationConcurrently; VoidTest MultiThreadLoadModel; VoidTest MultiThreadMultiSession; @@ -19,18 +20,26 @@ struct ConcurrencyTestsApi }; const ConcurrencyTestsApi& getapi(); -WINML_TEST_CLASS_BEGIN_WITH_SETUP(ConcurrencyTests, ConcurrencyTestsApiSetup) +WINML_TEST_CLASS_BEGIN(ConcurrencyTests) +WINML_TEST_CLASS_SETUP_CLASS(ConcurrencyTestsClassSetup) +WINML_TEST_CLASS_BEGIN_TESTS WINML_TEST(ConcurrencyTests, LoadBindEvalSqueezenetRealDataWithValidationConcurrently) WINML_TEST(ConcurrencyTests, MultiThreadLoadModel) WINML_TEST(ConcurrencyTests, MultiThreadMultiSession) -WINML_TEST(ConcurrencyTests, MultiThreadMultiSessionGpu) WINML_TEST(ConcurrencyTests, MultiThreadSingleSession) -WINML_TEST(ConcurrencyTests, MultiThreadSingleSessionGpu) WINML_TEST(ConcurrencyTests, EvalAsyncDifferentModels) WINML_TEST(ConcurrencyTests, EvalAsyncDifferentSessions) WINML_TEST(ConcurrencyTests, EvalAsyncDifferentBindings) WINML_TEST_CLASS_END() +WINML_TEST_CLASS_BEGIN(ConcurrencyTestsGpu) +WINML_TEST_CLASS_SETUP_CLASS(ConcurrencyTestsClassSetup) +WINML_TEST_CLASS_SETUP_METHOD(ConcurrencyTestsGpuMethodSetup) +WINML_TEST_CLASS_BEGIN_TESTS +WINML_TEST(ConcurrencyTestsGpu, MultiThreadMultiSessionGpu) +WINML_TEST(ConcurrencyTestsGpu, MultiThreadSingleSessionGpu) +WINML_TEST_CLASS_END() + // indices for imagenet label static constexpr uint32_t TABBY_CAT_INDEX = 281; static constexpr uint32_t TENCH_INDEX = 0; diff --git a/winml/test/image/imagetests.cpp b/winml/test/image/imagetests.cpp index 32efc43a29..24084f0c42 100644 --- a/winml/test/image/imagetests.cpp +++ b/winml/test/image/imagetests.cpp @@ -38,8 +38,8 @@ protected: winrt::Windows::AI::MachineLearning::LearningModelBinding m_model_binding = nullptr; winrt::Windows::AI::MachineLearning::LearningModelEvaluationResult m_result = nullptr; - void SetUp() override { - init_apartment(); + static void SetUpTestSuite() { + init_apartment(); } void LoadModel(const std::wstring& model_path) { diff --git a/winml/test/scenario/cppwinrt/CustomOps.cpp b/winml/test/scenario/cppwinrt/CustomOps.cpp index 2afd05d99b..77cfefe7f8 100644 --- a/winml/test/scenario/cppwinrt/CustomOps.cpp +++ b/winml/test/scenario/cppwinrt/CustomOps.cpp @@ -33,14 +33,13 @@ using namespace winrt::Windows::Graphics::Imaging; using namespace winrt::Windows::Storage; using namespace winrt::Windows::Storage::Streams; -static void CustomOpsScenarioTestSetup() +static void CustomOpsScenarioTestsClassSetup() { init_apartment(); } -static void CustomOpsScenarioGpuTestSetup() +static void CustomOpsScenarioTestsGpuMethodSetup() { - init_apartment(); GPUTEST; } @@ -724,11 +723,11 @@ static void CustomKernelWithCustomSchema() } } -const CustomOpsTestApi& getapi() { - static constexpr CustomOpsTestApi api = +const CustomOpsTestsApi& getapi() { + static constexpr CustomOpsTestsApi api = { - CustomOpsScenarioTestSetup, - CustomOpsScenarioGpuTestSetup, + CustomOpsScenarioTestsClassSetup, + CustomOpsScenarioTestsGpuMethodSetup, CustomOperatorFusion, CustomKernelWithBuiltInSchema, CustomKernelWithCustomSchema diff --git a/winml/test/scenario/cppwinrt/CustomOps.h b/winml/test/scenario/cppwinrt/CustomOps.h index 02447bac9d..4659234f0d 100644 --- a/winml/test/scenario/cppwinrt/CustomOps.h +++ b/winml/test/scenario/cppwinrt/CustomOps.h @@ -2,21 +2,26 @@ // Licensed under the MIT License. #include "test.h" -struct CustomOpsTestApi +struct CustomOpsTestsApi { - SetupTest CustomOpsScenarioTestSetup; - SetupTest CustomOpsScenarioGpuTestSetup; + SetupTest CustomOpsScenarioTestsClassSetup; + SetupTest CustomOpsScenarioTestsGpuMethodSetup; VoidTest CustomOperatorFusion; VoidTest CustomKernelWithBuiltInSchema; VoidTest CustomKernelWithCustomSchema; }; -const CustomOpsTestApi& getapi(); +const CustomOpsTestsApi& getapi(); -WINML_TEST_CLASS_BEGIN_WITH_SETUP(CustomOpsScenarioTest, CustomOpsScenarioTestSetup) -WINML_TEST(CustomOpsScenarioTest, CustomKernelWithBuiltInSchema) -WINML_TEST(CustomOpsScenarioTest, CustomKernelWithCustomSchema) +WINML_TEST_CLASS_BEGIN(CustomOpsScenarioTests) +WINML_TEST_CLASS_SETUP_CLASS(CustomOpsScenarioTestsClassSetup) +WINML_TEST_CLASS_BEGIN_TESTS +WINML_TEST(CustomOpsScenarioTests, CustomKernelWithBuiltInSchema) +WINML_TEST(CustomOpsScenarioTests, CustomKernelWithCustomSchema) WINML_TEST_CLASS_END() -WINML_TEST_CLASS_BEGIN_WITH_SETUP(CustomOpsScenarioGpuTest, CustomOpsScenarioGpuTestSetup) -WINML_TEST(CustomOpsScenarioGpuTest, CustomOperatorFusion) +WINML_TEST_CLASS_BEGIN(CustomOpsScenarioGpuTests) +WINML_TEST_CLASS_SETUP_CLASS(CustomOpsScenarioTestsClassSetup) +WINML_TEST_CLASS_SETUP_METHOD(CustomOpsScenarioTestsGpuMethodSetup) +WINML_TEST_CLASS_BEGIN_TESTS +WINML_TEST(CustomOpsScenarioGpuTests, CustomOperatorFusion) WINML_TEST_CLASS_END() \ No newline at end of file diff --git a/winml/test/scenario/cppwinrt/scenariotestscppwinrt.cpp b/winml/test/scenario/cppwinrt/scenariotestscppwinrt.cpp index 185d6cb19f..9c981bdd09 100644 --- a/winml/test/scenario/cppwinrt/scenariotestscppwinrt.cpp +++ b/winml/test/scenario/cppwinrt/scenariotestscppwinrt.cpp @@ -52,17 +52,20 @@ using namespace winrt::Windows::Storage; using namespace winrt::Windows::Storage::Streams; using namespace winrt::Windows::UI::Xaml::Media::Imaging; -static void ScenarioCppWinrtTestSetup() { +static void ScenarioCppWinrtTestsClassSetup() { init_apartment(); } -static void ScenarioCppWinrtGpuTestSetup() { - ScenarioCppWinrtTestSetup(); +static void ScenarioCppWinrtTestsGpuMethodSetup() { GPUTEST; }; -static void ScenarioCppWinrtGpuSkipEdgeCoreTestSetup() { - ScenarioCppWinrtGpuTestSetup(); +static void ScenarioCppWinrtTestsSkipEdgeCoreMethodSetup() { + SKIP_EDGECORE +}; + +static void ScenarioCppWinrtTestsGpuSkipEdgeCoreMethodSetup() { + ScenarioCppWinrtTestsGpuMethodSetup(); SKIP_EDGECORE }; @@ -1411,12 +1414,13 @@ static void D2DInterop() { renderTarget.put())); } -const ScenarioTestApi& getapi() { - static constexpr ScenarioTestApi api = +const ScenarioTestsApi& getapi() { + static constexpr ScenarioTestsApi api = { - ScenarioCppWinrtTestSetup, - ScenarioCppWinrtGpuTestSetup, - ScenarioCppWinrtGpuSkipEdgeCoreTestSetup, + ScenarioCppWinrtTestsClassSetup, + ScenarioCppWinrtTestsGpuMethodSetup, + ScenarioCppWinrtTestsSkipEdgeCoreMethodSetup, + ScenarioCppWinrtTestsGpuSkipEdgeCoreMethodSetup, Sample1, Scenario1LoadBindEvalDefault, Scenario2LoadModelFromStream, diff --git a/winml/test/scenario/cppwinrt/scenariotestscppwinrt.h b/winml/test/scenario/cppwinrt/scenariotestscppwinrt.h index 2409de5fd6..9999b4465c 100644 --- a/winml/test/scenario/cppwinrt/scenariotestscppwinrt.h +++ b/winml/test/scenario/cppwinrt/scenariotestscppwinrt.h @@ -2,11 +2,12 @@ // Licensed under the MIT License. #include "test.h" -struct ScenarioTestApi +struct ScenarioTestsApi { - SetupTest ScenarioCppWinrtTestSetup; - SetupTest ScenarioCppWinrtGpuTestSetup; - SetupTest ScenarioCppWinrtGpuSkipEdgeCoreTestSetup; + SetupClass ScenarioCppWinrtTestsClassSetup; + SetupTest ScenarioCppWinrtTestsGpuMethodSetup; + SetupTest ScenarioCppWinrtTestsSkipEdgeCoreMethodSetup; + SetupTest ScenarioCppWinrtTestsGpuSkipEdgeCoreMethodSetup; VoidTest Sample1; VoidTest Scenario1LoadBindEvalDefault; VoidTest Scenario2LoadModelFromStream; @@ -42,47 +43,61 @@ struct ScenarioTestApi VoidTest Scenario8SetDeviceSampleD3D11Device; VoidTest D2DInterop; }; -const ScenarioTestApi& getapi(); +const ScenarioTestsApi& getapi(); -WINML_TEST_CLASS_BEGIN_WITH_SETUP(ScenarioCppWinrtTest, ScenarioCppWinrtTestSetup) -WINML_TEST(ScenarioCppWinrtTest, Sample1) -WINML_TEST(ScenarioCppWinrtTest, Scenario1LoadBindEvalDefault) -WINML_TEST(ScenarioCppWinrtTest, Scenario2LoadModelFromStream) -WINML_TEST(ScenarioCppWinrtTest, Scenario5AsyncEval) -WINML_TEST(ScenarioCppWinrtTest, Scenario7EvalWithNoBind) -WINML_TEST(ScenarioCppWinrtTest, Scenario8SetDeviceSampleDefault) -WINML_TEST(ScenarioCppWinrtTest, Scenario8SetDeviceSampleCPU) -WINML_TEST(ScenarioCppWinrtTest, Scenario17DevDiagnostics) -WINML_TEST(ScenarioCppWinrtTest, DISABLED_Scenario22ImageBindingAsCPUTensor) -WINML_TEST(ScenarioCppWinrtTest, QuantizedModels) -WINML_TEST(ScenarioCppWinrtTest, EncryptedStream) +WINML_TEST_CLASS_BEGIN(ScenarioCppWinrtTests) +WINML_TEST_CLASS_SETUP_CLASS(ScenarioCppWinrtTestsClassSetup) +WINML_TEST_CLASS_BEGIN_TESTS +WINML_TEST(ScenarioCppWinrtTests, Sample1) +WINML_TEST(ScenarioCppWinrtTests, Scenario1LoadBindEvalDefault) +WINML_TEST(ScenarioCppWinrtTests, Scenario2LoadModelFromStream) +WINML_TEST(ScenarioCppWinrtTests, Scenario5AsyncEval) +WINML_TEST(ScenarioCppWinrtTests, Scenario7EvalWithNoBind) +WINML_TEST(ScenarioCppWinrtTests, Scenario8SetDeviceSampleDefault) +WINML_TEST(ScenarioCppWinrtTests, Scenario8SetDeviceSampleCPU) +WINML_TEST(ScenarioCppWinrtTests, Scenario17DevDiagnostics) +WINML_TEST(ScenarioCppWinrtTests, DISABLED_Scenario22ImageBindingAsCPUTensor) +WINML_TEST(ScenarioCppWinrtTests, QuantizedModels) +WINML_TEST(ScenarioCppWinrtTests, EncryptedStream) WINML_TEST_CLASS_END() -WINML_TEST_CLASS_BEGIN_WITH_SETUP(ScenarioCppWinrtGpuTest, ScenarioCppWinrtGpuTestSetup) -WINML_TEST(ScenarioCppWinrtGpuTest, Scenario3SoftwareBitmapInputBinding) -WINML_TEST(ScenarioCppWinrtGpuTest, Scenario6BindWithProperties) -WINML_TEST(ScenarioCppWinrtGpuTest, Scenario8SetDeviceSampleDefaultDirectX) -WINML_TEST(ScenarioCppWinrtGpuTest, Scenario8SetDeviceSampleMinPower) -WINML_TEST(ScenarioCppWinrtGpuTest, Scenario8SetDeviceSampleMaxPerf) -WINML_TEST(ScenarioCppWinrtGpuTest, Scenario8SetDeviceSampleCustomCommandQueue) -WINML_TEST(ScenarioCppWinrtGpuTest, DISABLED_Scenario9LoadBindEvalInputTensorGPU) -WINML_TEST(ScenarioCppWinrtGpuTest, Scenario13SingleModelOnCPUandGPU) -WINML_TEST(ScenarioCppWinrtGpuTest, Scenario11FreeDimensionsTensor) -WINML_TEST(ScenarioCppWinrtGpuTest, Scenario11FreeDimensionsImage) -WINML_TEST(ScenarioCppWinrtGpuTest, Scenario14RunModelSwapchain) -WINML_TEST(ScenarioCppWinrtGpuTest, Scenario20aLoadBindEvalCustomOperatorCPU) -WINML_TEST(ScenarioCppWinrtGpuTest, Scenario20bLoadBindEvalReplacementCustomOperatorCPU) -WINML_TEST(ScenarioCppWinrtGpuTest, DISABLED_Scenario21RunModel2ChainZ) -WINML_TEST(ScenarioCppWinrtGpuTest, DISABLED_Scenario22ImageBindingAsGPUTensor) -WINML_TEST(ScenarioCppWinrtGpuTest, MsftQuantizedModels) -WINML_TEST(ScenarioCppWinrtGpuTest, DISABLED_SyncVsAsync) -WINML_TEST(ScenarioCppWinrtGpuTest, DISABLED_CustomCommandQueueWithFence) -WINML_TEST(ScenarioCppWinrtGpuTest, DISABLED_ReuseVideoFrame) -WINML_TEST(ScenarioCppWinrtGpuTest, DeviceLostRecovery) +WINML_TEST_CLASS_BEGIN(ScenarioCppWinrtTestsGpu) +WINML_TEST_CLASS_SETUP_CLASS(ScenarioCppWinrtTestsClassSetup) +WINML_TEST_CLASS_SETUP_METHOD(ScenarioCppWinrtTestsGpuMethodSetup) +WINML_TEST_CLASS_BEGIN_TESTS +WINML_TEST(ScenarioCppWinrtTestsGpu, Scenario3SoftwareBitmapInputBinding) +WINML_TEST(ScenarioCppWinrtTestsGpu, Scenario6BindWithProperties) +WINML_TEST(ScenarioCppWinrtTestsGpu, Scenario8SetDeviceSampleDefaultDirectX) +WINML_TEST(ScenarioCppWinrtTestsGpu, Scenario8SetDeviceSampleMinPower) +WINML_TEST(ScenarioCppWinrtTestsGpu, Scenario8SetDeviceSampleMaxPerf) +WINML_TEST(ScenarioCppWinrtTestsGpu, Scenario8SetDeviceSampleCustomCommandQueue) +WINML_TEST(ScenarioCppWinrtTestsGpu, DISABLED_Scenario9LoadBindEvalInputTensorGPU) +WINML_TEST(ScenarioCppWinrtTestsGpu, Scenario13SingleModelOnCPUandGPU) +WINML_TEST(ScenarioCppWinrtTestsGpu, Scenario11FreeDimensionsTensor) +WINML_TEST(ScenarioCppWinrtTestsGpu, Scenario11FreeDimensionsImage) +WINML_TEST(ScenarioCppWinrtTestsGpu, Scenario14RunModelSwapchain) +WINML_TEST(ScenarioCppWinrtTestsGpu, Scenario20aLoadBindEvalCustomOperatorCPU) +WINML_TEST(ScenarioCppWinrtTestsGpu, Scenario20bLoadBindEvalReplacementCustomOperatorCPU) +WINML_TEST(ScenarioCppWinrtTestsGpu, DISABLED_Scenario21RunModel2ChainZ) +WINML_TEST(ScenarioCppWinrtTestsGpu, DISABLED_Scenario22ImageBindingAsGPUTensor) +WINML_TEST(ScenarioCppWinrtTestsGpu, MsftQuantizedModels) +WINML_TEST(ScenarioCppWinrtTestsGpu, DISABLED_SyncVsAsync) +WINML_TEST(ScenarioCppWinrtTestsGpu, DISABLED_CustomCommandQueueWithFence) +WINML_TEST(ScenarioCppWinrtTestsGpu, DISABLED_ReuseVideoFrame) +WINML_TEST(ScenarioCppWinrtTestsGpu, DeviceLostRecovery) WINML_TEST_CLASS_END() -WINML_TEST_CLASS_BEGIN_WITH_SETUP(ScenarioCppWinrtGpuSkipEdgeCoreTest, ScenarioCppWinrtGpuSkipEdgeCoreTestSetup) -WINML_TEST(ScenarioCppWinrtGpuSkipEdgeCoreTest, Scenario8SetDeviceSampleMyCameraDevice) -WINML_TEST(ScenarioCppWinrtGpuSkipEdgeCoreTest, Scenario8SetDeviceSampleD3D11Device ) -WINML_TEST(ScenarioCppWinrtGpuSkipEdgeCoreTest, D2DInterop) +WINML_TEST_CLASS_BEGIN(ScenarioCppWinrtTestsSkipEdgeCore) +WINML_TEST_CLASS_SETUP_CLASS(ScenarioCppWinrtTestsClassSetup) +WINML_TEST_CLASS_SETUP_METHOD(ScenarioCppWinrtTestsSkipEdgeCoreMethodSetup) +WINML_TEST_CLASS_BEGIN_TESTS +WINML_TEST(ScenarioCppWinrtTestsSkipEdgeCore, Scenario8SetDeviceSampleMyCameraDevice) +WINML_TEST_CLASS_END() + +WINML_TEST_CLASS_BEGIN(ScenarioCppWinrtTestsGpuSkipEdgeCore) +WINML_TEST_CLASS_SETUP_CLASS(ScenarioCppWinrtTestsClassSetup) +WINML_TEST_CLASS_SETUP_METHOD(ScenarioCppWinrtTestsGpuSkipEdgeCoreMethodSetup) +WINML_TEST_CLASS_BEGIN_TESTS +WINML_TEST(ScenarioCppWinrtTestsGpuSkipEdgeCore, Scenario8SetDeviceSampleD3D11Device) +WINML_TEST(ScenarioCppWinrtTestsGpuSkipEdgeCore, D2DInterop) WINML_TEST_CLASS_END() \ No newline at end of file