diff --git a/cmake/winml_unittests.cmake b/cmake/winml_unittests.cmake index 5c14e3521b..73d0ff7fde 100644 --- a/cmake/winml_unittests.cmake +++ b/cmake/winml_unittests.cmake @@ -83,6 +83,17 @@ function(get_winml_test_concurrency_src set(${output_winml_test_concurrency_src} ${winml_test_concurrency_src} PARENT_SCOPE) endfunction() +function(get_winml_test_image_src + winml_test_src_path + output_winml_test_image_src +) + if (onnxruntime_USE_DML) + set(${output_winml_test_scenario_libs} "onnxruntime_providers_dml" PARENT_SCOPE) + endif() + file(GLOB winml_test_image_src CONFIGURE_DEPENDS "${winml_test_src_path}/image/*.cpp") + set(${output_winml_test_image_src} ${winml_test_image_src} PARENT_SCOPE) +endfunction() + file(GLOB winml_test_common_src CONFIGURE_DEPENDS "${WINML_TEST_SRC_DIR}/common/*.cpp") add_library(winml_test_common STATIC ${winml_test_common_src}) add_dependencies(winml_test_common @@ -127,6 +138,18 @@ endif() # on dev machines but not on the aiinfra agent pool target_link_options(winml_test_scenario PRIVATE /ignore:4199) +get_winml_test_image_src(${WINML_TEST_SRC_DIR} winml_test_image_src winml_test_image_libs) +add_winml_test( + TARGET winml_test_image + SOURCES ${winml_test_image_src} + LIBS winml_test_common delayimp.lib ${winml_test_image_libs} +) +target_precompiled_header(winml_test_image testPch.h) + +target_link_options(winml_test_image PRIVATE /DELAYLOAD:d3d12.dll /DELAYLOAD:api-ms-win-core-file-l1-2-2.dll /DELAYLOAD:api-ms-win-core-synch-l1-2-1.dll) +if (EXISTS ${dxcore_header}) + target_link_options(winml_test_image PRIVATE /DELAYLOAD:ext-ms-win-dxcore-l1-*.dll) +endif() get_winml_test_concurrency_src(${WINML_TEST_SRC_DIR} winml_test_concurrency_src) add_winml_test( @@ -155,9 +178,15 @@ function(add_winml_collateral source) endfunction() add_winml_collateral("${WINML_TEST_SRC_DIR}/api/models/*.onnx") +add_winml_collateral("${WINML_TEST_SRC_DIR}/collateral/images/*.jpg") add_winml_collateral("${WINML_TEST_SRC_DIR}/collateral/images/*.png") add_winml_collateral("${WINML_TEST_SRC_DIR}/collateral/models/*.onnx") add_winml_collateral("${WINML_TEST_SRC_DIR}/common/testdata/squeezenet/*") +add_winml_collateral("${WINML_TEST_SRC_DIR}/image/images/*.jpg") +add_winml_collateral("${WINML_TEST_SRC_DIR}/image/images/*.png") +add_winml_collateral("${WINML_TEST_SRC_DIR}/image/groundTruth/*.jpg") +add_winml_collateral("${WINML_TEST_SRC_DIR}/image/groundTruth/*.png") +add_winml_collateral("${WINML_TEST_SRC_DIR}/image/models/*.onnx") add_winml_collateral("${WINML_TEST_SRC_DIR}/scenario/cppwinrt/*.onnx") add_winml_collateral("${WINML_TEST_SRC_DIR}/scenario/models/*.onnx") add_winml_collateral("${REPO_ROOT}/onnxruntime/test/testdata/sequence_length.onnx") diff --git a/winml/test/collateral/images/1080.jpg b/winml/test/collateral/images/1080.jpg new file mode 100644 index 0000000000..b501402f23 Binary files /dev/null and b/winml/test/collateral/images/1080.jpg differ diff --git a/winml/test/collateral/images/big.png b/winml/test/collateral/images/big.png new file mode 100644 index 0000000000..29825381bd Binary files /dev/null and b/winml/test/collateral/images/big.png differ diff --git a/winml/test/collateral/images/horizontal-crop.png b/winml/test/collateral/images/horizontal-crop.png new file mode 100644 index 0000000000..a2523d0ab2 Binary files /dev/null and b/winml/test/collateral/images/horizontal-crop.png differ diff --git a/winml/test/collateral/images/small.png b/winml/test/collateral/images/small.png new file mode 100644 index 0000000000..2970adf4b4 Binary files /dev/null and b/winml/test/collateral/images/small.png differ diff --git a/winml/test/collateral/images/vertical-crop.png b/winml/test/collateral/images/vertical-crop.png new file mode 100644 index 0000000000..23a4c1b822 Binary files /dev/null and b/winml/test/collateral/images/vertical-crop.png differ diff --git a/winml/test/collateral/models/README.md b/winml/test/collateral/models/README.md new file mode 100644 index 0000000000..5c438120f7 --- /dev/null +++ b/winml/test/collateral/models/README.md @@ -0,0 +1,4 @@ +The FNS_Candy machine learning models here are used with permission from Justin Johnson. +For additional information on these models, refer to: +* [Fast-neural-style GitHub repo](https://github.com/jcjohnson/fast-neural-style) +* [Perceptual Losses for Real-Time Style Transfer and Super-Resolution](https://cs.stanford.edu/people/jcjohns/papers/eccv16/JohnsonECCV16.pdf) diff --git a/winml/test/collateral/models/fns-candy.onnx b/winml/test/collateral/models/fns-candy.onnx new file mode 100644 index 0000000000..227285d302 Binary files /dev/null and b/winml/test/collateral/models/fns-candy.onnx differ diff --git a/winml/test/image/README.md b/winml/test/image/README.md new file mode 100644 index 0000000000..2b44b9a396 --- /dev/null +++ b/winml/test/image/README.md @@ -0,0 +1,127 @@ + +# Overview + +ImageTests are to test all kinds of scenarios that are related to images. + +# Purpose + +To make image related tests data driven and increase test code coverage. + +# Image Test Data Table + +|Table Name | Device |EvaluationStrategies|Models |Input Files |Output Binding Strategies|Input Image Sources |InputPixelFormats| +|--------------|--------------|--------------------|---------------------------|-----------------|-------------------------|---------------------|-----------------| +|Table Rows |GPU |Async |fns-candy_Bgr8_freeDimInput|fish_720.png |Bound |FromImageFeatureValue|Bgra8 +| |CPU |Sync |fns-candy_Bgr8 |fish_720_Gray.png|Unbound |FromVideoFrame |Rgba8 +| | | |fns-candy_Rgb8 | | |FromCPUResource |Gray8 +| | | |fns-candy_tensor | | |FromGPUResource | + +# Test Coverage + +### 1. Devices + +> Devices the Test uses. +* GPU +* CPU + +### 2. Evaluation Strategy +> The strategy to use during evalution. + +* EvalutateAsync +* EvaluateSync +* MultipleEvaluations (**TODO**) +* SwapChain (**TODO**) +* Chain (**TODO**) + +### 3. Output Binding Strategy + +* Bound + ``` + VideoFrame outputimage(BitmapPixelFormat::Rgba8, outputtensorshape.GetAt(3), outputtensorshape.GetAt(2)); + ImageFeatureValue outputTensor = ImageFeatureValue::CreateFromVideoFrame(outputimage); + ``` + +* Unbound + +### 4. Models +> Des: Currently, we are using four models, FNS-candy_Bgr8_freeDimInput, FNS-candy_Bgr8, FNS-candy_Rgb8, and FNS-candy_Tensor. We would try to get more models to cover the following listed cases. + +a. [Input Image Metadata](https://github.com/onnx/onnx/blob/master/docs/MetadataProps.md) + +1. No metadata provided (**TODO**) +2. BitmapPixelFormat + + * Bgr8 + * Rgb8 + * Gray8 + * Bgra8 (**TODO**, not supported in RS5) + * Rgba8 (**TODO**, not supported in RS5) + * yuv (**TODO**, not in ONNX1.2.2 spec) + +* ColorSpaceGamma + * sRGB + * Linear (**TODO**, not supported in RS5) + +* NominalPixelRange + * NominalRange_0_255 + * Normalized_0_1 (No such model) + * Normalized_1_1 (No such model) + * NominalRange_16_235 (No such model) + +b. Model Input Output Type +* free dimension +* image : corresponding to ImageFeatureDescriptor +* tensor : corresponding to TensorFeatureDescriptor + +c. Models we are using. + +* Model takes Gray8 image as input and also output image (**TODO**, MNIST outputs tensor) +* FNS-candy_Bgr8_freeDimInput +* FNS-candy_Bgr8 +* FNS-candy_Rgb8 +* FNS-candy_Tensor + + +### 5. Input Image Files + +* fish_720.png +* fish_720_gray.png + +### 6. Input + +a. input image source + +* FromImageFeatureValue + ``` + VideoFrame frame = VideoFrame::CreateWithSoftwareBitmap(softwareBitmap); + VERIFY_NO_THROW(m_modelBinding.Bind(inputFeature.Current().Name(), frame)); + ``` + +* FromVideoFrame + ``` + VideoFrame frame = VideoFrame::CreateWithSoftwareBitmap(softwareBitmap); + VERIFY_NO_THROW(m_modelBinding.Bind(inputFeature.Current().Name(), frame)); + ``` + +* FromGPUResource + > Bind input from GPU Resources as described in this [PR](https://mscodehub.visualstudio.com/_git/WindowsAI/pullrequest/4542?_a=overview) + +* FromCPUResource + > Bind input from GPU Resources as described in this [PR](https://mscodehub.visualstudio.com/_git/WindowsAI/pullrequest/4507?_a=overview) + +b. Multiple inputs (**TODO**) +1. test the case where 2 inputs requiring GPU processing are bound back to back forcing bind calls that potentially overlap GPU work from previous bind +2. Test models require multiple inputs + +c. Bind with Property (**TODO**) +* _If bind with Property, what properties should be tested? And how to make properties as parameters?_ + +### 7. InputPixelFormats +> We could convert the input image to the following PixelFormats, and then used as input of model. The pixel format expected from model input can be different from the pixel format of real input. But WinML can handle the conversion between them. + +* Bgra8 +* Rgba8 +* Gray8 + +# Implementation +Googletest's [value parameterized tests](https://github.com/google/googletest/blob/master/googletest/docs/advanced.md#value-parameterized-tests) are used. diff --git a/winml/test/image/batchGroundTruth/1080.jpg b/winml/test/image/batchGroundTruth/1080.jpg new file mode 100644 index 0000000000..245504f9a4 Binary files /dev/null and b/winml/test/image/batchGroundTruth/1080.jpg differ diff --git a/winml/test/image/batchGroundTruth/fish_720.png b/winml/test/image/batchGroundTruth/fish_720.png new file mode 100644 index 0000000000..1ce56f3714 Binary files /dev/null and b/winml/test/image/batchGroundTruth/fish_720.png differ diff --git a/winml/test/image/batchGroundTruth/fish_720_Gray.png b/winml/test/image/batchGroundTruth/fish_720_Gray.png new file mode 100644 index 0000000000..f68ed7336a Binary files /dev/null and b/winml/test/image/batchGroundTruth/fish_720_Gray.png differ diff --git a/winml/test/image/batchGroundTruth/kitten_224.png b/winml/test/image/batchGroundTruth/kitten_224.png new file mode 100644 index 0000000000..11e6d48829 Binary files /dev/null and b/winml/test/image/batchGroundTruth/kitten_224.png differ diff --git a/winml/test/image/groundTruth/groundTruth/Bgr8_Bgra8_1080.jpg b/winml/test/image/groundTruth/groundTruth/Bgr8_Bgra8_1080.jpg new file mode 100644 index 0000000000..245504f9a4 Binary files /dev/null and b/winml/test/image/groundTruth/groundTruth/Bgr8_Bgra8_1080.jpg differ diff --git a/winml/test/image/groundTruth/groundTruth/Bgr8_Bgra8_fish_720.png b/winml/test/image/groundTruth/groundTruth/Bgr8_Bgra8_fish_720.png new file mode 100644 index 0000000000..1ce56f3714 Binary files /dev/null and b/winml/test/image/groundTruth/groundTruth/Bgr8_Bgra8_fish_720.png differ diff --git a/winml/test/image/groundTruth/groundTruth/Bgr8_Bgra8_fish_720_Gray.png b/winml/test/image/groundTruth/groundTruth/Bgr8_Bgra8_fish_720_Gray.png new file mode 100644 index 0000000000..f68ed7336a Binary files /dev/null and b/winml/test/image/groundTruth/groundTruth/Bgr8_Bgra8_fish_720_Gray.png differ diff --git a/winml/test/image/groundTruth/groundTruth/Bgr8_Bgra8_kitten_224.png b/winml/test/image/groundTruth/groundTruth/Bgr8_Bgra8_kitten_224.png new file mode 100644 index 0000000000..11e6d48829 Binary files /dev/null and b/winml/test/image/groundTruth/groundTruth/Bgr8_Bgra8_kitten_224.png differ diff --git a/winml/test/image/groundTruth/groundTruth/Bgr8_Gray8_1080.jpg b/winml/test/image/groundTruth/groundTruth/Bgr8_Gray8_1080.jpg new file mode 100644 index 0000000000..551e6da51b Binary files /dev/null and b/winml/test/image/groundTruth/groundTruth/Bgr8_Gray8_1080.jpg differ diff --git a/winml/test/image/groundTruth/groundTruth/Bgr8_Gray8_fish_720.png b/winml/test/image/groundTruth/groundTruth/Bgr8_Gray8_fish_720.png new file mode 100644 index 0000000000..632734f425 Binary files /dev/null and b/winml/test/image/groundTruth/groundTruth/Bgr8_Gray8_fish_720.png differ diff --git a/winml/test/image/groundTruth/groundTruth/Bgr8_Gray8_fish_720_Gray.png b/winml/test/image/groundTruth/groundTruth/Bgr8_Gray8_fish_720_Gray.png new file mode 100644 index 0000000000..f68ed7336a Binary files /dev/null and b/winml/test/image/groundTruth/groundTruth/Bgr8_Gray8_fish_720_Gray.png differ diff --git a/winml/test/image/groundTruth/groundTruth/Bgr8_Gray8_kitten_224.png b/winml/test/image/groundTruth/groundTruth/Bgr8_Gray8_kitten_224.png new file mode 100644 index 0000000000..bd89fe99df Binary files /dev/null and b/winml/test/image/groundTruth/groundTruth/Bgr8_Gray8_kitten_224.png differ diff --git a/winml/test/image/groundTruth/groundTruth/Bgr8_Rgba8_1080.jpg b/winml/test/image/groundTruth/groundTruth/Bgr8_Rgba8_1080.jpg new file mode 100644 index 0000000000..245504f9a4 Binary files /dev/null and b/winml/test/image/groundTruth/groundTruth/Bgr8_Rgba8_1080.jpg differ diff --git a/winml/test/image/groundTruth/groundTruth/Bgr8_Rgba8_fish_720.png b/winml/test/image/groundTruth/groundTruth/Bgr8_Rgba8_fish_720.png new file mode 100644 index 0000000000..1ce56f3714 Binary files /dev/null and b/winml/test/image/groundTruth/groundTruth/Bgr8_Rgba8_fish_720.png differ diff --git a/winml/test/image/groundTruth/groundTruth/Bgr8_Rgba8_fish_720_Gray.png b/winml/test/image/groundTruth/groundTruth/Bgr8_Rgba8_fish_720_Gray.png new file mode 100644 index 0000000000..f68ed7336a Binary files /dev/null and b/winml/test/image/groundTruth/groundTruth/Bgr8_Rgba8_fish_720_Gray.png differ diff --git a/winml/test/image/groundTruth/groundTruth/Bgr8_Rgba8_kitten_224.png b/winml/test/image/groundTruth/groundTruth/Bgr8_Rgba8_kitten_224.png new file mode 100644 index 0000000000..11e6d48829 Binary files /dev/null and b/winml/test/image/groundTruth/groundTruth/Bgr8_Rgba8_kitten_224.png differ diff --git a/winml/test/image/groundTruth/groundTruth/Rgb8_Bgra8_1080.jpg b/winml/test/image/groundTruth/groundTruth/Rgb8_Bgra8_1080.jpg new file mode 100644 index 0000000000..a898ee3234 Binary files /dev/null and b/winml/test/image/groundTruth/groundTruth/Rgb8_Bgra8_1080.jpg differ diff --git a/winml/test/image/groundTruth/groundTruth/Rgb8_Bgra8_fish_720.png b/winml/test/image/groundTruth/groundTruth/Rgb8_Bgra8_fish_720.png new file mode 100644 index 0000000000..f70ecb6ad9 Binary files /dev/null and b/winml/test/image/groundTruth/groundTruth/Rgb8_Bgra8_fish_720.png differ diff --git a/winml/test/image/groundTruth/groundTruth/Rgb8_Bgra8_fish_720_Gray.png b/winml/test/image/groundTruth/groundTruth/Rgb8_Bgra8_fish_720_Gray.png new file mode 100644 index 0000000000..35cc4c906a Binary files /dev/null and b/winml/test/image/groundTruth/groundTruth/Rgb8_Bgra8_fish_720_Gray.png differ diff --git a/winml/test/image/groundTruth/groundTruth/Rgb8_Bgra8_kitten_224.png b/winml/test/image/groundTruth/groundTruth/Rgb8_Bgra8_kitten_224.png new file mode 100644 index 0000000000..26d76d587d Binary files /dev/null and b/winml/test/image/groundTruth/groundTruth/Rgb8_Bgra8_kitten_224.png differ diff --git a/winml/test/image/groundTruth/groundTruth/Rgb8_Gray8_1080.jpg b/winml/test/image/groundTruth/groundTruth/Rgb8_Gray8_1080.jpg new file mode 100644 index 0000000000..ff01b2cfce Binary files /dev/null and b/winml/test/image/groundTruth/groundTruth/Rgb8_Gray8_1080.jpg differ diff --git a/winml/test/image/groundTruth/groundTruth/Rgb8_Gray8_fish_720.png b/winml/test/image/groundTruth/groundTruth/Rgb8_Gray8_fish_720.png new file mode 100644 index 0000000000..e3d033cc89 Binary files /dev/null and b/winml/test/image/groundTruth/groundTruth/Rgb8_Gray8_fish_720.png differ diff --git a/winml/test/image/groundTruth/groundTruth/Rgb8_Gray8_fish_720_Gray.png b/winml/test/image/groundTruth/groundTruth/Rgb8_Gray8_fish_720_Gray.png new file mode 100644 index 0000000000..35cc4c906a Binary files /dev/null and b/winml/test/image/groundTruth/groundTruth/Rgb8_Gray8_fish_720_Gray.png differ diff --git a/winml/test/image/groundTruth/groundTruth/Rgb8_Gray8_kitten_224.png b/winml/test/image/groundTruth/groundTruth/Rgb8_Gray8_kitten_224.png new file mode 100644 index 0000000000..e847c18484 Binary files /dev/null and b/winml/test/image/groundTruth/groundTruth/Rgb8_Gray8_kitten_224.png differ diff --git a/winml/test/image/groundTruth/groundTruth/Rgb8_Rgba8_1080.jpg b/winml/test/image/groundTruth/groundTruth/Rgb8_Rgba8_1080.jpg new file mode 100644 index 0000000000..a898ee3234 Binary files /dev/null and b/winml/test/image/groundTruth/groundTruth/Rgb8_Rgba8_1080.jpg differ diff --git a/winml/test/image/groundTruth/groundTruth/Rgb8_Rgba8_fish_720.png b/winml/test/image/groundTruth/groundTruth/Rgb8_Rgba8_fish_720.png new file mode 100644 index 0000000000..f70ecb6ad9 Binary files /dev/null and b/winml/test/image/groundTruth/groundTruth/Rgb8_Rgba8_fish_720.png differ diff --git a/winml/test/image/groundTruth/groundTruth/Rgb8_Rgba8_fish_720_Gray.png b/winml/test/image/groundTruth/groundTruth/Rgb8_Rgba8_fish_720_Gray.png new file mode 100644 index 0000000000..35cc4c906a Binary files /dev/null and b/winml/test/image/groundTruth/groundTruth/Rgb8_Rgba8_fish_720_Gray.png differ diff --git a/winml/test/image/groundTruth/groundTruth/Rgb8_Rgba8_kitten_224.png b/winml/test/image/groundTruth/groundTruth/Rgb8_Rgba8_kitten_224.png new file mode 100644 index 0000000000..26d76d587d Binary files /dev/null and b/winml/test/image/groundTruth/groundTruth/Rgb8_Rgba8_kitten_224.png differ diff --git a/winml/test/image/imageTestHelper.cpp b/winml/test/image/imageTestHelper.cpp new file mode 100644 index 0000000000..6be4cab7c6 --- /dev/null +++ b/winml/test/image/imageTestHelper.cpp @@ -0,0 +1,284 @@ +#include "testPch.h" + +#include "imageTestHelper.h" +#include "robuffer.h" +#include "winrt/Windows.Storage.h" +#include "winrt/Windows.Storage.Streams.h" + +#include +#include +#include + +#define FENCE_SIGNAL_VALUE 1 + +using namespace winrt; +using namespace winrt::Windows::AI::MachineLearning; +using namespace winrt::Windows::Foundation::Collections; +using namespace winrt::Windows::Media; +using namespace winrt::Windows::Graphics::Imaging; + +namespace ImageTestHelper { + BitmapPixelFormat GetPixelFormat(const std::wstring& inputPixelFormat) { + // Return corresponding BitmapPixelFormat according to input string + if (L"Bgra8" == inputPixelFormat || L"Bgr8" == inputPixelFormat) { + return BitmapPixelFormat::Bgra8; + } else if (L"Rgba8" == inputPixelFormat || L"Rgb8" == inputPixelFormat) { + return BitmapPixelFormat::Rgba8; + } else if (L"Gray8" == inputPixelFormat) { + return BitmapPixelFormat::Gray8; + } else { + throw std::invalid_argument("Unsupported pixelFormat"); + } + } + + TensorFloat LoadInputImageFromCPU( + SoftwareBitmap softwareBitmap, + const std::wstring& modelPixelFormat) { + softwareBitmap = SoftwareBitmap::Convert(softwareBitmap, BitmapPixelFormat::Bgra8); + BYTE* pData = nullptr; + UINT32 size = 0; + winrt::Windows::Graphics::Imaging::BitmapBuffer spBitmapBuffer(softwareBitmap.LockBuffer(winrt::Windows::Graphics::Imaging::BitmapBufferAccessMode::Read)); + winrt::Windows::Foundation::IMemoryBufferReference reference = spBitmapBuffer.CreateReference(); + auto spByteAccess = reference.as<::Windows::Foundation::IMemoryBufferByteAccess>(); + spByteAccess->GetBuffer(&pData, &size); + + uint32_t height = softwareBitmap.PixelHeight(); + uint32_t width = softwareBitmap.PixelWidth(); + + // TODO: Need modification for Gray8 + std::vector shape = { 1, 3, height , width }; + float* pCPUTensor; + uint32_t uCapacity; + TensorFloat tf = TensorFloat::Create(shape); + com_ptr itn = tf.as(); + itn->GetBuffer(reinterpret_cast(&pCPUTensor), &uCapacity); + if (BitmapPixelFormat::Bgra8 == GetPixelFormat(modelPixelFormat)) { + for (UINT32 i = 0; i < size; i += 4) { + UINT32 pixelInd = i / 4; + pCPUTensor[pixelInd] = (float)pData[i]; + pCPUTensor[(height * width) + pixelInd] = (float)pData[i + 1]; + pCPUTensor[(height * width * 2) + pixelInd] = (float)pData[i + 2]; + } + } else if (BitmapPixelFormat::Rgba8 == GetPixelFormat(modelPixelFormat)) { + for (UINT32 i = 0; i < size; i += 4) { + UINT32 pixelInd = i / 4; + pCPUTensor[pixelInd] = (float)pData[i + 2]; + pCPUTensor[(height * width) + pixelInd] = (float)pData[i + 1]; + pCPUTensor[(height * width * 2) + pixelInd] = (float)pData[i]; + } + } + // else if() + // TODO: for Gray8 + else { + std::cerr << "Unsupportted pixelFormat"; + } + return tf; + } + + TensorFloat LoadInputImageFromGPU( + SoftwareBitmap softwareBitmap, + const std::wstring& modelPixelFormat) { + + softwareBitmap = SoftwareBitmap::Convert(softwareBitmap, BitmapPixelFormat::Bgra8); + BYTE* pData = nullptr; + UINT32 size = 0; + BitmapBuffer spBitmapBuffer(softwareBitmap.LockBuffer(winrt::Windows::Graphics::Imaging::BitmapBufferAccessMode::Read)); + winrt::Windows::Foundation::IMemoryBufferReference reference = spBitmapBuffer.CreateReference(); + com_ptr<::Windows::Foundation::IMemoryBufferByteAccess> spByteAccess = reference.as<::Windows::Foundation::IMemoryBufferByteAccess>(); + spByteAccess->GetBuffer(&pData, &size); + + std::vector shape = { 1, 3, softwareBitmap.PixelHeight() , softwareBitmap.PixelWidth() }; + float* pCPUTensor; + uint32_t uCapacity; + + // CPU tensorization + TensorFloat tf = TensorFloat::Create(shape); + com_ptr itn = tf.as(); + itn->GetBuffer(reinterpret_cast(&pCPUTensor), &uCapacity); + + uint32_t height = softwareBitmap.PixelHeight(); + uint32_t width = softwareBitmap.PixelWidth(); + if (BitmapPixelFormat::Bgra8 == GetPixelFormat(modelPixelFormat)) { + for (UINT32 i = 0; i < size; i += 4) { + UINT32 pixelInd = i / 4; + pCPUTensor[pixelInd] = (float)pData[i]; + pCPUTensor[(height * width) + pixelInd] = (float)pData[i + 1]; + pCPUTensor[(height * width * 2) + pixelInd] = (float)pData[i + 2]; + } + } else if (BitmapPixelFormat::Rgba8 == GetPixelFormat(modelPixelFormat)) { + for (UINT32 i = 0; i < size; i += 4) { + UINT32 pixelInd = i / 4; + pCPUTensor[pixelInd] = (float)pData[i + 2]; + pCPUTensor[(height * width) + pixelInd] = (float)pData[i + 1]; + pCPUTensor[(height * width * 2) + pixelInd] = (float)pData[i]; + } + } + // else if() + // TODO: for Gray8 + else { + std::cerr << "unsupported pixelFormat"; + } + + // create the d3d device. + com_ptr pD3D12Device = nullptr; + WINML_EXPECT_NO_THROW(D3D12CreateDevice(nullptr, D3D_FEATURE_LEVEL::D3D_FEATURE_LEVEL_11_0, __uuidof(ID3D12Device), reinterpret_cast(&pD3D12Device))); + + // create the command queue. + com_ptr dxQueue = nullptr; + D3D12_COMMAND_QUEUE_DESC commandQueueDesc = {}; + commandQueueDesc.Type = D3D12_COMMAND_LIST_TYPE_DIRECT; + pD3D12Device->CreateCommandQueue(&commandQueueDesc, __uuidof(ID3D12CommandQueue), reinterpret_cast(&dxQueue)); + com_ptr devicefactory = get_activation_factory(); + com_ptr tensorfactory = get_activation_factory(); + com_ptr<::IUnknown> spUnk; + devicefactory->CreateFromD3D12CommandQueue(dxQueue.get(), spUnk.put()); + + // Create ID3D12GraphicsCommandList and Allocator + D3D12_COMMAND_LIST_TYPE queuetype = dxQueue->GetDesc().Type; + com_ptr alloctor; + com_ptr cmdList; + + pD3D12Device->CreateCommandAllocator( + queuetype, + winrt::guid_of(), + alloctor.put_void()); + + pD3D12Device->CreateCommandList( + 0, + queuetype, + alloctor.get(), + nullptr, + winrt::guid_of(), + cmdList.put_void()); + + // Create Committed Resource + // 3 is number of channels we use. R G B without alpha. + UINT64 bufferbytesize = 3 * sizeof(float) * softwareBitmap.PixelWidth()*softwareBitmap.PixelHeight(); + D3D12_HEAP_PROPERTIES heapProperties = { + D3D12_HEAP_TYPE_DEFAULT, + D3D12_CPU_PAGE_PROPERTY_UNKNOWN, + D3D12_MEMORY_POOL_UNKNOWN, + 0, + 0 + }; + D3D12_RESOURCE_DESC resourceDesc = { + D3D12_RESOURCE_DIMENSION_BUFFER, + 0, + bufferbytesize, + 1, + 1, + 1, + DXGI_FORMAT_UNKNOWN, + { 1, 0 }, + D3D12_TEXTURE_LAYOUT_ROW_MAJOR, + D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS + }; + + com_ptr pGPUResource = nullptr; + com_ptr imageUploadHeap; + pD3D12Device->CreateCommittedResource( + &heapProperties, + D3D12_HEAP_FLAG_NONE, + &resourceDesc, + D3D12_RESOURCE_STATE_COMMON, + nullptr, + __uuidof(ID3D12Resource), + pGPUResource.put_void() + ); + + // Create the GPU upload buffer. + auto heap_properties = CD3DX12_HEAP_PROPERTIES(D3D12_HEAP_TYPE_UPLOAD); + WINML_EXPECT_NO_THROW(pD3D12Device->CreateCommittedResource( + &heap_properties, + D3D12_HEAP_FLAG_NONE, + &CD3DX12_RESOURCE_DESC::Buffer(bufferbytesize), + D3D12_RESOURCE_STATE_GENERIC_READ, + nullptr, + __uuidof(ID3D12Resource), + imageUploadHeap.put_void())); + + // Copy from Cpu to GPU + D3D12_SUBRESOURCE_DATA CPUData = {}; + CPUData.pData = reinterpret_cast(pCPUTensor); + CPUData.RowPitch = bufferbytesize; + CPUData.SlicePitch = bufferbytesize; + UpdateSubresources(cmdList.get(), pGPUResource.get(), imageUploadHeap.get(), 0, 0, 1, &CPUData); + + // Close the command list and execute it to begin the initial GPU setup. + WINML_EXPECT_NO_THROW(cmdList->Close()); + ID3D12CommandList* ppCommandLists[] = { cmdList.get() }; + dxQueue->ExecuteCommandLists(_countof(ppCommandLists), ppCommandLists); + + //Create Event + HANDLE directEvent = CreateEvent(nullptr, FALSE, FALSE, nullptr); + wil::unique_event hDirectEvent(directEvent); + + //Create Fence + Microsoft::WRL::ComPtr spDirectFence = nullptr; + WINML_EXPECT_HRESULT_SUCCEEDED(pD3D12Device->CreateFence( + 0, + D3D12_FENCE_FLAG_NONE, + IID_PPV_ARGS(spDirectFence.ReleaseAndGetAddressOf()))); + //Adds fence to queue + WINML_EXPECT_HRESULT_SUCCEEDED(dxQueue->Signal(spDirectFence.Get(), FENCE_SIGNAL_VALUE)); + WINML_EXPECT_HRESULT_SUCCEEDED(spDirectFence->SetEventOnCompletion(FENCE_SIGNAL_VALUE, hDirectEvent.get())); + + //Wait for signal + DWORD retVal = WaitForSingleObject(hDirectEvent.get(), INFINITE); + if (retVal != WAIT_OBJECT_0) { + WINML_EXPECT_HRESULT_SUCCEEDED(E_UNEXPECTED); + } + + // GPU tensorize + com_ptr<::IUnknown> spUnkTensor; + TensorFloat input1imagetensor(nullptr); + int64_t shapes[4] = { 1,3, softwareBitmap.PixelWidth(), softwareBitmap.PixelHeight() }; + tensorfactory->CreateFromD3D12Resource(pGPUResource.get(), shapes, 4, spUnkTensor.put()); + spUnkTensor.try_as(input1imagetensor); + + return input1imagetensor; + } + + bool VerifyHelper( + VideoFrame actual, + VideoFrame expected) { + // Verify two input ImageFeatureValues are identified. + auto softwareBitmapActual = actual.SoftwareBitmap(); + auto softwareBitmapExpected = expected.SoftwareBitmap(); + WINML_EXPECT_TRUE(softwareBitmapActual.PixelHeight() == softwareBitmapExpected.PixelHeight()); + WINML_EXPECT_TRUE(softwareBitmapActual.PixelWidth() == softwareBitmapExpected.PixelWidth()); + WINML_EXPECT_TRUE(softwareBitmapActual.BitmapPixelFormat() == softwareBitmapExpected.BitmapPixelFormat()); + + uint32_t size = 4 * softwareBitmapActual.PixelHeight() * softwareBitmapActual.PixelWidth(); + + winrt::Windows::Storage::Streams::Buffer actualOutputBuffer(size); + winrt::Windows::Storage::Streams::Buffer expectedOutputBuffer(size); + + softwareBitmapActual.CopyToBuffer(actualOutputBuffer); + softwareBitmapExpected.CopyToBuffer(expectedOutputBuffer); + + byte* actualBytes; + actualOutputBuffer.try_as<::Windows::Storage::Streams::IBufferByteAccess>()->Buffer(&actualBytes); + byte* expectedBytes; + expectedOutputBuffer.try_as<::Windows::Storage::Streams::IBufferByteAccess>()->Buffer(&expectedBytes); + + byte* pActualByte = actualBytes; + byte* pExpectedByte = expectedBytes; + + // hard code, might need to be modified later. + const float cMaxErrorRate = 0.06f; + int8_t epsilon = 20; + + // Even given two same ImageFeatureValues, the comparison cannot exactly match. + // So we use error rate. + UINT errors = 0; + for (uint32_t i = 0; i < size; i++, pActualByte++, pExpectedByte++) { + auto diff = (*pActualByte - *pExpectedByte); + if (diff > epsilon) { + errors++; + } + } + std::cerr << "total errors is " << errors << "/" << size << ", errors rate is " << (float)errors / size; + return (float)errors / size < cMaxErrorRate; + } +} diff --git a/winml/test/image/imageTestHelper.h b/winml/test/image/imageTestHelper.h new file mode 100644 index 0000000000..2754dffd7b --- /dev/null +++ b/winml/test/image/imageTestHelper.h @@ -0,0 +1,34 @@ +//----------------------------------------------------------------------------- +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// +//----------------------------------------------------------------------------- + +#pragma once + +#include +#include + +enum OutputBindingStrategy { Bound, Unbound, Empty }; +enum EvaluationStrategy { Async, Sync }; +enum ModelInputOutputType { Image, Tensor }; +enum InputImageSource { FromVideoFrame, FromImageFeatureValue, FromCPUResource, FromGPUResource }; +enum VideoFrameSource { FromSoftwareBitmap, FromDirect3DSurface, FromUnsupportedD3DSurface }; + +namespace ImageTestHelper +{ + winrt::Windows::Graphics::Imaging::BitmapPixelFormat GetPixelFormat(const std::wstring& inputPixelFormat); + + winrt::Windows::AI::MachineLearning::TensorFloat LoadInputImageFromCPU( + winrt::Windows::Graphics::Imaging::SoftwareBitmap softwareBitmap, + const std::wstring& modelPixelFormat); + + winrt::Windows::AI::MachineLearning::TensorFloat LoadInputImageFromGPU( + winrt::Windows::Graphics::Imaging::SoftwareBitmap softwareBitmap, + const std::wstring& modelPixelFormat); + + bool VerifyHelper( + winrt::Windows::Media::VideoFrame actual, + winrt::Windows::Media::VideoFrame expected); + +} diff --git a/winml/test/image/images/1080.jpg b/winml/test/image/images/1080.jpg new file mode 100644 index 0000000000..b501402f23 Binary files /dev/null and b/winml/test/image/images/1080.jpg differ diff --git a/winml/test/image/images/RGB_5.png b/winml/test/image/images/RGB_5.png new file mode 100644 index 0000000000..3645ab68fe Binary files /dev/null and b/winml/test/image/images/RGB_5.png differ diff --git a/winml/test/image/images/fish_720.png b/winml/test/image/images/fish_720.png new file mode 100644 index 0000000000..3d413cfc2a Binary files /dev/null and b/winml/test/image/images/fish_720.png differ diff --git a/winml/test/image/images/fish_720_gray.png b/winml/test/image/images/fish_720_gray.png new file mode 100644 index 0000000000..ff45e9ac9c Binary files /dev/null and b/winml/test/image/images/fish_720_gray.png differ diff --git a/winml/test/image/images/horizontal-crop.png b/winml/test/image/images/horizontal-crop.png new file mode 100644 index 0000000000..a2523d0ab2 Binary files /dev/null and b/winml/test/image/images/horizontal-crop.png differ diff --git a/winml/test/image/images/kitten_224.png b/winml/test/image/images/kitten_224.png new file mode 100644 index 0000000000..e47ca94863 Binary files /dev/null and b/winml/test/image/images/kitten_224.png differ diff --git a/winml/test/image/images/small.png b/winml/test/image/images/small.png new file mode 100644 index 0000000000..2970adf4b4 Binary files /dev/null and b/winml/test/image/images/small.png differ diff --git a/winml/test/image/images/vertical-crop.png b/winml/test/image/images/vertical-crop.png new file mode 100644 index 0000000000..23a4c1b822 Binary files /dev/null and b/winml/test/image/images/vertical-crop.png differ diff --git a/winml/test/image/imagetests.cpp b/winml/test/image/imagetests.cpp new file mode 100644 index 0000000000..32efc43a29 --- /dev/null +++ b/winml/test/image/imagetests.cpp @@ -0,0 +1,1029 @@ +#include "testPch.h" + +#include "filehelpers.h" +#include "imageTestHelper.h" +#include "robuffer.h" +#include "windows.ai.machinelearning.native.internal.h" +#include "winrt/Windows.Storage.h" +#include "winrt/Windows.Storage.Streams.h" + +#include +#include +#include +#include + +#ifndef BUILD_GOOGLE_TEST +#error Must use googletest for value-parameterized tests +#endif + +using namespace winrt; +using namespace winrt::Windows::AI::MachineLearning; +using namespace winrt::Windows::Foundation::Collections; +using namespace winrt::Windows::Media; +using namespace winrt::Windows::Graphics::Imaging; +using namespace winrt::Windows::Graphics::DirectX; +using namespace winrt::Windows::Storage; +using namespace winrt::Windows::Storage::Streams; + +enum BindingLocation { + CPU, + GPU +}; + +class ImageTests : public ::testing::Test { +protected: + winrt::Windows::AI::MachineLearning::LearningModel m_model = nullptr; + winrt::Windows::AI::MachineLearning::LearningModelDevice m_device = nullptr; + winrt::Windows::AI::MachineLearning::LearningModelSession m_session = nullptr; + winrt::Windows::AI::MachineLearning::LearningModelBinding m_model_binding = nullptr; + winrt::Windows::AI::MachineLearning::LearningModelEvaluationResult m_result = nullptr; + + void SetUp() override { + init_apartment(); + } + + void LoadModel(const std::wstring& model_path) { + std::wstring full_path = FileHelpers::GetModulePath() + model_path; + WINML_EXPECT_NO_THROW(m_model = LearningModel::LoadFromFilePath(full_path)); + } + + void ImageTests::PrepareModelSessionBinding( + const std::wstring& model_file_name, + LearningModelDeviceKind device_kind, + std::optional optimized_batch_size) { + LoadModel(std::wstring(model_file_name)); + WINML_EXPECT_NO_THROW(m_device = LearningModelDevice(device_kind)); + if (optimized_batch_size.has_value()) { + LearningModelSessionOptions options; + options.BatchSizeOverride(optimized_batch_size.value()); + WINML_EXPECT_NO_THROW(m_session = LearningModelSession(m_model, m_device, options)); + } + else { + WINML_EXPECT_NO_THROW(m_session = LearningModelSession(m_model, m_device)); + } + WINML_EXPECT_NO_THROW(m_model_binding = LearningModelBinding(m_session)); + } + + bool BindInputValue( + const std::wstring& image_file_name, + const std::wstring& input_pixel_format, + const std::wstring& model_pixel_format, + InputImageSource input_image_source, + LearningModelDeviceKind device_kind + ) { + std::wstring full_image_path = FileHelpers::GetModulePath() + image_file_name; + StorageFile image_file = StorageFile::GetFileFromPathAsync(full_image_path).get(); + IRandomAccessStream stream = image_file.OpenAsync(FileAccessMode::Read).get(); + BitmapDecoder bitmap_decoder = BitmapDecoder::CreateAsync(stream).get(); + SoftwareBitmap software_bitmap = bitmap_decoder.GetSoftwareBitmapAsync().get(); + + // Convert the input image to PixelFormat specified + software_bitmap = SoftwareBitmap::Convert( + software_bitmap, + ImageTestHelper::GetPixelFormat(input_pixel_format)); + + auto input_feature = m_model.InputFeatures().First(); + if (InputImageSource::FromImageFeatureValue == input_image_source) { + VideoFrame frame = VideoFrame::CreateWithSoftwareBitmap(software_bitmap); + ImageFeatureValue image_input_tensor = ImageFeatureValue::CreateFromVideoFrame(frame); + WINML_EXPECT_NO_THROW(m_model_binding.Bind(input_feature.Current().Name(), image_input_tensor)); + } + else if (InputImageSource::FromVideoFrame == input_image_source) { + VideoFrame frame = VideoFrame::CreateWithSoftwareBitmap(software_bitmap); + WINML_EXPECT_NO_THROW(m_model_binding.Bind(input_feature.Current().Name(), frame)); + } + else if (InputImageSource::FromCPUResource == input_image_source) { + TensorFloat tensor_float = ImageTestHelper::LoadInputImageFromCPU(software_bitmap, model_pixel_format); + WINML_EXPECT_NO_THROW(m_model_binding.Bind(input_feature.Current().Name(), tensor_float)); + } + else if (InputImageSource::FromGPUResource == input_image_source) { + TensorFloat tensor_float = ImageTestHelper::LoadInputImageFromGPU(software_bitmap, model_pixel_format); + if (LearningModelDeviceKind::Cpu == device_kind) { + WINML_EXPECT_THROW_SPECIFIC(m_model_binding.Bind(input_feature.Current().Name(), tensor_float), + winrt::hresult_error, + [](const winrt::hresult_error& e) -> bool { + return e.code() == WINML_ERR_INVALID_BINDING; + }); + return false; + } + WINML_EXPECT_NO_THROW(m_model_binding.Bind(input_feature.Current().Name(), tensor_float)); + } + return true; + } + + VideoFrame BindImageOutput( + ModelInputOutputType model_input_output_type, + OutputBindingStrategy output_binding_strategy, + const std::wstring& model_pixel_format) { + std::wstring output_data_binding_name = std::wstring(m_model.OutputFeatures().First().Current().Name()); + VideoFrame frame = nullptr; + if (OutputBindingStrategy::Bound == output_binding_strategy) { + if (ModelInputOutputType::Image == model_input_output_type) { + ImageFeatureDescriptor output_image_descriptor = nullptr; + WINML_EXPECT_NO_THROW(m_model.OutputFeatures().First().Current().as(output_image_descriptor)); + SoftwareBitmap bitmap( + ImageTestHelper::GetPixelFormat(model_pixel_format), + output_image_descriptor.Height(), + output_image_descriptor.Width()); + frame = VideoFrame::CreateWithSoftwareBitmap(bitmap); + + } + else if (ModelInputOutputType::Tensor == model_input_output_type) { + TensorFeatureDescriptor output_tensor_descriptor = nullptr; + WINML_EXPECT_NO_THROW(m_model.OutputFeatures().First().Current().as(output_tensor_descriptor)); + auto output_tensor_shape = output_tensor_descriptor.Shape(); + SoftwareBitmap bitmap( + ImageTestHelper::GetPixelFormat(model_pixel_format), + static_cast(output_tensor_shape.GetAt(3)), + static_cast(output_tensor_shape.GetAt(2))); + frame = VideoFrame::CreateWithSoftwareBitmap(bitmap); + } + auto output_tensor = ImageFeatureValue::CreateFromVideoFrame(frame); + WINML_EXPECT_NO_THROW(m_model_binding.Bind(output_data_binding_name, output_tensor)); + } + + // Else for Unbound + return frame; + } + + IVector BindImageOutput( + ModelInputOutputType model_input_output_type, + OutputBindingStrategy output_binding_strategy, + VideoFrameSource output_video_frame_source, + const std::wstring& model_pixel_format, + const uint32_t& batch_size) { + std::wstring output_data_binding_name = std::wstring(m_model.OutputFeatures().First().Current().Name()); + uint32_t width = 0, height = 0; + if (ModelInputOutputType::Image == model_input_output_type) { + ImageFeatureDescriptor output_image_descriptor = nullptr; + WINML_EXPECT_NO_THROW(m_model.OutputFeatures().First().Current().as(output_image_descriptor)); + width = output_image_descriptor.Width(); + height = output_image_descriptor.Height(); + } else { + TensorFeatureDescriptor output_tensor_descriptor = nullptr; + WINML_EXPECT_NO_THROW(m_model.OutputFeatures().First().Current().as(output_tensor_descriptor)); + auto output_tensor_shape = output_tensor_descriptor.Shape(); + width = static_cast(output_tensor_shape.GetAt(3)); + height = static_cast(output_tensor_shape.GetAt(2)); + } + IVector output_video_frames; + if (OutputBindingStrategy::Bound == output_binding_strategy) { + std::vector output_frames = {}; + for (uint32_t i = 0; i < batch_size; ++i) { + VideoFrame video_frame = nullptr; + if (VideoFrameSource::FromSoftwareBitmap == output_video_frame_source) { + video_frame = VideoFrame::CreateWithSoftwareBitmap(SoftwareBitmap(ImageTestHelper::GetPixelFormat(model_pixel_format), width, height)); + } + else if (VideoFrameSource::FromDirect3DSurface == output_video_frame_source) { + video_frame = VideoFrame::CreateAsDirect3D11SurfaceBacked(DirectXPixelFormat::B8G8R8A8UIntNormalized, width, height); + } + else if (VideoFrameSource::FromUnsupportedD3DSurface == output_video_frame_source) { + video_frame = VideoFrame::CreateAsDirect3D11SurfaceBacked(DirectXPixelFormat::B8G8R8X8UIntNormalized, width, height); + } + output_frames.emplace_back(video_frame); + } + output_video_frames = winrt::single_threaded_vector(std::move(output_frames)); + WINML_EXPECT_NO_THROW(m_model_binding.Bind(output_data_binding_name, output_video_frames)); + + } + + // Else for Unbound + return output_video_frames; + } + + void VerifyResults( + VideoFrame output_tensor, + const std::wstring& bm_image_path, + const std::wstring& model_pixel_format) { + SoftwareBitmap bm_software_bitmap = FileHelpers::GetSoftwareBitmapFromFile(bm_image_path); + bm_software_bitmap = SoftwareBitmap::Convert( + bm_software_bitmap, + ImageTestHelper::GetPixelFormat(model_pixel_format)); + VideoFrame bm_video_frame = VideoFrame::CreateWithSoftwareBitmap(bm_software_bitmap); + WINML_EXPECT_TRUE(ImageTestHelper::VerifyHelper(output_tensor, bm_video_frame)); + } + + void EvaluateTest(EvaluationStrategy strategy) { + if (EvaluationStrategy::Async == strategy) { + WINML_EXPECT_NO_THROW(m_result = m_session.EvaluateAsync(m_model_binding, L"").get()); + } + else if (EvaluationStrategy::Sync == strategy) { + WINML_EXPECT_NO_THROW(m_result = m_session.Evaluate(m_model_binding, L"")); + } + } + + bool ShouldSkip( + const std::wstring& model_file_name, + const std::wstring& image_file_name, + const InputImageSource input_image_source + ) { + // Case that the tensor's shape doesn't match model's shape should be skiped + if ((L"1080.jpg" == image_file_name || L"kitten_224.png" == image_file_name) && + (InputImageSource::FromGPUResource == input_image_source || InputImageSource::FromCPUResource == input_image_source)) { + return true; + } + + // Case that the images's shape doesn't match model's shape which expects free dimension should be skiped. + // Because the fns-candy is not real model that can handle free dimensional input + if ((L"1080.jpg" == image_file_name || L"kitten_224.png" == image_file_name) && + L"fns-candy_Bgr8_freeDimInput.onnx" == model_file_name) { + return true; + } + + return false; + } + + void ValidateOutputImageMetaData(const std::wstring& path, BitmapAlphaMode expected_mode, BitmapPixelFormat expected_format, bool supported) { + WINML_EXPECT_NO_THROW(LoadModel(path)); + //input does not have image metadata and output does + + WINML_EXPECT_TRUE(m_model.OutputFeatures().First().HasCurrent()); + + std::wstring name(m_model.OutputFeatures().First().Current().Name()); + std::wstring expected_tensor_name = L"add_3"; + WINML_EXPECT_EQUAL(name, expected_tensor_name); + + ImageFeatureDescriptor image_descriptor = nullptr; + if (supported) { + WINML_EXPECT_NO_THROW(m_model.OutputFeatures().First().Current().as(image_descriptor)); + WINML_EXPECT_TRUE(image_descriptor != nullptr); + + auto tensor_name = image_descriptor.Name(); + WINML_EXPECT_EQUAL(tensor_name, expected_tensor_name); + + auto model_data_kind = image_descriptor.Kind(); + WINML_EXPECT_EQUAL(model_data_kind, LearningModelFeatureKind::Image); + + WINML_EXPECT_TRUE(image_descriptor.IsRequired()); + + WINML_EXPECT_EQUAL(image_descriptor.Width(), 1920u); + WINML_EXPECT_EQUAL(image_descriptor.Height(), 1080u); + WINML_EXPECT_EQUAL(image_descriptor.BitmapAlphaMode(), expected_mode); + WINML_EXPECT_EQUAL(image_descriptor.BitmapPixelFormat(), expected_format); + } + else { + //not an image descriptor. a regular tensor + WINML_EXPECT_THROW_SPECIFIC(m_model.OutputFeatures().First().Current().as(image_descriptor), + winrt::hresult_no_interface + , [](const winrt::hresult_no_interface& e) -> bool { + return e.code() == E_NOINTERFACE; + }); + TensorFeatureDescriptor tensor_descriptor = nullptr; + WINML_EXPECT_NO_THROW(m_model.OutputFeatures().First().Current().as(tensor_descriptor)); + + // Make sure we fail binding ImageFeatureValue + LearningModelSession session(m_model); + LearningModelBinding binding(session); + auto ifv = FileHelpers::LoadImageFeatureValue(L"1080.jpg"); + WINML_EXPECT_THROW_SPECIFIC(binding.Bind(L"add_3", ifv), + winrt::hresult_error, + [](const winrt::hresult_error& e) -> bool { + return e.code() == WINML_ERR_INVALID_BINDING; + } + ); + } + } + + void RunConsecutiveImageBindingOnGpu(ImageFeatureValue & image1, ImageFeatureValue & image2) { + static const wchar_t* model_file_name = L"Add_ImageNet1920.onnx"; + std::wstring module_path = FileHelpers::GetModulePath(); + + // WinML model creation + LearningModel model(nullptr); + std::wstring full_model_path = module_path + model_file_name; + WINML_EXPECT_NO_THROW(model = LearningModel::LoadFromFilePath(full_model_path)); + LearningModelDeviceKind device_kind = LearningModelDeviceKind::DirectX; + LearningModelSession model_session(model, LearningModelDevice(device_kind)); + LearningModelBinding model_binding(model_session); + + //Input Binding + auto feature = model.InputFeatures().First(); + WINML_EXPECT_NO_THROW(model_binding.Bind(feature.Current().Name(), image1)); + feature.MoveNext(); + WINML_EXPECT_NO_THROW(model_binding.Bind(feature.Current().Name(), image2)); + } + + static ImageFeatureValue CreateImageFeatureValue(const std::wstring& full_image_path) { + StorageFile image_file = StorageFile::GetFileFromPathAsync(full_image_path).get(); + IRandomAccessStream stream = image_file.OpenAsync(FileAccessMode::Read).get(); + SoftwareBitmap software_bitmap = (BitmapDecoder::CreateAsync(stream).get()).GetSoftwareBitmapAsync().get(); + VideoFrame frame = VideoFrame::CreateWithSoftwareBitmap(software_bitmap); + ImageFeatureValue image_input_tensor = ImageFeatureValue::CreateFromVideoFrame(frame); + return image_input_tensor; + } +}; + +// MNIST model expects 28x28 data with +class MnistImageTest : public ImageTests, public testing::WithParamInterface> {}; +TEST_P(MnistImageTest, Evaluates) { + GPUTEST; + + PrepareModelSessionBinding(L"mnist.onnx", LearningModelDeviceKind::Cpu, std::nullopt); + + std::wstring filename; + unsigned int label; + std::tie(filename, label) = GetParam(); + auto image_feature_value = FileHelpers::LoadImageFeatureValue(std::wstring(filename)); + m_model_binding.Bind(L"Input3", image_feature_value); + + auto result = m_session.EvaluateAsync(m_model_binding, L"0").get(); + auto vector = result.Outputs().Lookup(L"Plus214_Output_0").as().GetAsVectorView(); + + unsigned int max_label = 0; + float max_val = 0; + for (unsigned int i = 0; i < vector.Size(); ++i) { + float val = vector.GetAt(i); + if (val > max_val) { + max_val = val; + max_label = i; + } + } + std::cerr << "Expected Label " << label; + std::cerr << "Evaluated Label " << max_label; + WINML_EXPECT_TRUE(max_label == label); +} +INSTANTIATE_TEST_SUITE_P(MnistInputOutput, MnistImageTest, + testing::Values( + std::make_pair(L"vertical-crop.png", 5), + std::make_pair(L"horizontal-crop.png", 2), + std::make_pair(L"big.png", 8), + std::make_pair(L"RGB_5.png", 5) + )); + +typedef std::tuple, std::wstring, std::wstring, InputImageSource, EvaluationStrategy, OutputBindingStrategy, LearningModelDeviceKind> ImageTestParamTuple; +struct ImageTestParam { + std::wstring model_file_name, model_pixel_format, image_file_name, input_pixel_format; + ModelInputOutputType model_input_output_type; + InputImageSource input_image_source; + EvaluationStrategy evaluation_strategy; + OutputBindingStrategy output_binding_strategy; + LearningModelDeviceKind device_kind; + + ImageTestParam(ImageTestParamTuple param) { + std::tuple model_info; + tie(model_info, image_file_name, input_pixel_format, input_image_source, evaluation_strategy, output_binding_strategy, device_kind) = param; + tie(model_file_name, model_input_output_type, model_pixel_format) = model_info; + } +}; +class ImageTest : public ImageTests, public testing::WithParamInterface {}; + +TEST_P(ImageTest, ImageTest) { + const auto param = ImageTestParam(GetParam()); + if (ShouldSkip(param.model_file_name, param.image_file_name, param.input_image_source)) { + GTEST_SKIP() << "This test is disabled"; + } + + if (LearningModelDeviceKind::Cpu != param.device_kind || InputImageSource::FromGPUResource == param.input_image_source) { + GPUTEST; + } + + PrepareModelSessionBinding(param.model_file_name, param.device_kind, {}); + + bool toContinue = BindInputValue( + param.image_file_name, + param.input_pixel_format, + param.model_pixel_format, + param.input_image_source, + param.device_kind); + + if (!toContinue) return; + + VideoFrame output_video_frame = BindImageOutput( + param.model_input_output_type, + param.output_binding_strategy, + std::wstring(param.model_pixel_format)); + + EvaluateTest(param.evaluation_strategy); + + // benchmark used to compare with the output from model + std::wstring benchmark_file_name = std::wstring( + param.model_pixel_format + L'_' + param.input_pixel_format + L'_' + param.image_file_name); + + // Verify the output by comparing with the benchmark image + std::wstring bm_image_path = FileHelpers::GetModulePath() + L"groundTruth\\" + benchmark_file_name; + if (OutputBindingStrategy::Unbound == param.output_binding_strategy) { + std::wstring output_data_binding_name = std::wstring(m_model.OutputFeatures().First().Current().Name()); + auto image_FV = m_result.Outputs().Lookup(output_data_binding_name).try_as(); + if (image_FV == nullptr) { + return; + } + output_video_frame = image_FV.VideoFrame(); + } + VerifyResults(output_video_frame, bm_image_path, param.model_pixel_format); +} +INSTANTIATE_TEST_SUITE_P(ImageTest, ImageTest, + testing::Combine( + testing::Values( + std::make_tuple(L"fns-candy_Bgr8.onnx", Image, L"Bgr8"), + std::make_tuple(L"fns-candy_Rgb8.onnx", Image, L"Rgb8"), + std::make_tuple(L"fns-candy_tensor.onnx", Tensor, L"Bgr8"), + std::make_tuple(L"fns-candy_Bgr8_freeDimInput.onnx", Image, L"Bgr8")), + testing::Values(L"1080.jpg", L"kitten_224.png", L"fish_720.png", L"fish_720_Gray.png"), + testing::Values(L"Bgra8", L"Rgba8", L"Gray8"), + testing::Values(FromVideoFrame, FromImageFeatureValue, FromCPUResource, FromGPUResource), + testing::Values(Async, Sync), + testing::Values(Bound, Unbound), + testing::Values(LearningModelDeviceKind::DirectX, LearningModelDeviceKind::Cpu) + )); + +typedef std::tuple, int, bool>, OutputBindingStrategy, EvaluationStrategy, VideoFrameSource, VideoFrameSource, LearningModelDeviceKind> BatchTestParamTuple; +struct BatchTestParam { + std::wstring model_file_name, model_pixel_format, image_file_name, input_pixel_format; + ModelInputOutputType model_input_output_type; + std::vector input_images; + int batch_size; + bool use_session_options; + OutputBindingStrategy output_binding_strategy; + EvaluationStrategy evaluation_strategy; + VideoFrameSource video_frame_source, output_video_frame_source; + LearningModelDeviceKind device_kind; + + BatchTestParam(BatchTestParamTuple param) { + std::tuple, int, bool> model_info; + tie(model_info, output_binding_strategy, evaluation_strategy, video_frame_source, output_video_frame_source, device_kind) = param; + tie(model_file_name, model_input_output_type, input_images, batch_size, use_session_options) = model_info; + } +}; +class BatchTest : public ImageTests, public testing::WithParamInterface {}; +TEST_P(BatchTest, BatchSupport) { + const auto param = BatchTestParam(GetParam()); + std::optional optimized_batch_size; + if (param.use_session_options) { + optimized_batch_size = param.use_session_options; + } + if (VideoFrameSource::FromDirect3DSurface == param.video_frame_source && LearningModelDeviceKind::Cpu == param.device_kind) { + return; + } + + // create model, device and session + PrepareModelSessionBinding(param.model_file_name, param.device_kind, optimized_batch_size); + + // create the input video_frames + std::vector input_frames = {}; + if (param.input_images.empty()) { + for (int i = 0; i < param.batch_size; ++i) { + if (VideoFrameSource::FromDirect3DSurface == param.video_frame_source) { + VideoFrame video_frame = VideoFrame::CreateAsDirect3D11SurfaceBacked(DirectXPixelFormat::B8G8R8X8UIntNormalized, 720, 720); + input_frames.emplace_back(video_frame); + } + else { + VideoFrame video_frame = VideoFrame::CreateWithSoftwareBitmap(SoftwareBitmap(BitmapPixelFormat::Bgra8, 720, 720)); + input_frames.emplace_back(video_frame); + } + } + } + else { + for (int i = 0; i < param.batch_size; ++i) { + std::wstring full_image_path = FileHelpers::GetModulePath() + param.input_images[i]; + StorageFile image_file = StorageFile::GetFileFromPathAsync(full_image_path).get(); + IRandomAccessStream stream = image_file.OpenAsync(FileAccessMode::Read).get(); + SoftwareBitmap software_bitmap = (BitmapDecoder::CreateAsync(stream).get()).GetSoftwareBitmapAsync().get(); + VideoFrame frame = VideoFrame::CreateWithSoftwareBitmap(software_bitmap); + + if (VideoFrameSource::FromDirect3DSurface == param.video_frame_source) { + uint32_t width = software_bitmap.PixelWidth(); + uint32_t height = software_bitmap.PixelHeight(); + auto D3D_video_frame = VideoFrame::CreateAsDirect3D11SurfaceBacked(DirectXPixelFormat::B8G8R8X8UIntNormalized, width, height); + frame.CopyToAsync(D3D_video_frame); + input_frames.emplace_back(D3D_video_frame); + } + else { + input_frames.emplace_back(frame); + } + } + } + auto video_frames = winrt::single_threaded_vector(std::move(input_frames)); + + auto input_feature_descriptor = m_model.InputFeatures().First(); + WINML_EXPECT_NO_THROW(m_model_binding.Bind(input_feature_descriptor.Current().Name(), video_frames)); + + auto output_video_frames = BindImageOutput( + param.model_input_output_type, + param.output_binding_strategy, + param.output_video_frame_source, + L"Bgra8", + param.batch_size); + + EvaluateTest(param.evaluation_strategy); + + // benchmark used to compare with the output from model + if (OutputBindingStrategy::Unbound == param.output_binding_strategy) { + std::wstring output_data_binding_name = std::wstring(m_model.OutputFeatures().First().Current().Name()); + output_video_frames = m_result.Outputs().Lookup(output_data_binding_name).try_as>(); + if (output_video_frames == nullptr) { + return; + } + } + if (!param.input_images.empty()) { + for (int i = 0; i < param.batch_size; ++i) { + std::wstring bm_image_path = FileHelpers::GetModulePath() + L"batchGroundTruth\\" + param.input_images[i]; + if (VideoFrameSource::FromSoftwareBitmap != param.output_video_frame_source && OutputBindingStrategy::Unbound != param.output_binding_strategy) { + VideoFrame D3D_video_frame = output_video_frames.GetAt(i); + VideoFrame SB_video_frame(BitmapPixelFormat::Bgra8, 720, 720); + D3D_video_frame.as().CopyToAsync(SB_video_frame).get(); + VerifyResults(SB_video_frame, bm_image_path, L"Bgra8"); + } + else { + VerifyResults(output_video_frames.GetAt(i), bm_image_path, L"Bgra8"); + } + } + } +} + +TEST_F(ImageTests, LoadBindEvalModelWithoutImageMetadata) { + GPUTEST; + + LoadModel(L"squeezenet_tensor_input.onnx"); + + auto feature_value = FileHelpers::LoadImageFeatureValue(L"227x227.png"); + + LearningModelSession model_session(m_model); + LearningModelBinding model_binding(model_session); + + model_binding.Bind(L"data", feature_value); + auto result = model_session.Evaluate(model_binding, L""); +} + + +TEST_F(ImageTests, LoadBindModelWithoutImageMetadata) { + GPUTEST; + + // Model expecting a tensor instead of an image + LoadModel(L"squeezenet_tensor_input.onnx"); + + LearningModelSession model_session(m_model); + LearningModelBinding model_binding(model_session); + + // Should work on images (by falling back to RGB8) + auto feature_value = FileHelpers::LoadImageFeatureValue(L"227x227.png"); + model_binding.Bind(L"data", feature_value); + + // Should work on tensors + auto tensor = TensorFloat::CreateFromIterable({ 1, 3, 227, 227 }, winrt::single_threaded_vector(std::vector(3 * 227 * 227))); + model_binding.Bind(L"data", tensor); +} + +TEST_F(ImageTests, LoadInvalidBindModelWithoutImageMetadata) { + GPUTEST; + + LoadModel(L"squeezenet_tensor_input.onnx"); + + LearningModelSession model_session(m_model); + LearningModelBinding model_binding(model_session); + + // expect not fail if image dimensions are bigger than required + auto feature_value = FileHelpers::LoadImageFeatureValue(L"1080.jpg"); + WINML_EXPECT_NO_THROW(model_binding.Bind(L"data", feature_value)); + + // expect fail if tensor is of wrong type + auto tensor_uint8 = TensorUInt8Bit::CreateFromIterable({ 1, 3, 227, 227 }, winrt::single_threaded_vector(std::vector(3 * 227 * 227))); + WINML_EXPECT_THROW_SPECIFIC(model_binding.Bind(L"data", tensor_uint8), + winrt::hresult_error, + [](const winrt::hresult_error& e) -> bool { + return e.code() == WINML_ERR_INVALID_BINDING; + }); + + // Should fail if tensor has smaller dimensions/type + auto tensor = TensorFloat::CreateFromIterable({ 1, 3, 22, 22 }, winrt::single_threaded_vector(std::vector(3 * 22 * 22))); + WINML_EXPECT_THROW_SPECIFIC(model_binding.Bind(L"data", tensor), + winrt::hresult_error, + [](const winrt::hresult_error& e) -> bool { + return e.code() == WINML_ERR_SIZE_MISMATCH; + }); +} + +TEST_F(ImageTests, ImageMetaDataTest) { + // supported image metadata + ValidateOutputImageMetaData(L"Add_ImageNet1920WithImageMetadataBgr8_SRGB_0_255.onnx", BitmapAlphaMode::Premultiplied, BitmapPixelFormat::Bgra8, true); + ValidateOutputImageMetaData(L"Add_ImageNet1920WithImageMetadataRgb8_SRGB_0_255.onnx", BitmapAlphaMode::Premultiplied, BitmapPixelFormat::Rgba8, true); + + // unsupported image metadata + ValidateOutputImageMetaData(L"Add_ImageNet1920WithImageMetadataBgra8_SRGB_0_255.onnx", BitmapAlphaMode::Straight, BitmapPixelFormat::Bgra8, false); + ValidateOutputImageMetaData(L"Add_ImageNet1920WithImageMetadataRgba8_SRGB_0_255.onnx", BitmapAlphaMode::Straight, BitmapPixelFormat::Rgba8, false); + ValidateOutputImageMetaData(L"Add_ImageNet1920WithImageMetadataBgr8_SRGB_0_1.onnx", BitmapAlphaMode::Straight, BitmapPixelFormat::Bgra8, false); + ValidateOutputImageMetaData(L"Add_ImageNet1920WithImageMetadataBgr8_SRGB_1_1.onnx", BitmapAlphaMode::Straight, BitmapPixelFormat::Bgra8, false); + ValidateOutputImageMetaData(L"Add_ImageNet1920WithImageMetadataBgr8_SRGB_16_235.onnx", BitmapAlphaMode::Straight, BitmapPixelFormat::Bgra8, false); + ValidateOutputImageMetaData(L"Add_ImageNet1920WithImageMetadataBgr8_LINEAR_0_255.onnx", BitmapAlphaMode::Straight, BitmapPixelFormat::Bgra8, false); +} + +//Tests if GPU will throw TDR if the same image feature value is binded back to back for two different inputs to a model +TEST_F(ImageTests, ImageBindingTwiceSameFeatureValueOnGpu) { + GPUTEST; + std::wstring module_path = FileHelpers::GetModulePath(); + static const wchar_t* input_data_image_filename = L"1080.jpg"; + + std::wstring full_image_path = module_path + input_data_image_filename; + ImageFeatureValue input_norm = CreateImageFeatureValue(full_image_path); + + RunConsecutiveImageBindingOnGpu(input_norm, input_norm); +} + +//Tests if GPU will throw TDR if 2 different image feature values are binded back to back for two different inputs to a model +TEST_F(ImageTests, ImageBindingTwiceDifferentFeatureValueOnGpu) { + GPUTEST; + std::wstring module_path = FileHelpers::GetModulePath(); + static const wchar_t* input_data_image_filename = L"1080.jpg"; + + std::wstring full_image_path = module_path + input_data_image_filename; + ImageFeatureValue input_norm = CreateImageFeatureValue(full_image_path); + ImageFeatureValue input_norm_1 = CreateImageFeatureValue(full_image_path); + + RunConsecutiveImageBindingOnGpu(input_norm, input_norm_1); +} + +static void RunImageBindingInputAndOutput(bool bindInputAsIInspectable) { + static const wchar_t* model_file_name = L"Add_ImageNet1920.onnx"; + std::wstring module_path = FileHelpers::GetModulePath(); + static const wchar_t* input_data_image_filename = L"1080.jpg"; + static const wchar_t* output_data_image_filename = L"out_Add_ImageNet_1080.jpg"; + + // WinML model creation + LearningModel model(nullptr); + std::wstring full_model_path = module_path + model_file_name; + WINML_EXPECT_NO_THROW(model = LearningModel::LoadFromFilePath(full_model_path)); + LearningModelDeviceKind device_kind = LearningModelDeviceKind::DirectX; + LearningModelSession model_session(model, LearningModelDevice(device_kind)); + LearningModelBinding model_binding(model_session); + + std::wstring full_image_path = module_path + input_data_image_filename; + + StorageFile image_file = StorageFile::GetFileFromPathAsync(full_image_path).get(); + IRandomAccessStream stream = image_file.OpenAsync(FileAccessMode::Read).get(); + SoftwareBitmap software_bitmap = (BitmapDecoder::CreateAsync(stream).get()).GetSoftwareBitmapAsync().get(); + VideoFrame frame = VideoFrame::CreateWithSoftwareBitmap(software_bitmap); + + if (bindInputAsIInspectable) { + auto feature = model.InputFeatures().First(); + WINML_EXPECT_NO_THROW(model_binding.Bind(feature.Current().Name(), frame)); + feature.MoveNext(); + WINML_EXPECT_NO_THROW(model_binding.Bind(feature.Current().Name(), frame)); + } + else { + ImageFeatureValue input_image_tensor = ImageFeatureValue::CreateFromVideoFrame(frame); + auto feature = model.InputFeatures().First(); + WINML_EXPECT_NO_THROW(model_binding.Bind(feature.Current().Name(), input_image_tensor)); + feature.MoveNext(); + WINML_EXPECT_NO_THROW(model_binding.Bind(feature.Current().Name(), input_image_tensor)); + } + + auto output_tensor_descriptor = model.OutputFeatures().First().Current().as(); + auto output_tensor_shape = output_tensor_descriptor.Shape(); + VideoFrame output_image(BitmapPixelFormat::Rgba8, static_cast(output_tensor_shape.GetAt(3)), static_cast(output_tensor_shape.GetAt(2))); + ImageFeatureValue output_tensor = ImageFeatureValue::CreateFromVideoFrame(output_image); + + WINML_EXPECT_NO_THROW(model_binding.Bind(model.OutputFeatures().First().Current().Name(), output_tensor)); + + // Evaluate the model + winrt::hstring correlation_id; + model_session.EvaluateAsync(model_binding, correlation_id).get(); + + //check the output video frame object + StorageFolder current_folder = StorageFolder::GetFolderFromPathAsync(module_path).get(); + StorageFile out_image_file = current_folder.CreateFileAsync(output_data_image_filename, CreationCollisionOption::ReplaceExisting).get(); + IRandomAccessStream write_stream = out_image_file.OpenAsync(FileAccessMode::ReadWrite).get(); + + BitmapEncoder encoder = BitmapEncoder::CreateAsync(BitmapEncoder::JpegEncoderId(), write_stream).get(); + // Set the software bitmap + encoder.SetSoftwareBitmap(output_image.SoftwareBitmap()); + + encoder.FlushAsync().get(); + + BYTE* data = nullptr; + UINT32 ui_capacity = 0; + winrt::Windows::Graphics::Imaging::BitmapBuffer bitmap_buffer(output_image.SoftwareBitmap().LockBuffer(winrt::Windows::Graphics::Imaging::BitmapBufferAccessMode::Read)); + winrt::Windows::Foundation::IMemoryBufferReference reference = bitmap_buffer.CreateReference(); + auto spByteAccess = reference.as<::Windows::Foundation::IMemoryBufferByteAccess>(); + WINML_EXPECT_HRESULT_SUCCEEDED(spByteAccess->GetBuffer(&data, &ui_capacity)); + WINML_EXPECT_NOT_EQUAL(data[0], 0); +} + +TEST_F(ImageTests, ImageBindingInputAndOutput) { + GPUTEST; + RunImageBindingInputAndOutput(false /*bindInputAsIInspectable*/); +} + +TEST_F(ImageTests, ImageBindingInputAndOutput_BindInputTensorAsInspectable) { + GPUTEST; + RunImageBindingInputAndOutput(true /*bindInputAsIInspectable*/); +} + +static void TestImageBindingStyleTransfer(const wchar_t* model_file_name, const wchar_t* input_data_image_filename, wchar_t* output_data_image_filename) { + GPUTEST; + + //this test only checks that the operation completed successfully without crashing + + std::wstring module_path = FileHelpers::GetModulePath(); + + // WinML model creation + LearningModel model(nullptr); + std::wstring full_model_path = module_path + model_file_name; + WINML_EXPECT_NO_THROW(model = LearningModel::LoadFromFilePath(full_model_path)); + LearningModelDeviceKind device_kind = LearningModelDeviceKind::DirectX; + LearningModelDevice device = nullptr; + WINML_EXPECT_NO_THROW(device = LearningModelDevice(device_kind)); + LearningModelSession model_session = nullptr; + WINML_EXPECT_NO_THROW(model_session = LearningModelSession(model, device)); + LearningModelBinding model_binding = nullptr; + WINML_EXPECT_NO_THROW(model_binding = LearningModelBinding(model_session)); + + std::wstring full_image_path = module_path + input_data_image_filename; + + StorageFile image_file = StorageFile::GetFileFromPathAsync(full_image_path).get(); + IRandomAccessStream stream = image_file.OpenAsync(FileAccessMode::Read).get(); + SoftwareBitmap software_bitmap = (BitmapDecoder::CreateAsync(stream).get()).GetSoftwareBitmapAsync().get(); + VideoFrame frame = VideoFrame::CreateWithSoftwareBitmap(software_bitmap); + //aizBUG:3762 Cannot bind the same tensor to 2 different input. will deal with this in a later check in + ImageFeatureValue input_1_image_tensor = ImageFeatureValue::CreateFromVideoFrame(frame); + + auto feature = model.InputFeatures().First(); + WINML_EXPECT_NO_THROW(model_binding.Bind(feature.Current().Name(), input_1_image_tensor)); + + auto output_tensor_descriptor = model.OutputFeatures().First().Current().as(); + auto output_tensor_shape = output_tensor_descriptor.Shape(); + VideoFrame output_image(BitmapPixelFormat::Rgba8, static_cast(output_tensor_shape.GetAt(3)), static_cast(output_tensor_shape.GetAt(2))); + ImageFeatureValue output_tensor = ImageFeatureValue::CreateFromVideoFrame(output_image); + + WINML_EXPECT_NO_THROW(model_binding.Bind(model.OutputFeatures().First().Current().Name(), output_tensor)); + + // Evaluate the model + winrt::hstring correlation_id; + WINML_EXPECT_NO_THROW(model_session.EvaluateAsync(model_binding, correlation_id).get()); + + //check the output video frame object + StorageFolder current_folder = StorageFolder::GetFolderFromPathAsync(module_path).get(); + StorageFile out_image_file = current_folder.CreateFileAsync(output_data_image_filename, CreationCollisionOption::ReplaceExisting).get(); + IRandomAccessStream write_stream = out_image_file.OpenAsync(FileAccessMode::ReadWrite).get(); + + BitmapEncoder encoder = BitmapEncoder::CreateAsync(BitmapEncoder::JpegEncoderId(), write_stream).get(); + // Set the software bitmap + encoder.SetSoftwareBitmap(output_image.SoftwareBitmap()); + + encoder.FlushAsync().get(); + +} + +TEST_F(ImageTests, ImageBindingStyleTransfer) { + //this test only checks that the operation completed successfully without crashing + TestImageBindingStyleTransfer(L"fns-candy.onnx", L"fish_720.png", L"out_fish_720_StyleTransfer.jpg"); +} + +TEST_F(ImageTests, ImageBindingAsGPUTensor) { + GPUTEST; + + static const wchar_t* model_file_name = L"fns-candy.onnx"; + std::wstring module_path = FileHelpers::GetModulePath(); + static const wchar_t* input_data_image_filename = L"fish_720.png"; + static const wchar_t* output_data_image_filename = L"out_fish_720_StyleTransfer.jpg"; + + // WinML model creation + LearningModel model(nullptr); + std::wstring full_model_path = module_path + model_file_name; + WINML_EXPECT_NO_THROW(model = LearningModel::LoadFromFilePath(full_model_path)); + + ID3D12Device* D3D12_device = nullptr; + WINML_EXPECT_NO_THROW(D3D12CreateDevice(nullptr, D3D_FEATURE_LEVEL::D3D_FEATURE_LEVEL_11_0, __uuidof(ID3D12Device), reinterpret_cast(&D3D12_device))); + ID3D12CommandQueue* dx_queue = nullptr; + D3D12_COMMAND_QUEUE_DESC command_queue_desc = {}; + command_queue_desc.Type = D3D12_COMMAND_LIST_TYPE_DIRECT; + D3D12_device->CreateCommandQueue(&command_queue_desc, __uuidof(ID3D12CommandQueue), reinterpret_cast(&dx_queue)); + auto device_factory = get_activation_factory(); + auto tensor_factory = get_activation_factory(); + + + com_ptr<::IUnknown> p_unk; + device_factory->CreateFromD3D12CommandQueue(dx_queue, p_unk.put()); + + LearningModelDevice dml_device_custom = nullptr; + WINML_EXPECT_NO_THROW(p_unk.as(dml_device_custom)); + LearningModelSession dml_session_custom = nullptr; + WINML_EXPECT_NO_THROW(dml_session_custom = LearningModelSession(model, dml_device_custom)); + + LearningModelBinding model_binding(dml_session_custom); + + std::wstring full_image_path = module_path + input_data_image_filename; + + StorageFile image_file = StorageFile::GetFileFromPathAsync(full_image_path).get(); + IRandomAccessStream stream = image_file.OpenAsync(FileAccessMode::Read).get(); + SoftwareBitmap software_bitmap = (BitmapDecoder::CreateAsync(stream).get()).GetSoftwareBitmapAsync().get(); + + UINT64 buffer_byte_size = software_bitmap.PixelWidth()*software_bitmap.PixelHeight() * 3 * sizeof(float); + D3D12_HEAP_PROPERTIES heap_properties = { + D3D12_HEAP_TYPE_DEFAULT, + D3D12_CPU_PAGE_PROPERTY_UNKNOWN, + D3D12_MEMORY_POOL_UNKNOWN, + 0, + 0 + }; + D3D12_RESOURCE_DESC resource_desc = { + D3D12_RESOURCE_DIMENSION_BUFFER, + 0, + buffer_byte_size, + 1, + 1, + 1, + DXGI_FORMAT_UNKNOWN, + { 1, 0 }, + D3D12_TEXTURE_LAYOUT_ROW_MAJOR, + D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS + }; + + com_ptr GPU_resource = nullptr; + D3D12_device->CreateCommittedResource( + &heap_properties, + D3D12_HEAP_FLAG_NONE, + &resource_desc, + D3D12_RESOURCE_STATE_COMMON, + nullptr, + __uuidof(ID3D12Resource), + GPU_resource.put_void() + ); + com_ptr<::IUnknown> sp_unk_tensor; + TensorFloat input_1_image_tensor(nullptr); + __int64 shape[4] = { 1,3, software_bitmap.PixelWidth(), software_bitmap.PixelHeight() }; + tensor_factory->CreateFromD3D12Resource(GPU_resource.get(), shape, 4, sp_unk_tensor.put()); + sp_unk_tensor.try_as(input_1_image_tensor); + + auto feature = model.InputFeatures().First(); + WINML_EXPECT_NO_THROW(model_binding.Bind(feature.Current().Name(), input_1_image_tensor)); + + auto output_tensor_descriptor = model.OutputFeatures().First().Current().as(); + auto output_tensor_shape = output_tensor_descriptor.Shape(); + VideoFrame output_image(BitmapPixelFormat::Rgba8, static_cast(output_tensor_shape.GetAt(3)), static_cast(output_tensor_shape.GetAt(2))); + ImageFeatureValue output_tensor = ImageFeatureValue::CreateFromVideoFrame(output_image); + + WINML_EXPECT_NO_THROW(model_binding.Bind(model.OutputFeatures().First().Current().Name(), output_tensor)); + + // Evaluate the model + winrt::hstring correlation_id; + dml_session_custom.EvaluateAsync(model_binding, correlation_id).get(); + + //check the output video frame object + StorageFolder current_folder = StorageFolder::GetFolderFromPathAsync(module_path).get(); + StorageFile out_image_file = current_folder.CreateFileAsync(output_data_image_filename, CreationCollisionOption::ReplaceExisting).get(); + IRandomAccessStream write_stream = out_image_file.OpenAsync(FileAccessMode::ReadWrite).get(); + + BitmapEncoder encoder = BitmapEncoder::CreateAsync(BitmapEncoder::JpegEncoderId(), write_stream).get(); + // Set the software bitmap + encoder.SetSoftwareBitmap(output_image.SoftwareBitmap()); + + encoder.FlushAsync().get(); +} + +static void GetCleanSession(LearningModelDeviceKind device_kind, std::wstring modelFilePath, LearningModelDevice &device, LearningModelSession &session) { + LearningModel model(nullptr); + std::wstring full_model_path = modelFilePath; + WINML_EXPECT_NO_THROW(model = LearningModel::LoadFromFilePath(full_model_path)); + WINML_EXPECT_NO_THROW(device = LearningModelDevice(device_kind)); + WINML_EXPECT_NO_THROW(session = LearningModelSession(model, device)); +} + +static void BindInputToSession(BindingLocation bind_location, std::wstring input_data_location, LearningModelSession& session, LearningModelBinding& binding) { + StorageFile image_file = StorageFile::GetFileFromPathAsync(input_data_location).get(); + IRandomAccessStream stream = image_file.OpenAsync(FileAccessMode::Read).get(); + SoftwareBitmap software_bitmap = (BitmapDecoder::CreateAsync(stream).get()).GetSoftwareBitmapAsync().get(); + VideoFrame cpu_video_frame = VideoFrame::CreateWithSoftwareBitmap(software_bitmap); + if (bind_location == BindingLocation::CPU) { + ImageFeatureValue input_image_feature_value = ImageFeatureValue::CreateFromVideoFrame(cpu_video_frame); + WINML_EXPECT_NO_THROW(binding.Bind(session.Model().InputFeatures().First().Current().Name(), input_image_feature_value)); + } else { + DirectXPixelFormat format = DirectXPixelFormat::B8G8R8X8UIntNormalized; + VideoFrame gpu_video_frame = VideoFrame::CreateAsDirect3D11SurfaceBacked(format, + software_bitmap.PixelWidth(), + software_bitmap.PixelHeight(), + session.Device().Direct3D11Device()); + cpu_video_frame.CopyToAsync(gpu_video_frame).get(); + ImageFeatureValue input_image_feature_value = ImageFeatureValue::CreateFromVideoFrame(gpu_video_frame); + WINML_EXPECT_NO_THROW(binding.Bind(session.Model().InputFeatures().First().Current().Name(), input_image_feature_value)); + } +} + +static void BindOutputToSession(BindingLocation bind_location, LearningModelSession& session, LearningModelBinding& binding) { + auto output_tensor_descriptor = session.Model().OutputFeatures().First().Current().as(); + auto output_tensor_shape = output_tensor_descriptor.Shape(); + if (bind_location == BindingLocation::CPU) { + VideoFrame output_image(BitmapPixelFormat::Rgba8, static_cast(output_tensor_shape.GetAt(3)), static_cast(output_tensor_shape.GetAt(2))); + ImageFeatureValue output_tensor = ImageFeatureValue::CreateFromVideoFrame(output_image); + WINML_EXPECT_NO_THROW(binding.Bind(session.Model().OutputFeatures().First().Current().Name(), output_tensor)); + } + else { + VideoFrame output_image = VideoFrame::CreateAsDirect3D11SurfaceBacked(DirectXPixelFormat::B8G8R8X8UIntNormalized, + static_cast(output_tensor_shape.GetAt(3)), + static_cast(output_tensor_shape.GetAt(2))); + ImageFeatureValue output_tensor = ImageFeatureValue::CreateFromVideoFrame(output_image); + WINML_EXPECT_NO_THROW(binding.Bind(session.Model().OutputFeatures().First().Current().Name(), output_tensor)); + } +} + +static void SynchronizeGPUWorkloads(const wchar_t* model_file_name, const wchar_t* input_data_image_filename) { + //this test only checks that the operations complete successfully without crashing + GPUTEST; + std::wstring module_path = FileHelpers::GetModulePath(); + LearningModelDevice device = nullptr; + LearningModelSession session = nullptr; + LearningModelBinding binding = nullptr; + + /* + * lazy dx11 loading scenarios: + */ + + // Scenario 1 + GetCleanSession(LearningModelDeviceKind::DirectX, module_path + model_file_name, device, session); + + // input: CPU, output: CPU + WINML_EXPECT_NO_THROW(binding = LearningModelBinding(session)); + BindInputToSession(BindingLocation::CPU, module_path + input_data_image_filename, session, binding); + BindOutputToSession(BindingLocation::CPU, session, binding); + WINML_EXPECT_NO_THROW(session.Evaluate(binding, L"")); + + // ---> verify that 11 stack is not initialized + WINML_EXPECT_FALSE(device.as()->SharedHandleInitialized()); + + // input: CPU, output: GPU + WINML_EXPECT_NO_THROW(binding = LearningModelBinding(session)); + BindInputToSession(BindingLocation::CPU, module_path + input_data_image_filename, session, binding); + BindOutputToSession(BindingLocation::GPU, session, binding); + WINML_EXPECT_NO_THROW(session.Evaluate(binding, L"")); + + // Scenario 2 + GetCleanSession(LearningModelDeviceKind::DirectX, module_path + model_file_name, device, session); + + // input: CPU, output: CPU + WINML_EXPECT_NO_THROW(binding = LearningModelBinding(session)); + BindInputToSession(BindingLocation::CPU, module_path + input_data_image_filename, session, binding); + BindOutputToSession(BindingLocation::CPU, session, binding); + WINML_EXPECT_NO_THROW(session.Evaluate(binding, L"")); + + // ---> verify that 11 stack is not initialized + WINML_EXPECT_FALSE(device.as()->SharedHandleInitialized()); + + // input: GPU, output: CPU + WINML_EXPECT_NO_THROW(binding = LearningModelBinding(session)); + BindInputToSession(BindingLocation::GPU, module_path + input_data_image_filename, session, binding); + BindOutputToSession(BindingLocation::CPU, session, binding); + WINML_EXPECT_NO_THROW(session.Evaluate(binding, L"")); + + // ---> verify that 11 stack is initialized + WINML_EXPECT_TRUE(device.as()->SharedHandleInitialized()); + + // Scenario 3 + GetCleanSession(LearningModelDeviceKind::DirectX, module_path + model_file_name, device, session); + + // input: CPU, output: CPU + WINML_EXPECT_NO_THROW(binding = LearningModelBinding(session)); + BindInputToSession(BindingLocation::CPU, module_path + input_data_image_filename, session, binding); + BindOutputToSession(BindingLocation::CPU, session, binding); + WINML_EXPECT_NO_THROW(session.Evaluate(binding, L"")); + + // ---> verify that 11 stack is not initialized + WINML_EXPECT_FALSE(device.as()->SharedHandleInitialized()); + + // input: GPU, output: GPU + WINML_EXPECT_NO_THROW(binding = LearningModelBinding(session)); + BindInputToSession(BindingLocation::GPU, module_path + input_data_image_filename, session, binding); + BindOutputToSession(BindingLocation::GPU, session, binding); + WINML_EXPECT_NO_THROW(session.Evaluate(binding, L"")); + + // ---> verify that 11 stack is initialized + WINML_EXPECT_TRUE(device.as()->SharedHandleInitialized()); + + /* + * non lazy dx11 loading scenarios: + */ + + // Scenario 1 + GetCleanSession(LearningModelDeviceKind::DirectX, module_path + model_file_name, device, session); + + // input: GPU, output: CPU + WINML_EXPECT_NO_THROW(binding = LearningModelBinding(session)); + BindInputToSession(BindingLocation::GPU, module_path + input_data_image_filename, session, binding); + BindOutputToSession(BindingLocation::CPU, session, binding); + WINML_EXPECT_NO_THROW(session.Evaluate(binding, L"")); + + // ---> verify that 11 stack is initialized + WINML_EXPECT_TRUE(device.as()->SharedHandleInitialized()); + + // input : CPU, output : CPU + WINML_EXPECT_NO_THROW(binding = LearningModelBinding(session)); + BindInputToSession(BindingLocation::CPU, module_path + input_data_image_filename, session, binding); + BindOutputToSession(BindingLocation::CPU, session, binding); + WINML_EXPECT_NO_THROW(session.Evaluate(binding, L"")); + + // Scenario 2 + GetCleanSession(LearningModelDeviceKind::DirectX, module_path + model_file_name, device, session); + + // input: CPU, output: GPU + WINML_EXPECT_NO_THROW(binding = LearningModelBinding(session)); + BindInputToSession(BindingLocation::CPU, module_path + input_data_image_filename, session, binding); + BindOutputToSession(BindingLocation::GPU, session, binding); + WINML_EXPECT_NO_THROW(session.Evaluate(binding, L"")); + + // input : CPU, output : CPU + WINML_EXPECT_NO_THROW(binding = LearningModelBinding(session)); + BindInputToSession(BindingLocation::CPU, module_path + input_data_image_filename, session, binding); + BindOutputToSession(BindingLocation::CPU, session, binding); + WINML_EXPECT_NO_THROW(session.Evaluate(binding, L"")); +} + +TEST_F(ImageTests, SynchronizeGPUWorkloads) { + SynchronizeGPUWorkloads(L"fns-candy.onnx", L"fish_720.png"); +} diff --git a/winml/test/image/models/README.md b/winml/test/image/models/README.md new file mode 100644 index 0000000000..5c438120f7 --- /dev/null +++ b/winml/test/image/models/README.md @@ -0,0 +1,4 @@ +The FNS_Candy machine learning models here are used with permission from Justin Johnson. +For additional information on these models, refer to: +* [Fast-neural-style GitHub repo](https://github.com/jcjohnson/fast-neural-style) +* [Perceptual Losses for Real-Time Style Transfer and Super-Resolution](https://cs.stanford.edu/people/jcjohns/papers/eccv16/JohnsonECCV16.pdf) diff --git a/winml/test/image/models/fns-candy_Bgr8.onnx b/winml/test/image/models/fns-candy_Bgr8.onnx new file mode 100644 index 0000000000..edca1e6914 Binary files /dev/null and b/winml/test/image/models/fns-candy_Bgr8.onnx differ diff --git a/winml/test/image/models/fns-candy_Bgr8_Batch2.onnx b/winml/test/image/models/fns-candy_Bgr8_Batch2.onnx new file mode 100644 index 0000000000..0170a96b5b Binary files /dev/null and b/winml/test/image/models/fns-candy_Bgr8_Batch2.onnx differ diff --git a/winml/test/image/models/fns-candy_Bgr8_Batch3.onnx b/winml/test/image/models/fns-candy_Bgr8_Batch3.onnx new file mode 100644 index 0000000000..3cb4938875 Binary files /dev/null and b/winml/test/image/models/fns-candy_Bgr8_Batch3.onnx differ diff --git a/winml/test/image/models/fns-candy_Bgr8_freeDimInput.onnx b/winml/test/image/models/fns-candy_Bgr8_freeDimInput.onnx new file mode 100644 index 0000000000..f78b64648f Binary files /dev/null and b/winml/test/image/models/fns-candy_Bgr8_freeDimInput.onnx differ diff --git a/winml/test/image/models/fns-candy_Bgr8_freeDimInput_Batch10.onnx b/winml/test/image/models/fns-candy_Bgr8_freeDimInput_Batch10.onnx new file mode 100644 index 0000000000..f7ef5555fd Binary files /dev/null and b/winml/test/image/models/fns-candy_Bgr8_freeDimInput_Batch10.onnx differ diff --git a/winml/test/image/models/fns-candy_Bgr8_tensor_Batch3.onnx b/winml/test/image/models/fns-candy_Bgr8_tensor_Batch3.onnx new file mode 100644 index 0000000000..9bbdb8f52a Binary files /dev/null and b/winml/test/image/models/fns-candy_Bgr8_tensor_Batch3.onnx differ diff --git a/winml/test/image/models/fns-candy_Rgb8.onnx b/winml/test/image/models/fns-candy_Rgb8.onnx new file mode 100644 index 0000000000..1f7b380009 Binary files /dev/null and b/winml/test/image/models/fns-candy_Rgb8.onnx differ diff --git a/winml/test/image/models/fns-candy_tensor.onnx b/winml/test/image/models/fns-candy_tensor.onnx new file mode 100644 index 0000000000..56541a5c85 Binary files /dev/null and b/winml/test/image/models/fns-candy_tensor.onnx differ