From bccbdd03f10ad3d99261d6fa758667fea87ab798 Mon Sep 17 00:00:00 2001
From: Xiang Zhang <xianz@microsoft.com>
Date: Tue, 12 May 2020 15:46:46 -0700
Subject: [PATCH] User/xianz/enable batch tests (#3914)

* enable batch tests in winml_image_test

* copy batchGroundTruth folder

* skip GPU tests when GPU is unavailable
---
 cmake/winml_unittests.cmake                   |   2 ++
 .../{ => batchGroundTruth}/1080.jpg           | Bin
 .../{ => batchGroundTruth}/fish_720.png       | Bin
 .../{ => batchGroundTruth}/fish_720_Gray.png  | Bin
 .../{ => batchGroundTruth}/kitten_224.png     | Bin
 winml/test/image/imagetests.cpp               |  25 +++++++++++++++++-
 6 files changed, 26 insertions(+), 1 deletion(-)
 rename winml/test/image/batchGroundTruth/{ => batchGroundTruth}/1080.jpg (100%)
 rename winml/test/image/batchGroundTruth/{ => batchGroundTruth}/fish_720.png (100%)
 rename winml/test/image/batchGroundTruth/{ => batchGroundTruth}/fish_720_Gray.png (100%)
 rename winml/test/image/batchGroundTruth/{ => batchGroundTruth}/kitten_224.png (100%)

diff --git a/cmake/winml_unittests.cmake b/cmake/winml_unittests.cmake
index b809ea98a5..4ef243af1d 100644
--- a/cmake/winml_unittests.cmake
+++ b/cmake/winml_unittests.cmake
@@ -272,6 +272,8 @@ add_winml_collateral("${WINML_TEST_SRC_DIR}/image/images/*.jpg")
 add_winml_collateral("${WINML_TEST_SRC_DIR}/image/images/*.png")
 add_winml_collateral("${WINML_TEST_SRC_DIR}/image/groundTruth/*.jpg")
 add_winml_collateral("${WINML_TEST_SRC_DIR}/image/groundTruth/*.png")
+add_winml_collateral("${WINML_TEST_SRC_DIR}/image/batchGroundTruth/*.jpg")
+add_winml_collateral("${WINML_TEST_SRC_DIR}/image/batchGroundTruth/*.png")
 add_winml_collateral("${WINML_TEST_SRC_DIR}/image/models/*.onnx")
 add_winml_collateral("${WINML_TEST_SRC_DIR}/scenario/cppwinrt/*.onnx")
 add_winml_collateral("${WINML_TEST_SRC_DIR}/scenario/models/*.onnx")
diff --git a/winml/test/image/batchGroundTruth/1080.jpg b/winml/test/image/batchGroundTruth/batchGroundTruth/1080.jpg
similarity index 100%
rename from winml/test/image/batchGroundTruth/1080.jpg
rename to winml/test/image/batchGroundTruth/batchGroundTruth/1080.jpg
diff --git a/winml/test/image/batchGroundTruth/fish_720.png b/winml/test/image/batchGroundTruth/batchGroundTruth/fish_720.png
similarity index 100%
rename from winml/test/image/batchGroundTruth/fish_720.png
rename to winml/test/image/batchGroundTruth/batchGroundTruth/fish_720.png
diff --git a/winml/test/image/batchGroundTruth/fish_720_Gray.png b/winml/test/image/batchGroundTruth/batchGroundTruth/fish_720_Gray.png
similarity index 100%
rename from winml/test/image/batchGroundTruth/fish_720_Gray.png
rename to winml/test/image/batchGroundTruth/batchGroundTruth/fish_720_Gray.png
diff --git a/winml/test/image/batchGroundTruth/kitten_224.png b/winml/test/image/batchGroundTruth/batchGroundTruth/kitten_224.png
similarity index 100%
rename from winml/test/image/batchGroundTruth/kitten_224.png
rename to winml/test/image/batchGroundTruth/batchGroundTruth/kitten_224.png
diff --git a/winml/test/image/imagetests.cpp b/winml/test/image/imagetests.cpp
index 0f6eb76214..c69f90401d 100644
--- a/winml/test/image/imagetests.cpp
+++ b/winml/test/image/imagetests.cpp
@@ -451,6 +451,12 @@ TEST_P(BatchTest, BatchSupport) {
     if (VideoFrameSource::FromDirect3DSurface == param.video_frame_source && LearningModelDeviceKind::Cpu == param.device_kind) {
         return;
     }
+    if (LearningModelDeviceKind::Cpu != param.device_kind ||
+        VideoFrameSource::FromDirect3DSurface == param.video_frame_source ||
+        VideoFrameSource::FromDirect3DSurface == param.output_video_frame_source ||
+        VideoFrameSource::FromUnsupportedD3DSurface == param.output_video_frame_source) {
+        GPUTEST;
+    }
 
     // create model, device and session
     PrepareModelSessionBinding(param.model_file_name, param.device_kind, optimized_batch_size);
@@ -526,7 +532,24 @@ TEST_P(BatchTest, BatchSupport) {
         }
     }
 }
-
+INSTANTIATE_TEST_SUITE_P(BatchTest, BatchTest,
+    testing::Combine(
+        testing::Values(
+            std::make_tuple(L"fns-candy_Bgr8_Batch2.onnx", Image, std::vector<std::wstring>({L"fish_720.png", L"fish_720.png"}), 2, false),
+            std::make_tuple(L"fns-candy_Bgr8_Batch2.onnx", Image, std::vector<std::wstring>({L"1080.jpg", L"fish_720.png"}), 2, false),
+            std::make_tuple(L"fns-candy_Bgr8_Batch2.onnx", Image, std::vector<std::wstring>({L"fish_720_Gray.png", L"fish_720.png"}), 2, false),
+            std::make_tuple(L"fns-candy_Bgr8_Batch3.onnx", Image, std::vector<std::wstring>({L"1080.jpg", L"fish_720_Gray.png", L"fish_720.png"}), 3, false),
+            std::make_tuple(L"fns-candy_Bgr8_Batch3.onnx", Image, std::vector<std::wstring>({L"1080.jpg", L"kitten_224.png", L"fish_720.png"}), 3, false),
+            std::make_tuple(L"fns-candy_Bgr8_tensor_Batch3.onnx", Tensor, std::vector<std::wstring>({L"1080.jpg", L"fish_720_Gray.png", L"fish_720.png"}), 3, false),
+            std::make_tuple(L"fns-candy_Bgr8_freeDimInput_Batch10.onnx", Image, std::vector<std::wstring>({}), 10, false),
+            std::make_tuple(L"fns-candy_Bgr8.onnx", Image, std::vector<std::wstring>({L"1080.jpg", L"fish_720_Gray.png", L"fish_720.png"}), 3, false),
+            std::make_tuple(L"fns-candy_Bgr8.onnx", Image, std::vector<std::wstring>({L"1080.jpg", L"fish_720_Gray.png", L"fish_720.png"}), 3, true)),
+        testing::Values(Bound, Unbound),
+        testing::Values(Async, Sync),
+        testing::Values(FromSoftwareBitmap, FromDirect3DSurface),
+        testing::Values(FromSoftwareBitmap, FromDirect3DSurface, FromUnsupportedD3DSurface),
+        testing::Values(LearningModelDeviceKind::DirectX, LearningModelDeviceKind::Cpu)
+    ));
 TEST_F(ImageTests, LoadBindEvalModelWithoutImageMetadata) {
     GPUTEST;