diff --git a/winml/test/image/imageTestHelper.cpp b/winml/test/image/imageTestHelper.cpp
index 1f769eeb46..6d0c536fbe 100644
--- a/winml/test/image/imageTestHelper.cpp
+++ b/winml/test/image/imageTestHelper.cpp
@@ -274,6 +274,8 @@ namespace ImageTestHelper {
         // So we use error rate.
         UINT errors = 0;
         for (uint32_t i = 0; i < size; i++, pActualByte++, pExpectedByte++) {
+            // Only the check the first three channels, which are (B, G, R)
+            if((i + 1) % 4 == 0) continue;
             auto diff = (*pActualByte - *pExpectedByte);
             if (diff > epsilon) {
                 errors++;
diff --git a/winml/test/scenario/cppwinrt/scenariotestscppwinrt.cpp b/winml/test/scenario/cppwinrt/scenariotestscppwinrt.cpp
index 4a2c5b33d2..d4e6e832c4 100644
--- a/winml/test/scenario/cppwinrt/scenariotestscppwinrt.cpp
+++ b/winml/test/scenario/cppwinrt/scenariotestscppwinrt.cpp
@@ -768,9 +768,10 @@ bool VerifyHelper(ImageFeatureValue actual, ImageFeatureValue expected) {
   // hard code, might need to be modified later.
   const float cMaxErrorRate = 0.06f;
   byte epsilon = 20;
-
   UINT errors = 0;
   for (uint32_t i = 0; i < size; i++, pActualByte++, pExpectedByte++) {
+    // Only the check the first three channels, which are (B, G, R)
+    if((i + 1) % 4 == 0) continue;
     auto diff = std::abs(*pActualByte - *pExpectedByte);
     if (diff > epsilon) {
       errors++;