From 559bd52252f2db17e849c9101da4a22ad6e69f8b Mon Sep 17 00:00:00 2001 From: Adrian Lizarraga Date: Wed, 6 Dec 2023 11:05:41 -0800 Subject: [PATCH] [QNN EP] Update QNN SDK to version 2.17.0 (#18684) ### Description - Update QNN CI Pipelines to use QNN SDK version 2.17.0 - **Print warning if unit test requires adjusted tolerance to pass** - **Temporarily disable unloading QnnCpu.dll for windows x64 due to crash when calling FreeLibrary** - Enable fixed HTP tests - QnnHTPBackendTests.LayerNorm1D_LastAxis_DynamicScale - QnnHTPBackendTests.GlobalMaxPool_LargeInput2_u8 - QnnHTPBackendTests.ReduceSumS8Opset13_Rank5 - QnnHTPBackendTests.ReduceSumU8Opset13_Rank5_LastAxis - QnnHTPBackendTests.WhereLargeDataBroadcastU8 - QnnHTPBackendTests.WhereLargeDataBroadcastTransformedU8 - Enabled fixed CPU tests - QnnCPUBackendTests.Resize_DownSample_Linear_AlignCorners_scales - Increased tolerance for HTP tests that are less accurate on QNN SDK 2.17.0 - QnnHTPBackendTests.AveragePool_CountIncludePad_HTP_u8 - QnnHTPBackendTests.AveragePool_AutopadSameUpper_HTP_u8 - QnnHTPBackendTests.AveragePool_AutopadSameLower_HTP_u8 - QnnHTPBackendTests.ConvU8U8S32_bias_dynamic_input - QnnHTPBackendTests.ConvU8U8S32_bias_initializer - QnnHTPBackendTests.ConvU8U8S32_large_input1_padding_bias_initializer - QnnHTPBackendTests.LRNSize3 - QnnHTPBackendTests.LRNSize5 - QnnHTPBackendTests.MaxPool_Large_Input_HTP_u8 - QnnHTPBackendTests.MaxPool_LargeInput_1Pads - QnnHTPBackendTests.Resize_DownSample_Linear_HalfPixel - QnnHTPBackendTests.ResizeU8_2xLinearPytorchHalfPixel - QnnHTPBackendTests.ResizeU8_2xLinearHalfPixel - QnnHTPBackendTests.ResizeU8_2xLinearAlignCorners - QnnHTPBackendTests.ResizeU8_2xLinearAsymmetric - Disabled ONNX model tests - averagepool_2d_ceil: Accuracy issues **only on Windows x64 QnnCpu.dll** - Disabled QDQ model tests (onnx_test_runner) - facedetection_op8_qdq: Accuracy issues - Disabled CPU EP tests (these use QnnCpu.dll) - ActivationOpTest.Relu: QNN SDK 2.17 Relu treats inf as FLT_MAX - GemmOpTypedTests/0.TestGemmBroadcast: Inaccuracy when weight is initializer and bias is not - MathOpTest.MatMulFloatType "test padding and broadcast B > A": Inaccuracy (**only linux**) - Fix Gemm translation bugs in QNN EP: - Do not skip processing of inputs that need to be transposed. ### Motivation and Context - Allow testing with newest QNN SDK version - Take advantage of improvements to enable new models. --- .../qnn/builder/opbuilder/gemm_op_builder.cc | 8 +- .../qnn/builder/qnn_backend_manager.cc | 7 +- .../providers/qnn/builder/qnn_model_wrapper.h | 2 +- onnxruntime/test/onnx/TestCase.cc | 9 ++ .../cpu/activation/activation_op_test.h | 5 +- .../test/providers/cpu/math/gemm_test.cc | 13 +- .../test/providers/cpu/math/matmul_test.cc | 6 + .../providers/cpu/tensor/resize_op_test.cc | 4 +- .../test/providers/qnn/argmaxmin_op_test.cc | 3 +- .../test/providers/qnn/average_pool_test.cc | 18 ++- .../test/providers/qnn/batch_norm_htp_test.cc | 3 +- onnxruntime/test/providers/qnn/conv_test.cc | 55 +++++--- .../test/providers/qnn/gemm_op_test.cc | 130 +++++++++++++++--- .../test/providers/qnn/layer_norm_test.cc | 47 ++++--- onnxruntime/test/providers/qnn/lrn_op_test.cc | 33 ++++- .../test/providers/qnn/matmul_test.cpp | 29 ++-- .../test/providers/qnn/pad_op_test.cpp | 3 +- .../test/providers/qnn/pool_op_test.cpp | 76 +++++++--- .../test/providers/qnn/qnn_test_utils.cc | 22 +++ .../test/providers/qnn/qnn_test_utils.h | 103 +++++++++++--- .../test/providers/qnn/reduce_op_test.cc | 72 +++------- onnxruntime/test/providers/qnn/resize_test.cc | 41 ++++-- .../test/providers/qnn/simple_op_htp_test.cc | 37 +++-- .../test/providers/qnn/transpose_htp_test.cc | 3 +- .../test/providers/qnn/where_htp_test.cc | 16 +-- ...arm64-v8a-QNN-crosscompile-ci-pipeline.yml | 2 +- .../azure-pipelines/linux-qnn-ci-pipeline.yml | 2 +- .../qnn-ep-nuget-packaging-pipeline.yml | 4 +- .../win-qnn-arm64-ci-pipeline.yml | 2 +- .../azure-pipelines/win-qnn-ci-pipeline.yml | 2 +- 30 files changed, 524 insertions(+), 233 deletions(-) diff --git a/onnxruntime/core/providers/qnn/builder/opbuilder/gemm_op_builder.cc b/onnxruntime/core/providers/qnn/builder/opbuilder/gemm_op_builder.cc index 5ce10dc524..338e467657 100644 --- a/onnxruntime/core/providers/qnn/builder/opbuilder/gemm_op_builder.cc +++ b/onnxruntime/core/providers/qnn/builder/opbuilder/gemm_op_builder.cc @@ -92,7 +92,10 @@ Status GemmOpBuilder::ProcessInputs(QnnModelWrapper& qnn_model_wrapper, utils::InitializeQuantizeParam(quantize_param, is_quantized_tensor); const auto& input_name = inputs[input_i].node_arg.Name(); - if (qnn_model_wrapper.IsQnnTensorWrapperExist(input_name)) { + + // Only skip if the input tensor has already been added (by producer op) *and* we don't need + // to transpose it. + if (qnn_model_wrapper.IsQnnTensorWrapperExist(input_name) && input_trans_flag[input_i] == 0) { LOGS(logger, VERBOSE) << "Tensor already added, skip it: " << input_name; input_names.push_back(input_name); continue; @@ -134,7 +137,8 @@ Status GemmOpBuilder::ProcessInputs(QnnModelWrapper& qnn_model_wrapper, std::vector perm{1, 0}; ORT_RETURN_IF_ERROR(qnn_model_wrapper.AddTransposeNode(node_unit.Index(), node_input_name, input_tensor_name, old_input_shape, perm, input_shape, - qnn_data_type, quantize_param, do_op_validation)); + qnn_data_type, quantize_param, do_op_validation, + qnn_model_wrapper.IsGraphInput(node_input_name))); } if (2 == input_i && 2 == input_shape.size()) { diff --git a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc index ab0ea042ea..38d74909db 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc +++ b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc @@ -1160,16 +1160,21 @@ Status QnnBackendManager::UnloadLib(void* handle) { #ifdef _WIN32 HMODULE mod = static_cast(handle); + +// TODO: QNN SDK 2.17 crashes for some models/tests on Windows x64 when unloading library. +// Example: ReductionOpTest.ArgMax +#if !defined(_M_AMD64) if (FreeLibrary(mod) == 0) { return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Failed to free library."); } +#endif // !defined(_M_AMD64) mod_handles_.erase(mod); #else auto rt = ::dlclose(handle); if (rt != 0) { return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Failed to free library."); } -#endif +#endif // defined(_WIN32) return Status::OK(); } diff --git a/onnxruntime/core/providers/qnn/builder/qnn_model_wrapper.h b/onnxruntime/core/providers/qnn/builder/qnn_model_wrapper.h index 2765556243..8ae489c749 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_model_wrapper.h +++ b/onnxruntime/core/providers/qnn/builder/qnn_model_wrapper.h @@ -178,7 +178,7 @@ class QnnModelWrapper { Status UnpackInitializerData(const ONNX_NAMESPACE::TensorProto& initializer, std::vector& unpacked_tensor) const; - QnnBackendType GetQnnBackendType() { return qnn_backend_type_; } + QnnBackendType GetQnnBackendType() const { return qnn_backend_type_; } const GraphViewer& GetGraphViewer() const { return graph_viewer_; } diff --git a/onnxruntime/test/onnx/TestCase.cc b/onnxruntime/test/onnx/TestCase.cc index 636c0bbfa9..6d07ddde5c 100644 --- a/onnxruntime/test/onnx/TestCase.cc +++ b/onnxruntime/test/onnx/TestCase.cc @@ -1352,6 +1352,15 @@ std::unique_ptr> GetBrokenTests(const std::string& provider broken_tests->insert({"gridsample_volumetric_nearest_align_corners_0", "unknown version"}); broken_tests->insert({"gridsample_volumetric_nearest_align_corners_1", "unknown version"}); broken_tests->insert({"spacetodepth", "result differs"}); + // Fails with QNN SDK 2.17.0: + // expected 7.70947 (40f6b3f3), got 7.84096 (40fae920), diff: 0.131491, tol=0.00870947 idx=419. 100 of 1715 differ + broken_tests->insert({"facedetection_op8_qdq", "result differs"}); + +#if defined(_WIN32) && defined(_M_AMD64) + // Fails with QNN SDK 2.17.0 on Windows x64: + // expected 13.5 (41580000), got 0 (0), diff: 13.5, tol=0.0145 idx=3. 3 of 4 differ + broken_tests->insert({"averagepool_2d_ceil", "result differs"}); +#endif } #ifdef DISABLE_CONTRIB_OPS diff --git a/onnxruntime/test/providers/cpu/activation/activation_op_test.h b/onnxruntime/test/providers/cpu/activation/activation_op_test.h index c78443eaf8..b5ec140258 100644 --- a/onnxruntime/test/providers/cpu/activation/activation_op_test.h +++ b/onnxruntime/test/providers/cpu/activation/activation_op_test.h @@ -46,11 +46,12 @@ inline void TestActivationOp(const char* szOp, const std::vector> } #endif -// Disabled because of NNAPI treat float::inf as float::max -#if defined(USE_NNAPI) +// Disabled because NNAPI and QNN EP (SDK 2.17) treat float::inf as float::max +#if defined(USE_NNAPI) || defined(USE_QNN) int relu = strcmp(szOp, "Relu"); if (relu == 0) { excluded_providers.insert(kNnapiExecutionProvider); + excluded_providers.insert(kQnnExecutionProvider); } #endif // Use relative error because of computation error for float::max diff --git a/onnxruntime/test/providers/cpu/math/gemm_test.cc b/onnxruntime/test/providers/cpu/math/gemm_test.cc index 36ab867f1b..bf089e083d 100644 --- a/onnxruntime/test/providers/cpu/math/gemm_test.cc +++ b/onnxruntime/test/providers/cpu/math/gemm_test.cc @@ -357,10 +357,19 @@ TYPED_TEST(GemmOpTypedTests, TestGemmBroadcast) { test.AddOutput("Y", {2, 3}, {static_cast(11.0f), static_cast(12.0f), static_cast(13.0f), static_cast(-9.0f), static_cast(-8.0f), static_cast(-7.0f)}); + + std::unordered_set excluded_providers; #if defined(OPENVINO_CONFIG_GPU_FP16) || defined(OPENVINO_CONFIG_GPU_FP32) - test.ConfigExcludeEps({kOpenVINOExecutionProvider}); // OpenVINO: Temporarily disabled due to accuracy issues + excluded_providers.insert(kOpenVINOExecutionProvider); // OpenVINO: Temporarily disabled due to accuracy issues #endif - test.Config(run_with_tunable_op) + + if (b_is_initializer && !c_is_initializer) { + // Accuracy issues on QNN's CPU backend with QNN SDK version 2.17 + excluded_providers.insert(kQnnExecutionProvider); + } + + test.ConfigExcludeEps(excluded_providers) + .Config(run_with_tunable_op) .RunWithConfig(); }; diff --git a/onnxruntime/test/providers/cpu/math/matmul_test.cc b/onnxruntime/test/providers/cpu/math/matmul_test.cc index 9bf71c1328..24340e69c1 100644 --- a/onnxruntime/test/providers/cpu/math/matmul_test.cc +++ b/onnxruntime/test/providers/cpu/math/matmul_test.cc @@ -173,6 +173,12 @@ void RunMatMulTest(int32_t opset_version, bool is_a_constant, bool is_b_constant // QNN can't handle 0 shap excluded_providers.insert(kQnnExecutionProvider); } +#if defined(__linux__) + if (t.name == "test padding and broadcast B > A") { + // Accuracy error with QNN SDK 2.17.0 on CPU backend. + excluded_providers.insert(kQnnExecutionProvider); + } +#endif test.ConfigExcludeEps(excluded_providers) .Config(run_with_tunable_op) .RunWithConfig(); diff --git a/onnxruntime/test/providers/cpu/tensor/resize_op_test.cc b/onnxruntime/test/providers/cpu/tensor/resize_op_test.cc index 2ead9ec91f..3ea7295aef 100644 --- a/onnxruntime/test/providers/cpu/tensor/resize_op_test.cc +++ b/onnxruntime/test/providers/cpu/tensor/resize_op_test.cc @@ -397,9 +397,7 @@ TEST(ResizeOpTest, ResizeOpLinearDownSampleTest_4DBilinear_align_corners) { std::vector Y = {1.0f, 4.0f}; test.AddOutput("Y", {N, C, static_cast(H * scales[2]), static_cast(W * scales[3])}, Y); - - // QNN: result mismatch ("NaN" instead of 1.0f on QNN CPU backend) - test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kQnnExecutionProvider}); + test.Run(); }; run_test(false); diff --git a/onnxruntime/test/providers/qnn/argmaxmin_op_test.cc b/onnxruntime/test/providers/qnn/argmaxmin_op_test.cc index eaeebba5be..e86151008e 100644 --- a/onnxruntime/test/providers/qnn/argmaxmin_op_test.cc +++ b/onnxruntime/test/providers/qnn/argmaxmin_op_test.cc @@ -102,8 +102,7 @@ static void RunQDQArgMxxOpTest(const std::string& op_type, TestInputDef i BuildQDQArgMxxTestCase(op_type, input_def, attrs), // QDQ model provider_options, opset, - expected_ep_assignment, - 1e-5f); + expected_ep_assignment); } // diff --git a/onnxruntime/test/providers/qnn/average_pool_test.cc b/onnxruntime/test/providers/qnn/average_pool_test.cc index 0ee52f7fec..1a0f9bfcba 100644 --- a/onnxruntime/test/providers/qnn/average_pool_test.cc +++ b/onnxruntime/test/providers/qnn/average_pool_test.cc @@ -45,7 +45,8 @@ static void RunQDQAveragePoolOpTest(const std::string& op_type, const std::vector>& input_defs, const std::vector& attrs, ExpectedEPNodeAssignment expected_ep_assignment, - int opset = 18) { + int opset = 18, + QDQTolerance tolerance = QDQTolerance()) { ProviderOptions provider_options; #if defined(_WIN32) provider_options["backend_path"] = "QnnHtp.dll"; @@ -57,7 +58,8 @@ static void RunQDQAveragePoolOpTest(const std::string& op_type, BuildQDQOpTestCase(op_type, input_defs, {}, attrs), provider_options, opset, - expected_ep_assignment); + expected_ep_assignment, + tolerance); } // @@ -146,7 +148,9 @@ TEST_F(QnnHTPBackendTests, AveragePool_CountIncludePad_HTP_u8) { {utils::MakeAttribute("kernel_shape", std::vector{1, 1}), utils::MakeAttribute("count_include_pad", static_cast(1))}, ExpectedEPNodeAssignment::All, - 18); + 18, + // Need tolerance of 0.414% of output range after QNN SDK 2.17 + QDQTolerance(0.00414f)); } // QDQ AveragePool that use auto_pad 'SAME_UPPER'. @@ -159,7 +163,9 @@ TEST_F(QnnHTPBackendTests, AveragePool_AutopadSameUpper_HTP_u8) { {utils::MakeAttribute("kernel_shape", std::vector{1, 1}), utils::MakeAttribute("auto_pad", "SAME_UPPER")}, ExpectedEPNodeAssignment::All, - 18); + 18, + // Need to use tolerance of 0.414% of output range after QNN SDK 2.17 + QDQTolerance(0.00414f)); } // QDQ AveragePool that use auto_pad 'SAME_LOWER'. @@ -172,7 +178,9 @@ TEST_F(QnnHTPBackendTests, AveragePool_AutopadSameLower_HTP_u8) { {utils::MakeAttribute("kernel_shape", std::vector{1, 1}), utils::MakeAttribute("auto_pad", "SAME_LOWER")}, ExpectedEPNodeAssignment::All, - 18); + 18, + // Need to use tolerance of 0.414% of output range after QNN SDK 2.17 + QDQTolerance(0.00414f)); } #endif // defined(__aarch64__) || defined(_M_ARM64) || defined(__linux__) diff --git a/onnxruntime/test/providers/qnn/batch_norm_htp_test.cc b/onnxruntime/test/providers/qnn/batch_norm_htp_test.cc index b4e8f53907..bf36922f88 100644 --- a/onnxruntime/test/providers/qnn/batch_norm_htp_test.cc +++ b/onnxruntime/test/providers/qnn/batch_norm_htp_test.cc @@ -168,8 +168,7 @@ static void RunBatchNormQDQTest(const TestInputDef& input_def, BuildQDQBatchNormTestCase(input_def, scale_def, bias_def), provider_options, 11, - expected_ep_assignment, - 1e-5f); + expected_ep_assignment); } // TODO: FIX TRANSLATION!!! diff --git a/onnxruntime/test/providers/qnn/conv_test.cc b/onnxruntime/test/providers/qnn/conv_test.cc index 0549051bc2..1cd8498ea1 100644 --- a/onnxruntime/test/providers/qnn/conv_test.cc +++ b/onnxruntime/test/providers/qnn/conv_test.cc @@ -148,7 +148,7 @@ static void RunHTPConvOpTest(const std::string& conv_op_type, const TestInputDef ExpectedEPNodeAssignment expected_ep_assignment, bool use_contrib_qdq = false, int opset = 13, - float fp32_abs_err = 1e-5f) { + QDQTolerance tolerance = QDQTolerance()) { ProviderOptions provider_options; #if defined(_WIN32) @@ -165,7 +165,7 @@ static void RunHTPConvOpTest(const std::string& conv_op_type, const TestInputDef provider_options, opset, expected_ep_assignment, - fp32_abs_err); + tolerance); } // Check that QNN compiles DQ -> Conv -> Q as a single unit. @@ -405,7 +405,9 @@ TEST_F(QnnHTPBackendTests, Test_QDQConvWithDynamicWeightsFromMul) { RunQnnModelTest(BuildConvMulGraph, provider_options, 13, - ExpectedEPNodeAssignment::All); + ExpectedEPNodeAssignment::All, + 4e-4f); // Accuracy decreased slightly in QNN SDK 2.17. + // Expected: 9.94500065, Actual: 9.94537735 } // Check that QNN compiles DQ -> Conv -> Q as a single unit. @@ -419,7 +421,11 @@ TEST_F(QnnHTPBackendTests, ConvU8U8S32_bias_dynamic_input) { {0, 0, 0, 0}, // Pads {1, 1}, // Dilations "NOTSET", - ExpectedEPNodeAssignment::All); + ExpectedEPNodeAssignment::All, + false, // use_qdq_contrib_ops + 13, // opset + // Need tolerance of 0.413% of output range after QNN SDK 2.17 + QDQTolerance(0.00413f)); } // Tests 16-bit QDQ Conv with dynamic weights and bias (uses QNN's Conv2d) @@ -518,8 +524,7 @@ TEST_F(QnnHTPBackendTests, DepthwiseConvU16U8S32_StaticBias) { "NOTSET", ExpectedEPNodeAssignment::All, true, // Use com.microsoft QDQ ops for 16-bit - 13, - 0.2f); + 13); } // Tests 16-bit activations, 8-bit static weights QDQ Conv with static bias. @@ -541,8 +546,7 @@ TEST_F(QnnHTPBackendTests, ConvU16U8S32_StaticBias) { "NOTSET", ExpectedEPNodeAssignment::All, true, // Use com.microsoft QDQ ops for 16-bit - 13, - 0.6f); + 13); } // Tests 16-bit activations, 8-bit static weights QDQ Conv with dynamic bias. @@ -565,8 +569,7 @@ TEST_F(QnnHTPBackendTests, DepthwiseConvU16U8S32_DynamicBias) { "NOTSET", ExpectedEPNodeAssignment::All, true, // Use com.microsoft QDQ ops for 16-bit - 13, - 0.2f); + 13); } // Tests 16-bit activations, 8-bit static weights QDQ Conv with dynamic bias. @@ -588,8 +591,7 @@ TEST_F(QnnHTPBackendTests, ConvU16U8S32_DynamicBias) { "NOTSET", ExpectedEPNodeAssignment::All, true, // Use com.microsoft QDQ ops for 16-bit - 13, - 0.57f); + 13); } // Tests 16-bit activations, 8-bit static weights QDQ Conv with no bias @@ -611,8 +613,7 @@ TEST_F(QnnHTPBackendTests, ConvU16U8S32_NoBias) { "NOTSET", ExpectedEPNodeAssignment::All, true, // Use com.microsoft QDQ ops for 16-bit - 13, - 0.58f); + 13); } // Tests 16-bit activations, 8-bit static weights QDQ Conv with no bias @@ -635,8 +636,7 @@ TEST_F(QnnHTPBackendTests, DepthwiseConvU16U8S32_NoBias) { "NOTSET", ExpectedEPNodeAssignment::All, true, // Use com.microsoft QDQ ops for 16-bit - 13, - 0.2f); + 13); } // Test that dynamic weights with default bias works for Conv. This was previously not working @@ -678,7 +678,11 @@ TEST_F(QnnHTPBackendTests, ConvU8U8S32_bias_initializer) { {0, 0, 0, 0}, // Pads {1, 1}, // Dilations "NOTSET", - ExpectedEPNodeAssignment::All); + ExpectedEPNodeAssignment::All, + false, // use_qdq_contrib_ops + 13, // opset + // Need tolerance of 0.413% of output range after QNN SDK 2.17 + QDQTolerance(0.00413f)); } // Tests 1D Conv with bias as an initializer. @@ -827,10 +831,20 @@ TEST_F(QnnHTPBackendTests, ConvU8U8S32_large_input1_padding_bias_initializer) { {1, 1, 1, 1}, {1, 1}, "NOTSET", - ExpectedEPNodeAssignment::All); + ExpectedEPNodeAssignment::All, + false, // use_qdq_contrib_ops + 13, // opset + // Need tolerance of 0.73% of output range after QNN SDK 2.17 + QDQTolerance(0.00730f)); } TEST_F(QnnHTPBackendTests, ConvU8U8S32_large_input2_bias_initializer) { +#ifdef __linux__ + // On Linux QNN SDK 2.17: Need a tolerance of 0.785% of output range to pass. + QDQTolerance tolerance = QDQTolerance(0.00785f); +#else + QDQTolerance tolerance = QDQTolerance(); +#endif RunHTPConvOpTest("Conv", TestInputDef({1, 128, 8, 56}, false, 0.f, 10.f), // Dynamic input TestInputDef({32, 128, 1, 1}, true, -1.f, 1.f), // Random static weights @@ -839,7 +853,10 @@ TEST_F(QnnHTPBackendTests, ConvU8U8S32_large_input2_bias_initializer) { {0, 0, 0, 0}, {1, 1}, "NOTSET", - ExpectedEPNodeAssignment::All); + ExpectedEPNodeAssignment::All, + false, + 13, + tolerance); } TEST_F(QnnHTPBackendTests, ConvU8U8S32_LargeInput_Dilations_Pads) { diff --git a/onnxruntime/test/providers/qnn/gemm_op_test.cc b/onnxruntime/test/providers/qnn/gemm_op_test.cc index 15f26717b0..959d637753 100644 --- a/onnxruntime/test/providers/qnn/gemm_op_test.cc +++ b/onnxruntime/test/providers/qnn/gemm_op_test.cc @@ -126,6 +126,57 @@ TEST_F(QnnCPUBackendTests, Gemm_TransAB_Dynamic_B_And_Bias) { ExpectedEPNodeAssignment::All); } +TEST_F(QnnCPUBackendTests, Gemm_Broadcast_Bias_DynamicInputs) { + std::vector input_a_data = {1.0f, 2.0f, 3.0f, 4.0f, -1.0f, -2.0f, -3.0f, -4.0f}; + std::vector input_b_data(12, 1.0f); + std::vector input_c_data = {1.0f, 2.0f, 3.0f}; + // Expected output (2,3): + // 11.0f, 12.0f, 13.0f, + // -9.0f, -8.0f, -7.0f + + // All dynamic inputs + RunGemmTestOnCPU({TestInputDef({2, 4}, false, input_a_data), + TestInputDef({4, 3}, false, input_b_data), + TestInputDef({3}, false, input_c_data)}, + {}, + ExpectedEPNodeAssignment::All); +} + +// TODO: When this is fixed, enable GemmOpTypedTests/0.TestGemmBroadcast test in cpu/math/gemm_test.cc +// This began failing in QNN SDK 2.17 for the CPU backend. +// Log: the value pair (11, 10) at index #0 don't match, which is -1 from 11 +TEST_F(QnnCPUBackendTests, DISABLED_Gemm_Broadcast_Bias_DynamicA_StaticB_DynamicC) { + std::vector input_a_data = {1.0f, 2.0f, 3.0f, 4.0f, -1.0f, -2.0f, -3.0f, -4.0f}; + std::vector input_b_data(12, 1.0f); + std::vector input_c_data = {1.0f, 2.0f, 3.0f}; + // Expected output (2,3): + // 11.0f, 12.0f, 13.0f, + // -9.0f, -8.0f, -7.0f + + // Dynamic A, static B, dynamic C + RunGemmTestOnCPU({TestInputDef({2, 4}, false, input_a_data), + TestInputDef({4, 3}, true, input_b_data), + TestInputDef({3}, false, input_c_data)}, + {}, + ExpectedEPNodeAssignment::All); +} + +TEST_F(QnnCPUBackendTests, Gemm_Broadcast_Bias_DynamicA_StaticB_StaticC) { + std::vector input_a_data = {1.0f, 2.0f, 3.0f, 4.0f, -1.0f, -2.0f, -3.0f, -4.0f}; + std::vector input_b_data(12, 1.0f); + std::vector input_c_data = {1.0f, 2.0f, 3.0f}; + // Expected output (2,3): + // 11.0f, 12.0f, 13.0f, + // -9.0f, -8.0f, -7.0f + + // Dynamic A, static B, static C + RunGemmTestOnCPU({TestInputDef({2, 4}, false, input_a_data), + TestInputDef({4, 3}, true, input_b_data), + TestInputDef({3}, true, input_c_data)}, + {}, + ExpectedEPNodeAssignment::All); +} + #if defined(__aarch64__) || defined(_M_ARM64) || defined(__linux__) // // HTP tests: @@ -186,8 +237,8 @@ static void RunQDQGemmTestOnHTP(const std::vector>& input_de const std::vector& attrs, ExpectedEPNodeAssignment expected_ep_assignment, int opset = 13, - float f32_abs_err = 1e-4f, - bool use_contrib_qdq = false) { + bool use_contrib_qdq = false, + QDQTolerance tolerance = QDQTolerance()) { ProviderOptions provider_options; #if defined(_WIN32) @@ -202,7 +253,7 @@ static void RunQDQGemmTestOnHTP(const std::vector>& input_de provider_options, opset, expected_ep_assignment, - f32_abs_err); + tolerance); } // Test 8-bit QDQ Gemm with dynamic inputs A and Bias. The B input is an initializer. @@ -217,6 +268,64 @@ TEST_F(QnnHTPBackendTests, Gemm_Dynamic_A_Static_B_Dynamic_Bias_U8) { ExpectedEPNodeAssignment::All); } +// Test broadcasting of bias input. All inputs are dynamic. +TEST_F(QnnHTPBackendTests, Gemm_Broadcast_Bias_DynamicInputs) { + std::vector input_a_data = {1.0f, 2.0f, 3.0f, 4.0f, -1.0f, -2.0f, -3.0f, -4.0f}; + std::vector input_b_data(12, 1.0f); + std::vector input_c_data = {1.0f, 2.0f, 3.0f}; + // Expected output (2,3): + // 11.0f, 12.0f, 13.0f, + // -9.0f, -8.0f, -7.0f + + // All dynamic inputs + RunQDQGemmTestOnHTP({TestInputDef({2, 4}, false, input_a_data), + TestInputDef({4, 3}, false, input_b_data), + TestInputDef({3}, false, input_c_data)}, + {}, + ExpectedEPNodeAssignment::All, + 13, + false, + QDQTolerance(0.00410f)); +} + +TEST_F(QnnHTPBackendTests, Gemm_Broadcast_Bias_DynamicA_StaticB_DynamicC) { + std::vector input_a_data = {1.0f, 2.0f, 3.0f, 4.0f, -1.0f, -2.0f, -3.0f, -4.0f}; + std::vector input_b_data(12, 1.0f); + std::vector input_c_data = {1.0f, 2.0f, 3.0f}; + // Expected output (2,3): + // 11.0f, 12.0f, 13.0f, + // -9.0f, -8.0f, -7.0f + + // Dynamic A, static B, dynamic C + RunQDQGemmTestOnHTP({TestInputDef({2, 4}, false, input_a_data), + TestInputDef({4, 3}, true, input_b_data), + TestInputDef({3}, false, input_c_data)}, + {}, + ExpectedEPNodeAssignment::All, + 13, + false, + QDQTolerance(0.00410f)); +} + +TEST_F(QnnHTPBackendTests, Gemm_Broadcast_Bias_DynamicA_StaticB_StaticC) { + std::vector input_a_data = {1.0f, 2.0f, 3.0f, 4.0f, -1.0f, -2.0f, -3.0f, -4.0f}; + std::vector input_b_data(12, 1.0f); + std::vector input_c_data = {1.0f, 2.0f, 3.0f}; + // Expected output (2,3): + // 11.0f, 12.0f, 13.0f, + // -9.0f, -8.0f, -7.0f + + // Dynamic A, static B, static C + RunQDQGemmTestOnHTP({TestInputDef({2, 4}, false, input_a_data), + TestInputDef({4, 3}, true, input_b_data), + TestInputDef({3}, true, input_c_data)}, + {}, + ExpectedEPNodeAssignment::All, + 13, + false, + QDQTolerance(0.00410f)); +} + // Test 16-bit QDQ Gemm with dynamic inputs A and Bias. The B input is an initializer. // TODO: Inaccuracy detected for output 'output_0', element 0. // Output quant params: scale=0.001872879103757441, zero_point=0. @@ -233,17 +342,10 @@ TEST_F(QnnHTPBackendTests, DISABLED_Gemm_Dynamic_A_Static_B_Dynamic_Bias_U16) { {}, ExpectedEPNodeAssignment::All, 13, // opset - 1e-4f, // f32_abs_err true); // Use com.microsoft Q/DQ ops } // Test QDQ Gemm (16bit act, 8bit weight) with dynamic inputs A and Bias. The B input is an initializer. -// TODO: Allow small inaccuracies based on % of expected value. -// Inaccuracy detected for output 'output_0', element 0. -// Output quant params: scale=0.001872879103757441, zero_point=0. -// Expected val: 120.73912048339844 -// QNN QDQ val: 120.48043823242188 (err 0.2586822509765625) -// CPU QDQ val: 120.48980712890625 (err 0.2493133544921875) TEST_F(QnnHTPBackendTests, Gemm_Dynamic_A_Static_B_Dynamic_Bias_U16Act_U8Weight) { std::vector input_a_data = GetFloatDataInRange(-10.0f, 10.0f, 6); std::vector input_b_data = GetFloatDataInRange(-5.0f, 5.0f, 24); @@ -254,7 +356,6 @@ TEST_F(QnnHTPBackendTests, Gemm_Dynamic_A_Static_B_Dynamic_Bias_U16Act_U8Weight) {}, ExpectedEPNodeAssignment::All, 13, // opset - 0.15f, // f32_abs_err true); // Use com.microsoft Q/DQ ops } @@ -301,12 +402,6 @@ TEST_F(QnnHTPBackendTests, Gemm_TransAB_Static_B_And_Bias_U8) { } // Test QDQ Gemm (16bit activation, 8bit weight) with transposed A/B and static B and Bias inputs. -// TODO: Allow small inaccuracies based on % of expected value. -// Inaccuracy detected for output 'output_0', element 0. -// Output quant params: scale=0.00047966410056687891, zero_point=0. -// Expected val: 29.434776306152344 -// QNN QDQ val: 29.191877365112305 (err 0.24289894104003906) -// CPU QDQ val: 29.197153091430664 (err 0.23762321472167969) TEST_F(QnnHTPBackendTests, Gemm_TransAB_Static_B_And_Bias_U16Act_U8Weight) { std::vector input_a_data = GetFloatDataInRange(-10.0f, 10.0f, 6); std::vector input_b_data = GetFloatDataInRange(-5.0f, 5.0f, 24); @@ -318,7 +413,6 @@ TEST_F(QnnHTPBackendTests, Gemm_TransAB_Static_B_And_Bias_U16Act_U8Weight) { utils::MakeAttribute("transB", static_cast(1))}, ExpectedEPNodeAssignment::All, 13, // opset - 0.15f, // f32_abs_err true); // Use com.microsoft Q/DQ ops } diff --git a/onnxruntime/test/providers/qnn/layer_norm_test.cc b/onnxruntime/test/providers/qnn/layer_norm_test.cc index 085454004e..8cebdd813d 100644 --- a/onnxruntime/test/providers/qnn/layer_norm_test.cc +++ b/onnxruntime/test/providers/qnn/layer_norm_test.cc @@ -35,7 +35,13 @@ static void RunLayerNormCpuTest(const TestInputDef& input_def, expected_ep_assignment); } +#ifdef __linux__ +// This CPU test fails on Linux, QNN SDK 2.17 +// the value pair (-1.75661933, 0) at index #1 don't match, which is 1.75662 from -1.75662 +TEST_F(QnnCPUBackendTests, DISABLED_LayerNorm) { +#else TEST_F(QnnCPUBackendTests, LayerNorm) { +#endif RunLayerNormCpuTest(TestInputDef({2, 3}, false, GetFloatDataInRange(0.0f, 10.0f, 6)), TestInputDef({2, 3}, false, GetFloatDataInRange(0.0f, 10.0f, 6)), {utils::MakeAttribute("axis", static_cast(0))}, @@ -73,18 +79,21 @@ TEST_F(QnnCPUBackendTests, LayerNorm3D) { template GetTestQDQModelFn BuildQDQLayerNormTestCase(const TestInputDef& input_def, const TestInputDef& scale_def, - const std::vector& attrs) { - return [input_def, scale_def, attrs](ModelTestBuilder& builder, - std::vector>& output_qparams) { + const std::vector& attrs, + bool use_contrib_qdq_ops) { + return [input_def, scale_def, attrs, use_contrib_qdq_ops](ModelTestBuilder& builder, + std::vector>& output_qparams) { // input -> Q -> DQ -> NodeArg* input = MakeTestInput(builder, input_def); QuantParams input_qparams = GetTestInputQuantParams(input_def); - NodeArg* input_qdq = AddQDQNodePair(builder, input, input_qparams.scale, input_qparams.zero_point); + NodeArg* input_qdq = AddQDQNodePair(builder, input, input_qparams.scale, input_qparams.zero_point, + use_contrib_qdq_ops); // scale input -> Q -> DQ -> NodeArg* scale = MakeTestInput(builder, scale_def); QuantParams scale_qparams = GetTestInputQuantParams(scale_def); - NodeArg* scale_qdq = AddQDQNodePair(builder, scale, scale_qparams.scale, scale_qparams.zero_point); + NodeArg* scale_qdq = AddQDQNodePair(builder, scale, scale_qparams.scale, scale_qparams.zero_point, + use_contrib_qdq_ops); // LayerNormalization NodeArg* layer_norm_output = builder.MakeIntermediate(); @@ -96,7 +105,7 @@ GetTestQDQModelFn BuildQDQLayerNormTestCase(const TestInputDef Q -> DQ -> output AddQDQNodePairWithOutputAsGraphOutput(builder, layer_norm_output, output_qparams[0].scale, - output_qparams[0].zero_point); + output_qparams[0].zero_point, use_contrib_qdq_ops); }; } @@ -106,7 +115,8 @@ template static void RunLayerNormQDQTest(const TestInputDef& input_def, const TestInputDef& scale_def, const std::vector& attrs, - ExpectedEPNodeAssignment expected_ep_assignment) { + ExpectedEPNodeAssignment expected_ep_assignment, + bool use_contrib_qdq_ops = false) { ProviderOptions provider_options; #if defined(_WIN32) provider_options["backend_path"] = "QnnHtp.dll"; @@ -115,7 +125,8 @@ static void RunLayerNormQDQTest(const TestInputDef& input_def, #endif TestQDQModelAccuracy(BuildOpTestCase("LayerNormalization", {input_def, scale_def}, {}, attrs), - BuildQDQLayerNormTestCase(input_def, scale_def, attrs), + BuildQDQLayerNormTestCase(input_def, scale_def, attrs, + use_contrib_qdq_ops), provider_options, 17, // opset expected_ep_assignment); @@ -129,21 +140,25 @@ TEST_F(QnnHTPBackendTests, LayerNorm1D_Axis0_Unsupported) { ExpectedEPNodeAssignment::None); } -// Test accuracy of 8-bit QDQ LayerNorm with a static scale input. This used to fail on QNN DK 2.13, -// but was fixed in QNN SDK 2.14. -TEST_F(QnnHTPBackendTests, LayerNorm1D_LastAxis_StaticScale) { +// Test accuracy of 8-bit QDQ LayerNorm with a static scale input. +TEST_F(QnnHTPBackendTests, LayerNorm1D_LastAxis_StaticScale_AU8_WU8) { RunLayerNormQDQTest(TestInputDef({1, 2, 3}, false, GetFloatDataInRange(0.0f, 10.0f, 6)), TestInputDef({3}, true, GetFloatDataInRange(0.0f, 1.0f, 3)), // Static {utils::MakeAttribute("axis", static_cast(-1))}, // Last axis ExpectedEPNodeAssignment::All); } +// Test accuracy of 16-bit QDQ LayerNorm with a static scale input. +TEST_F(QnnHTPBackendTests, LayerNorm1D_LastAxis_StaticScale_AU16_WU8) { + RunLayerNormQDQTest(TestInputDef({1, 2, 3}, false, GetFloatDataInRange(0.0f, 10.0f, 6)), + TestInputDef({3}, true, GetFloatDataInRange(0.0f, 1.0f, 3)), // Static + {utils::MakeAttribute("axis", static_cast(-1))}, // Last axis + ExpectedEPNodeAssignment::All, + true); // Use 'com.microsoft' Q/DQ ops +} + // Test accuracy of 8-bit QDQ LayerNorm with a dynamic scale input. -// TODO(adrianlizarraga): Investigate graph finalization error in QNN SDK 2.14.1 -// Failed QNN FinalizeGraphs: QnnDsp Failed to finalize graph (id: 1) with err 1002 -// C:\qnn_src\QNN\HTP\HTP\src\hexagon\prepare\graph_prepare.cc:232:ERROR:could not create op: q::flat_from_vtcm -// C:\qnn_src\QNN\HTP\HTP\src\hexagon\prepare\graph_prepare.cc:1021:ERROR:Op 0x103d00000002 preparation failed with err:-1 -TEST_F(QnnHTPBackendTests, DISABLED_LayerNorm1D_LastAxis_DynamicScale) { +TEST_F(QnnHTPBackendTests, LayerNorm1D_LastAxis_DynamicScale) { RunLayerNormQDQTest(TestInputDef({1, 2, 3}, false, GetFloatDataInRange(0.0f, 10.0f, 6)), TestInputDef({3}, false, GetFloatDataInRange(0.0f, 1.0f, 3)), // Dynamic {utils::MakeAttribute("axis", static_cast(-1))}, // Last axis diff --git a/onnxruntime/test/providers/qnn/lrn_op_test.cc b/onnxruntime/test/providers/qnn/lrn_op_test.cc index 4f64b4a7e0..751db5049f 100644 --- a/onnxruntime/test/providers/qnn/lrn_op_test.cc +++ b/onnxruntime/test/providers/qnn/lrn_op_test.cc @@ -84,7 +84,7 @@ template static void RunQDQLRNOpTest(const TestInputDef& input_def, int64_t size, ExpectedEPNodeAssignment expected_ep_assignment, float alpha = 0.0001f, float beta = 0.75f, float bias = 1.0f, - int opset = 13) { + int opset = 13, QDQTolerance tolerance = QDQTolerance()) { ProviderOptions provider_options; #if defined(_WIN32) provider_options["backend_path"] = "QnnHtp.dll"; @@ -97,7 +97,7 @@ static void RunQDQLRNOpTest(const TestInputDef& input_def, int64_t size, provider_options, opset, expected_ep_assignment, - 1e-5f); + tolerance); } // @@ -130,19 +130,42 @@ TEST_F(QnnCPUBackendTests, LRN_size_larger_than_channel) { TEST_F(QnnHTPBackendTests, LRNSize3) { RunQDQLRNOpTest(TestInputDef({1, 128, 4, 5}, false, -10.0f, 10.0f), 3, // Size - ExpectedEPNodeAssignment::All); + ExpectedEPNodeAssignment::All, + 0.0001f, // alpha + 0.75f, // beta + 1.0f, // bias + 13, // opset + // Need to use tolerance of 0.405% of output range after QNN SDK 2.17 + QDQTolerance(0.00405f)); } TEST_F(QnnHTPBackendTests, LRNSize5) { RunQDQLRNOpTest(TestInputDef({1, 128, 4, 5}, false, -10.0f, 10.0f), 5, // Size - ExpectedEPNodeAssignment::All); + ExpectedEPNodeAssignment::All, + 0.0001f, // alpha + 0.75f, // beta + 1.0f, // bias + 13, // opset + // Need to use tolerance of 0.407% of output range after QNN SDK 2.17 + QDQTolerance(0.00407f)); } TEST_F(QnnHTPBackendTests, LRN_size_larger_than_channel) { +#ifdef __linux__ + // On Linux QNN SDK 2.17: Need a tolerance of 0.407% of output range to pass. + QDQTolerance tolerance = QDQTolerance(0.00407f); +#else + QDQTolerance tolerance = QDQTolerance(); +#endif RunQDQLRNOpTest(TestInputDef({1, 128, 4, 5}, false, -10.0f, 10.0f), 255, // Size - ExpectedEPNodeAssignment::All); + ExpectedEPNodeAssignment::All, + 0.0001f, // alpha + 0.75f, // beta + 1.0f, // bias + 13, // opset + tolerance); } #endif // defined(__aarch64__) || defined(_M_ARM64) || defined(__linux__) diff --git a/onnxruntime/test/providers/qnn/matmul_test.cpp b/onnxruntime/test/providers/qnn/matmul_test.cpp index 3da3dc8581..f26af7c79f 100644 --- a/onnxruntime/test/providers/qnn/matmul_test.cpp +++ b/onnxruntime/test/providers/qnn/matmul_test.cpp @@ -83,8 +83,7 @@ static void RunQDQMatMulOpOpTest(const TestInputDef& input1_def, const TestInputDef& input2_def, ExpectedEPNodeAssignment expected_ep_assignment, int opset = 18, - bool use_contrib_qdq = false, - float fp32_abs_err = 1e-4f) { + bool use_contrib_qdq = false) { ProviderOptions provider_options; #if defined(_WIN32) provider_options["backend_path"] = "QnnHtp.dll"; @@ -97,8 +96,7 @@ static void RunQDQMatMulOpOpTest(const TestInputDef& input1_def, use_contrib_qdq), provider_options, opset, - expected_ep_assignment, - fp32_abs_err); + expected_ep_assignment); } // @@ -128,6 +126,20 @@ TEST_F(QnnCPUBackendTests, DISABLED_MatMulOp_Broadcast) { ExpectedEPNodeAssignment::All, 18, 0.0004f); } +#if defined(__linux__) +TEST_F(QnnCPUBackendTests, DISABLED_MatMulOp_PaddingAndBroadcast_BLargerThanA) { +#else +// TODO: When fixed, enable MathOpTest.MatMulFloatType from cpu/mat/matmul_test.cc +// QNN SDK 2.17: Accuracy errors +TEST_F(QnnCPUBackendTests, MatMulOp_PaddingAndBroadcast_BLargerThanA) { +#endif + std::vector input0_shape = {2, 3, 2}; + std::vector input1_shape = {3, 2, 2, 1}; + RunMatMulOpOpTest(TestInputDef(input0_shape, false, GetSequentialFloatData(input0_shape)), + TestInputDef(input1_shape, false, GetSequentialFloatData(input1_shape)), + ExpectedEPNodeAssignment::All, 7); +} + #if defined(__aarch64__) || defined(_M_ARM64) || defined(__linux__) // // HTP tests: @@ -149,8 +161,7 @@ TEST_F(QnnHTPBackendTests, MatMulOp_HTP_A16_W8Static) { TestInputDef({3, 2}, true, input1_data), ExpectedEPNodeAssignment::All, 18, - true, // Use com.microsoft Q/DQ ops - 7e-3f); + true); // Use com.microsoft Q/DQ ops } // Test QDQ MatMul with uint16 activation uint16 weights, both dynamic @@ -166,8 +177,7 @@ TEST_F(QnnHTPBackendTests, DISABLED_MatMulOp_HTP_A16_W16Dynamic) { TestInputDef({3, 2}, false, input1_data), ExpectedEPNodeAssignment::All, 18, - true, // Use com.microsoft Q/DQ ops - 7e-3f); + true); // Use com.microsoft Q/DQ ops } // Test QDQ MatMul with uint16 activation uint16 weights, both dynamic @@ -183,8 +193,7 @@ TEST_F(QnnHTPBackendTests, DISABLED_MatMulOp_HTP_A16_W16DynamicLarge) { TestInputDef({1, 12, 512, 96}, false, input1_data), ExpectedEPNodeAssignment::All, 18, - true, // Use com.microsoft Q/DQ ops - 7e-3f); + true); // Use com.microsoft Q/DQ ops } // Test 16-bit QDQ MatMul with static weights diff --git a/onnxruntime/test/providers/qnn/pad_op_test.cpp b/onnxruntime/test/providers/qnn/pad_op_test.cpp index 792dbeadfa..4ef71457d5 100644 --- a/onnxruntime/test/providers/qnn/pad_op_test.cpp +++ b/onnxruntime/test/providers/qnn/pad_op_test.cpp @@ -135,8 +135,7 @@ static void RunQDQPadOpTest(const TestInputDef& data_def, has_constant_value, constant_value_quantized), provider_options, opset, - expected_ep_assignment, - 1e-5f); + expected_ep_assignment); } // diff --git a/onnxruntime/test/providers/qnn/pool_op_test.cpp b/onnxruntime/test/providers/qnn/pool_op_test.cpp index 7ed9072a95..5dd3a6aaa3 100644 --- a/onnxruntime/test/providers/qnn/pool_op_test.cpp +++ b/onnxruntime/test/providers/qnn/pool_op_test.cpp @@ -21,13 +21,15 @@ namespace test { template GetTestQDQModelFn BuildPoolQDQTestCase(const std::string& op_type, const TestInputDef& input_def, - const std::vector& attrs) { - return [op_type, input_def, attrs](ModelTestBuilder& builder, - std::vector>& output_qparams) { + const std::vector& attrs, + bool use_contrib_qdq_ops) { + return [op_type, input_def, attrs, use_contrib_qdq_ops](ModelTestBuilder& builder, + std::vector>& output_qparams) { // input -> Q -> DQ -> NodeArg* input = MakeTestInput(builder, input_def); QuantParams input_qparams = GetTestInputQuantParams(input_def); - NodeArg* input_qdq = AddQDQNodePair(builder, input, input_qparams.scale, input_qparams.zero_point); + NodeArg* input_qdq = AddQDQNodePair(builder, input, input_qparams.scale, input_qparams.zero_point, + use_contrib_qdq_ops); // MaxPool NodeArg* pool_output = builder.MakeIntermediate(); @@ -41,7 +43,7 @@ GetTestQDQModelFn BuildPoolQDQTestCase(const std::string& op_type, // NOTE: Input and output quantization parameters must be equal for MaxPool. output_qparams[0] = input_qparams; // Overwrite! AddQDQNodePairWithOutputAsGraphOutput(builder, pool_output, input_qparams.scale, - input_qparams.zero_point); + input_qparams.zero_point, use_contrib_qdq_ops); }; } @@ -72,7 +74,9 @@ static void RunQDQPoolOpTest(const std::string& op_type, const TestInputDef& input_def, const std::vector& attrs, ExpectedEPNodeAssignment expected_ep_assignment, - int opset = 18) { + int opset = 18, + bool use_contrib_qdq_ops = false, + QDQTolerance tolerance = QDQTolerance()) { ProviderOptions provider_options; #if defined(_WIN32) provider_options["backend_path"] = "QnnHtp.dll"; @@ -81,11 +85,11 @@ static void RunQDQPoolOpTest(const std::string& op_type, #endif TestQDQModelAccuracy(BuildOpTestCase(op_type, {input_def}, {}, attrs), - BuildPoolQDQTestCase(op_type, input_def, attrs), + BuildPoolQDQTestCase(op_type, input_def, attrs, use_contrib_qdq_ops), provider_options, opset, expected_ep_assignment, - 1e-5f); + tolerance); } // @@ -119,7 +123,7 @@ TEST_F(QnnCPUBackendTests, MaxPool_Large_Input) { ExpectedEPNodeAssignment::All); } -// QNN v2.13, backendValidateOpConfig() failed for node `MaxPool` of type `PoolMax2d` with error code 4003 +// Fails on QNN v2.17, QNN.graphAddNode() failed for node `MaxPool` of type `PoolMax2d` with error code 6000 TEST_F(QnnCPUBackendTests, DISABLED_MaxPool_Ceil) { RunPoolOpTest("MaxPool", TestInputDef({1, 2, 3, 3}, false, -10.0f, 10.0f), // Dynamic input with range [-10, 10] @@ -133,7 +137,7 @@ TEST_F(QnnCPUBackendTests, DISABLED_MaxPool_Ceil) { ExpectedEPNodeAssignment::All); } -// QNN v2.13, backendValidateOpConfig() failed for node `MaxPool` of type `PoolMax2d` with error code 4003 +// Fails on QNN v2.17, QNN.graphAddNode() failed for node `MaxPool` of type `PoolMax2d` with error code 6000 TEST_F(QnnCPUBackendTests, DISABLED_MaxPool_Large_Input2_Ceil) { RunPoolOpTest("MaxPool", TestInputDef({1, 128, 16, 113}, false, -10.0f, 10.0f), // Dynamic input with range [-10, 10] @@ -183,7 +187,11 @@ TEST_F(QnnHTPBackendTests, MaxPool_Large_Input_HTP_u8) { utils::MakeAttribute("ceil_mode", static_cast(0)), utils::MakeAttribute("storage_order", static_cast(0)), utils::MakeAttribute("auto_pad", "NOTSET")}, - ExpectedEPNodeAssignment::All); + ExpectedEPNodeAssignment::All, + 18, // opset + false, // use_contrib_qdq_ops + // Need a tolerance of 0.417% of output range after QNN SDK 2.17 + QDQTolerance(0.00417f)); } TEST_F(QnnHTPBackendTests, MaxPool_Ceil_HTP_u8) { @@ -219,7 +227,7 @@ TEST_F(QnnHTPBackendTests, DISABLED_MaxPool_Large_Input2_Ceil_HTP_u8) { // QNN v2.13: Certain large input sizes cause the QNN graph to fail to finalize with error 1002 (QNN_COMMON_ERROR_MEM_ALLOC). // Fixed in QNN v2.14.1. -TEST_F(QnnHTPBackendTests, MaxPool_LargeInput_1Pads) { +TEST_F(QnnHTPBackendTests, MaxPool_LargeInput_1Pads_u8) { RunQDQPoolOpTest("MaxPool", TestInputDef({1, 64, 384, 576}, false, -10.0f, 10.0f), // Dynamic input with range [-10, 10] {utils::MakeAttribute("kernel_shape", std::vector{3, 3}), @@ -229,17 +237,48 @@ TEST_F(QnnHTPBackendTests, MaxPool_LargeInput_1Pads) { utils::MakeAttribute("ceil_mode", static_cast(0)), utils::MakeAttribute("storage_order", static_cast(0)), utils::MakeAttribute("auto_pad", "NOTSET")}, - ExpectedEPNodeAssignment::All); + ExpectedEPNodeAssignment::All, + 18, // opset + false, // use_contrib_qdq_ops + // Need a tolerance of 0.417% of output range after QNN SDK 2.17 + QDQTolerance(0.00417f)); +} + +// Test uint16 QDQ MaxPool with large inputs. +TEST_F(QnnHTPBackendTests, MaxPool_LargeInput_1Pads_u16) { + RunQDQPoolOpTest("MaxPool", + TestInputDef({1, 64, 384, 576}, false, -10.0f, 10.0f), // Dynamic input with range [-10, 10] + {utils::MakeAttribute("kernel_shape", std::vector{3, 3}), + utils::MakeAttribute("strides", std::vector{2, 2}), + utils::MakeAttribute("pads", std::vector{1, 1, 1, 1}), + utils::MakeAttribute("dilations", std::vector{1, 1}), + utils::MakeAttribute("ceil_mode", static_cast(0)), + utils::MakeAttribute("storage_order", static_cast(0)), + utils::MakeAttribute("auto_pad", "NOTSET")}, + ExpectedEPNodeAssignment::All, + 18, // opset + true); // use_contrib_qdq_ops } // QDQ GlobalMaxPool test TEST_F(QnnHTPBackendTests, GlobalMaxPool_u8) { + std::vector input_data = GetFloatDataInRange(-10.0f, 10.0f, 18); RunQDQPoolOpTest("GlobalMaxPool", - TestInputDef({1, 2, 3, 3}, false, -10.0f, 10.0f), // Dynamic input with range [-10, 10] + TestInputDef({1, 2, 3, 3}, false, input_data), // Dynamic input with range [-10, 10] {}, ExpectedEPNodeAssignment::All); } +TEST_F(QnnHTPBackendTests, GlobalMaxPool_u16) { + std::vector input_data = GetFloatDataInRange(-10.0f, 10.0f, 18); + RunQDQPoolOpTest("GlobalMaxPool", + TestInputDef({1, 2, 3, 3}, false, input_data), // Dynamic input with range [-10, 10] + {}, + ExpectedEPNodeAssignment::All, + 18, + true); // Use 'com.microsoft' domain Q/DQ ops +} + TEST_F(QnnHTPBackendTests, GlobalMaxPool_Large_Input_u8) { RunQDQPoolOpTest("GlobalMaxPool", TestInputDef({1, 128, 16, 113}, false, -10.0f, 10.0f), // Dynamic input with range [-10, 10] @@ -247,14 +286,7 @@ TEST_F(QnnHTPBackendTests, GlobalMaxPool_Large_Input_u8) { ExpectedEPNodeAssignment::All); } -// initial_sequencer_dp.cc:156:ERROR:A single op, "q::MaxPool_valid.tcm" (Op ID: 277700000016), requires 0x6c0800 bytes of TCM, which is greater than the TCM size of 0x400000! -// QnnDsp graph prepare failed 13 -// QnnDsp Failed to finalize graph QNN_983391626356502531_0 with err: 1002 -// QnnDsp Failed to finalize graph (id: 1) with err 1002 -// QnnDsp Wake up free backend 1 thread(s) -// QnnDsp QnnGraph_finalize done. status 0x3ea -// Failed to finalize QNN graph. -TEST_F(QnnHTPBackendTests, DISABLED_GlobalMaxPool_LargeInput2_u8) { +TEST_F(QnnHTPBackendTests, GlobalMaxPool_LargeInput2_u8) { RunQDQPoolOpTest("GlobalMaxPool", TestInputDef({1, 64, 384, 576}, false, -10.0f, 10.0f), // Dynamic input with range [-10, 10] {}, diff --git a/onnxruntime/test/providers/qnn/qnn_test_utils.cc b/onnxruntime/test/providers/qnn/qnn_test_utils.cc index a067c9c53e..665a838b43 100644 --- a/onnxruntime/test/providers/qnn/qnn_test_utils.cc +++ b/onnxruntime/test/providers/qnn/qnn_test_utils.cc @@ -42,6 +42,28 @@ std::vector GetFloatDataInRange(float min_val, float max_val, size_t num_ return data; } +std::vector GetSequentialFloatData(const std::vector& shape, float start, float step) { + if (shape.empty()) { + return {}; + } + + int64_t count = 1; + for (auto dim : shape) { + count *= dim; + } + + std::vector data; + data.reserve(static_cast(count)); + + float val = start; + for (int64_t i = 0; i < count; i++) { + data.push_back(val); + val += step; + } + + return data; +} + void TryEnableQNNSaver(ProviderOptions& qnn_options) { // Allow dumping QNN API calls to file by setting an environment variable that enables the QNN Saver backend. constexpr auto kEnableQNNSaverEnvironmentVariableName = "ORT_UNIT_TEST_ENABLE_QNN_SAVER"; diff --git a/onnxruntime/test/providers/qnn/qnn_test_utils.h b/onnxruntime/test/providers/qnn/qnn_test_utils.h index 396fc193bf..fe77c6bdba 100644 --- a/onnxruntime/test/providers/qnn/qnn_test_utils.h +++ b/onnxruntime/test/providers/qnn/qnn_test_utils.h @@ -84,6 +84,16 @@ inline QuantParams GetDataQuantParams(gsl::span data) { */ std::vector GetFloatDataInRange(float min_val, float max_val, size_t num_elems); +/** + * Returns a float vector with sequential data. + * + * \param shape The tensor shape used to determine the number of values. + * \param start The starting value. + * \param step The step size. + * \return A vector of sequential floats. + */ +std::vector GetSequentialFloatData(const std::vector& shape, float start = 0.0f, float step = 1.0f); + // Class that defines an input that can be created with ModelTestBuilder. // Defines whether the input is an initializer and if the data should be randomized or if // set to an explicit value. @@ -239,6 +249,19 @@ void InferenceModel(const std::string& model_data, const char* log_id, */ void TryEnableQNNSaver(ProviderOptions& qnn_options); +struct QDQTolerance { + // When comparing output activations between QNN EP and CPU EP (both running the QDQ model), + // this value defines the maximum tolerable difference as a percentage of the output range. + // Ex: (qdq@QNN_EP - qdq@CPU_EP) / (rmax_output - rmin_output) <= DEFAULT_QDQ_TOLERANCE. + static constexpr float DEFAULT_QDQ_TOLERANCE = 0.004f; // 0.4% is equivalent to 1 int8 quantization unit + // or 262 int16 quantization units. + + QDQTolerance() : value(DEFAULT_QDQ_TOLERANCE) {} + explicit QDQTolerance(float tolerance) : value(tolerance) {} + + float value; +}; + /** * Tests the accuracy of a QDQ model on QNN EP by runnning 3 inferences: * @@ -254,13 +277,15 @@ void TryEnableQNNSaver(ProviderOptions& qnn_options); * \param qnn_options QNN EP provider options. * \param opset_version The opset version. * \param expected_ep_assignment Describes "which nodes" should be assigned to the EP. - * \param fp32_abs_err Small tolerance used for floating-point comparisons. + * \param tolerance The percent tolerance (as fraction) QNN EP results are allowed to differ from the QDQ model on CPU EP. + * This tolerance is a percentage of the output range. * \param log_severity The logger's severity setting. */ template inline void TestQDQModelAccuracy(const GetTestModelFn& f32_model_fn, const GetTestQDQModelFn& qdq_model_fn, ProviderOptions qnn_options, int opset_version, - ExpectedEPNodeAssignment expected_ep_assignment, float fp32_abs_err = 1e-4f, + ExpectedEPNodeAssignment expected_ep_assignment, + QDQTolerance tolerance = QDQTolerance(), logging::Severity log_severity = logging::Severity::kERROR, const std::string& qnn_ctx_model_path = "") { // Add kMSDomain to cover contrib op like Gelu @@ -366,37 +391,71 @@ inline void TestQDQModelAccuracy(const GetTestModelFn& f32_model_fn, const GetTe gsl::span cpu_f32_vals = output_vals[i]; gsl::span cpu_qdq_vals = cpu_qdq_tensor.DataAsSpan(); gsl::span qnn_qdq_vals = qnn_qdq_tensor.DataAsSpan(); + constexpr QuantType qmin = std::numeric_limits::min(); + constexpr QuantType qmax = std::numeric_limits::max(); + const float output_range = output_qparams[i].scale * static_cast(qmax - qmin); ASSERT_EQ(num_vals, cpu_qdq_vals.size()); ASSERT_EQ(num_vals, qnn_qdq_vals.size()); - for (size_t j = 0; j < num_vals && error_count < max_error_count; j++) { - const float expected_val = cpu_f32_vals[j]; // "ground-truth" - const float qnn_qdq_val = qnn_qdq_vals[j]; - const float cpu_qdq_val = cpu_qdq_vals[j]; - const float cpu_err = std::fabs(expected_val - cpu_qdq_val); - const float qnn_err = std::fabs(expected_val - qnn_qdq_val); + float max_f32_err = 0.0f; + float max_qdq_err = 0.0f; + bool print_accuracy_warning = false; - // Case 1 (qnn_err <= cpu_err): QNN EP is *more* accurate, which makes (qnn_err - cpu_err) zero or - // a negative value. - // Case 2 (qnn_err > cpu_err): QNN EP is less accurate, but the error difference is within 1 - // quantization unit (i.e., scale). This can occur due to rounding differences. - const bool is_as_accurate_as_cpu_qdq = (qnn_err - cpu_err) <= (output_qparams[i].scale + fp32_abs_err); - if (!is_as_accurate_as_cpu_qdq) { + for (size_t j = 0; j < num_vals && error_count < max_error_count; j++) { + const float expected_val = cpu_f32_vals[j]; // f32@CPU_EP val ("ground-truth") + const float qnn_qdq_val = qnn_qdq_vals[j]; // qdq@QNN_EP val + const float cpu_qdq_val = cpu_qdq_vals[j]; // qdq@CPU_EP val + + // Get errors of qdq@CPU_EP and qdq@QNN_EP against f32@CPU_EP. + const float cpu_err = std::fabs(expected_val - cpu_qdq_val); + const float cpu_err_norm = cpu_err / output_range; + const float qnn_err = std::fabs(expected_val - qnn_qdq_val); + const float qnn_err_norm = qnn_err / output_range; + + // Also compare the QDQ values against each other. + // This is equivalent to abs(qdq@QNN_EP - qdq@CPU_EP) / output_range + const float qdq_vals_err_norm = std::fabs(qnn_err_norm - cpu_err_norm); + + // True if qdq@QNN_EP is at least as accurate as qdq@CPU_EP when compared to expected f32@CPU_EP value. + const bool is_as_accurate_as_cpu_ep = qnn_err_norm <= cpu_err_norm; + + // True if the normalized difference between qdq@QNN_EP and qdq@CPU_EP is within tolerance. + const bool qdq_vals_diff_within_tolerance = qdq_vals_err_norm <= tolerance.value; + + const bool passed_test = is_as_accurate_as_cpu_ep || qdq_vals_diff_within_tolerance; + if (!passed_test) { ++error_count; } - - EXPECT_TRUE(is_as_accurate_as_cpu_qdq) + EXPECT_TRUE(passed_test) << "Inaccuracy detected for output '" << debug_output_name << "', element " << j - << ".\nOutput quant params: scale=" << output_qparams[i].scale - << ", zero_point=" << static_cast(output_qparams[i].zero_point) - << ".\nExpected val: " << expected_val << "\n" - << "QNN QDQ val: " << qnn_qdq_val << " (err " << qnn_err << ")\n" - << "CPU QDQ val: " << cpu_qdq_val << " (err " << cpu_err << ")"; + << "\noutput_range=" << output_range << ", tolerance=" << (tolerance.value * 100) << "%" + << ".\nExpected val (f32@CPU_EP): " << expected_val << "\n" + << "qdq@QNN_EP val: " << qnn_qdq_val << " (err: " << qnn_err << ", err/output_range: " + << qnn_err_norm * 100.0f << "%)\n" + << "qdq@CPU_EP val: " << cpu_qdq_val << " (err: " << cpu_err << ", err/output_range: " + << cpu_err_norm * 100.0f << "%)\n" + << "abs(qdq@QNN_EP - qdq@CPU_EP) / output_range = " << qdq_vals_err_norm * 100.0f << "%"; + + max_f32_err = std::max(max_f32_err, qnn_err_norm); + max_qdq_err = std::max(max_qdq_err, qdq_vals_err_norm); + if (passed_test && !is_as_accurate_as_cpu_ep && (qdq_vals_err_norm > QDQTolerance::DEFAULT_QDQ_TOLERANCE)) { + print_accuracy_warning = true; + } + } + + if (print_accuracy_warning) { + std::cerr << std::endl + << "[WARNING]: Output " << i + << " required larger tolerance to pass accuracy checks" << std::endl + << "Max normalized error against f32@CPU_EP = " << max_f32_err * 100.0f << "%" << std::endl + << "Max normalized error against qdq@CPU_EP = " << max_qdq_err * 100.0f << "%" << std::endl + << "Default tolerance = " << QDQTolerance::DEFAULT_QDQ_TOLERANCE * 100.0f << "%" << std::endl + << "Tolerance used = " << tolerance.value * 100.0f << "%" << std::endl; } } else { - VerifyOutput(debug_output_name, cpu_f32_outputs[i].Get(), qnn_qdq_tensor, fp32_abs_err); + VerifyOutput(debug_output_name, cpu_f32_outputs[i].Get(), qnn_qdq_tensor, 1e-4f); } } } diff --git a/onnxruntime/test/providers/qnn/reduce_op_test.cc b/onnxruntime/test/providers/qnn/reduce_op_test.cc index 1403197cd6..e39ba5fb40 100644 --- a/onnxruntime/test/providers/qnn/reduce_op_test.cc +++ b/onnxruntime/test/providers/qnn/reduce_op_test.cc @@ -365,8 +365,7 @@ static void RunReduceOpQDQTest(const std::string& op_type, const std::vector& axes, bool keepdims, int opset, - ExpectedEPNodeAssignment expected_ep_assignment, - float fp32_abs_err = 1e-4f) { + ExpectedEPNodeAssignment expected_ep_assignment) { ProviderOptions provider_options; #if defined(_WIN32) provider_options["backend_path"] = "QnnHtp.dll"; @@ -383,8 +382,7 @@ static void RunReduceOpQDQTest(const std::string& op_type, noop_with_empty_axes), provider_options, opset, - expected_ep_assignment, - fp32_abs_err); + expected_ep_assignment); } // @@ -405,22 +403,14 @@ TEST_F(QnnHTPBackendTests, ReduceSumU8Opset13) { ExpectedEPNodeAssignment::All); } -// TODO: Investigate inaccuracy -// Input values: 3.21289 -5.9981 -1.72799 6.27263 -// Input quantization params [-10, 10]: scale=0.0784313753, zero_point=127 -// -// Inaccuracy detected for output 'output', element 0. -// Output quant params: scale=0.0068997270427644253, zero_point=0. -// Expected val: 1.7594304084777832 -// QNN QDQ val: 1.731831431388855 (err 0.027598977088928223) -// CPU QDQ val: 1.7594304084777832 (err 0) -TEST_F(QnnHTPBackendTests, DISABLED_ReduceSumU8Opset13_Inaccurate) { +// Test 8-bit QDQ ReduceSum of last axis. +TEST_F(QnnHTPBackendTests, ReduceSumU8Opset13_LastAxis) { const std::vector input_data = {3.21289f, -5.9981f, -1.72799f, 6.27263f}; RunReduceOpQDQTest("ReduceSum", - TestInputDef({2, 2}, false, input_data).OverrideValueRange(-10.0f, 10.0f), - {0, 1}, // axes - true, // keepdims - 13, // opset + TestInputDef({2, 2}, false, input_data), + {1}, // axes + true, // keepdims + 13, // opset ExpectedEPNodeAssignment::All); } // Test creates a Q -> DQ -> ReduceSum -> Q -> DQ graph, and checks that all @@ -443,7 +433,8 @@ TEST_F(QnnHTPBackendTests, ReduceSumU8Opset11) { // - Uses int8 as the quantization type. // - Uses opset 13, which has "axes" as an input. TEST_F(QnnHTPBackendTests, ReduceSumS8Opset13) { - std::vector input_data = GetFloatDataInRange(-10.0f, 10.0f, 9); + // non-symmetrical input range so output sum is not trivially zero. + std::vector input_data = GetFloatDataInRange(-10.0f, 20.0f, 9); RunReduceOpQDQTest("ReduceSum", TestInputDef({3, 3}, false, input_data), @@ -466,14 +457,7 @@ TEST_F(QnnHTPBackendTests, ReduceSumS8Opset13_NoKeepDims) { } // Test rank 5 ReduceSum (s8 quant) with axes = [0, 1, 2, 3, 4], keep_dims = true -// TODO: QNN 2.15.1 Graph finalization error: -// graph_prepare.cc:234:ERROR:could not create op: q::Sum -// graph_prepare.cc:1093:ERROR:Op 0x102500000011 preparation failed with err:-1 -// Completed stage: Graph Transformations and Optimizations (17163 us) -// QnnDsp "node_token_3" generated: could not create op -// QnnDsp RouterWindows graph prepare failed 12 -// QnnDsp Failed to finalize graph (id: 1) with err 1002{} -TEST_F(QnnHTPBackendTests, DISABLED_ReduceSumS8Opset13_Rank5) { +TEST_F(QnnHTPBackendTests, ReduceSumS8Opset13_Rank5) { RunReduceOpQDQTest("ReduceSum", TestInputDef({1, 3, 4, 4, 2}, false, GetFloatDataInRange(-10.0f, 10.0f, 96)), {0, 1, 2, 3, 4}, // axes @@ -493,8 +477,7 @@ TEST_F(QnnHTPBackendTests, ReduceSumS8Opset13_Rank6_Unsupported) { } // Test rank 5 ReduceSum (u8 quant) with axes = [-1], keep_dims = false -// TODO: Enable on QNN 2.15.1 (works fine) -TEST_F(QnnHTPBackendTests, DISABLED_ReduceSumU8Opset13_Rank5_LastAxis) { +TEST_F(QnnHTPBackendTests, ReduceSumU8Opset13_Rank5_LastAxis) { constexpr size_t num_elems = 2ULL * 12 * 124 * 2 * 4; std::vector input_data = GetFloatDataInRange(-100.0f, 100.0f, num_elems); RunReduceOpQDQTest("ReduceSum", @@ -618,22 +601,14 @@ TEST_F(QnnHTPBackendTests, ReduceMeanU8Opset18) { ExpectedEPNodeAssignment::All); } -// TODO: Investigate inaccuracy -// Input values: 3.21289 -5.9981 -1.72799 6.27263 -// Input quantization params [-10, 10]: scale=0.0784313753, zero_point=127 -// -// Inaccuracy detected for output 'output', element 0. -// Output quant params: scale=0.0017249317606911063, zero_point=0. -// Expected val: 0.4398576021194458 -// QNN QDQ val: 0.43295785784721375 (err 0.0068997442722320557) -// CPU QDQ val: 0.4398576021194458 (err 0) -TEST_F(QnnHTPBackendTests, DISABLED_ReduceMeanU8Opset18_Inaccurate) { +// Test 8-bit QDQ ReduceMean of last axis +TEST_F(QnnHTPBackendTests, ReduceMeanU8Opset18_LastAxis) { const std::vector input_data = {3.21289f, -5.9981f, -1.72799f, 6.27263f}; RunReduceOpQDQTest("ReduceMean", - TestInputDef({2, 2}, false, input_data).OverrideValueRange(-10.0f, 10.0f), - {0, 1}, // axes - true, // keepdims - 18, // opset + TestInputDef({2, 2}, false, input_data), + {1}, // axes + true, // keepdims + 18, // opset ExpectedEPNodeAssignment::All); } @@ -656,22 +631,15 @@ TEST_F(QnnHTPBackendTests, ReduceMeanU8Opset13) { // // - Uses int8 as the quantization type. // - Uses opset 18, which has "axes" as an input. -// -// TODO(adrianlizarraga): Inaccuracy detected for output 'output', element 0. -// Output quant params: scale=0.0007829521200619638, zero_point=127. -// Expected val: -0.19965279102325439 -// QNN QDQ val: -0.19730393588542938 (err 0.0023488551378250122) -// CPU QDQ val: -0.19965279102325439 (err 0) TEST_F(QnnHTPBackendTests, ReduceMeanS8Opset18) { - std::vector input_data = GetFloatDataInRange(-10.0f, 10.0f, 48); + std::vector input_data = GetFloatDataInRange(-10.0f, 20.0f, 48); RunReduceOpQDQTest("ReduceMean", TestInputDef({1, 3, 4, 4}, false, input_data), {0, 1, 2, 3}, // axes true, // keepdims 18, // opset - ExpectedEPNodeAssignment::All, - 0.0016f); // TODO: Remove additional tolerance needed for inaccuracy + ExpectedEPNodeAssignment::All); } #endif // defined(__aarch64__) || defined(_M_ARM64) || defined(__linux__) diff --git a/onnxruntime/test/providers/qnn/resize_test.cc b/onnxruntime/test/providers/qnn/resize_test.cc index cd6865d443..14df171140 100644 --- a/onnxruntime/test/providers/qnn/resize_test.cc +++ b/onnxruntime/test/providers/qnn/resize_test.cc @@ -158,7 +158,8 @@ static void RunQDQResizeOpTest(const TestInputDef& input_def, const std::string& mode, const std::string& coordinate_transformation_mode, const std::string& nearest_mode, ExpectedEPNodeAssignment expected_ep_assignment, - int opset = 19) { + int opset = 19, + QDQTolerance tolerance = QDQTolerance()) { ProviderOptions provider_options; #if defined(_WIN32) provider_options["backend_path"] = "QnnHtp.dll"; @@ -171,7 +172,8 @@ static void RunQDQResizeOpTest(const TestInputDef& input_def, nearest_mode), provider_options, opset, - expected_ep_assignment); + expected_ep_assignment, + tolerance); } // @@ -295,12 +297,7 @@ TEST_F(QnnCPUBackendTests, Resize2xLinearAlignCorners_scales) { } // Test Resize downsample with mode: "linear", coordinate_transformation_mode: "align_corners" -// TODO: Enable ResizeOpTest.ResizeOpLinearDownSampleTest_4DBilinear_align_corners in cpu resize_op tests when fixed. -// -// Input f32[1,1,2,4]: 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0 -// Expected output f32[1, 1, 1, 2]: 1.0, 4.0 -// Actual output f32[1, 1, 1, 2]: NaN, NaN -TEST_F(QnnCPUBackendTests, DISABLED_Resize_DownSample_Linear_AlignCorners_scales) { +TEST_F(QnnCPUBackendTests, Resize_DownSample_Linear_AlignCorners_scales) { std::vector input_data = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f}; RunCPUResizeOpTestWithScales(TestInputDef({1, 1, 2, 4}, false, input_data), {1.0f, 1.0f, 0.6f, 0.6f}, "linear", "align_corners", "", @@ -308,11 +305,12 @@ TEST_F(QnnCPUBackendTests, DISABLED_Resize_DownSample_Linear_AlignCorners_scales } // Test Resize downsample with mode: "linear", coordinate_transformation_mode: "half_pixel" +// Fails on QNN v2.17, the value pair (2.66666651, 3.5) at index #0 don't match, which is 0.833333 from 2.66667 // TODO: Enable ResizeOpTest.ResizeOpLinearDownSampleTest_4DBilinear cpu resize_op tests when fixed. // // Input f32[1,1,2,4]: 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0 // Expected output f32[1, 1, 1, 2]: 2.6666 4.3333 -// Actual output f32[1, 1, 1, 2]: NaN, NaN +// Actual output f32[1, 1, 1, 2]: 3.5, 5.5 TEST_F(QnnCPUBackendTests, DISABLED_Resize_DownSample_Linear_HalfPixel_scales) { std::vector input_data = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f}; RunCPUResizeOpTestWithScales(TestInputDef({1, 1, 2, 4}, false, input_data), @@ -338,7 +336,10 @@ TEST_F(QnnHTPBackendTests, Resize_DownSample_Linear_HalfPixel) { std::vector input_data = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f}; RunQDQResizeOpTest(TestInputDef({1, 1, 2, 4}, false, input_data), {1, 1, 1, 2}, "linear", "half_pixel", "", - ExpectedEPNodeAssignment::All); + ExpectedEPNodeAssignment::All, + 19, + // Need tolerance of 0.539% of output range after QNN SDK 2.17 + QDQTolerance(0.00539f)); } // Test 2x QDQ Resize mode: "linear", coordinate_transformation_mode: "pytorch_half_pixel" @@ -347,7 +348,10 @@ TEST_F(QnnHTPBackendTests, ResizeU8_2xLinearPytorchHalfPixel) { std::vector input_data = GetFloatDataInRange(-10.0f, 10.0f, 48); RunQDQResizeOpTest(TestInputDef({1, 3, 4, 4}, false, input_data), {1, 3, 8, 8}, "linear", "pytorch_half_pixel", "", - ExpectedEPNodeAssignment::All); + ExpectedEPNodeAssignment::All, + 19, + // Need tolerance of 0.609% of output range after QNN SDK 2.17 + QDQTolerance(0.00609f)); } // Test 2x QDQ Resize mode: "linear", coordinate_transformation_mode: "half_pixel" @@ -356,7 +360,10 @@ TEST_F(QnnHTPBackendTests, ResizeU8_2xLinearHalfPixel) { std::vector input_data = GetFloatDataInRange(-10.0f, 10.0f, 48); RunQDQResizeOpTest(TestInputDef({1, 3, 4, 4}, false, input_data), {1, 3, 8, 8}, "linear", "half_pixel", "", - ExpectedEPNodeAssignment::All); + ExpectedEPNodeAssignment::All, + 19, + // Need tolerance of 0.609% of output range after QNN SDK 2.17 + QDQTolerance(0.00609f)); } // Test 2x QDQ Resize mode: "linear", coordinate_transformation_mode: "align_corners" @@ -365,7 +372,10 @@ TEST_F(QnnHTPBackendTests, ResizeU8_2xLinearAlignCorners) { std::vector input_data = GetFloatDataInRange(-10.0f, 10.0f, 48); RunQDQResizeOpTest(TestInputDef({1, 3, 4, 4}, false, input_data), {1, 3, 8, 8}, "linear", "align_corners", "", - ExpectedEPNodeAssignment::All); + ExpectedEPNodeAssignment::All, + 19, + // Need tolerance of 0.533% of output range after QNN SDK 2.17 + QDQTolerance(0.00533f)); } // Test 2x QDQ Resize mode: "linear", coordinate_transformation_mode: "asymmetric" @@ -374,7 +384,10 @@ TEST_F(QnnHTPBackendTests, ResizeU8_2xLinearAsymmetric) { std::vector input_data = GetFloatDataInRange(-10.0f, 10.0f, 48); RunQDQResizeOpTest(TestInputDef({1, 3, 4, 4}, false, input_data), {1, 3, 8, 8}, "linear", "asymmetric", "", - ExpectedEPNodeAssignment::All); + ExpectedEPNodeAssignment::All, + 19, + // Need tolerance of 0.619% of output range after QNN SDK 2.17 + QDQTolerance(0.00619f)); } // Test 2x QDQ Resize mode: "nearest", coordinate_transformation_mode: "half_pixel", nearest_mode: "round_prefer_floor" diff --git a/onnxruntime/test/providers/qnn/simple_op_htp_test.cc b/onnxruntime/test/providers/qnn/simple_op_htp_test.cc index 3435bd71aa..39733f5048 100644 --- a/onnxruntime/test/providers/qnn/simple_op_htp_test.cc +++ b/onnxruntime/test/providers/qnn/simple_op_htp_test.cc @@ -93,6 +93,22 @@ TEST_F(QnnCPUBackendTests, DISABLED_SpaceToDepth_Flaky2) { } } +// Test f32 Relu on the CPU backend. +// TODO: When this is fixed, enable ActivationOpTest.Relu test in cpu/activation/activation_op_test tests. +// Disabled because QNN SDK 2.17 Relu treats inf as FLT_MAX. +// Log: the value pair (inf, 3.40282347e+38) at index #12 don't match +TEST_F(QnnCPUBackendTests, DISABLED_UnaryOp_Relu) { + std::vector input_data{-1.0f, 0, 1.0f, + 100.0f, -100.0f, 1000.0f, -1000.0f, + FLT_MIN, FLT_MIN / 10, -FLT_MIN / 10, + FLT_MAX, -FLT_MAX, std::numeric_limits::infinity()}; + RunOpTestOnCPU("Relu", + {TestInputDef({13}, false, input_data)}, + {}, + 14, + ExpectedEPNodeAssignment::All); +} + #if defined(__aarch64__) || defined(_M_ARM64) || defined(__linux__) // Tests the accuracy of a QDQ model on QNN EP by comparing to CPU EP, which runs both the fp32 model @@ -105,7 +121,7 @@ static void RunQDQOpTest(const std::string& op_type, ExpectedEPNodeAssignment expected_ep_assignment, const std::string& op_domain = kOnnxDomain, bool use_contrib_qdq = false, - float fp32_abs_err = 1e-4f) { + QDQTolerance tolerance = QDQTolerance()) { ProviderOptions provider_options; #if defined(_WIN32) provider_options["backend_path"] = "QnnHtp.dll"; @@ -118,7 +134,7 @@ static void RunQDQOpTest(const std::string& op_type, provider_options, opset_version, expected_ep_assignment, - fp32_abs_err); + tolerance); } // Runs a non-QDQ model on HTP and compares output to CPU EP. @@ -208,8 +224,7 @@ TEST_F(QnnHTPBackendTests, UnaryOp_Gelu_U16) { 11, ExpectedEPNodeAssignment::All, kMSDomain, // GeLu is a contrib op. - true, // Use MS domain Q/DQ ops. - 0.0025f); // TODO(adrianlizarraga): Accuracy + true); // Use MS domain Q/DQ ops. } // Check that QNN compiles DQ -> Elu -> Q as a single unit. @@ -280,8 +295,7 @@ TEST_F(QnnHTPBackendTests, UnaryOp_HardSwish_U16) { 14, ExpectedEPNodeAssignment::All, kOnnxDomain, - true, - 0.001f); // TODO(adrianlizarraga): Remove additional tolerance needed for inaccuracy + true); } // Check that QNN compiles DQ -> Atan -> Q as a single unit. @@ -308,8 +322,7 @@ TEST_F(QnnHTPBackendTests, UnaryOp_Atan_U16) { 14, ExpectedEPNodeAssignment::All, kOnnxDomain, // Atan domain - true, // Q/DQ op domain is com.microsoft - 1.8e-4f); + true); // Q/DQ op domain is com.microsoft } // Check that QNN compiles DQ -> Asin -> Q as a single unit. @@ -751,7 +764,7 @@ TEST_F(QnnHTPBackendTests, ContextBinaryCacheEmbedModeTest) { provider_options, 14, ExpectedEPNodeAssignment::All, - 1e-4f, + QDQTolerance(), logging::Severity::kERROR, context_binary_file); } @@ -801,7 +814,7 @@ TEST_F(QnnHTPBackendTests, ContextBinaryCacheNonEmbedModeTest) { provider_options, 14, ExpectedEPNodeAssignment::All, - 1e-4f, + QDQTolerance(), logging::Severity::kERROR, context_binary_file); } @@ -905,7 +918,7 @@ TEST_F(QnnHTPBackendTests, ContextBinary2InputsTest) { provider_options, 14, ExpectedEPNodeAssignment::All, - 1e-4f, + QDQTolerance(), logging::Severity::kERROR, context_binary_file); } @@ -1147,7 +1160,7 @@ TEST_F(QnnHTPBackendTests, BinaryOp_HTP_Or_Unsupported) { TestInputDef({1, 4}, false, {false, true, false, true})}, {}, 17, - ExpectedEPNodeAssignment::None); + ExpectedEPNodeAssignment::All); } // Test 8-bit QDQ GridSample with bilinear diff --git a/onnxruntime/test/providers/qnn/transpose_htp_test.cc b/onnxruntime/test/providers/qnn/transpose_htp_test.cc index 8d8c1ebb0f..119b8301f3 100644 --- a/onnxruntime/test/providers/qnn/transpose_htp_test.cc +++ b/onnxruntime/test/providers/qnn/transpose_htp_test.cc @@ -76,8 +76,7 @@ static void RunTransposeQDQTest(const TestInputDef& input_def, BuildQDQTransposeTestCase(input_def, attrs), provider_options, 18, - expected_ep_assignment, - 1e-5f); + expected_ep_assignment); } /** diff --git a/onnxruntime/test/providers/qnn/where_htp_test.cc b/onnxruntime/test/providers/qnn/where_htp_test.cc index 2d2aa23c28..ec525ef4eb 100644 --- a/onnxruntime/test/providers/qnn/where_htp_test.cc +++ b/onnxruntime/test/providers/qnn/where_htp_test.cc @@ -85,8 +85,7 @@ static void RunWhereQDQTest(const TestInputDef& condition_def, BuildQDQWhereTestCase(condition_def, x_def, y_def), provider_options, 18, - expected_ep_assignment, - 1e-5f); + expected_ep_assignment); } // Check that QNN compiles DQ -> Where -> Q as a single unit. @@ -121,24 +120,15 @@ TEST_F(QnnHTPBackendTests, WhereLargeDataU8) { // Check that QNN compiles DQ -> Where -> Q as a single unit. // Large data broadcast, QNN v2.13 failed to finalize graph -// C:\qnn_src\QNN\HTP\HTP\src\hexagon\prepare\seq\initial_sequencer_dp.cc:156:ERROR:A single op, -// "q::Broadcast" (Op ID: 19c700000012), requires 0x500800 bytes of TCM, which is greater than the TCM size of 0x400000! -// QnnDsp graph prepare failed 13 -// QnnDsp Failed to finalize graph QNN_4851394333842096633_1 with err: 1002 -// QnnDsp Failed to finalize graph (id: 1) with err 1002 // Worked with QNN v2.16 -TEST_F(QnnHTPBackendTests, DISABLED_WhereLargeDataBroadcastU8) { +TEST_F(QnnHTPBackendTests, WhereLargeDataBroadcastU8) { RunWhereQDQTest(TestInputDef({5120}, false, false, true), TestInputDef({1, 16, 64, 5120}, true, 0.0f, 1.0f), TestInputDef({1}, true, {3.0f}), ExpectedEPNodeAssignment::All); } -// .\hexagon\prepare\seq\initial_sequencer_dp.cc:149:ERROR:A single op, -// "q::Broadcast" (Op ID: 19a200000012), requires 0xb40000 bytes of TCM, which is greater than the TCM size of 0x400000! -// .\hexagon\prepare\seq\initial_sequencer_dp.cc : 156 : ERROR : -// The name of the failing op before optimization is : "q::QNN_ElementWiseSelect"(Op ID : 12). -TEST_F(QnnHTPBackendTests, DISABLED_WhereLargeDataBroadcastTransformedU8) { +TEST_F(QnnHTPBackendTests, WhereLargeDataBroadcastTransformedU8) { RunWhereQDQTest(TestInputDef({1, 1, 5120, 1}, false, false, true), TestInputDef({1, 64, 5120, 16}, true, 0.0f, 1.0f), TestInputDef({1, 1, 1, 1}, true, {3.0f}), diff --git a/tools/ci_build/github/azure-pipelines/android-arm64-v8a-QNN-crosscompile-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/android-arm64-v8a-QNN-crosscompile-ci-pipeline.yml index 4ebc6ea510..e2ca4f64a0 100644 --- a/tools/ci_build/github/azure-pipelines/android-arm64-v8a-QNN-crosscompile-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/android-arm64-v8a-QNN-crosscompile-ci-pipeline.yml @@ -31,7 +31,7 @@ parameters: - name: QnnSdk displayName: QNN SDK version type: string - default: qnn-v2.14.1.230828 + default: qnn-v2.17.0.231124 jobs: - job: Build_QNN_EP diff --git a/tools/ci_build/github/azure-pipelines/linux-qnn-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-qnn-ci-pipeline.yml index 491c896de8..d21b917cbd 100644 --- a/tools/ci_build/github/azure-pipelines/linux-qnn-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-qnn-ci-pipeline.yml @@ -32,7 +32,7 @@ parameters: - name: QnnSdk displayName: QNN SDK version type: string - default: qnn-v2.14.1.230828 + default: qnn-v2.17.0.231124 jobs: - job: Build_QNN_EP diff --git a/tools/ci_build/github/azure-pipelines/qnn-ep-nuget-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/qnn-ep-nuget-packaging-pipeline.yml index 654ccad3af..d9aff36c4a 100644 --- a/tools/ci_build/github/azure-pipelines/qnn-ep-nuget-packaging-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/qnn-ep-nuget-packaging-pipeline.yml @@ -2,12 +2,12 @@ parameters: - name: qnn_sdk_path_win displayName: QNN Windows SDK path type: string - default: C:\data\qnnsdk\qnn-v2.14.1.230828_win + default: C:\data\qnnsdk\qnn-v2.17.0.231124_win - name: qnn_sdk_info displayName: QNN SDK Version Information type: string - default: qnn-v2.14.1.230828_win + default: qnn-v2.17.0.231124_win - name: ort_package_version displayName: OnnxRuntime Nuget package version diff --git a/tools/ci_build/github/azure-pipelines/win-qnn-arm64-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-qnn-arm64-ci-pipeline.yml index b36a25034b..5e35cbfed6 100644 --- a/tools/ci_build/github/azure-pipelines/win-qnn-arm64-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-qnn-arm64-ci-pipeline.yml @@ -32,7 +32,7 @@ parameters: - name: QnnSdk displayName: QNN SDK version type: string - default: qnn-v2.14.1.230828_win + default: qnn-v2.17.0.231124_win jobs: - job: 'build' diff --git a/tools/ci_build/github/azure-pipelines/win-qnn-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-qnn-ci-pipeline.yml index 68e0d51480..65b2924c8b 100644 --- a/tools/ci_build/github/azure-pipelines/win-qnn-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-qnn-ci-pipeline.yml @@ -32,7 +32,7 @@ parameters: - name: QnnSdk displayName: QNN SDK version type: string - default: qnn-v2.14.1.230828_win + default: qnn-v2.17.0.231124_win jobs: - job: 'build'