From b47e1e64d7b5e0f6a5c77704e9677d301bb1a55a Mon Sep 17 00:00:00 2001 From: Adrian Lizarraga Date: Tue, 4 Feb 2025 11:42:46 -0800 Subject: [PATCH] [QNN EP] Make offloading graph input/output quantization (to CPU) the default (#23368) ### Description Makes the QNN provider option `offload_graph_io_quantization` enabled by default. It was previously disabled by default. ### Motivation and Context Enabling this option significantly decreases inference latency for many models. --- .../core/session/onnxruntime_c_api.h | 4 ++-- .../providers/qnn/qnn_execution_provider.cc | 10 ++++++---- .../test/providers/qnn/argmaxmin_op_test.cc | 1 + .../test/providers/qnn/average_pool_test.cc | 2 ++ .../test/providers/qnn/batch_norm_htp_test.cc | 2 ++ onnxruntime/test/providers/qnn/cast_test.cc | 1 + onnxruntime/test/providers/qnn/clip_op_test.cc | 2 ++ onnxruntime/test/providers/qnn/conv_test.cc | 5 +++++ .../test/providers/qnn/flatten_op_test.cc | 2 ++ .../test/providers/qnn/gather_elems_op_test.cc | 3 +++ .../test/providers/qnn/gather_op_htp_test.cc | 2 ++ onnxruntime/test/providers/qnn/gemm_op_test.cc | 3 +++ .../providers/qnn/instance_norm_htp_test.cc | 1 + .../test/providers/qnn/layer_norm_test.cc | 2 ++ .../test/providers/qnn/leakyrelu_op_htp_test.cc | 2 ++ .../test/providers/qnn/logical_comp_ops_test.cc | 3 +++ onnxruntime/test/providers/qnn/lrn_op_test.cc | 2 ++ onnxruntime/test/providers/qnn/matmul_test.cpp | 4 ++++ .../test/providers/qnn/max_min_op_test.cc | 2 ++ onnxruntime/test/providers/qnn/pad_op_test.cpp | 2 ++ onnxruntime/test/providers/qnn/pool_op_test.cpp | 2 ++ .../test/providers/qnn/qnn_basic_test.cc | 13 +++++++++++++ .../test/providers/qnn/qnn_ep_context_test.cc | 17 +++++++++++++++++ .../test/providers/qnn/qnn_test_utils.cc | 4 ++-- .../test/providers/qnn/reduce_op_test.cc | 2 ++ .../providers/qnn/reshape_expand_op_test.cc | 3 +++ onnxruntime/test/providers/qnn/resize_test.cc | 3 +++ .../test/providers/qnn/simple_op_htp_test.cc | 5 +++++ .../test/providers/qnn/slice_htp_test.cc | 1 + onnxruntime/test/providers/qnn/split_op_test.cc | 1 + .../providers/qnn/squeeze_unsqueeze_op_test.cc | 3 +++ onnxruntime/test/providers/qnn/tile_op_test.cc | 1 + onnxruntime/test/providers/qnn/topk_op_test.cc | 1 + .../test/providers/qnn/transpose_htp_test.cc | 1 + .../test/providers/qnn/where_htp_test.cc | 1 + 35 files changed, 105 insertions(+), 8 deletions(-) diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index 6fef2448be..c0392caf18 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -3665,8 +3665,8 @@ struct OrtApi { * - "1": Enabled. * "offload_graph_io_quantization": Offload graph input quantization and graph output dequantization to another * execution provider (typically CPU EP). - * - "0": Default. Disabled. QNN EP will handle quantization and dequantization of graph I/O. - * - "1": Enabled. + * - "0": Disabled. QNN EP will handle quantization and dequantization of graph I/O. + * - "1": Enabled. This is the default value. * "enable_htp_spill_fill_buffer": Enable HTP spill fill buffer setting. The flag is used while generating context binary. * - "0": Default. Disabled. * - "1": Enabled. diff --git a/onnxruntime/core/providers/qnn/qnn_execution_provider.cc b/onnxruntime/core/providers/qnn/qnn_execution_provider.cc index b1555b6050..9e61583e4e 100644 --- a/onnxruntime/core/providers/qnn/qnn_execution_provider.cc +++ b/onnxruntime/core/providers/qnn/qnn_execution_provider.cc @@ -351,13 +351,15 @@ QNNExecutionProvider::QNNExecutionProvider(const ProviderOptions& provider_optio // Add this option because this feature requires QnnSystem lib and it's no supported for Windows x86_64 platform enable_spill_fill_buffer_ = ParseBoolOption("enable_htp_spill_fill_buffer", false, provider_options_map); - model_settings_.offload_graph_io_quantization = ParseBoolOption("offload_graph_io_quantization", false, + model_settings_.offload_graph_io_quantization = ParseBoolOption("offload_graph_io_quantization", true, provider_options_map); if (disable_cpu_ep_fallback_ && model_settings_.offload_graph_io_quantization) { - LOGS_DEFAULT(WARNING) << "Fallback to CPU EP is disabled, but user configured QNN EP to offload graph I/O " - << "quantization/dequantization to another EP. Session creation will fail if the CPU EP " - << "handles the graph I/O quantization/dequantization."; + LOGS_DEFAULT(INFO) << "Fallback to CPU EP is disabled, but user tried to configure QNN EP to offload graph I/O " + << "quantization/dequantization to another EP. These are conflicting options. Fallback to CPU " + << "EP will remain disabled and graph I/O quantization/dequantization will not be offloaded " + << "to another EP."; + model_settings_.offload_graph_io_quantization = false; } static const std::string QNN_HTP_SHARED_MEMORY_ALLOCATOR_ENABLED = "enable_htp_shared_memory_allocator"; diff --git a/onnxruntime/test/providers/qnn/argmaxmin_op_test.cc b/onnxruntime/test/providers/qnn/argmaxmin_op_test.cc index da6eda1317..0dd223bca6 100644 --- a/onnxruntime/test/providers/qnn/argmaxmin_op_test.cc +++ b/onnxruntime/test/providers/qnn/argmaxmin_op_test.cc @@ -70,6 +70,7 @@ static void RunQDQArgMxxOpTest(const std::string& op_type, TestInputDef i #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; TestQDQModelAccuracy(BuildOpTestCase(op_type, {input_def}, {}, attrs), // baseline float32 model BuildQDQArgMxxTestCase(op_type, input_def, attrs), // QDQ model diff --git a/onnxruntime/test/providers/qnn/average_pool_test.cc b/onnxruntime/test/providers/qnn/average_pool_test.cc index f897a08da6..16dd84ace2 100644 --- a/onnxruntime/test/providers/qnn/average_pool_test.cc +++ b/onnxruntime/test/providers/qnn/average_pool_test.cc @@ -31,6 +31,7 @@ static void RunAveragePoolOpTest(const std::string& op_type, #else provider_options["backend_path"] = "libQnnCpu.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; RunQnnModelTest(BuildOpTestCase(op_type, input_defs, {}, attrs), provider_options, @@ -53,6 +54,7 @@ static void RunQDQAveragePoolOpTest(const std::string& op_type, #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; TestQDQModelAccuracy(BuildOpTestCase(op_type, input_defs, {}, attrs), BuildQDQOpTestCase(op_type, input_defs, {}, attrs), diff --git a/onnxruntime/test/providers/qnn/batch_norm_htp_test.cc b/onnxruntime/test/providers/qnn/batch_norm_htp_test.cc index 0a39413a4e..7471b44faf 100644 --- a/onnxruntime/test/providers/qnn/batch_norm_htp_test.cc +++ b/onnxruntime/test/providers/qnn/batch_norm_htp_test.cc @@ -160,6 +160,7 @@ static void RunBatchNormQDQTest(const TestInputDef& input_def, #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; // Runs model with DQ-> InstanceNorm -> Q and compares the outputs of the CPU and QNN EPs. TestQDQModelAccuracy(BuildBatchNormTestCase(input_def, scale_def, bias_def), @@ -180,6 +181,7 @@ static void RunBatchNormFP16Test(const TestInputDef& input_def, #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; TestInputDef input_fp16_def = ConvertToFP16InputDef(input_def); TestInputDef scale_fp16_def = ConvertToFP16InputDef(scale_def); diff --git a/onnxruntime/test/providers/qnn/cast_test.cc b/onnxruntime/test/providers/qnn/cast_test.cc index e2e4b0d714..3a90a6db6d 100644 --- a/onnxruntime/test/providers/qnn/cast_test.cc +++ b/onnxruntime/test/providers/qnn/cast_test.cc @@ -57,6 +57,7 @@ static void RunCastOpTest(const std::vector& shape, ONNX_NAMESPACE::Ten #else provider_options["backend_path"] = use_htp ? "libQnnHtp.so" : "libQnnCpu.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; if (use_htp && enable_fp16_precision) { provider_options["enable_htp_fp16_precision"] = "1"; diff --git a/onnxruntime/test/providers/qnn/clip_op_test.cc b/onnxruntime/test/providers/qnn/clip_op_test.cc index 21bd6fcc98..2ac34acbf1 100644 --- a/onnxruntime/test/providers/qnn/clip_op_test.cc +++ b/onnxruntime/test/providers/qnn/clip_op_test.cc @@ -117,6 +117,7 @@ static void RunQDQClipTestOnHTP(const TestInputDef& input_def, #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; auto f32_model_builder = BuildOpTestCase("Clip", {input_def}, {min_max_defs}, {}); auto qdq_model_builder = BuildQDQOpTestCase("Clip", {input_def}, {min_max_defs}, {}, @@ -205,6 +206,7 @@ TEST_F(QnnHTPBackendTests, Clip_U8_Rank5) { #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; RunQnnModelTest(model_fn, provider_options, diff --git a/onnxruntime/test/providers/qnn/conv_test.cc b/onnxruntime/test/providers/qnn/conv_test.cc index 446e6eab6b..d0cd14e802 100644 --- a/onnxruntime/test/providers/qnn/conv_test.cc +++ b/onnxruntime/test/providers/qnn/conv_test.cc @@ -93,6 +93,8 @@ static void RunCPUConvOpTest(const std::string& conv_op_type, const TestInputDef #else provider_options["backend_path"] = "libQnnCpu.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; + auto build_fn = BuildF32ConvTestCase(conv_op_type, input_def, weights_def, bias_def, strides, pads, dilations, group, auto_pad); RunQnnModelTest(build_fn, @@ -317,6 +319,7 @@ static void RunHTPConvOpTest(const std::string& conv_op_type, const TestInputDef #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; TestQDQModelAccuracy(BuildF32ConvTestCase(conv_op_type, input_def, weights_def, bias_def, strides, pads, dilations, group, auto_pad, output_activation), @@ -354,6 +357,7 @@ static void RunHTPConvOpPerChannelTest(const std::string& conv_op_type, const Te #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; auto f32_fn = BuildF32ConvTestCase(conv_op_type, input_def, weights_def, bias_def, strides, pads, dilations, group, auto_pad, output_activation); @@ -665,6 +669,7 @@ TEST_F(QnnHTPBackendTests, Test_QDQConvWithDynamicWeightsFromMul) { #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; auto BuildConvMulGraph = [](ModelTestBuilder& builder) { // DQ node for Conv input diff --git a/onnxruntime/test/providers/qnn/flatten_op_test.cc b/onnxruntime/test/providers/qnn/flatten_op_test.cc index b33f8f9c00..9c87fae512 100644 --- a/onnxruntime/test/providers/qnn/flatten_op_test.cc +++ b/onnxruntime/test/providers/qnn/flatten_op_test.cc @@ -101,6 +101,7 @@ static void RunQDQFlattenTestOnHTP(const TestInputDef& input_def, #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; auto f32_model_builder = BuildOpTestCase("Flatten", {input_def}, {}, attrs); auto qdq_model_builder = BuildQDQOpTestCase("Flatten", {input_def}, {}, attrs, kOnnxDomain, use_contrib_qdq); @@ -172,6 +173,7 @@ TEST_F(QnnHTPBackendTests, Flatten_QDQ8bit_Rank5) { #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; RunQnnModelTest(model_fn, provider_options, diff --git a/onnxruntime/test/providers/qnn/gather_elems_op_test.cc b/onnxruntime/test/providers/qnn/gather_elems_op_test.cc index 85dc792666..4cce75d692 100644 --- a/onnxruntime/test/providers/qnn/gather_elems_op_test.cc +++ b/onnxruntime/test/providers/qnn/gather_elems_op_test.cc @@ -67,6 +67,7 @@ static void RunCPUGatherElemsOpTest(const TestInputDef& input_def, #else provider_options["backend_path"] = "libQnnCpu.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; RunQnnModelTest(BuildOpTestCase("GatherElements", {input_def}, {indices_def}, attrs), provider_options, @@ -91,6 +92,7 @@ static void RunHTPQDQGatherElemsOpTest(const TestInputDef& input_def, #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; auto f32_model_builder = BuildOpTestCase("GatherElements", {input_def}, {indices_def}, attrs); auto qdq_model_builder = BuildQDQGatherElemsTestCase(input_def, indices_def, attrs, @@ -119,6 +121,7 @@ static void RunHTPGatherElemsOpTest(const TestInputDef& input_def, #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; RunQnnModelTest(BuildOpTestCase("GatherElements", {input_def}, {indices_def}, attrs), provider_options, diff --git a/onnxruntime/test/providers/qnn/gather_op_htp_test.cc b/onnxruntime/test/providers/qnn/gather_op_htp_test.cc index 1d6fd53d06..31ac98a60b 100644 --- a/onnxruntime/test/providers/qnn/gather_op_htp_test.cc +++ b/onnxruntime/test/providers/qnn/gather_op_htp_test.cc @@ -63,6 +63,7 @@ static void RunQDQGatherOpTest(const TestInputDef& input_def, #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; auto f32_model_builder = BuildOpTestCase("Gather", {input_def}, {indices_def}, attrs); auto qdq_model_builder = BuildQDQGatherTestCase(input_def, indices_def, attrs, @@ -178,6 +179,7 @@ static void RunOpTest(const std::string& op_type, #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; // Runs model with a Q/DQ binary op and compares the outputs of the CPU and QNN EPs. RunQnnModelTest(BuildOpTestCase(op_type, {input_def_1}, {input_defs_2}, attrs, op_domain), diff --git a/onnxruntime/test/providers/qnn/gemm_op_test.cc b/onnxruntime/test/providers/qnn/gemm_op_test.cc index 0c1146ba22..4f6af0a9a4 100644 --- a/onnxruntime/test/providers/qnn/gemm_op_test.cc +++ b/onnxruntime/test/providers/qnn/gemm_op_test.cc @@ -29,6 +29,7 @@ static void RunGemmTestOnCPU(const std::vector>& input_de #else provider_options["backend_path"] = "libQnnCpu.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; RunQnnModelTest(BuildOpTestCase("Gemm", input_defs, {}, attrs), provider_options, @@ -246,6 +247,8 @@ static void RunQDQGemmTestOnHTP(const std::vector>& input_de #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; + auto f32_model_builder = BuildOpTestCase("Gemm", input_defs, {}, attrs); auto qdq_model_builder = BuildQDQGemmTestCase(input_defs, attrs, use_contrib_qdq); TestQDQModelAccuracy(f32_model_builder, diff --git a/onnxruntime/test/providers/qnn/instance_norm_htp_test.cc b/onnxruntime/test/providers/qnn/instance_norm_htp_test.cc index 3598ba1ac8..d4f66b72e0 100644 --- a/onnxruntime/test/providers/qnn/instance_norm_htp_test.cc +++ b/onnxruntime/test/providers/qnn/instance_norm_htp_test.cc @@ -79,6 +79,7 @@ static void RunInstanceNormQDQTest(const TestInputDef& input_def, #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; // Runs model with DQ-> InstanceNorm -> Q and compares the outputs of the CPU and QNN EPs. TestQDQModelAccuracy(BuildOpTestCase("InstanceNormalization", {input_def, scale_def, bias_def}, {}, attrs), diff --git a/onnxruntime/test/providers/qnn/layer_norm_test.cc b/onnxruntime/test/providers/qnn/layer_norm_test.cc index 947ac19be4..b2997c6278 100644 --- a/onnxruntime/test/providers/qnn/layer_norm_test.cc +++ b/onnxruntime/test/providers/qnn/layer_norm_test.cc @@ -28,6 +28,7 @@ static void RunLayerNormCpuTest(const TestInputDef& input_def, #else provider_options["backend_path"] = "libQnnCpu.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; RunQnnModelTest(BuildOpTestCase("LayerNormalization", {input_def, scale_def}, {}, attrs), provider_options, @@ -152,6 +153,7 @@ static void RunLayerNormQDQTest(const TestInputDef& input_def, #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; TestQDQModelAccuracy(BuildOpTestCase("LayerNormalization", {input_def, scale_def}, {}, attrs), BuildQDQLayerNormTestCase(input_def, scale_def, bias_def, attrs, diff --git a/onnxruntime/test/providers/qnn/leakyrelu_op_htp_test.cc b/onnxruntime/test/providers/qnn/leakyrelu_op_htp_test.cc index ece8d91d53..77d96b56d2 100644 --- a/onnxruntime/test/providers/qnn/leakyrelu_op_htp_test.cc +++ b/onnxruntime/test/providers/qnn/leakyrelu_op_htp_test.cc @@ -28,6 +28,7 @@ static void RunLeakyReluOpQDQTest(const TestInputDef& input_def, #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; TestQDQModelAccuracy(BuildOpTestCase("LeakyRelu", {input_def}, {}, attrs), BuildQDQOpTestCase("LeakyRelu", {input_def}, {}, attrs), @@ -66,6 +67,7 @@ TEST_F(QnnHTPBackendTests, LeakyReluFP16OpSet16) { #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; auto input_def = TestInputDef({1, 2, 3}, false, {-40.0f, -20.0f, 1.0f, 10.0f, 30.0f, 40.0f}); TestInputDef input_fp16_def = ConvertToFP16InputDef(input_def); diff --git a/onnxruntime/test/providers/qnn/logical_comp_ops_test.cc b/onnxruntime/test/providers/qnn/logical_comp_ops_test.cc index 522b781379..eca018c4b5 100644 --- a/onnxruntime/test/providers/qnn/logical_comp_ops_test.cc +++ b/onnxruntime/test/providers/qnn/logical_comp_ops_test.cc @@ -73,6 +73,7 @@ static void RunCPULogicalOpTest(const std::string& op_type, const std::vector(op_type, shape), provider_options, @@ -157,6 +159,7 @@ TEST_F(QnnHTPBackendTests, EqualToCast4D) { #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; // Model building function that creates a QDQ graph with an Equal node followed by // a Cast to float32. diff --git a/onnxruntime/test/providers/qnn/lrn_op_test.cc b/onnxruntime/test/providers/qnn/lrn_op_test.cc index 4b26ed0da9..1744be53d5 100644 --- a/onnxruntime/test/providers/qnn/lrn_op_test.cc +++ b/onnxruntime/test/providers/qnn/lrn_op_test.cc @@ -70,6 +70,7 @@ static void RunCPULRNOpTest(const TestInputDef& input_def, int64_t size, provider_options["backend_path"] = "libQnnCpu.so"; fp32_abs_err = 1.5e-5f; // On linux we need slightly larger tolerance. #endif + provider_options["offload_graph_io_quantization"] = "0"; RunQnnModelTest(BuildLRNTestCase(input_def, size, alpha, beta, bias), provider_options, @@ -91,6 +92,7 @@ static void RunQDQLRNOpTest(const TestInputDef& input_def, int64_t size, #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; TestQDQModelAccuracy(BuildLRNTestCase(input_def, size, alpha, beta, bias), BuildQDQLRNTestCase(input_def, size, alpha, beta, bias), diff --git a/onnxruntime/test/providers/qnn/matmul_test.cpp b/onnxruntime/test/providers/qnn/matmul_test.cpp index dec9369b81..acc653d8bd 100644 --- a/onnxruntime/test/providers/qnn/matmul_test.cpp +++ b/onnxruntime/test/providers/qnn/matmul_test.cpp @@ -44,6 +44,7 @@ static void RunMatMulOpTest(bool is_htp_backend, const std::vector& sha provider_options["backend_path"] = "libQnnCpu.so"; #endif } + provider_options["offload_graph_io_quantization"] = "0"; RunQnnModelTest(BuildMatMulOpTestCase( TestInputDef(shape_0, is_initializer_0, GetSequentialFloatData(shape_0, 0.01f, 0.02f)), @@ -142,6 +143,7 @@ static void RunQDQMatMulOpTest(const std::vector& shape_0, const std::v #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; TestInputDef input0_def( shape_0, is_initializer_0, @@ -172,6 +174,7 @@ static void RunQDQPerChannelMatMulOpTest( #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; if (enable_fp16_precision) { provider_options["enable_htp_fp16_precision"] = "1"; @@ -314,6 +317,7 @@ TEST_F(QnnHTPBackendTests, MatMulOp_QDQ_Regression_uint16_dynamic_inputs) { #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; // Test with rank 4 inputs { diff --git a/onnxruntime/test/providers/qnn/max_min_op_test.cc b/onnxruntime/test/providers/qnn/max_min_op_test.cc index 9a45d11b7e..c7f5eaa0a0 100644 --- a/onnxruntime/test/providers/qnn/max_min_op_test.cc +++ b/onnxruntime/test/providers/qnn/max_min_op_test.cc @@ -26,6 +26,7 @@ static void RunCPUMinOrMaxOpTest(const std::string& op_type, #else provider_options["backend_path"] = "libQnnCpu.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; RunQnnModelTest(BuildOpTestCase(op_type, input_defs, {}, {}, kOnnxDomain), provider_options, @@ -47,6 +48,7 @@ static void RunQDQMinOrMaxOpTest(const std::string& op_type, #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; TestQDQModelAccuracy(BuildOpTestCase(op_type, input_defs, {}, {}, kOnnxDomain), // baseline float32 model BuildQDQOpTestCase(op_type, input_defs, {}, {}, kOnnxDomain), // QDQ model diff --git a/onnxruntime/test/providers/qnn/pad_op_test.cpp b/onnxruntime/test/providers/qnn/pad_op_test.cpp index 4ce6db7fac..ae8c4428e2 100644 --- a/onnxruntime/test/providers/qnn/pad_op_test.cpp +++ b/onnxruntime/test/providers/qnn/pad_op_test.cpp @@ -116,6 +116,7 @@ static void RunPadOpTest(const TestInputDef& data_def, provider_options["backend_path"] = "libQnnCpu.so"; #endif } + provider_options["offload_graph_io_quantization"] = "0"; if (enable_fp16_precision) { provider_options["enable_htp_fp16_precision"] = "1"; @@ -144,6 +145,7 @@ static void RunQDQPadOpTest(const TestInputDef& data_def, #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; TestQDQModelAccuracy(BuildPadTestCase(data_def, pads_def, constant_value_def, attrs), BuildPadQDQTestCase(data_def, pads_def, constant_value_def, attrs, diff --git a/onnxruntime/test/providers/qnn/pool_op_test.cpp b/onnxruntime/test/providers/qnn/pool_op_test.cpp index f0ca355719..76399b413f 100644 --- a/onnxruntime/test/providers/qnn/pool_op_test.cpp +++ b/onnxruntime/test/providers/qnn/pool_op_test.cpp @@ -60,6 +60,7 @@ static void RunPoolOpTest(const std::string& op_type, #else provider_options["backend_path"] = "libQnnCpu.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; RunQnnModelTest(BuildOpTestCase(op_type, {input_def}, {}, attrs), provider_options, @@ -83,6 +84,7 @@ static void RunQDQPoolOpTest(const std::string& op_type, #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; TestQDQModelAccuracy(BuildOpTestCase(op_type, {input_def}, {}, attrs), BuildPoolQDQTestCase(op_type, input_def, attrs, use_contrib_qdq_ops), diff --git a/onnxruntime/test/providers/qnn/qnn_basic_test.cc b/onnxruntime/test/providers/qnn/qnn_basic_test.cc index a6fb664728..b58610a258 100644 --- a/onnxruntime/test/providers/qnn/qnn_basic_test.cc +++ b/onnxruntime/test/providers/qnn/qnn_basic_test.cc @@ -120,6 +120,7 @@ TEST(QnnEP, TestDisableCPUFallback_ModelNotFullySupported) { #else options["backend_path"] = "libQnnCpu.so"; #endif + options["offload_graph_io_quantization"] = "0"; so.AppendExecutionProvider("QNN", options); @@ -151,6 +152,7 @@ TEST(QnnEP, TestDisableCPUFallback_TryingToRunOnQnnCPU) { #else options["backend_path"] = "libQnnCpu.so"; #endif + options["offload_graph_io_quantization"] = "0"; auto input_defs = {TestInputDef({1, 2, 2, 2}, false, -10.0f, 10.0f), TestInputDef({1, 2, 2, 2}, false, -10.0f, 10.0f)}; @@ -199,6 +201,7 @@ TEST(QnnEP, TestDisableCPUFallback_ConflictingConfig) { #else options["backend_path"] = "libQnnCpu.so"; #endif + options["offload_graph_io_quantization"] = "0"; so.AppendExecutionProvider("QNN", options); @@ -229,6 +232,7 @@ TEST_F(QnnHTPBackendTests, TestConvWithExternalData) { #else options["backend_path"] = "libQnnHtp.so"; #endif + options["offload_graph_io_quantization"] = "0"; so.AppendExecutionProvider("QNN", options); @@ -304,6 +308,7 @@ static void RunNHWCResizeModel(const ORTCHAR_T* ort_model_path, bool use_htp, bo so.SetGraphOptimizationLevel(ORT_ENABLE_ALL); onnxruntime::ProviderOptions options; + options["offload_graph_io_quantization"] = "0"; #if defined(_WIN32) options["backend_path"] = use_htp ? "QnnHtp.dll" : "QnnCpu.dll"; @@ -594,6 +599,7 @@ TEST_F(QnnHTPBackendTests, MultithreadSessionRun) { #else options["backend_path"] = "libQnnHtp.so"; #endif + options["offload_graph_io_quantization"] = "0"; auto qnn_ep = QnnExecutionProviderWithOptions(options, &session_opts); EXPECT_TRUE(session_obj.RegisterExecutionProvider(std::move(qnn_ep)).IsOK()); @@ -643,6 +649,7 @@ TEST_F(QnnHTPBackendTests, MultithreadHtpPowerCfgSessionRunOption) { #else options["backend_path"] = "libQnnHtp.so"; #endif + options["offload_graph_io_quantization"] = "0"; auto qnn_ep = QnnExecutionProviderWithOptions(options, &session_opts); EXPECT_TRUE(session_obj.RegisterExecutionProvider(std::move(qnn_ep)).IsOK()); @@ -708,6 +715,7 @@ TEST_F(QnnHTPBackendTests, MultithreadDefaultHtpPowerCfgFromEpOption) { #else options["backend_path"] = "libQnnHtp.so"; #endif + options["offload_graph_io_quantization"] = "0"; options["htp_performance_mode"] = "burst"; auto qnn_ep = QnnExecutionProviderWithOptions(options, &session_opts); @@ -759,6 +767,7 @@ TEST_F(QnnHTPBackendTests, MultithreadHtpPowerCfgDefaultAndRunOption) { #else options["backend_path"] = "libQnnHtp.so"; #endif + options["offload_graph_io_quantization"] = "0"; options["htp_performance_mode"] = "burst"; auto qnn_ep = QnnExecutionProviderWithOptions(options, &session_opts); @@ -923,6 +932,7 @@ TEST_F(QnnHTPBackendTests, ProfilingTest) { #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; provider_options["enable_htp_fp16_precision"] = "1"; provider_options["profiling_level"] = "detailed"; provider_options["profiling_file_path"] = "detailed_profile.csv"; @@ -943,6 +953,7 @@ TEST_F(QnnHTPBackendTests, CastAddHTPAccuracyTest) { #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; RunQnnModelTest(BuildCastAddTestCase(), provider_options, @@ -1013,6 +1024,7 @@ TEST_F(QnnHTPBackendTests, EPRejectsDynamicShapesF32) { #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; provider_options["enable_htp_fp16_precision"] = "1"; // QNN EP will use fp16 precision. // CPU EP will use fp32, so we can relax accuracy requirements. @@ -1111,6 +1123,7 @@ TEST_F(QnnHTPBackendTests, UseHtpSharedMemoryAllocatorForInputs) { #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; provider_options["enable_htp_shared_memory_allocator"] = "1"; std::unique_ptr qnn_ep; diff --git a/onnxruntime/test/providers/qnn/qnn_ep_context_test.cc b/onnxruntime/test/providers/qnn/qnn_ep_context_test.cc index a4ecf2a240..8670fbbe2b 100644 --- a/onnxruntime/test/providers/qnn/qnn_ep_context_test.cc +++ b/onnxruntime/test/providers/qnn/qnn_ep_context_test.cc @@ -101,6 +101,7 @@ void QnnContextBinaryMultiPartitionTestBody(bool single_ep_node = true) { #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; const std::unordered_map domain_to_version = {{"", 13}, {kMSDomain, 1}}; @@ -314,6 +315,7 @@ TEST_F(QnnHTPBackendTests, QnnContextBinaryGeneration2InputTypes) { #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; const std::unordered_map domain_to_version = {{"", 13}, {kMSDomain, 1}}; @@ -364,6 +366,7 @@ TEST_F(QnnHTPBackendTests, QnnContextGeneration2InputsOrderIssue) { #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; // Add kMSDomain to cover contrib op like Gelu const std::unordered_map domain_to_version = {{"", 13}, {kMSDomain, 1}}; @@ -400,6 +403,7 @@ TEST_F(QnnHTPBackendTests, QnnContextGenerationNodeNamePrefix) { #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; std::string node_name_prefix = "node_name_prefix_test"; // Add kMSDomain to cover contrib op like Gelu @@ -442,6 +446,7 @@ TEST_F(QnnHTPBackendTests, QnnContextBinaryCacheEmbedModeTest) { #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; const std::string context_binary_file = "./qnn_context_binary_test.onnx"; std::remove(context_binary_file.c_str()); @@ -490,6 +495,7 @@ TEST_F(QnnHTPBackendTests, QnnContextBinaryCacheNonEmbedModeTest) { #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; const std::string context_binary_file = "./testdata/qnn_context_cache_non_embed.onnx"; std::string qnn_ctx_bin = "./testdata/qnn_context_cache_non_embed.onnx_QNNExecutionProvider_QNN_8283143575221199085_1_0.bin"; @@ -571,6 +577,7 @@ TEST_F(QnnHTPBackendTests, QnnContextBinaryCache_InvalidGraph) { #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; const std::string context_binary_file = "./qnn_context_cache_non_embed.onnx"; std::filesystem::path context_bin = "qnn_context_cache_non_embed.onnx_QNNExecutionProvider_QNN_8283143575221199085_1_0.bin"; std::remove(context_binary_file.c_str()); @@ -668,6 +675,7 @@ TEST_F(QnnHTPBackendTests, QnnContextBinaryRelativePathTest) { #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(QnnExecutionProviderWithOptions(provider_options))); ASSERT_STATUS_OK(session_object.Load(model_data.data(), static_cast(model_data.size()))); @@ -698,6 +706,7 @@ TEST_F(QnnHTPBackendTests, QnnContextBinaryAbsolutePathTest) { #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(QnnExecutionProviderWithOptions(provider_options))); ASSERT_STATUS_OK(session_object.Load(model_data.data(), static_cast(model_data.size()))); @@ -723,6 +732,7 @@ TEST_F(QnnHTPBackendTests, QnnContextBinaryFileNotExistTest) { #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(QnnExecutionProviderWithOptions(provider_options))); ASSERT_STATUS_OK(session_object.Load(model_data.data(), static_cast(model_data.size()))); @@ -748,6 +758,7 @@ TEST_F(QnnHTPBackendTests, QnnContextBinaryFileEmptyStringTest) { #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(QnnExecutionProviderWithOptions(provider_options))); ASSERT_STATUS_OK(session_object.Load(model_data.data(), static_cast(model_data.size()))); @@ -765,6 +776,7 @@ TEST_F(QnnHTPBackendTests, QnnContextBinary2InputsTest) { #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; const std::string context_binary_file = "./qnn_context_binary_2inputs_test.onnx"; std::remove(context_binary_file.c_str()); @@ -816,6 +828,7 @@ TEST_F(QnnHTPBackendTests, QnnContextBinaryCache_SingleNodeNameNotMatchGraphName #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; const std::string context_binary_file = "./qnn_context_cache_non_embed.onnx"; std::filesystem::path context_bin = "qnn_context_cache_non_embed.onnx_QNNExecutionProvider_QNN_8283143575221199085_1_0.bin"; std::remove(context_binary_file.c_str()); @@ -893,6 +906,7 @@ TEST_F(QnnHTPBackendTests, QnnMultiContextEmbeded) { #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; Ort::SessionOptions so; so.AppendExecutionProvider("QNN", provider_options); @@ -908,6 +922,7 @@ TEST_F(QnnHTPBackendTests, QnnMultiContextExternal) { #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; Ort::SessionOptions so; so.AppendExecutionProvider("QNN", provider_options); @@ -1037,6 +1052,7 @@ TEST_F(QnnHTPBackendTests, QnnContextShareAcrossSessions1) { #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; // Create QDQ models std::vector onnx_model_paths{"./weight_share1.onnx", "./weight_share2.onnx"}; @@ -1134,6 +1150,7 @@ TEST_F(QnnHTPBackendTests, QnnContextShareAcrossSessions2) { #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; // Create QDQ models std::vector onnx_model_paths{"./weight_share21.onnx", "./weight_share22.onnx"}; diff --git a/onnxruntime/test/providers/qnn/qnn_test_utils.cc b/onnxruntime/test/providers/qnn/qnn_test_utils.cc index 4feeb5f830..cb385267ba 100644 --- a/onnxruntime/test/providers/qnn/qnn_test_utils.cc +++ b/onnxruntime/test/providers/qnn/qnn_test_utils.cc @@ -278,7 +278,7 @@ static BackendSupport GetHTPSupport(const onnxruntime::logging::Logger& logger) MockKernelLookup kernel_lookup; onnxruntime::GraphViewer graph_viewer(graph); std::unique_ptr qnn_ep = QnnExecutionProviderWithOptions( - {{"backend_path", "QnnHtp.dll"}}); + {{"backend_path", "QnnHtp.dll"}, {"offload_graph_io_quantization", "0"}}); qnn_ep->SetLogger(&logger); auto result = qnn_ep->GetCapability(graph_viewer, kernel_lookup); @@ -341,7 +341,7 @@ static BackendSupport GetCPUSupport(const onnxruntime::logging::Logger& logger) MockKernelLookup kernel_lookup; onnxruntime::GraphViewer graph_viewer(graph); std::unique_ptr qnn_ep = QnnExecutionProviderWithOptions( - {{"backend_path", "QnnCpu.dll"}}); + {{"backend_path", "QnnCpu.dll"}, {"offload_graph_io_quantization", "0"}}); qnn_ep->SetLogger(&logger); auto result = qnn_ep->GetCapability(graph_viewer, kernel_lookup); diff --git a/onnxruntime/test/providers/qnn/reduce_op_test.cc b/onnxruntime/test/providers/qnn/reduce_op_test.cc index b7b1960ab2..f36909d362 100644 --- a/onnxruntime/test/providers/qnn/reduce_op_test.cc +++ b/onnxruntime/test/providers/qnn/reduce_op_test.cc @@ -82,6 +82,7 @@ static void RunReduceTest(const std::string& op_type, float fp32_abs_err = 1e-5f, bool enable_fp16 = false) { ProviderOptions provider_options; + provider_options["offload_graph_io_quantization"] = "0"; if (enable_fp16) { #if defined(_WIN32) provider_options["backend_path"] = "QnnHtp.dll"; @@ -427,6 +428,7 @@ static void RunReduceOpQDQTest(const std::string& op_type, #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; constexpr bool noop_with_empty_axes = false; const bool axes_as_input = ReduceOpHasAxesInput(op_type, opset); // Later opsets have "axes" as an input. diff --git a/onnxruntime/test/providers/qnn/reshape_expand_op_test.cc b/onnxruntime/test/providers/qnn/reshape_expand_op_test.cc index b66547a939..ea55cb4609 100644 --- a/onnxruntime/test/providers/qnn/reshape_expand_op_test.cc +++ b/onnxruntime/test/providers/qnn/reshape_expand_op_test.cc @@ -30,6 +30,7 @@ static void RunReshapeExpandTestOnCPU(const std::string& op_type, #else provider_options["backend_path"] = "libQnnCpu.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; RunQnnModelTest(BuildOpTestCase(op_type, {input_def}, {shape_def}, attrs), provider_options, @@ -161,6 +162,7 @@ static void RunReshapeExpandTestOnHTP(const std::string& op_type, #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; RunQnnModelTest(BuildOpTestCase(op_type, {input_def}, {shape_def}, attrs), provider_options, @@ -185,6 +187,7 @@ static void RunQDQReshapeExpandTestOnHTP(const std::string& op_type, #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; auto f32_model_builder = BuildOpTestCase(op_type, {input_def}, {shape_def}, attrs); auto qdq_model_builder = BuildQDQReshapeExpandTestCase(op_type, input_def, shape_def, attrs, use_contrib_qdq); diff --git a/onnxruntime/test/providers/qnn/resize_test.cc b/onnxruntime/test/providers/qnn/resize_test.cc index 651f55bc05..f323713d60 100644 --- a/onnxruntime/test/providers/qnn/resize_test.cc +++ b/onnxruntime/test/providers/qnn/resize_test.cc @@ -127,6 +127,7 @@ static void RunCPUResizeOpTest(const TestInputDef& input_def, const std:: #else provider_options["backend_path"] = "libQnnCpu.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; RunQnnModelTest(GetResizeModelBuilder(input_def, sizes_data, mode, coordinate_transformation_mode, nearest_mode), provider_options, @@ -145,6 +146,7 @@ static void RunCPUResizeOpTestWithScales(const TestInputDef& input_def, c #else provider_options["backend_path"] = "libQnnCpu.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; RunQnnModelTest(GetResizeModelBuilderWithScales(input_def, scales_data, mode, coordinate_transformation_mode, nearest_mode), provider_options, @@ -166,6 +168,7 @@ static void RunQDQResizeOpTest(const TestInputDef& input_def, #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; TestQDQModelAccuracy(GetResizeModelBuilder(input_def, sizes_data, mode, coordinate_transformation_mode, nearest_mode), GetQDQResizeModelBuilder(input_def, sizes_data, mode, coordinate_transformation_mode, diff --git a/onnxruntime/test/providers/qnn/simple_op_htp_test.cc b/onnxruntime/test/providers/qnn/simple_op_htp_test.cc index 83a02a61b1..36f3d3845c 100644 --- a/onnxruntime/test/providers/qnn/simple_op_htp_test.cc +++ b/onnxruntime/test/providers/qnn/simple_op_htp_test.cc @@ -32,6 +32,7 @@ static void RunOpTestOnCPU(const std::string& op_type, #else provider_options["backend_path"] = "libQnnCpu.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; RunQnnModelTest(BuildOpTestCase(op_type, input_defs, {}, attrs, op_domain), provider_options, @@ -129,6 +130,7 @@ static void RunQDQOpTest(const std::string& op_type, #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; TestQDQModelAccuracy(BuildOpTestCase(op_type, input_defs, {}, attrs, op_domain), BuildQDQOpTestCase(op_type, input_defs, {}, attrs, op_domain, use_contrib_qdq), @@ -787,6 +789,7 @@ TEST_F(QnnHTPBackendTests, QuantAccuracyTest) { #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; // Note: a graph input -> Q -> DQ -> is optimized by Qnn to have a perfectly accurate output. // ORT's CPU EP, on the otherhand, actually quantizes and dequantizes the input, which leads to different outputs. @@ -1218,6 +1221,7 @@ TEST_F(QnnHTPBackendTests, Add_U8_U16_Convert) { #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; TestQDQModelAccuracy(BuildOpTestCase("Add", {input0_def, input1_def}, {}, {}, kOnnxDomain), BuildQDQConvertAddTestCase(input0_def, input1_def), @@ -1283,6 +1287,7 @@ TEST_F(QnnHTPBackendTests, DQ_Q_ConvertFusion_SameType) { #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; QuantParams out_qparams_u8 = {1.0f, 128}; QuantParams out_qparams_u16 = {1.0f, 32768}; diff --git a/onnxruntime/test/providers/qnn/slice_htp_test.cc b/onnxruntime/test/providers/qnn/slice_htp_test.cc index 07c97d2d7b..dc16192188 100644 --- a/onnxruntime/test/providers/qnn/slice_htp_test.cc +++ b/onnxruntime/test/providers/qnn/slice_htp_test.cc @@ -84,6 +84,7 @@ static void RunSliceQDQTest(const TestInputDef& data_def, #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; const std::vector> f32_inputs = {data_def}; const std::vector> int64_inputs = {starts_def, ends_def, axes_def, steps_def}; diff --git a/onnxruntime/test/providers/qnn/split_op_test.cc b/onnxruntime/test/providers/qnn/split_op_test.cc index 23682f7e93..52603b1be5 100644 --- a/onnxruntime/test/providers/qnn/split_op_test.cc +++ b/onnxruntime/test/providers/qnn/split_op_test.cc @@ -276,6 +276,7 @@ static void RunQDQSplitOpTestOnHTP(const TestInputDef& input_def, #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; const bool split_is_input = opset >= 13; auto f32_model_builder = BuildSplitTestCase(input_def, split, split_is_input, axis, num_outputs); diff --git a/onnxruntime/test/providers/qnn/squeeze_unsqueeze_op_test.cc b/onnxruntime/test/providers/qnn/squeeze_unsqueeze_op_test.cc index abc1b3a89d..ead1843e84 100644 --- a/onnxruntime/test/providers/qnn/squeeze_unsqueeze_op_test.cc +++ b/onnxruntime/test/providers/qnn/squeeze_unsqueeze_op_test.cc @@ -161,6 +161,7 @@ static void RunQDQSqueezeTestOnHTP(const std::string& op_type, #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; auto f32_model_builder = BuildOpTestCase(op_type, {input_def}, {axes_def}, {}); auto qdq_model_builder = BuildQDQSqueezeTestCase(op_type, input_def, axes_def, use_contrib_qdq); @@ -219,6 +220,7 @@ TEST_F(QnnHTPBackendTests, Squeeze_Rank5_Rank2_f32) { #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; RunQnnModelTest(model_fn, provider_options, @@ -275,6 +277,7 @@ TEST_F(QnnHTPBackendTests, Unsqueeze_Rank3_Rank5_f32) { #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; RunQnnModelTest(model_fn, provider_options, diff --git a/onnxruntime/test/providers/qnn/tile_op_test.cc b/onnxruntime/test/providers/qnn/tile_op_test.cc index 85541efe56..023d6562c6 100644 --- a/onnxruntime/test/providers/qnn/tile_op_test.cc +++ b/onnxruntime/test/providers/qnn/tile_op_test.cc @@ -98,6 +98,7 @@ static void RunQDQTileTestOnHTP(const TestInputDef& input_def, #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; auto f32_model_builder = BuildOpTestCase("Tile", {input_def}, {repeats_def}, {}); auto qdq_model_builder = BuildQDQTileTestCase(input_def, repeats_def, use_contrib_qdq); diff --git a/onnxruntime/test/providers/qnn/topk_op_test.cc b/onnxruntime/test/providers/qnn/topk_op_test.cc index 354a5d1e3b..b338dd727b 100644 --- a/onnxruntime/test/providers/qnn/topk_op_test.cc +++ b/onnxruntime/test/providers/qnn/topk_op_test.cc @@ -154,6 +154,7 @@ static void RunQDQTopKTestOnHTP(const TestInputDef& input_def, #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; auto f32_model_builder = BuildTopKTestCase(input_def, k_def, attrs); auto qdq_model_builder = BuildQDQTopKTestCase(input_def, k_def, attrs, use_contrib_qdq); diff --git a/onnxruntime/test/providers/qnn/transpose_htp_test.cc b/onnxruntime/test/providers/qnn/transpose_htp_test.cc index 63746e22d2..b7bec34f7d 100644 --- a/onnxruntime/test/providers/qnn/transpose_htp_test.cc +++ b/onnxruntime/test/providers/qnn/transpose_htp_test.cc @@ -70,6 +70,7 @@ static void RunTransposeQDQTest(const TestInputDef& input_def, #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; // Runs model with DQ-> Transpose -> Q and compares the outputs of the CPU and QNN EPs. TestQDQModelAccuracy(BuildTransposeTestCase(input_def, attrs), diff --git a/onnxruntime/test/providers/qnn/where_htp_test.cc b/onnxruntime/test/providers/qnn/where_htp_test.cc index ec525ef4eb..e1b0604b31 100644 --- a/onnxruntime/test/providers/qnn/where_htp_test.cc +++ b/onnxruntime/test/providers/qnn/where_htp_test.cc @@ -79,6 +79,7 @@ static void RunWhereQDQTest(const TestInputDef& condition_def, #else provider_options["backend_path"] = "libQnnHtp.so"; #endif + provider_options["offload_graph_io_quantization"] = "0"; // Runs model with DQ-> Where -> Q and compares the outputs of the CPU and QNN EPs. TestQDQModelAccuracy(BuildWhereTestCase(condition_def, x_def, y_def),