From e5404231799a07e731ed0e4d8f0825e8ccc2a069 Mon Sep 17 00:00:00 2001
From: Hector Li <hecli@microsoft.com>
Date: Fri, 3 May 2024 08:55:31 -0700
Subject: [PATCH] [QNN EP] Conv ConvTranspose 3D support (#20507)

### Description
Support Conv ConvTranspose 3D for QNN EP
---
 onnxruntime/core/providers/common.h           |  19 +
 .../cpu/nn/conv_transpose_attributes.h        |  19 -
 .../qnn/builder/opbuilder/base_op_builder.h   |  56 +-
 .../qnn/builder/opbuilder/conv_op_builder.cc  | 245 ++++-----
 .../builder/opbuilder/simple_op_builder.cc    |  11 -
 .../core/providers/qnn/builder/qnn_def.h      |   5 +
 .../providers/qnn/builder/qnn_model_wrapper.h |  26 +-
 .../core/providers/qnn/builder/qnn_utils.cc   |   6 +
 onnxruntime/test/onnx/main.cc                 |   3 +-
 onnxruntime/test/providers/qnn/conv_test.cc   | 510 ++++++++++++++++--
 10 files changed, 684 insertions(+), 216 deletions(-)

diff --git a/onnxruntime/core/providers/common.h b/onnxruntime/core/providers/common.h
index 16c8e2045f..564c7690c8 100644
--- a/onnxruntime/core/providers/common.h
+++ b/onnxruntime/core/providers/common.h
@@ -146,6 +146,25 @@ inline Status ComputePadAndOutputShape(const int64_t in_dim,
   return Status::OK();
 }
 
+constexpr inline int64_t ComputeTotalPad(int64_t in_size, int64_t stride, int64_t adj,
+                                         int64_t kernel, int64_t dilation, int64_t out_size) {
+  return std::max<int64_t>(0, (in_size - 1) * stride + adj + (kernel - 1) * dilation + 1 - out_size);
+}
+
+inline void DistributePadding(AutoPadType pad_type, const int64_t& total_pad,
+                              int64_t& pad_head, int64_t& pad_tail) {
+  if (pad_type == AutoPadType::SAME_UPPER) {
+    // pad more on tail when total_pad is odd.
+    pad_head = total_pad / 2;
+    pad_tail = total_pad - total_pad / 2;
+  } else {
+    // When pad_type is NOTSET, SAME_LOWER or VALID,
+    // pad more on head when total_pad is odd.
+    pad_head = total_pad - total_pad / 2;
+    pad_tail = total_pad / 2;
+  }
+}
+
 // Note: This helper function will not have overflow protection
 template <template <typename...> class Container, typename T>
 T Product(const Container<T>& c) {
diff --git a/onnxruntime/core/providers/cpu/nn/conv_transpose_attributes.h b/onnxruntime/core/providers/cpu/nn/conv_transpose_attributes.h
index 7bd37f2828..973743d711 100644
--- a/onnxruntime/core/providers/cpu/nn/conv_transpose_attributes.h
+++ b/onnxruntime/core/providers/cpu/nn/conv_transpose_attributes.h
@@ -192,25 +192,6 @@ struct ConvTransposeAttributes : public ConvAttributes {
   TensorShapeVector output_shape;
 
  private:
-  int64_t ComputeTotalPad(int64_t in_size, int64_t stride, int64_t adj,
-                          int64_t kernel, int64_t dilation, int64_t out_size) const {
-    return std::max<int64_t>(0, (in_size - 1) * stride + adj + (kernel - 1) * dilation + 1 - out_size);
-  }
-
-  void DistributePadding(AutoPadType pad_type, const int64_t& total_pad,
-                         int64_t& pad_head, int64_t& pad_tail) const {
-    if (pad_type == AutoPadType::SAME_UPPER) {
-      // pad more on tail when total_pad is odd.
-      pad_head = total_pad / 2;
-      pad_tail = total_pad - total_pad / 2;
-    } else {
-      // When pad_type is NOTSET, SAME_LOWER or VALID,
-      // pad more on head when total_pad is odd.
-      pad_head = total_pad - total_pad / 2;
-      pad_tail = total_pad / 2;
-    }
-  }
-
   void ComputeTransposePadAndOutputShape(
       const int64_t in_size,
       const int64_t stride,
diff --git a/onnxruntime/core/providers/qnn/builder/opbuilder/base_op_builder.h b/onnxruntime/core/providers/qnn/builder/opbuilder/base_op_builder.h
index 0c6939ceb6..616354ce31 100644
--- a/onnxruntime/core/providers/qnn/builder/opbuilder/base_op_builder.h
+++ b/onnxruntime/core/providers/qnn/builder/opbuilder/base_op_builder.h
@@ -105,7 +105,6 @@ class BaseOpBuilder : public IOpBuilder {
                                                   QnnQuantParamsWrapper& quant_param) const ORT_MUST_USE_RESULT;
 
   static const std::string& GetQnnOpType(const std::string& onnx_op_type) {
-    // TODO: Use QNN operator names defined in "QnnOpDef.h"
     static const std::unordered_map<std::string, std::string> onnx_op_type_to_qnn_op_type = {
         {"Add", QNN_OP_ELEMENT_WISE_ADD},
         {"Mul", QNN_OP_ELEMENT_WISE_MULTIPLY},
@@ -218,22 +217,40 @@ class BaseOpBuilder : public IOpBuilder {
 
   // NCHW shape to HWCN shape, required for Conv weight
   Status NchwShapeToHwcn(const std::vector<uint32_t>& nchw_shape, std::vector<uint32_t>& hwcn_shape) const {
-    ORT_ENFORCE(nchw_shape.size() == 4, "shape should have 4 dimension NCHW.");
-    hwcn_shape[0] = nchw_shape[2];
-    hwcn_shape[1] = nchw_shape[3];
-    hwcn_shape[2] = nchw_shape[1];
-    hwcn_shape[3] = nchw_shape[0];
+    if (nchw_shape.size() == 4) {
+      hwcn_shape[0] = nchw_shape[2];
+      hwcn_shape[1] = nchw_shape[3];
+      hwcn_shape[2] = nchw_shape[1];
+      hwcn_shape[3] = nchw_shape[0];
+    } else if (nchw_shape.size() == 5) {
+      hwcn_shape[0] = nchw_shape[2];
+      hwcn_shape[1] = nchw_shape[3];
+      hwcn_shape[2] = nchw_shape[4];
+      hwcn_shape[3] = nchw_shape[1];
+      hwcn_shape[4] = nchw_shape[0];
+    } else {
+      return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Unsupported rank! only support 4 or 5.");
+    }
 
     return Status::OK();
   }
 
   // CNHW shape to HWCN shape, required for Conv weight
   Status CnhwShapeToHwcn(const std::vector<uint32_t>& cnhw_shape, std::vector<uint32_t>& hwcn_shape) const {
-    ORT_ENFORCE(cnhw_shape.size() == 4, "shape should have 4 dimension CNHW.");
-    hwcn_shape[0] = cnhw_shape[2];
-    hwcn_shape[1] = cnhw_shape[3];
-    hwcn_shape[2] = cnhw_shape[0];
-    hwcn_shape[3] = cnhw_shape[1];
+    if (cnhw_shape.size() == 4) {
+      hwcn_shape[0] = cnhw_shape[2];
+      hwcn_shape[1] = cnhw_shape[3];
+      hwcn_shape[2] = cnhw_shape[0];
+      hwcn_shape[3] = cnhw_shape[1];
+    } else if (cnhw_shape.size() == 5) {
+      hwcn_shape[0] = cnhw_shape[2];
+      hwcn_shape[1] = cnhw_shape[3];
+      hwcn_shape[2] = cnhw_shape[4];
+      hwcn_shape[3] = cnhw_shape[0];
+      hwcn_shape[4] = cnhw_shape[1];
+    } else {
+      return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Unsupported rank! only support 4 or 5.");
+    }
 
     return Status::OK();
   }
@@ -244,14 +261,18 @@ class BaseOpBuilder : public IOpBuilder {
 
   Status TransposeFromNchwToHwcn(const QnnModelWrapper& qnn_model_wrapper,
                                  const onnx::TensorProto& initializer,
-                                 std::vector<uint8_t>& transposed_data) const {
-    return TransposeInitializer(qnn_model_wrapper, initializer, nchw2hwcn_perm, transposed_data);
+                                 std::vector<uint8_t>& transposed_data,
+                                 bool is_3d = false) const {
+    auto& perm = is_3d ? nchw2hwcn_perm_3d : nchw2hwcn_perm;
+    return TransposeInitializer(qnn_model_wrapper, initializer, perm, transposed_data);
   }
 
   Status TransposeFromCnhwToHwcn(const QnnModelWrapper& qnn_model_wrapper,
                                  const onnx::TensorProto& initializer,
-                                 std::vector<uint8_t>& transposed_data) const {
-    return TransposeInitializer(qnn_model_wrapper, initializer, cnhw2hwcn_perm, transposed_data);
+                                 std::vector<uint8_t>& transposed_data,
+                                 bool is_3d = false) const {
+    auto& perm = is_3d ? cnhw2hwcn_perm_3d : cnhw2hwcn_perm;
+    return TransposeInitializer(qnn_model_wrapper, initializer, perm, transposed_data);
   }
 
   Status TwoDimensionTranspose(const QnnModelWrapper& qnn_model_wrapper,
@@ -311,11 +332,6 @@ class BaseOpBuilder : public IOpBuilder {
 
  private:
   std::string op_builder_type_;
-
- protected:
-  const std::vector<size_t> nchw2nhwc_perm{0, 2, 3, 1};
-  const std::vector<size_t> nchw2hwcn_perm{2, 3, 1, 0};
-  const std::vector<size_t> cnhw2hwcn_perm{2, 3, 0, 1};
 };
 
 // Type that holds information about an ONNX attribute.
diff --git a/onnxruntime/core/providers/qnn/builder/opbuilder/conv_op_builder.cc b/onnxruntime/core/providers/qnn/builder/opbuilder/conv_op_builder.cc
index 9b92987f06..4eeca98454 100644
--- a/onnxruntime/core/providers/qnn/builder/opbuilder/conv_op_builder.cc
+++ b/onnxruntime/core/providers/qnn/builder/opbuilder/conv_op_builder.cc
@@ -53,11 +53,11 @@ class ConvOpBuilder : public BaseOpBuilder {
                              const logging::Logger& logger,
                              std::vector<std::string>& input_names,
                              bool do_op_validation) const ORT_MUST_USE_RESULT;
-  Status ProcessConv2DInputs(QnnModelWrapper& qnn_model_wrapper,
-                             const NodeUnit& node_unit,
-                             const logging::Logger& logger,
-                             std::vector<std::string>& input_names,
-                             bool do_op_validation) const ORT_MUST_USE_RESULT;
+  Status ProcessConv2D3DInputs(QnnModelWrapper& qnn_model_wrapper,
+                               const NodeUnit& node_unit,
+                               const logging::Logger& logger,
+                               std::vector<std::string>& input_names,
+                               bool do_op_validation) const ORT_MUST_USE_RESULT;
   Status ProcessAttributesAndOutputs(QnnModelWrapper& qnn_model_wrapper,
                                      const NodeUnit& node_unit,
                                      std::vector<std::string>&& input_names,
@@ -87,8 +87,8 @@ Status ConvOpBuilder::IsOpSupported(QnnModelWrapper& qnn_model_wrapper,
   const auto& input_0 = inputs[0];
   std::vector<uint32_t> input_shape;
   ORT_RETURN_IF_NOT(qnn_model_wrapper.GetOnnxShape(input_0.node_arg, input_shape), "Cannot get shape");
-  if (input_shape.size() != 4 && input_shape.size() != 3) {
-    return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "QNN Conv only supports 2D (rank 4) or 1D (rank 3) inputs.");
+  if (input_shape.size() != 5 && input_shape.size() != 4 && input_shape.size() != 3) {
+    return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "QNN Conv only supports 3D(rank 5), 2D (rank 4) or 1D (rank 3) inputs.");
   }
 
   ONNX_NAMESPACE::DataType input_data_type = input_0.node_arg.Type();
@@ -160,20 +160,20 @@ Status ConvOpBuilder::ProcessInputs(QnnModelWrapper& qnn_model_wrapper,
   ORT_RETURN_IF_NOT(qnn_model_wrapper.GetOnnxShape(inputs[0].node_arg, input0_shape),
                     "QNN EP: Cannot get shape for first input");
 
-  const bool is_1d_conv = input0_shape.size() == 3;
-
-  if (is_1d_conv) {
+  if (input0_shape.size() == 3) {
     return ProcessConv1DInputs(qnn_model_wrapper, node_unit, logger, input_names, do_op_validation);
+  } else if (input0_shape.size() == 4 || input0_shape.size() == 5) {
+    return ProcessConv2D3DInputs(qnn_model_wrapper, node_unit, logger, input_names, do_op_validation);
   }
 
-  return ProcessConv2DInputs(qnn_model_wrapper, node_unit, logger, input_names, do_op_validation);
+  return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "QNN Conv only supports 3D(rank 5), 2D (rank 4) or 1D (rank 3) inputs.");
 }
 
-Status ConvOpBuilder::ProcessConv2DInputs(QnnModelWrapper& qnn_model_wrapper,
-                                          const NodeUnit& node_unit,
-                                          const logging::Logger& logger,
-                                          std::vector<std::string>& input_names,
-                                          bool do_op_validation) const {
+Status ConvOpBuilder::ProcessConv2D3DInputs(QnnModelWrapper& qnn_model_wrapper,
+                                            const NodeUnit& node_unit,
+                                            const logging::Logger& logger,
+                                            std::vector<std::string>& input_names,
+                                            bool do_op_validation) const {
   const auto& inputs = node_unit.Inputs();
   const size_t num_inputs = inputs.size();
   OnnxConvType conv_type = {};
@@ -209,20 +209,27 @@ Status ConvOpBuilder::ProcessConv2DInputs(QnnModelWrapper& qnn_model_wrapper,
       return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "QNN EP: Unexpected convolution op type: ", node_unit.OpType().c_str());
     }
 
+    bool is_3d = (input_info.shape.size() == 5);
+
     std::vector<uint8_t> unpacked_tensor;
     if (input_info.is_initializer) {
       // Get transposed initializer bytes.
       if (conv_type == OnnxConvType::kConv) {
-        ORT_RETURN_IF_ERROR(TransposeFromNchwToHwcn(qnn_model_wrapper, *input_info.initializer_tensor, unpacked_tensor));
+        ORT_RETURN_IF_ERROR(TransposeFromNchwToHwcn(qnn_model_wrapper, *input_info.initializer_tensor, unpacked_tensor, is_3d));
       } else if (conv_type == OnnxConvType::kConvTranspose) {
-        ORT_RETURN_IF_ERROR(TransposeFromCnhwToHwcn(qnn_model_wrapper, *input_info.initializer_tensor, unpacked_tensor));
+        ORT_RETURN_IF_ERROR(TransposeFromCnhwToHwcn(qnn_model_wrapper, *input_info.initializer_tensor, unpacked_tensor, is_3d));
       } else {
         return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "QNN EP: Unexpected convolution op type: ", node_unit.OpType().c_str());
       }
 
       // Transpose quantization parameter's axis if this is using per-channel quantization.
       if (input_info.quant_param.IsPerChannel()) {
-        const std::vector<size_t>& perm = conv_type == OnnxConvType::kConv ? nchw2hwcn_perm : cnhw2hwcn_perm;
+        std::vector<size_t> perm;
+        if (is_3d) {
+          perm = conv_type == OnnxConvType::kConv ? nchw2hwcn_perm_3d : cnhw2hwcn_perm_3d;
+        } else {
+          perm = conv_type == OnnxConvType::kConv ? nchw2hwcn_perm : cnhw2hwcn_perm;
+        }
         std::vector<size_t> perm_inv(perm.size());
         ORT_RETURN_IF_ERROR(utils::InvertPerm<size_t>(perm, perm_inv));
         ORT_RETURN_IF_ERROR(input_info.quant_param.HandleTranspose<size_t>(perm_inv));
@@ -243,7 +250,9 @@ Status ConvOpBuilder::ProcessConv2DInputs(QnnModelWrapper& qnn_model_wrapper,
                                                                      input_info.qnn_data_type,
                                                                      input_info.quant_param,
                                                                      do_op_validation,
-                                                                     is_graph_input));
+                                                                     is_graph_input,
+                                                                     false,
+                                                                     is_3d));
       } else if (conv_type == OnnxConvType::kConvTranspose) {
         ORT_RETURN_IF_ERROR(qnn_model_wrapper.AddCnhwToHwcnTranspose(node_unit.Index(),
                                                                      input1_name,
@@ -253,7 +262,9 @@ Status ConvOpBuilder::ProcessConv2DInputs(QnnModelWrapper& qnn_model_wrapper,
                                                                      input_info.qnn_data_type,
                                                                      input_info.quant_param,
                                                                      do_op_validation,
-                                                                     is_graph_input));
+                                                                     is_graph_input,
+                                                                     false,
+                                                                     is_3d));
       } else {
         return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "QNN EP: Unexpected convolution op type: ", node_unit.OpType().c_str());
       }
@@ -484,80 +495,48 @@ Status ConvOpBuilder::ProcessConv1DInputs(QnnModelWrapper& qnn_model_wrapper,
   return Status::OK();
 }
 
-static Status GetAutoPadding(std::vector<uint32_t>& pads, const std::string& auto_pad, OnnxConvType conv_type,
-                             const std::array<uint32_t, 2>& strides, const std::array<uint32_t, 2>& dilations,
-                             const std::array<uint32_t, 2>& input_dims, const std::array<uint32_t, 2>& filter_dims,
-                             const std::array<uint32_t, 2>& output_dims, const std::array<uint32_t, 2>& output_padding) {
-  constexpr size_t HEIGHT_IDX = 0;
-  constexpr size_t WIDTH_IDX = 1;
-
-  std::array<uint32_t, 2> total_padding = {};
-
-  if (conv_type == OnnxConvType::kConv) {
-    // dilated_filter_height = (shape(in[1])[height] - 1) * dilation[0] + 1
-    // height_out = floor((pad_amount[0,0] + shape(in[0])[height] + pad_amount[0,1] - dilated_filter_height) / stride[0] + 1)
-    //
-    // Set total_height_padding equal to pad_amount[0,0] + pad_amount[0,1] and solve for it.
-    uint32_t dilated_filter_height = (filter_dims[HEIGHT_IDX] - 1) * dilations[HEIGHT_IDX] + 1;
-    total_padding[HEIGHT_IDX] = (output_dims[HEIGHT_IDX] - 1) * strides[HEIGHT_IDX] + dilated_filter_height - input_dims[HEIGHT_IDX];  // Total height padding
-
-    // dilated_filter_width = (shape(in[1])[width] - 1) * dilation[1] + 1
-    // width_out = floor((pad_amount[1,0] + shape(in[0])[width] + pad_amount[1,1] - dilated_filter_width) / stride[1] + 1)
-    //
-    // Set total_width_padding equal to pad_amount[1,0] + pad_amount[1,1] and solve for it.
-    uint32_t dilated_filter_width = (filter_dims[WIDTH_IDX] - 1) * dilations[WIDTH_IDX] + 1;
-    total_padding[WIDTH_IDX] = (output_dims[WIDTH_IDX] - 1) * strides[WIDTH_IDX] + dilated_filter_width - input_dims[WIDTH_IDX];  // Total width padding
-  } else if (conv_type == OnnxConvType::kConvTranspose) {
-    // height_out = floor(stride[0] * (shape(in[0])[height] - 1) + shape(in[1])[height] - pad_amount[0,0] - pad_amount[0,1] + output_padding[0])
-    //
-    // Set total_height_padding equal to pad_amount[0,0] + pad_amount[0,1] and solve for it.
-    total_padding[HEIGHT_IDX] = strides[HEIGHT_IDX] * (input_dims[HEIGHT_IDX] - 1) + output_padding[HEIGHT_IDX] + filter_dims[HEIGHT_IDX] - output_dims[HEIGHT_IDX];
-
-    // width_out = floor(stride[1] * (shape(in[0])[width] - 1) + shape(in[1])[width] - pad_amount[1,0] - pad_amount[1,1] + output_padding[1])
-    //
-    // Set total_width_padding equal to pad_amount[1,0] + pad_amount[1,1] and solve for it.
-    total_padding[WIDTH_IDX] = strides[WIDTH_IDX] * (input_dims[WIDTH_IDX] - 1) + output_padding[WIDTH_IDX] + filter_dims[WIDTH_IDX] - output_dims[WIDTH_IDX];
-  } else {
-    return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "QNN EP: Unexpected conv op type when computing auto-padding");
-  }
-
-  pads.resize(4);  // Make room.
-
-  if (auto_pad == "SAME_UPPER") {
-    pads[0] = total_padding[0] / 2;
-    pads[1] = total_padding[1] / 2;
-    pads[2] = total_padding[0] - pads[0];
-    pads[3] = total_padding[1] - pads[1];
-  } else if (auto_pad == "SAME_LOWER") {
-    pads[2] = total_padding[0] / 2;
-    pads[3] = total_padding[1] / 2;
-    pads[0] = total_padding[0] - pads[2];
-    pads[1] = total_padding[1] - pads[3];
-  } else {
-    return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "QNN EP: Cannot calculate auto-padding for unsupported auto_pad setting: ",
-                           auto_pad.c_str());
-  }
-
-  return Status::OK();
-}
-
 Status ConvOpBuilder::ProcessAttributesAndOutputs(QnnModelWrapper& qnn_model_wrapper,
                                                   const NodeUnit& node_unit,
                                                   std::vector<std::string>&& input_names,
                                                   const logging::Logger& logger,
                                                   bool do_op_validation) const {
-  ORT_UNUSED_PARAMETER(do_op_validation);
+  const auto& outputs = node_unit.Outputs();
+
+  std::vector<uint32_t> output_shape;
+  ORT_RETURN_IF_NOT(qnn_model_wrapper.GetOnnxShape(outputs[0].node_arg, output_shape), "Cannot get shape");
+  const bool is_1d_conv = output_shape.size() == 3;
+  const bool is_3d_conv = output_shape.size() == 5;
+
   OnnxConvType conv_type = {};
   ORT_RETURN_IF_ERROR(GetOnnxConvType(node_unit.OpType(), conv_type));
 
   NodeAttrHelper node_helper(node_unit);
   std::vector<std::string> param_tensor_names;
 
+  const auto& input_0 = node_unit.Inputs()[0];
+  const auto& input_1 = node_unit.Inputs()[1];
+  std::vector<uint32_t> input_0_shape;  // NHW[D]C
+  std::vector<uint32_t> input_1_shape;  // NCHW[D]
+  ORT_RETURN_IF_NOT(qnn_model_wrapper.GetOnnxShape(input_0.node_arg, input_0_shape), "Cannot get shape");
+  ORT_RETURN_IF_NOT(qnn_model_wrapper.GetOnnxShape(input_1.node_arg, input_1_shape), "Cannot get shape");
+
+  // Kernel shape
+  std::vector<uint32_t> kernel_shape;
+  kernel_shape = node_helper.Get("kernel_shape", kernel_shape);
+  if (kernel_shape.empty()) {  // infer from weight shape
+    kernel_shape.assign(input_1_shape.begin() + 2, input_1_shape.end());
+  }
+  if (is_1d_conv) {
+    // insert Hight = 1 for 1D
+    kernel_shape.insert(kernel_shape.begin(), 1);
+  }
+
   // Dilations parameter
-  std::vector<uint32_t> dilations = {1, 1};
+  std::vector<uint32_t> dilations;
+  dilations.assign(kernel_shape.size(), 1);
 
   if (conv_type == OnnxConvType::kConv) {
-    dilations = node_helper.Get("dilations", std::vector<uint32_t>{1, 1});
+    dilations = node_helper.Get("dilations", dilations);
 
     // Handle 1D conv by setting height dilation to 1.
     if (dilations.size() == 1) {
@@ -574,7 +553,9 @@ Status ConvOpBuilder::ProcessAttributesAndOutputs(QnnModelWrapper& qnn_model_wra
   }
 
   // Strides parameter.
-  auto strides = node_helper.Get("strides", std::vector<uint32_t>{1, 1});
+  std::vector<uint32_t> strides;
+  strides.assign(kernel_shape.size(), 1);
+  strides = node_helper.Get("strides", strides);
   {
     // Handle 1D conv by setting the height stride to 1.
     if (strides.size() == 1) {
@@ -591,10 +572,10 @@ Status ConvOpBuilder::ProcessAttributesAndOutputs(QnnModelWrapper& qnn_model_wra
   }
 
   // Output padding parameter. (Only for ConvTranspose)
-  std::vector<uint32_t> output_padding = {0, 0};
-
+  std::vector<uint32_t> output_padding;
+  output_padding.assign(kernel_shape.size(), 0);
   if (conv_type == OnnxConvType::kConvTranspose) {
-    output_padding = node_helper.Get("output_padding", std::vector<uint32_t>{0, 0});
+    output_padding = node_helper.Get("output_padding", output_padding);
 
     // Handle 1D conv.
     if (output_padding.size() == 1) {
@@ -610,54 +591,45 @@ Status ConvOpBuilder::ProcessAttributesAndOutputs(QnnModelWrapper& qnn_model_wra
     qnn_model_wrapper.AddParamWrapper(std::move(output_padding_paramwrapper));
   }
 
-  const auto& outputs = node_unit.Outputs();
-  const auto& output_name = outputs[0].node_arg.Name();
-
-  std::vector<uint32_t> output_shape;
-  ORT_RETURN_IF_NOT(qnn_model_wrapper.GetOnnxShape(outputs[0].node_arg, output_shape), "Cannot get shape");
-  const bool is_1d_conv = output_shape.size() == 3;
-
   // Pads attribute
   {
-    std::vector<uint32_t> pads = node_helper.Get("pads", std::vector<uint32_t>({0, 0, 0, 0}));
+    std::vector<uint32_t> pads;
+    pads.assign(kernel_shape.size() * 2, 0);
+    pads = node_helper.Get("pads", pads);
     auto auto_pad = node_helper.Get("auto_pad", std::string("NOTSET"));
     ORT_RETURN_IF(auto_pad != "NOTSET" && auto_pad != "SAME_LOWER" && auto_pad != "SAME_UPPER",
                   "QNN Conv operators do not support 'auto_pad' value: ", auto_pad.c_str());
 
     if (auto_pad != "NOTSET") {
-      const auto& input_0 = node_unit.Inputs()[0];
-      const auto& input_1 = node_unit.Inputs()[1];
-      std::vector<uint32_t> input_0_shape;  // NHWC
-      std::vector<uint32_t> input_1_shape;  // NCHW
-      ORT_RETURN_IF_NOT(qnn_model_wrapper.GetOnnxShape(input_0.node_arg, input_0_shape), "Cannot get shape");
-      ORT_RETURN_IF_NOT(qnn_model_wrapper.GetOnnxShape(input_1.node_arg, input_1_shape), "Cannot get shape");
-
-      std::array<uint32_t, 2> input_dims = {};
-      std::array<uint32_t, 2> filter_dims = {};
-      std::array<uint32_t, 2> output_dims = {};
-
+      auto pad_type = StringToAutoPadType(auto_pad);
+      // skip N, C, input0 shape NHWC
+      std::vector<uint32_t> input_dims(input_0_shape.begin() + 1, input_0_shape.end() - 1);
+      std::vector<uint32_t> output_dims(output_shape.begin() + 1, output_shape.end() - 1);
       if (is_1d_conv) {
-        input_dims[0] = 1;
-        input_dims[1] = input_0_shape[1];
-
-        filter_dims[0] = 1;
-        filter_dims[1] = input_1_shape[2];
-
-        output_dims[0] = 1;
-        output_dims[1] = output_shape[1];
-      } else {
-        input_dims[0] = input_0_shape[1];
-        input_dims[1] = input_0_shape[2];
-
-        filter_dims[0] = input_1_shape[2];
-        filter_dims[1] = input_1_shape[3];
-
-        output_dims[0] = output_shape[1];
-        output_dims[1] = output_shape[2];
+        // insert Hight = 1 for 1D
+        input_dims.insert(input_dims.begin(), 1);
+        output_dims.insert(output_dims.begin(), 1);
+      }
+      size_t rank = input_dims.size();
+      for (size_t dim = 0; dim < rank; ++dim) {
+        int64_t pad_head = pads[dim];
+        int64_t pad_tail = pads[rank + dim];
+        if (conv_type == OnnxConvType::kConv) {
+          ORT_RETURN_IF_ERROR(onnxruntime::ComputePad(input_dims[dim],
+                                                      strides[dim],
+                                                      kernel_shape[dim],
+                                                      dilations[dim],
+                                                      pad_type,
+                                                      pad_head,
+                                                      pad_tail));
+        } else if (conv_type == OnnxConvType::kConvTranspose) {
+          auto total_pad = ComputeTotalPad(input_dims[dim], strides[dim], output_padding[dim],
+                                           kernel_shape[dim], dilations[dim], output_dims[dim]);
+          DistributePadding(pad_type, total_pad, pad_head, pad_tail);
+        }
+        pads[dim] = narrow<uint32_t>(pad_head);
+        pads[rank + dim] = narrow<uint32_t>(pad_tail);
       }
-
-      ORT_RETURN_IF_ERROR(GetAutoPadding(pads, auto_pad, conv_type, {strides[0], strides[1]}, {dilations[0], dilations[1]},
-                                         input_dims, filter_dims, output_dims, {output_padding[0], output_padding[1]}));
     } else {
       // Handle 1D conv by setting padding for height to 0.
       if (pads.size() == 2) {
@@ -672,8 +644,9 @@ Status ConvOpBuilder::ProcessAttributesAndOutputs(QnnModelWrapper& qnn_model_wra
     }
 
     ReArranagePads(pads);
+    uint32_t pad_size = narrow<uint32_t>(pads.size() / 2);
     QnnParamWrapper pad_amount_paramwrapper(node_unit.Index(), node_unit.Name(), QNN_OP_CONV_2D_PARAM_PAD_AMOUNT,
-                                            {2, 2}, std::move(pads));
+                                            {pad_size, 2}, std::move(pads));
     param_tensor_names.push_back(pad_amount_paramwrapper.GetParamTensorName());
     qnn_model_wrapper.AddParamWrapper(std::move(pad_amount_paramwrapper));
   }
@@ -683,7 +656,9 @@ Status ConvOpBuilder::ProcessAttributesAndOutputs(QnnModelWrapper& qnn_model_wra
   uint32_t num_input_channels = 0;
   ORT_RETURN_IF_ERROR(GetInputChannelNumber(qnn_model_wrapper, node_unit, num_input_channels));
 
-  const bool is_depthwise_conv2d = (conv_type == OnnxConvType::kConv) && (num_input_channels == num_output_channels) &&
+  // There's DepthWiseConv2d, but no DepthWiseConv3d
+  const bool is_depthwise_conv2d = (!is_3d_conv) && (conv_type == OnnxConvType::kConv) &&
+                                   (num_input_channels == num_output_channels) &&
                                    (group == num_output_channels);
 
   if (!is_depthwise_conv2d) {  // DepthWiseConv2d does not need a group parameter.
@@ -697,7 +672,16 @@ Status ConvOpBuilder::ProcessAttributesAndOutputs(QnnModelWrapper& qnn_model_wra
     LOGS(logger, VERBOSE) << "Using DepthWiseConv2d instead of Conv2d for node " << node_unit.Name();
   }
 
-  const std::string& output_node_type = is_depthwise_conv2d ? QNN_OP_DEPTH_WISE_CONV_2D : GetQnnOpType(node_unit.OpType());
+  std::string output_node_type;
+  if (is_3d_conv) {
+    if (conv_type == OnnxConvType::kConv) {
+      output_node_type = QNN_OP_CONV_3D;
+    } else {
+      output_node_type = QNN_OP_TRANSPOSE_CONV_3D;
+    }
+  } else {
+    output_node_type = is_depthwise_conv2d ? QNN_OP_DEPTH_WISE_CONV_2D : GetQnnOpType(node_unit.OpType());
+  }
 
   QnnQuantParamsWrapper output_quantize_param;
   ORT_RETURN_IF_ERROR(output_quantize_param.Init(qnn_model_wrapper, outputs[0]));
@@ -707,6 +691,7 @@ Status ConvOpBuilder::ProcessAttributesAndOutputs(QnnModelWrapper& qnn_model_wra
   Qnn_DataType_t qnn_data_type = QNN_DATATYPE_FLOAT_32;
   ORT_RETURN_IF_ERROR(utils::GetQnnDataType(is_quantized_tensor, type_proto, qnn_data_type));
 
+  const auto& output_name = outputs[0].node_arg.Name();
   if (is_1d_conv) {
     const bool is_graph_output = qnn_model_wrapper.IsGraphOutput(output_name);
     std::vector<uint32_t> output_shape_2d = {
@@ -724,7 +709,8 @@ Status ConvOpBuilder::ProcessAttributesAndOutputs(QnnModelWrapper& qnn_model_wra
                                                       output_node_type,
                                                       std::move(input_names),
                                                       {conv_output_name},
-                                                      std::move(param_tensor_names)),
+                                                      std::move(param_tensor_names),
+                                                      do_op_validation),
                       "Failed to add node.");
 
     // Add Reshape to convert QNN Conv2d/TransposeConv2d/DepthWiseConv2d output back to 1D.
@@ -748,7 +734,8 @@ Status ConvOpBuilder::ProcessAttributesAndOutputs(QnnModelWrapper& qnn_model_wra
                                                       output_node_type,
                                                       std::move(input_names),
                                                       {output_name},
-                                                      std::move(param_tensor_names)),
+                                                      std::move(param_tensor_names),
+                                                      do_op_validation),
                       "Failed to add node.");
   }
 
diff --git a/onnxruntime/core/providers/qnn/builder/opbuilder/simple_op_builder.cc b/onnxruntime/core/providers/qnn/builder/opbuilder/simple_op_builder.cc
index 5e2b3f6113..185276d116 100644
--- a/onnxruntime/core/providers/qnn/builder/opbuilder/simple_op_builder.cc
+++ b/onnxruntime/core/providers/qnn/builder/opbuilder/simple_op_builder.cc
@@ -323,17 +323,6 @@ Status SimpleOpBuilder::ProcessAttributesAndOutputs(QnnModelWrapper& qnn_model_w
     if (node_unit.Domain() != kMSInternalNHWCDomain && (op_type == "DepthToSpace" || op_type == "SpaceToDepth" || op_type == "GridSample")) {
       return Status::OK();
     }
-
-    // Explicitly skip the Op validation for Q & DQ node with 5D because of QNN bug.
-    // TODO (hecli), remove once QNN v2.17 is ready
-    if (op_type == "QuantizeLinear" || op_type == "DequantizeLinear") {
-      std::vector<uint32_t> input_shape;
-      ORT_RETURN_IF_NOT(qnn_model_wrapper.GetOnnxShape(node_unit.Inputs()[0].node_arg, input_shape),
-                        "QNN EP: Cannot get input shape");
-      if (input_shape.size() == 5) {
-        return Status::OK();
-      }
-    }
   }
 
   std::vector<std::string> param_tensor_names;
diff --git a/onnxruntime/core/providers/qnn/builder/qnn_def.h b/onnxruntime/core/providers/qnn/builder/qnn_def.h
index cf62d863fa..ffd2dc9b11 100644
--- a/onnxruntime/core/providers/qnn/builder/qnn_def.h
+++ b/onnxruntime/core/providers/qnn/builder/qnn_def.h
@@ -497,5 +497,10 @@ typedef struct GraphConfigInfo {
   const QnnGraph_Config_t** graphConfigs;
 } GraphConfigInfo_t;
 
+static const std::vector<size_t> nchw2hwcn_perm{2, 3, 1, 0};
+static const std::vector<size_t> nchw2hwcn_perm_3d{2, 3, 4, 1, 0};
+static const std::vector<size_t> cnhw2hwcn_perm{2, 3, 0, 1};
+static const std::vector<size_t> cnhw2hwcn_perm_3d{2, 3, 4, 0, 1};
+
 }  // namespace qnn
 }  // namespace onnxruntime
diff --git a/onnxruntime/core/providers/qnn/builder/qnn_model_wrapper.h b/onnxruntime/core/providers/qnn/builder/qnn_model_wrapper.h
index d4c9113a07..3fd268d1b3 100644
--- a/onnxruntime/core/providers/qnn/builder/qnn_model_wrapper.h
+++ b/onnxruntime/core/providers/qnn/builder/qnn_model_wrapper.h
@@ -165,10 +165,18 @@ class QnnModelWrapper {
                                 const QnnQuantParamsWrapper& quantize_param,
                                 bool do_op_validation,
                                 bool is_for_input = true,
-                                bool is_for_output = false) {
+                                bool is_for_output = false,
+                                bool is_3d = false) {
     LOGS(logger_, VERBOSE) << "Add NCHW->HWCN Transpose node after Conv weight input: " << input_name
                            << " -> " << output_name;
-    return AddTransposeNode(node_index, input_name, output_name, input_shape, nchw2hwcn_perm_, output_shape,
+    auto perm = is_3d ? nchw2hwcn_perm_3d : nchw2hwcn_perm;
+    std::vector<uint32_t> transpose_perm;
+    transpose_perm.resize(perm.size());
+    std::transform(perm.begin(), perm.end(),
+                   transpose_perm.begin(), [](size_t item) -> uint32_t {
+                     return narrow<uint32_t>(item);
+                   });
+    return AddTransposeNode(node_index, input_name, output_name, input_shape, transpose_perm, output_shape,
                             tensor_data_type, quantize_param, do_op_validation, is_for_input, is_for_output);
   }
 
@@ -182,10 +190,18 @@ class QnnModelWrapper {
                                 const QnnQuantParamsWrapper& quantize_param,
                                 bool do_op_validation,
                                 bool is_for_input = true,
-                                bool is_for_output = false) {
+                                bool is_for_output = false,
+                                bool is_3d = false) {
     LOGS(logger_, VERBOSE) << "Add CNHW->HWCN Transpose node after ConvTranspose weight input: " << input_name
                            << " -> " << output_name;
-    return AddTransposeNode(node_index, input_name, output_name, input_shape, cnhw2hwcn_perm_, output_shape,
+    auto perm = is_3d ? cnhw2hwcn_perm_3d : cnhw2hwcn_perm;
+    std::vector<uint32_t> transpose_perm;
+    transpose_perm.resize(perm.size());
+    std::transform(perm.begin(), perm.end(),
+                   transpose_perm.begin(), [](size_t item) -> uint32_t {
+                     return narrow<uint32_t>(item);
+                   });
+    return AddTransposeNode(node_index, input_name, output_name, input_shape, transpose_perm, output_shape,
                             tensor_data_type, quantize_param, do_op_validation, is_for_input, is_for_output);
   }
 
@@ -258,8 +274,6 @@ class QnnModelWrapper {
   const std::unordered_map<std::string, size_t>& input_index_map_;
   const std::unordered_map<std::string, size_t>& output_index_map_;
   const std::unordered_set<std::string>& initializer_lookup_;
-  const std::vector<uint32_t> nchw2hwcn_perm_{2, 3, 1, 0};
-  const std::vector<uint32_t> cnhw2hwcn_perm_{2, 3, 0, 1};
   QnnBackendType qnn_backend_type_ = QnnBackendType::CPU;
 };  // QnnModelWrapper
 
diff --git a/onnxruntime/core/providers/qnn/builder/qnn_utils.cc b/onnxruntime/core/providers/qnn/builder/qnn_utils.cc
index 43a46e1788..19362daee6 100644
--- a/onnxruntime/core/providers/qnn/builder/qnn_utils.cc
+++ b/onnxruntime/core/providers/qnn/builder/qnn_utils.cc
@@ -328,6 +328,12 @@ std::ostream& operator<<(std::ostream& out, const Qnn_Tensor_t& tensor) {
     } else if (GetQnnTensorDataType(tensor) == QNN_DATATYPE_INT_32 ||
                GetQnnTensorDataType(tensor) == QNN_DATATYPE_SFIXED_POINT_32) {
       operator<< <int32_t>(out, GetQnnTensorClientBuf(tensor));
+    } else if (GetQnnTensorDataType(tensor) == QNN_DATATYPE_UINT_16 ||
+               GetQnnTensorDataType(tensor) == QNN_DATATYPE_UFIXED_POINT_16) {
+      operator<< <uint16_t>(out, GetQnnTensorClientBuf(tensor));
+    } else if (GetQnnTensorDataType(tensor) == QNN_DATATYPE_INT_16 ||
+               GetQnnTensorDataType(tensor) == QNN_DATATYPE_SFIXED_POINT_16) {
+      operator<< <int16_t>(out, GetQnnTensorClientBuf(tensor));
     } else if (GetQnnTensorDataType(tensor) == QNN_DATATYPE_UINT_8 ||
                GetQnnTensorDataType(tensor) == QNN_DATATYPE_UFIXED_POINT_8) {
       operator<< <uint8_t>(out, GetQnnTensorClientBuf(tensor));
diff --git a/onnxruntime/test/onnx/main.cc b/onnxruntime/test/onnx/main.cc
index c190577611..0356bf5218 100644
--- a/onnxruntime/test/onnx/main.cc
+++ b/onnxruntime/test/onnx/main.cc
@@ -812,7 +812,8 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)");
         ORT_TSTR("sce_NCd1d2d3_sum_weight_high_ii"),
         ORT_TSTR("sce_NCd1d2d3_sum_weight_high_ii_expanded"),
         ORT_TSTR("sce_none_weights_log_prob_expanded"),
-        ORT_TSTR("sce_none_weights_expanded")};
+        ORT_TSTR("sce_none_weights_expanded"),
+        ORT_TSTR("convtranspose_3d")};
 
     std::unordered_set<std::basic_string<ORTCHAR_T>> all_disabled_tests(std::begin(immutable_broken_tests), std::end(immutable_broken_tests));
 
diff --git a/onnxruntime/test/providers/qnn/conv_test.cc b/onnxruntime/test/providers/qnn/conv_test.cc
index 0eeeccf4fe..a469cccbbd 100644
--- a/onnxruntime/test/providers/qnn/conv_test.cc
+++ b/onnxruntime/test/providers/qnn/conv_test.cc
@@ -322,6 +322,17 @@ TEST_F(QnnCPUBackendTests, DISABLED_Convf32_dynamic_bias) {
                    1,                                                      // default group
                    "NOTSET",                                               // No auto-padding
                    ExpectedEPNodeAssignment::All);
+
+  RunCPUConvOpTest("Conv",
+                   TestInputDef<float>({1, 1, 3, 3, 3}, false, 0.0f, 10.0f),  // Random dynamic input
+                   TestInputDef<float>({2, 1, 2, 2, 2}, true, 0.0f, 1.0f),    // Random static weights
+                   TestInputDef<float>({2}, false, -1.0f, 1.0f),              // Random dynamic bias
+                   {1, 1, 1},                                                 // default strides
+                   {0, 0, 0, 0, 0, 0},                                        // default pads
+                   {1, 1, 1},                                                 // default dilations
+                   1,                                                         // default group
+                   "NOTSET",                                                  // No auto-padding
+                   ExpectedEPNodeAssignment::All);
 }
 
 // Check that QNN compiles DQ -> Conv -> Q as a single unit.
@@ -337,6 +348,17 @@ TEST_F(QnnCPUBackendTests, Convf32_bias_initializer) {
                    1,                                                      // default group
                    "NOTSET",                                               // No auto-padding
                    ExpectedEPNodeAssignment::All);
+
+  RunCPUConvOpTest("Conv",
+                   TestInputDef<float>({1, 1, 3, 3, 3}, false, 0.0f, 10.0f),  // Random dynamic input
+                   TestInputDef<float>({2, 1, 2, 2, 2}, true, 0.0f, 1.0f),    // Random static weights
+                   TestInputDef<float>({2}, true, -1.0f, 1.0f),               // Random static bias
+                   {1, 1, 1},                                                 // default strides
+                   {0, 0, 0, 0, 0, 0},                                        // default pads
+                   {1, 1, 1},                                                 // default dilations
+                   1,                                                         // default group
+                   "NOTSET",                                                  // No auto-padding
+                   ExpectedEPNodeAssignment::All);
 }
 
 // Tests Conv's auto_pad value "SAME_UPPER" (compares to CPU EP).
@@ -351,6 +373,17 @@ TEST_F(QnnCPUBackendTests, Convf32_AutoPadUpper) {
                    1,                                                      // default group
                    "SAME_UPPER",                                           // auto_pad
                    ExpectedEPNodeAssignment::All);
+
+  RunCPUConvOpTest("Conv",
+                   TestInputDef<float>({1, 1, 3, 3, 3}, false, -3.0f, 3.0f),  // Random dynamic input
+                   TestInputDef<float>({2, 1, 2, 2, 2}, true, -1.0f, 1.0f),   // Random static weights
+                   TestInputDef<float>({2}, true, -1.0f, 1.0f),               // Random static bias
+                   {1, 1, 1},                                                 // strides
+                   {},                                                        // pads
+                   {1, 1, 1},                                                 // dilations
+                   1,                                                         // default group
+                   "SAME_UPPER",                                              // auto_pad
+                   ExpectedEPNodeAssignment::All);
 }
 
 // Tests ConvTranspose's auto_pad value "SAME_UPPER" (compares to CPU EP).
@@ -365,6 +398,17 @@ TEST_F(QnnCPUBackendTests, ConvTransposef32_AutoPadUpper) {
                    1,                                                      // default group
                    "SAME_UPPER",                                           // auto_pad
                    ExpectedEPNodeAssignment::All);
+
+  RunCPUConvOpTest("ConvTranspose",
+                   TestInputDef<float>({1, 1, 3, 3, 3}, false, -3.0f, 3.0f),  // Random dynamic input
+                   TestInputDef<float>({1, 2, 2, 2, 2}, true, -1.0f, 1.0f),   // Random static weights
+                   TestInputDef<float>({2}, true, -1.0f, 1.0f),               // Random static bias
+                   {1, 1, 1},                                                 // strides
+                   {},                                                        // pads
+                   {1, 1, 1},                                                 // dilations
+                   1,                                                         // default group
+                   "SAME_UPPER",                                              // auto_pad
+                   ExpectedEPNodeAssignment::All);
 }
 
 // Tests Conv's auto_pad value "SAME_LOWER" (compares to CPU EP).
@@ -379,6 +423,17 @@ TEST_F(QnnCPUBackendTests, Convf32_AutoPadLower) {
                    1,                                                      // default group
                    "SAME_LOWER",                                           // auto_pad
                    ExpectedEPNodeAssignment::All);
+
+  RunCPUConvOpTest("Conv",
+                   TestInputDef<float>({1, 1, 3, 3, 3}, false, -3.0f, 3.0f),  // Random dynamic input
+                   TestInputDef<float>({2, 1, 2, 2, 2}, false, -1.0f, 1.0f),  // Random dynamic weights
+                   TestInputDef<float>({2}, true, -1.0f, 1.0f),               // Random static bias
+                   {1, 1, 1},                                                 // strides
+                   {},                                                        // pads
+                   {1, 1, 1},                                                 // dilations
+                   1,                                                         // default group
+                   "SAME_LOWER",                                              // auto_pad
+                   ExpectedEPNodeAssignment::All);
 }
 
 // Tests ConvTranspose's auto_pad value "SAME_LOWER" (compares to CPU EP).
@@ -395,6 +450,23 @@ TEST_F(QnnCPUBackendTests, ConvTransposef32_AutoPadLower) {
                    ExpectedEPNodeAssignment::All);
 }
 
+// Tests ConvTranspose's auto_pad value "SAME_LOWER" (compares to CPU EP).
+// Exception from graphFinalize
+// Exception thrown at 0x00007FFFB7651630 (QnnCpu.dll) in onnxruntime_test_all.exe:
+// 0xC0000005: Access violation reading location 0x0000000000000000.
+TEST_F(QnnCPUBackendTests, DISABLED_ConvTranspose3D_f32_AutoPadLower) {
+  RunCPUConvOpTest("ConvTranspose",
+                   TestInputDef<float>({1, 1, 3, 3, 3}, false, -3.0f, 3.0f),  // Random dynamic input
+                   TestInputDef<float>({1, 2, 2, 2, 2}, false, -1.0f, 1.0f),  // Random dynamic weights
+                   TestInputDef<float>({2}, true, -1.0f, 1.0f),               // Random static bias
+                   {1, 1, 1},                                                 // strides
+                   {},                                                        // pads
+                   {1, 1, 1},                                                 // dilations
+                   1,                                                         // default group
+                   "SAME_LOWER",                                              // auto_pad
+                   ExpectedEPNodeAssignment::All);
+}
+
 // large input,output, pads
 TEST_F(QnnCPUBackendTests, Convf32_large_input1_pad_bias_initializer) {
   RunCPUConvOpTest("Conv",
@@ -409,6 +481,19 @@ TEST_F(QnnCPUBackendTests, Convf32_large_input1_pad_bias_initializer) {
                    ExpectedEPNodeAssignment::All,
                    13,
                    1e-4f);
+
+  RunCPUConvOpTest("Conv",
+                   TestInputDef<float>({1, 3, 60, 452, 20}, false, 0.0f, 10.0f),  // Random dynamic input
+                   TestInputDef<float>({16, 3, 3, 3, 3}, true, 0.0f, 1.0f),       // Random dynamic weights
+                   TestInputDef<float>({16}, true, -1.0f, 1.0f),                  // Random static bias
+                   {1, 1, 1},
+                   {1, 1, 1, 1, 1, 1},
+                   {1, 1, 1},
+                   1,  // default group
+                   "NOTSET",
+                   ExpectedEPNodeAssignment::All,
+                   13,
+                   2e-4f);
 }
 
 TEST_F(QnnCPUBackendTests, Convf32_large_input2_nopad_bias_initializer) {
@@ -432,6 +517,19 @@ TEST_F(QnnCPUBackendTests, Convf32_large_input2_nopad_bias_initializer) {
                    ExpectedEPNodeAssignment::All,
                    13,  // opset
                    fp32_abs_err);
+
+  RunCPUConvOpTest("Conv",
+                   TestInputDef<float>({1, 32, 16, 113, 12}, false, -3.0f, 3.0f),  // Random dynamic input
+                   TestInputDef<float>({16, 32, 1, 1, 1}, false, -1.0f, 1.0f),     // Random dynamic weights
+                   TestInputDef<float>({16}, true, -1.0f, 1.0f),                   // Random static bias
+                   {1, 1, 1},
+                   {0, 0, 0, 0, 0, 0},
+                   {1, 1, 1},
+                   1,  // default group
+                   "NOTSET",
+                   ExpectedEPNodeAssignment::All,
+                   13,  // opset
+                   fp32_abs_err);
 }
 
 // Test 1D Conv with static weights (implemented in QNN EP as 2D convolution with height of 1).
@@ -581,6 +679,21 @@ TEST_F(QnnHTPBackendTests, ConvU8U8S32_bias_dynamic_input) {
                                      13,     // opset
                                      // Need tolerance of 0.413% of output range after QNN SDK 2.17
                                      QDQTolerance(0.00413f));
+
+  RunHTPConvOpTest<uint8_t, uint8_t>("Conv",
+                                     TestInputDef<float>({1, 1, 5, 5, 5}, false, 0.0f, 10.0f),   // Random dynamic input
+                                     TestInputDef<float>({1, 1, 3, 3, 3}, true, -10.0f, 10.0f),  // Random static input
+                                     TestInputDef<float>({1}, false, {2.0f}),                    // Dynamic bias
+                                     {1, 1, 1},                                                  // Strides
+                                     {0, 0, 0, 0, 0, 0},                                         // Pads
+                                     {1, 1, 1},                                                  // Dilations
+                                     1,                                                          // default group
+                                     "NOTSET",
+                                     ExpectedEPNodeAssignment::All,
+                                     false,  // use_qdq_contrib_ops
+                                     13,     // opset
+                                     // Need tolerance of 0.413% of output range after QNN SDK 2.17
+                                     QDQTolerance(0.00413f));
 }
 
 // Test per-channel QDQ Conv. in0: u8, in1 (weight): s8, in2 (bias): s32, out: u8
@@ -610,6 +723,41 @@ TEST_F(QnnHTPBackendTests, ConvU8S8S32_PerChannel) {
                                               13);    // opset
 }
 
+// Test per-channel QDQ Conv. in0: u8, in1 (weight): s8, in2 (bias): s32, out: u8
+// \QNN\HTP\HTP\src\hexagon\prepare\graph_prepare.cc:203:ERROR:could not create op: q::QNN_Conv3d_w_scale
+// \QNN\HTP\HTP\src\hexagon\prepare\graph_prepare.cc:1187:ERROR:Op 0x1a preparation failed with err:-1
+// QnnDsp <E> "Conv" generated: could not create op
+// QnnDsp <E> RouterWindows graph prepare failed 12
+// QnnDsp <E> Failed to finalize graph (id: 1) with err 1002
+// QnnDsp <V> Wake up free backend 1 thread(s)
+// QnnDsp <I> QnnGraph_finalize done. status 0x3ea
+// onnxruntime::qnn::QnnModel::FinalizeGraphs] Failed to finalize QNN graph.
+TEST_F(QnnHTPBackendTests, DISABLED_Conv3D_U8S8S32_PerChannel) {
+  std::vector<int64_t> input_shape = {1, 2, 4, 4, 4};
+  std::vector<int64_t> weight_shape = {3, 2, 2, 2, 2};
+  std::vector<int64_t> bias_shape = {3};
+
+  TestInputDef<float> input_def(input_shape, false,
+                                GetFloatDataInRange(-10.0f, 10.0f, TensorShape(input_shape).Size()));
+  TestInputDef<float> weight_def(weight_shape, true,
+                                 GetFloatDataInRange(-1.0f, 5.0f, TensorShape(weight_shape).Size()));
+  TestInputDef<float> bias_def(bias_shape, true,
+                               GetFloatDataInRange(-1.0f, 1.0f, TensorShape(bias_shape).Size()));
+
+  RunHTPConvOpPerChannelTest<uint8_t, int8_t>("Conv",
+                                              input_def,
+                                              weight_def,
+                                              bias_def,
+                                              {1, 1, 1},           // Strides
+                                              {0, 0, 0, 0, 0, 0},  // Pads
+                                              {1, 1, 1},           // Dilations
+                                              1,                   // default group
+                                              "NOTSET",
+                                              ExpectedEPNodeAssignment::All,
+                                              false,
+                                              13);
+}
+
 // Test per-channel QDQ Conv that maps to QNN's DepthwiseConv2d (input_chans == output_chans == group).
 // in0: u8, in1 (weight): s8, in2 (bias): s32, out: u8
 TEST_F(QnnHTPBackendTests, ConvDepthwiseU8S8S32_PerChannel) {
@@ -638,6 +786,41 @@ TEST_F(QnnHTPBackendTests, ConvDepthwiseU8S8S32_PerChannel) {
                                               13);    // opset
 }
 
+// Conv3D per-channel
+// \QNN\HTP\HTP\src\hexagon\prepare\graph_prepare.cc:203:ERROR:could not create op: q::QNN_Conv3d_w_scale
+// \QNN\HTP\HTP\src\hexagon\prepare\graph_prepare.cc:1187:ERROR:Op 0x1a preparation failed with err:-1
+// QnnDsp <E> "Conv" generated: could not create op
+// QnnDsp <E> RouterWindows graph prepare failed 12
+// QnnDsp <E> Failed to finalize graph (id: 1) with err 1002
+// QnnDsp <V> Wake up free backend 1 thread(s)
+// QnnDsp <I> QnnGraph_finalize done. status 0x3ea
+// onnxruntime::qnn::QnnModel::FinalizeGraphs] Failed to finalize QNN graph.
+TEST_F(QnnHTPBackendTests, DISABLED_Conv3D_U8S8S32_PerChannel2) {
+  std::vector<int64_t> input_shape = {1, 2, 4, 4, 4};
+  std::vector<int64_t> weight_shape = {2, 1, 2, 2, 2};
+  std::vector<int64_t> bias_shape = {2};
+
+  TestInputDef<float> input_def(input_shape, false,
+                                GetFloatDataInRange(-10.0f, 10.0f, TensorShape(input_shape).Size()));
+  TestInputDef<float> weight_def(weight_shape, true,
+                                 GetFloatDataInRange(-1.0f, 5.0f, TensorShape(weight_shape).Size()));
+  TestInputDef<float> bias_def(bias_shape, true,
+                               GetFloatDataInRange(-1.0f, 1.0f, TensorShape(bias_shape).Size()));
+
+  RunHTPConvOpPerChannelTest<uint8_t, int8_t>("Conv",
+                                              input_def,
+                                              weight_def,
+                                              bias_def,
+                                              {1, 1, 1},           // Strides
+                                              {0, 0, 0, 0, 0, 0},  // Pads
+                                              {1, 1, 1},           // Dilations
+                                              2,                   // group
+                                              "NOTSET",
+                                              ExpectedEPNodeAssignment::All,
+                                              false,
+                                              13);
+}
+
 // Test per-channel QDQ ConvTranspose. in0: u8, in1 (weight): s8, in2 (bias): s32, out: u8
 TEST_F(QnnHTPBackendTests, ConvTransposeU8S8S32_PerChannel) {
   std::vector<int64_t> input_shape = {1, 2, 4, 4};
@@ -665,6 +848,34 @@ TEST_F(QnnHTPBackendTests, ConvTransposeU8S8S32_PerChannel) {
                                               13);    // opset
 }
 
+// ConvTranspose3D per-channel
+// Disable it for 2.21 since it failed, re-enabled it for 2.22
+TEST_F(QnnHTPBackendTests, DISABLED_ConvTranspose3D_U8S8S32_PerChannel) {
+  std::vector<int64_t> input_shape = {1, 2, 4, 4, 4};
+  std::vector<int64_t> weight_shape = {2, 3, 2, 2, 2};
+  std::vector<int64_t> bias_shape = {3};
+
+  TestInputDef<float> input_def(input_shape, false,
+                                GetFloatDataInRange(-10.0f, 10.0f, TensorShape(input_shape).Size()));
+  TestInputDef<float> weight_def(weight_shape, true,
+                                 GetFloatDataInRange(-1.0f, 5.0f, TensorShape(weight_shape).Size()));
+  TestInputDef<float> bias_def(bias_shape, true,
+                               GetFloatDataInRange(-1.0f, 1.0f, TensorShape(bias_shape).Size()));
+
+  RunHTPConvOpPerChannelTest<uint8_t, int8_t>("ConvTranspose",
+                                              input_def,
+                                              weight_def,
+                                              bias_def,
+                                              {1, 1, 1},           // Strides
+                                              {0, 0, 0, 0, 0, 0},  // Pads
+                                              {1, 1, 1},           // Dilations
+                                              1,                   // default group
+                                              "NOTSET",
+                                              ExpectedEPNodeAssignment::All,
+                                              false,
+                                              13);
+}
+
 // Test per-channel QDQ Conv. in0: u16, in1 (weight): s8, in2 (bias): s32, out: u16
 TEST_F(QnnHTPBackendTests, ConvU16S8S32_PerChannel) {
   std::vector<int64_t> input_shape = {1, 2, 4, 4};
@@ -692,6 +903,41 @@ TEST_F(QnnHTPBackendTests, ConvU16S8S32_PerChannel) {
                                                13);   // opset
 }
 
+// Conv3D per-channel
+// \QNN\HTP\HTP\src\hexagon\prepare\graph_prepare.cc:203:ERROR:could not create op: q::QNN_Conv3d_w_scale
+// \QNN\HTP\HTP\src\hexagon\prepare\graph_prepare.cc:1187:ERROR:Op 0x1a preparation failed with err:-1
+// QnnDsp <E> "Conv" generated: could not create op
+// QnnDsp <E> RouterWindows graph prepare failed 12
+// QnnDsp <E> Failed to finalize graph (id: 1) with err 1002
+// QnnDsp <V> Wake up free backend 1 thread(s)
+// QnnDsp <I> QnnGraph_finalize done. status 0x3ea
+// onnxruntime::qnn::QnnModel::FinalizeGraphs] Failed to finalize QNN graph.
+TEST_F(QnnHTPBackendTests, DISABLED_Conv3D_U16S8S32_PerChannel) {
+  std::vector<int64_t> input_shape = {1, 2, 4, 4, 4};
+  std::vector<int64_t> weight_shape = {3, 2, 2, 2, 2};
+  std::vector<int64_t> bias_shape = {3};
+
+  TestInputDef<float> input_def(input_shape, false,
+                                GetFloatDataInRange(-10.0f, 10.0f, TensorShape(input_shape).Size()));
+  TestInputDef<float> weight_def(weight_shape, true,
+                                 GetFloatDataInRange(-1.0f, 5.0f, TensorShape(weight_shape).Size()));
+  TestInputDef<float> bias_def(bias_shape, true,
+                               GetFloatDataInRange(-1.0f, 1.0f, TensorShape(bias_shape).Size()));
+
+  RunHTPConvOpPerChannelTest<uint16_t, int8_t>("Conv",
+                                               input_def,
+                                               weight_def,
+                                               bias_def,
+                                               {1, 1, 1},           // Strides
+                                               {0, 0, 0, 0, 0, 0},  // Pads
+                                               {1, 1, 1},           // Dilations
+                                               1,                   // default group
+                                               "NOTSET",
+                                               ExpectedEPNodeAssignment::All,
+                                               true,
+                                               13);
+}
+
 // Test per-channel QDQ ConvTranspose. in0: u16, in1 (weight): s8, in2 (bias): s32, out: u16
 TEST_F(QnnHTPBackendTests, ConvTransposeU16S8S32_PerChannel) {
   std::vector<int64_t> input_shape = {1, 2, 4, 4};
@@ -719,6 +965,33 @@ TEST_F(QnnHTPBackendTests, ConvTransposeU16S8S32_PerChannel) {
                                                13);   // opset
 }
 
+// Disable it for 2.21, re-enable it for 2.22
+TEST_F(QnnHTPBackendTests, DISABLED_ConvTranspose3D_U16S8S32_PerChannel) {
+  std::vector<int64_t> input_shape = {1, 2, 4, 4, 4};
+  std::vector<int64_t> weight_shape = {2, 3, 2, 2, 2};
+  std::vector<int64_t> bias_shape = {3};
+
+  TestInputDef<float> input_def(input_shape, false,
+                                GetFloatDataInRange(-10.0f, 10.0f, TensorShape(input_shape).Size()));
+  TestInputDef<float> weight_def(weight_shape, true,
+                                 GetFloatDataInRange(-1.0f, 5.0f, TensorShape(weight_shape).Size()));
+  TestInputDef<float> bias_def(bias_shape, true,
+                               GetFloatDataInRange(-1.0f, 1.0f, TensorShape(bias_shape).Size()));
+
+  RunHTPConvOpPerChannelTest<uint16_t, int8_t>("ConvTranspose",
+                                               input_def,
+                                               weight_def,
+                                               bias_def,
+                                               {1, 1, 1},           // Strides
+                                               {0, 0, 0, 0, 0, 0},  // Pads
+                                               {1, 1, 1},           // Dilations
+                                               1,                   // default group
+                                               "NOTSET",
+                                               ExpectedEPNodeAssignment::All,
+                                               true,
+                                               13);
+}
+
 // Test per-channel QDQ Conv that maps to QNN's DepthwiseConv2d (input_chans == output_chans == group).
 // in0: u16, in1 (weight): s8, in2 (bias): s32, out: u16
 TEST_F(QnnHTPBackendTests, ConvDepthwiseU16S8S32_PerChannel) {
@@ -747,6 +1020,41 @@ TEST_F(QnnHTPBackendTests, ConvDepthwiseU16S8S32_PerChannel) {
                                                13);   // opset
 }
 
+// Test per-channel QDQ Conv3D
+// \QNN\HTP\HTP\src\hexagon\prepare\graph_prepare.cc:203:ERROR:could not create op: q::QNN_Conv3d_w_scale
+// \QNN\HTP\HTP\src\hexagon\prepare\graph_prepare.cc:1187:ERROR:Op 0x1a preparation failed with err:-1
+// QnnDsp <E> "Conv" generated: could not create op
+// QnnDsp <E> RouterWindows graph prepare failed 12
+// QnnDsp <E> Failed to finalize graph (id: 1) with err 1002
+// QnnDsp <V> Wake up free backend 1 thread(s)
+// QnnDsp <I> QnnGraph_finalize done. status 0x3ea
+// onnxruntime::qnn::QnnModel::FinalizeGraphs] Failed to finalize QNN graph.
+TEST_F(QnnHTPBackendTests, DISABLED_Conv3D_U16S8S32_PerChannel2) {
+  std::vector<int64_t> input_shape = {1, 2, 4, 4, 4};
+  std::vector<int64_t> weight_shape = {2, 1, 2, 2, 2};
+  std::vector<int64_t> bias_shape = {2};
+
+  TestInputDef<float> input_def(input_shape, false,
+                                GetFloatDataInRange(-10.0f, 10.0f, TensorShape(input_shape).Size()));
+  TestInputDef<float> weight_def(weight_shape, true,
+                                 GetFloatDataInRange(-1.0f, 5.0f, TensorShape(weight_shape).Size()));
+  TestInputDef<float> bias_def(bias_shape, true,
+                               GetFloatDataInRange(-1.0f, 1.0f, TensorShape(bias_shape).Size()));
+
+  RunHTPConvOpPerChannelTest<uint16_t, int8_t>("Conv",
+                                               input_def,
+                                               weight_def,
+                                               bias_def,
+                                               {1, 1, 1},           // Strides
+                                               {0, 0, 0, 0, 0, 0},  // Pads
+                                               {1, 1, 1},           // Dilations
+                                               2,                   // default group
+                                               "NOTSET",
+                                               ExpectedEPNodeAssignment::All,
+                                               true,
+                                               13);
+}
+
 // Tests 16-bit QDQ Conv with dynamic weights and bias (uses QNN's Conv2d)
 // TODO: Inaccuracy detected for output 'output', element 0.
 // Output quant params: scale=0.0040235077030956745, zero_point=0.
@@ -829,11 +1137,6 @@ TEST_F(QnnHTPBackendTests, DISABLED_DepthwiseConvU16S16S32_NoBias) {
 
 // Tests 16-bit activations, 8-bit static weights QDQ Conv with static bias.
 // Uses QNN's DepthwiseConv2d operator.
-// TODO: Inaccuracy detected for output 'output', element 8.
-// Output quant params: scale=0.0027466239407658577, zero_point=10194.
-// Expected val: 152
-// QNN QDQ val: 151.8004150390625 (err 0.1995849609375)
-// CPU QDQ val: 151.9981689453125 (err 0.0018310546875)
 TEST_F(QnnHTPBackendTests, DepthwiseConvU16U8S32_StaticBias) {
   std::vector<float> input_data = GetFloatDataInRange(-10.0f, 10.0f, 25);
   std::vector<float> weight_data = GetFloatDataInRange(-1.0f, 5.0f, 9);
@@ -849,14 +1152,24 @@ TEST_F(QnnHTPBackendTests, DepthwiseConvU16U8S32_StaticBias) {
                                       ExpectedEPNodeAssignment::All,
                                       true,  // Use com.microsoft QDQ ops for 16-bit
                                       13);
+
+  std::vector<float> input_data_3d = GetFloatDataInRange(-10.0f, 10.0f, 125);
+  std::vector<float> weight_data_3d = GetFloatDataInRange(-1.0f, 5.0f, 27);
+  RunHTPConvOpTest<uint16_t, uint8_t>("Conv",
+                                      TestInputDef<float>({1, 1, 5, 5, 5}, false, input_data_3d),  // Input
+                                      TestInputDef<float>({1, 1, 3, 3, 3}, true, weight_data_3d),  // Weights
+                                      TestInputDef<float>({1}, true, {2.0f}),                      // Bias
+                                      {1, 1, 1},                                                   // Strides
+                                      {0, 0, 0, 0, 0, 0},                                          // Pads
+                                      {1, 1, 1},                                                   // Dilations
+                                      1,                                                           // default group
+                                      "NOTSET",
+                                      ExpectedEPNodeAssignment::All,
+                                      true,  // Use com.microsoft QDQ ops for 16-bit
+                                      13);
 }
 
 // Tests 16-bit activations, 8-bit static weights QDQ Conv with static bias.
-// TODO: Inaccuracy detected for output 'output', element 0.
-// Output quant params: scale=0.0040235077030956745, zero_point=0.
-// Expected val: 87.354057312011719
-// QNN QDQ val: 87.559577941894531 (err 0.2055206298828125)
-// CPU QDQ val: 87.398635864257812 (err 0.04457855224609375)
 TEST_F(QnnHTPBackendTests, ConvU16U8S32_StaticBias) {
   std::vector<float> input_data = GetFloatDataInRange(-10.0f, 10.0f, 50);
   std::vector<float> weight_data = GetFloatDataInRange(-1.0f, 5.0f, 18);
@@ -872,15 +1185,25 @@ TEST_F(QnnHTPBackendTests, ConvU16U8S32_StaticBias) {
                                       ExpectedEPNodeAssignment::All,
                                       true,  // Use com.microsoft QDQ ops for 16-bit
                                       13);
+
+  std::vector<float> input_data_3d = GetFloatDataInRange(-10.0f, 10.0f, 150);
+  std::vector<float> weight_data_3d = GetFloatDataInRange(-1.0f, 5.0f, 36);
+  RunHTPConvOpTest<uint16_t, uint8_t>("Conv",
+                                      TestInputDef<float>({1, 2, 5, 5, 3}, false, input_data_3d),  // Input
+                                      TestInputDef<float>({1, 2, 3, 3, 2}, true, weight_data_3d),  // Weights
+                                      TestInputDef<float>({1}, true, {2.0f}),                      // Bias
+                                      {1, 1, 1},                                                   // Strides
+                                      {0, 0, 0, 0, 0, 0},                                          // Pads
+                                      {1, 1, 1},                                                   // Dilations
+                                      1,                                                           // default group
+                                      "NOTSET",
+                                      ExpectedEPNodeAssignment::All,
+                                      true,  // Use com.microsoft QDQ ops for 16-bit
+                                      13);
 }
 
 // Tests 16-bit activations, 8-bit static weights QDQ Conv with dynamic bias.
 // Uses QNN's DepthwiseConv2d operator.
-// TODO: Inaccuracy detected for output 'output', element 1.
-// Output quant params: scale=0.0027466239407658577, zero_point=10194.
-// Expected val: -13.000001907348633
-// QNN QDQ val: -13.095903396606445 (err 0.0959014892578125)
-// CPU QDQ val: -12.999771118164062 (err 0.0002307891845703125)
 TEST_F(QnnHTPBackendTests, DepthwiseConvU16U8S32_DynamicBias) {
   std::vector<float> input_data = GetFloatDataInRange(-10.0f, 10.0f, 25);
   std::vector<float> weight_data = GetFloatDataInRange(-1.0f, 5.0f, 9);
@@ -896,14 +1219,24 @@ TEST_F(QnnHTPBackendTests, DepthwiseConvU16U8S32_DynamicBias) {
                                       ExpectedEPNodeAssignment::All,
                                       true,  // Use com.microsoft QDQ ops for 16-bit
                                       13);
+
+  std::vector<float> input_data_3d = GetFloatDataInRange(-10.0f, 10.0f, 75);
+  std::vector<float> weight_data_3d = GetFloatDataInRange(-1.0f, 5.0f, 27);
+  RunHTPConvOpTest<uint16_t, uint8_t>("Conv",
+                                      TestInputDef<float>({1, 1, 5, 5, 3}, false, input_data_3d),  // Input
+                                      TestInputDef<float>({1, 1, 3, 3, 3}, true, weight_data_3d),  // Weights
+                                      TestInputDef<float>({1}, false, {2.0f}),                     // Bias
+                                      {1, 1, 1},                                                   // Strides
+                                      {0, 0, 0, 0, 0, 0},                                          // Pads
+                                      {1, 1, 1},                                                   // Dilations
+                                      1,                                                           // default group
+                                      "NOTSET",
+                                      ExpectedEPNodeAssignment::All,
+                                      true,  // Use com.microsoft QDQ ops for 16-bit
+                                      13);
 }
 
 // Tests 16-bit activations, 8-bit static weights QDQ Conv with dynamic bias.
-// TODO: Inaccuracy detected for output 'output', element 0.
-// Output quant params: scale=0.0040235077030956745, zero_point=0.
-// Expected val: 87.354057312011719
-// QNN QDQ val: 87.559577941894531 (err 0.2055206298828125)
-// CPU QDQ val: 87.398635864257812 (err 0.04457855224609375)
 TEST_F(QnnHTPBackendTests, ConvU16U8S32_DynamicBias) {
   std::vector<float> input_data = GetFloatDataInRange(-10.0f, 10.0f, 50);
   std::vector<float> weight_data = GetFloatDataInRange(-1.0f, 5.0f, 18);
@@ -919,14 +1252,24 @@ TEST_F(QnnHTPBackendTests, ConvU16U8S32_DynamicBias) {
                                       ExpectedEPNodeAssignment::All,
                                       true,  // Use com.microsoft QDQ ops for 16-bit
                                       13);
+
+  std::vector<float> input_data_3d = GetFloatDataInRange(-10.0f, 10.0f, 150);
+  std::vector<float> weight_data_3d = GetFloatDataInRange(-1.0f, 5.0f, 36);
+  RunHTPConvOpTest<uint16_t, uint8_t>("Conv",
+                                      TestInputDef<float>({1, 2, 5, 5, 3}, false, input_data_3d),  // Input
+                                      TestInputDef<float>({1, 2, 3, 3, 2}, true, weight_data_3d),  // Weights
+                                      TestInputDef<float>({1}, false, {2.0f}),                     // Bias
+                                      {1, 1, 1},                                                   // Strides
+                                      {0, 0, 0, 0, 0, 0},                                          // Pads
+                                      {1, 1, 1},                                                   // Dilations
+                                      1,                                                           // default group
+                                      "NOTSET",
+                                      ExpectedEPNodeAssignment::All,
+                                      true,  // Use com.microsoft QDQ ops for 16-bit
+                                      13);
 }
 
 // Tests 16-bit activations, 8-bit static weights QDQ Conv with no bias
-// TODO: Inaccuracy detected for output 'output', element 7.
-// Output quant params: scale=0.0039929896593093872, zero_point=0.
-// Expected val: 246.98667907714844
-// QNN QDQ val: 247.82090759277344 (err 0.834228515625)
-// CPU QDQ val: 247.24192810058594 (err 0.2552490234375)
 TEST_F(QnnHTPBackendTests, ConvU16U8S32_NoBias) {
   std::vector<float> input_data = GetFloatDataInRange(-10.0f, 10.0f, 50);
   std::vector<float> weight_data = GetFloatDataInRange(-1.0f, 5.0f, 18);
@@ -942,15 +1285,25 @@ TEST_F(QnnHTPBackendTests, ConvU16U8S32_NoBias) {
                                       ExpectedEPNodeAssignment::All,
                                       true,  // Use com.microsoft QDQ ops for 16-bit
                                       13);
+
+  std::vector<float> input_data_3d = GetFloatDataInRange(-10.0f, 10.0f, 150);
+  std::vector<float> weight_data_3d = GetFloatDataInRange(-1.0f, 5.0f, 36);
+  RunHTPConvOpTest<uint16_t, uint8_t>("Conv",
+                                      TestInputDef<float>({1, 2, 5, 5, 3}, false, input_data_3d),  // Input
+                                      TestInputDef<float>({1, 2, 3, 3, 2}, true, weight_data_3d),  // Weights
+                                      TestInputDef<float>(),                                       // Bias
+                                      {1, 1, 1},                                                   // Strides
+                                      {0, 0, 0, 0, 0, 0},                                          // Pads
+                                      {1, 1, 1},                                                   // Dilations
+                                      1,                                                           // default group
+                                      "NOTSET",
+                                      ExpectedEPNodeAssignment::All,
+                                      true,  // Use com.microsoft QDQ ops for 16-bit
+                                      13);
 }
 
 // Tests 16-bit activations, 8-bit static weights QDQ Conv with no bias
 // Uses QNN's DepthwiseConv2d operator.
-// TODO: Inaccuracy detected for output 'output', element 8.
-// Output quant params: scale=0.0027466239407658577, zero_point=10923.
-// Expected val: 150
-// QNN QDQ val: 149.80087280273438 (err 0.199127197265625)
-// CPU QDQ val: 149.99862670898438 (err 0.001373291015625)
 TEST_F(QnnHTPBackendTests, DepthwiseConvU16U8S32_NoBias) {
   std::vector<float> input_data = GetFloatDataInRange(-10.0f, 10.0f, 25);
   std::vector<float> weight_data = GetFloatDataInRange(-1.0f, 5.0f, 9);
@@ -966,6 +1319,21 @@ TEST_F(QnnHTPBackendTests, DepthwiseConvU16U8S32_NoBias) {
                                       ExpectedEPNodeAssignment::All,
                                       true,  // Use com.microsoft QDQ ops for 16-bit
                                       13);
+
+  std::vector<float> input_data_3d = GetFloatDataInRange(-10.0f, 10.0f, 75);
+  std::vector<float> weight_data_3d = GetFloatDataInRange(-1.0f, 5.0f, 18);
+  RunHTPConvOpTest<uint16_t, uint8_t>("Conv",
+                                      TestInputDef<float>({1, 1, 5, 5, 3}, false, input_data_3d),  // Input
+                                      TestInputDef<float>({1, 1, 3, 3, 2}, true, weight_data_3d),  // Weights
+                                      TestInputDef<float>(),                                       // Bias
+                                      {1, 1, 1},                                                   // Strides
+                                      {0, 0, 0, 0, 0, 0},                                          // Pads
+                                      {1, 1, 1},                                                   // Dilations
+                                      1,                                                           // default group
+                                      "NOTSET",
+                                      ExpectedEPNodeAssignment::All,
+                                      true,  // Use com.microsoft QDQ ops for 16-bit
+                                      13);
 }
 
 // Test that dynamic weights with default bias works for Conv. This was previously not working
@@ -981,6 +1349,17 @@ TEST_F(QnnHTPBackendTests, ConvU8U8S32_DynamicWeight_NoBias) {
                                      1,                                                          // default group
                                      "NOTSET",
                                      ExpectedEPNodeAssignment::All);
+
+  RunHTPConvOpTest<uint8_t, uint8_t>("Conv",
+                                     TestInputDef<float>({1, 3, 32, 32, 32}, false, -10.0f, 10.0f),  // Input
+                                     TestInputDef<float>({1, 3, 4, 4, 4}, false, -10.0f, 10.0f),     // Weights
+                                     TestInputDef<float>(),                                          // Bias
+                                     {1, 1, 1},                                                      // Strides
+                                     {0, 0, 0, 0, 0, 0},                                             // Pads
+                                     {1, 1, 1},                                                      // Dilations
+                                     1,                                                              // default group
+                                     "NOTSET",
+                                     ExpectedEPNodeAssignment::All);
 }
 
 // Test that dynamic weights with default bias works for ConvTranspose. This was previously not working
@@ -998,6 +1377,23 @@ TEST_F(QnnHTPBackendTests, ConvTransposeU8U8S32_DynamicWeight_NoBias) {
                                      ExpectedEPNodeAssignment::All);
 }
 
+// QNN op validation crash. Run correctly if by pass the QNN op validation
+// Exception from backendValidateOpConfig:
+// Exception thrown at 0x00007FFF9E0128B0 (QnnHtpPrepare.dll) in onnxruntime_test_all.exe:
+// 0xC0000005: Access violation reading location 0x7079745F656C706D.
+TEST_F(QnnHTPBackendTests, DISABLED_ConvTranspose3D_U8U8S32_DynamicWeight_NoBias) {
+  RunHTPConvOpTest<uint8_t, uint8_t>("ConvTranspose",
+                                     TestInputDef<float>({1, 3, 32, 32, 32}, false, -10.0f, 10.0f),  // Input
+                                     TestInputDef<float>({3, 1, 4, 4, 4}, false, -10.0f, 10.0f),     // Weights
+                                     TestInputDef<float>(),                                          // Bias
+                                     {1, 1, 1},                                                      // Strides
+                                     {0, 0, 0, 0, 0, 0},                                             // Pads
+                                     {1, 1, 1},                                                      // Dilations
+                                     1,                                                              // default group
+                                     "NOTSET",
+                                     ExpectedEPNodeAssignment::All);
+}
+
 // Check that QNN compiles DQ -> Conv -> Q as a single unit.
 // Tests bias as an initializer.
 TEST_F(QnnHTPBackendTests, ConvU8U8S32_bias_initializer) {
@@ -1015,6 +1411,21 @@ TEST_F(QnnHTPBackendTests, ConvU8U8S32_bias_initializer) {
                                      13,     // opset
                                      // Need tolerance of 0.413% of output range after QNN SDK 2.17
                                      QDQTolerance(0.00413f));
+
+  RunHTPConvOpTest<uint8_t, uint8_t>("Conv",
+                                     TestInputDef<float>({1, 1, 5, 5, 5}, false, 0.0f, 10.0f),   // Random dynamic input
+                                     TestInputDef<float>({1, 1, 3, 3, 3}, true, -10.0f, 10.0f),  // Random static weight
+                                     TestInputDef<float>({1}, true, {2.0f}),                     // Initializer bias
+                                     {1, 1, 1},                                                  // Strides
+                                     {0, 0, 0, 0, 0, 0},                                         // Pads
+                                     {1, 1, 1},                                                  // Dilations
+                                     1,                                                          // default group
+                                     "NOTSET",
+                                     ExpectedEPNodeAssignment::All,
+                                     false,  // use_qdq_contrib_ops
+                                     13,     // opset
+                                     // Need tolerance of 0.413% of output range after QNN SDK 2.17
+                                     QDQTolerance(0.00413f));
 }
 
 // Tests 1D Conv with bias as an initializer.
@@ -1061,6 +1472,19 @@ TEST_F(QnnHTPBackendTests, ConvU8U8S32_AutoPadUpper) {
                                      ExpectedEPNodeAssignment::All,
                                      false,  // use_contrib_qdq
                                      13);
+
+  RunHTPConvOpTest<uint8_t, uint8_t>("Conv",
+                                     TestInputDef<float>({1, 1, 5, 5, 5}, false, 0.f, 10.f),  // Dynamic input
+                                     TestInputDef<float>({1, 1, 4, 4, 4}, true, -1.f, 1.f),   // Static weights
+                                     TestInputDef<float>({1}, true, {1.0f}),                  // Initializer bias
+                                     {1, 1, 1},                                               // strides
+                                     {},                                                      // pads
+                                     {1, 1, 1},                                               // dilations
+                                     1,                                                       // default group
+                                     "SAME_UPPER",                                            // auto_pad
+                                     ExpectedEPNodeAssignment::All,
+                                     false,  // use_contrib_qdq
+                                     13);
 }
 
 // Tests Conv1d auto_pad value "SAME_UPPER" on HTP backend (compares to CPU EP).
@@ -1111,6 +1535,19 @@ TEST_F(QnnHTPBackendTests, ConvU8U8S32_AutoPadLower) {
                                      ExpectedEPNodeAssignment::All,
                                      false,  // use_contrib_qdq
                                      13);
+
+  RunHTPConvOpTest<uint8_t, uint8_t>("Conv",
+                                     TestInputDef<float>({1, 1, 5, 5, 5}, false, 0.f, 10.f),  // Dynamic input
+                                     TestInputDef<float>({1, 1, 4, 4, 4}, true, -1.f, 1.f),   // Static weights
+                                     TestInputDef<float>({1}, true, {1.0f}),                  // Initializer bias
+                                     {1, 1, 1},                                               // strides
+                                     {},                                                      // pads
+                                     {1, 1, 1},                                               // dilations
+                                     1,                                                       // default group
+                                     "SAME_LOWER",                                            // auto_pad
+                                     ExpectedEPNodeAssignment::All,
+                                     false,  // use_contrib_qdq
+                                     13);
 }
 
 // Tests ConvTranspose's auto_pad value "SAME_LOWER" on HTP backend (compares to CPU EP).
@@ -1127,6 +1564,19 @@ TEST_F(QnnHTPBackendTests, ConvTransposeU8U8S32_AutoPadLower) {
                                      ExpectedEPNodeAssignment::All,
                                      false,  // use_contrib_qdq
                                      13);
+
+  RunHTPConvOpTest<uint8_t, uint8_t>("ConvTranspose",
+                                     TestInputDef<float>({1, 1, 5, 5, 5}, false, 0.f, 10.f),  // Dynamic input
+                                     TestInputDef<float>({1, 1, 4, 4, 4}, true, -1.f, 1.f),   // Static weights
+                                     TestInputDef<float>({1}, true, {1.0f}),                  // Initializer bias
+                                     {1, 1, 1},                                               // strides
+                                     {},                                                      // pads
+                                     {1, 1, 1},                                               // dilations
+                                     1,                                                       // default group
+                                     "SAME_LOWER",                                            // auto_pad
+                                     ExpectedEPNodeAssignment::All,
+                                     false,  // use_contrib_qdq
+                                     13);
 }
 
 // Tests Conv1d auto_pad value "SAME_LOWER" on HTP backend (compares to CPU EP).