saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-28 20:11:22 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Resize and EP specific transpose optimization updates (#17664)

Browse source 
### Description
<!-- Describe your changes. -->
- Treat Resize as layout sensitive by default
- whilst the ONNX spec does not specify a layout, EPs tend to implement
only one
- add second usage in L2 of TransposeOptimizer to plugin the ability to
push a Transpose through a Resize assigned to the CPU EP
- Allow EP specific logic for changes the ops considered to be layout
sensitive to be plugged in
  - expected usage is for #17200 


### Motivation and Context
<!-- - Why is this change required? What problem does it solve?
- If it fixes an open issue, please link to the issue here. -->
Finish simplifying/clarifying transpose optimization and layout
transformation that was proposed in #15552. This PR along with #17618
should complete the changes.

---------

Co-authored-by: Edward Chen <18449977+edgchen1@users.noreply.github.com>
This commit is contained in:
Scott McKay 2023-09-29 08:11:36 +10:00 committed by GitHub
parent 20f96fd096
commit 9cb60c5b86
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 238 additions and 118 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
onnxruntime/core/framework/graph_partitioner.cc
View file

@ -177,9 +177,9 @@ static Status GetCapabilityForEP(const GetCapabilityForEPParams& params) {
}
#if !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD)
// Run layout transformer only for EPs other than CPU EP and provided the preferred layout is NHWC
// Run layout transformer for EPs with preferred layout of NHWC
// CPU EP layout transformation happens later when level 3 transformers are run.
if (params.mode != GraphPartitioner::Mode::kAssignOnly &&
if (params.mode != GraphPartitioner::Mode::kAssignOnly && params.transform_layout.get() &&
current_ep.GetPreferredLayout() == DataLayout::NHWC) {
for (auto& capability : capabilities) {
TryAssignNodes(graph, *capability->sub_graph, ep_type);

5
onnxruntime/core/framework/kernel_registry_manager.cc
View file

@ -63,8 +63,9 @@ Status KernelRegistryManager::SearchKernelRegistry(const Node& node,
auto create_error_message = [&node, &status](const std::string& prefix) {
std::ostringstream errormsg;
errormsg << prefix << node.OpType() << "(" << node.SinceVersion() << ")";
if (!node.Name().empty()) errormsg << " (node " << node.Name() << "). ";
if (!status.IsOK()) errormsg << status.ErrorMessage();
errormsg << " (node:'" << node.Name() << "' ep:'" << node.GetExecutionProviderType() << "'). ";
if (!status.IsOK())
errormsg << status.ErrorMessage();
return errormsg.str();
};

2
onnxruntime/core/graph/contrib_ops/internal_nhwc_onnx_schemas.cc
View file

@ -94,7 +94,7 @@ void OpSet_Internal_NHWC_ONNX::ForEachSchema(const std::function<void(ONNX_NAMES
// so supporting older opsets is unnecessary.
// NOTE: This should be in sync with GetLayoutSensitiveOps in
// /onnxruntime/core/optimizer/transpose_optimizer/transpose_optimizer.cc
// /onnxruntime/core/optimizer/transpose_optimization/transpose_optimizer.cc
REGISTER_NHWC_SCHEMA_WITH_ACTIVATION(fn, AveragePool, 11);
REGISTER_NHWC_SCHEMA_WITH_ACTIVATION(fn, BatchNormalization, 9);

19
onnxruntime/core/optimizer/graph_transformer_utils.cc
View file

@ -189,6 +189,8 @@ InlinedVector<std::unique_ptr<GraphTransformer>> GenerateTransformers(
const InlinedHashSet<std::string_view> cpu_ep = {onnxruntime::kCpuExecutionProvider};
#endif
const InlinedHashSet<std::string_view> dml_ep = {onnxruntime::kDmlExecutionProvider};
AllocatorPtr cpu_allocator = std::make_shared<CPUAllocator>();
switch (level) {
case TransformerLevel::Level1: {
// RewriteRule optimizations are the simplest (they generally remove unnecessary nodes and are cheap to run)
@ -240,13 +242,14 @@ InlinedVector<std::unique_ptr<GraphTransformer>> GenerateTransformers(
// run TransposeOptimizer last as it works in a slightly different way by moving Transpose nodes around.
// shouldn't affect the end result - just easier to debug any issue if it's last.
// local CPU allocator is enough as this allocator is finally passed to a local tensor.
// We will also benefit by using a local allocator as we don't need to pass allocator as parameter for EP API refactor
AllocatorPtr cpu_allocator = std::make_shared<CPUAllocator>();
transformers.emplace_back(std::make_unique<TransposeOptimizer>(std::move(cpu_allocator)));
} break;
case TransformerLevel::Level2: {
// we run TransposeOptimizer again in Level2 for some CPU EP specific optimizations that can only be
// applied once nodes are assigned to the CPU EP (which happens between level 1 and level 2).
transformers.emplace_back(std::make_unique<TransposeOptimizer>(std::move(cpu_allocator), kCpuExecutionProvider));
const bool enable_quant_qdq_cleanup =
session_options.config_options.GetConfigOrDefault(kOrtSessionOptionsEnableQuantQDQCleanup, "0") == "1";
#if !defined(DISABLE_CONTRIB_OPS)
@ -366,16 +369,16 @@ InlinedVector<std::unique_ptr<GraphTransformer>> GenerateTransformers(
if (MlasNchwcGetBlockSize() > 1) {
transformers.emplace_back(std::make_unique<NchwcTransformer>());
}
AllocatorPtr cpu_allocator = std::make_shared<CPUAllocator>();
auto cpu_registry = cpu_execution_provider.GetKernelRegistry();
auto nhwc_transformer = std::make_unique<NhwcTransformer>(std::move(cpu_allocator), std::move(cpu_registry));
if (nhwc_transformer->IsActive()) {
transformers.emplace_back(std::move(nhwc_transformer));
}
// NCHWCtransformer should have a higher priority versus this. Because NCHWCtransformer also do the similar things
// of fusion patterns and target on CPU. However, NCHWCtransformer will reorder the layout to nchwc which is only available for
// x86-64 cpu, not edge cpu like arm. But This transformer could be used by opencl-ep/cpu-ep. So
// we will prefer NhwcTransformer once ort runs on x86-64 CPU, otherwise ConvAddActivationFusion is enabled.
// NchwcTransformer must have a higher priority than ConvAddActivationFusion. NchwcTransformer does similar
// fusions targeting CPU but also reorders the layout to NCHWc which is expected to be more efficient but is
// only available on x86-64.
// PR #6351 implemented similar fusion-pattern for CUDA only, and can only fuse conv-add-relu,
// while we can fuse more activation.
transformers.emplace_back(std::make_unique<ConvAddActivationFusion>(cpu_ep));

131
onnxruntime/core/optimizer/layout_transformation/layout_transformation.cc
View file

@ -13,40 +13,11 @@ using namespace onnx_transpose_optimization;
namespace onnxruntime {
namespace layout_transformation {
// Layout sensitive NCHW ops. TransformLayoutForEP will wrap these with Transpose nodes to convert the input
// data to NHWC and output data back to NCHW, and move the op to the internal NHWC domain (kMSInternalNHWCDomain).
// The EP requesting these ops MUST be able to handle the node with the operator in the kMSInternalNHWCDomain.
// Once all the layout sensitive ops requested by the EP are wrapped the transpose optimizer will attempt to remove
// as many of the layout transposes as possible.
const std::unordered_set<std::string_view>& GetORTLayoutSensitiveOps() {
static std::unordered_set<std::string_view> ort_layout_sensitive_ops = []() {
const auto& layout_sensitive_ops = onnx_transpose_optimization::GetLayoutSensitiveOps();
std::unordered_set<std::string_view> ort_specific_ops =
{ "FusedConv",
"QLinearAveragePool",
"QLinearGlobalAveragePool"
#if defined(USE_CUDA) || defined(USE_ROCM) || defined(USE_QNN) || defined(USE_WEBNN)
// The CUDA/ROCM Resize kernel is layout sensitive as it only handles NCHW input.
// The CPU kernel and ONNX spec are not limited to handling NCHW input so are not layout sensitive, and
// onnx_layout_transformation::HandleResize is used.
,
"Resize"
#endif
};
ort_specific_ops.insert(layout_sensitive_ops.cbegin(), layout_sensitive_ops.cend());
return ort_specific_ops;
}();
return ort_layout_sensitive_ops;
}
namespace {
// Cost check for aggressively pushing the Transpose nodes involved in the layout transformation further out.
static CostCheckResult
PostLayoutTransformCostCheck(const api::GraphRef& graph, const api::NodeRef& node,
const std::vector<int64_t>& perm,
const std::unordered_set<std::string>& outputs_leading_to_transpose) {
CostCheckResult PostLayoutTransformCostCheck(const api::GraphRef& graph, const api::NodeRef& node,
const std::vector<int64_t>& perm,
const std::unordered_set<std::string>& outputs_leading_to_transpose) {
// we aggressively push the layout transpose nodes.
// Exception: pushing through a Concat can result in Transpose nodes being added to multiple other inputs which
// can potentially be worse for performance. Use the cost check in that case.
@ -59,28 +30,97 @@ PostLayoutTransformCostCheck(const api::GraphRef& graph, const api::NodeRef& nod
return OrtEPCostCheck(graph, node, perm, outputs_leading_to_transpose);
}
/// <summary>
/// Default function for checking if a node should have its layout changed. Allows EP specific adjustments to the
/// default set of layout sensitive operators if required.
///
/// Longer term, if required, the EP API could allow the EP to provide a delegate to plugin EP specific logic so we
/// don't hardcode it here.
/// </summary>
/// <param name="node">Node to check</param>
/// <returns>true if the node should have its layout converted to NHWC.</returns>
bool ConvertNodeLayout(const api::NodeRef& node) {
// skip if op is not an ONNX or contrib op
auto domain = node.Domain();
if (domain != kOnnxDomain && domain != kMSDomain) {
return false;
}
const auto& layout_sensitive_ops = GetORTLayoutSensitiveOps();
// handle special cases
#if defined(USE_XNNPACK)
if (node.GetExecutionProviderType() == kXnnpackExecutionProvider) {
if (node.OpType() == "Resize") {
// XNNPACK supports NCHW and NHWC for Resize so we don't need to use the internal NHWC domain and wrap the Resize
// with Transpose nodes. EPAwareHandleResize will allow an NCHW <-> NHWC Transpose to be pushed through
// the Resize during transpose optimization.
return false;
}
}
#endif
#if defined(USE_JSEP)
// TODO(fs-eire): Remove special case handing of JSEP once NHWC Resize implementation is fixed
if (node.GetExecutionProviderType() == kJsExecutionProvider) {
if (node.OpType() == "Resize") {
// leave Resize as-is pending bugfix for NHWC implementation. this means the node will remain in the ONNX domain
// with the original input layout.
return false;
}
}
#endif
// #if defined(USE_CUDA)
// if (node.GetExecutionProviderType() == kCudaExecutionProvider) {
// Update as per https://github.com/microsoft/onnxruntime/pull/17200 with CUDA ops that support NHWC
// }
// #endif
return layout_sensitive_ops.count(node.OpType()) != 0;
}
} // namespace
// Layout sensitive NCHW ops. TransformLayoutForEP will wrap these with Transpose nodes to convert the input
// data to NHWC and output data back to NCHW, and move the op to the internal NHWC domain (kMSInternalNHWCDomain).
// The EP requesting these ops MUST be able to handle the node with the operator in the kMSInternalNHWCDomain domain.
// Once all the layout sensitive ops requested by the EP are wrapped the transpose optimizer will attempt to remove
// as many of the layout transposes as possible.
const std::unordered_set<std::string_view>& GetORTLayoutSensitiveOps() {
static std::unordered_set<std::string_view> ort_layout_sensitive_ops = []() {
const auto& layout_sensitive_ops = onnx_transpose_optimization::GetLayoutSensitiveOps();
std::unordered_set<std::string_view> ort_specific_ops =
{
"FusedConv",
"QLinearAveragePool",
"QLinearGlobalAveragePool",
// Whilst the ONNX spec doesn't specify a layout for Resize, we treat it as layout sensitive by default
// as EPs tend to only support one layout.
"Resize",
};
ort_specific_ops.insert(layout_sensitive_ops.cbegin(), layout_sensitive_ops.cend());
return ort_specific_ops;
}();
return ort_layout_sensitive_ops;
}
Status TransformLayoutForEP(Graph& graph, bool& modified, const IExecutionProvider& execution_provider,
AllocatorPtr cpu_allocator,
const DebugGraphFn& debug_graph_fn) {
// We pass in nullptr for the new_node_ep param as new nodes will be assigned by the graph partitioner after
// TransformLayoutForEP returns.
// sub graph recurse will be added later.
// sub graph recurse will be added later
auto api_graph = MakeApiGraph(graph, cpu_allocator, /*new_node_ep*/ nullptr);
const auto& layout_sensitive_ops = GetORTLayoutSensitiveOps();
// to convert to NHWC we need to wrap layout sensitive nodes to Transpose from NCHW to NHWC and back.
for (auto& node : api_graph->Nodes()) {
if (layout_sensitive_ops.count(node->OpType())) {
if (node->GetExecutionProviderType() != execution_provider.Type()) {
continue;
}
auto domain = node->Domain();
// Skip if domain is incorrect
if (domain != kOnnxDomain && domain != kMSDomain) {
continue;
}
if (node->GetExecutionProviderType() != execution_provider.Type()) {
continue;
}
if (ConvertNodeLayout(*node)) {
// if already transformed then change the domain to kMSInternalNHWCDomain this way the EP
// knows this op is in the expected format.
if (node->GetAttributeIntDefault("channels_last", 0) == 1) {
@ -137,7 +177,6 @@ Status TransformLayoutForEP(Graph& graph, bool& modified, const IExecutionProvid
WrapTransposesAroundNode(*api_graph, *node, {&input_perm}, {&output_perm});
}
// TODO: Technically Resize doesn't need to change domain as the ONNX Resize spec is not layout sensitive.
SwapNodeOpTypeAndDomain(*api_graph, *node, node->OpType(), kMSInternalNHWCDomain);
modified = true;
}

27
onnxruntime/core/optimizer/transpose_optimization/onnx_transpose_optimization.cc
View file

@ -1242,18 +1242,7 @@ static void PermuteInput(api::GraphRef& graph, api::NodeRef& node, size_t i, con
node.SetInput(i, gather_output);
}
static bool HandleResize([[maybe_unused]] HandlerArgs& args) {
#if defined(USE_CUDA) || defined(USE_ROCM) || defined(USE_QNN) || defined(USE_WEBNN)
// The CUDA Resize kernel requires that the input is NCHW, so we can't push a Transpose through a Resize
// in ORT builds with CUDA enabled.
// The ROCm EP is generated from the CUDA EP kernel so the same applies to builds with ROCm enabled.
// The QNN EP requires the input to be NHWC, so the Resize handler is also not enabled for QNN builds.
//
// TODO: Remove this special case once the CUDA Resize kernel is implemented "generically" (i.e.) aligning with the
// generic nature of the ONNX spec.
// See https://github.com/microsoft/onnxruntime/pull/10824 for a similar fix applied to the CPU Resize kernel.
return false;
#else
bool HandleResize([[maybe_unused]] HandlerArgs& args) {
auto inputs = args.node.Inputs();
int64_t rank_int = gsl::narrow_cast<int64_t>(args.perm.size());
@ -1279,10 +1268,10 @@ static bool HandleResize([[maybe_unused]] HandlerArgs& args) {
TransposeOutputs(args.ctx, args.node, args.perm);
return true;
#endif
}
constexpr HandlerInfo resize_handler = {&FirstInput, &HandleResize};
// Not currently registered by default.
// constexpr HandlerInfo resize_handler = {&FirstInput, &HandleResize};
static bool HandlePad(HandlerArgs& args) {
size_t rank = args.perm.size();
@ -2034,8 +2023,11 @@ static const std::unordered_map<std::string_view, const HandlerInfo&> handler_ma
{"Split", split_handler},
{"Shape", shape_handler},
{"Pad", pad_handler},
{"Resize", resize_handler},
{"ReduceSum", reduce_op_handler},
// Execution providers tend to only implement Resize for specific layouts. Due to that, it's safer to not
// push a Transpose through a Resize unless the EP specifically checks that it can handle the change via an
// extended handler.
// {"Resize", resize_handler},
{"ReduceLogSum", reduce_op_handler},
{"ReduceLogSumExp", reduce_op_handler},
@ -2043,6 +2035,7 @@ static const std::unordered_map<std::string_view, const HandlerInfo&> handler_ma
{"ReduceMean", reduce_op_handler},
{"ReduceMin", reduce_op_handler},
{"ReduceProd", reduce_op_handler},
{"ReduceSum", reduce_op_handler},
{"ReduceSumSquare", reduce_op_handler},
{"ReduceL1", reduce_op_handler},
{"ReduceL2", reduce_op_handler},
@ -2385,6 +2378,8 @@ OptimizeResult OptimizeImpl(OptimizerCtx& ctx) {
continue;
}
// NOTE: this bleeds ORT specific logic into the base optimizer, however we justify that for now because we expect
// the types that the ORT DQ provides to be added to the ONNX spec, at which point this special case can go away.
if (IsMSDomain(dq_domain) && !TransposeQuantizeDequantizeAxis(ctx.graph, perm_inv, *dq_node)) {
continue;
}

1
onnxruntime/core/optimizer/transpose_optimization/onnx_transpose_optimization.h
View file

@ -98,6 +98,7 @@ bool HandleSimpleNodeWithAxis(HandlerArgs& args, std::optional<int64_t> default_
// base handlers that are used by extended handlers. add from transpose_optimizer.cc as needed.
bool HandleReduceOps(HandlerArgs& args);
bool HandleResize([[maybe_unused]] HandlerArgs& args);
void TransposeInput(api::GraphRef& graph, api::NodeRef& node, size_t i,
const std::vector<int64_t>& perm,

27
onnxruntime/core/optimizer/transpose_optimization/ort_transpose_optimization.cc
View file

@ -5,12 +5,35 @@
#include <algorithm>
#include "core/graph/constants.h"
#include "core/framework/utils.h"
#include "core/optimizer/transpose_optimization/ort_optimizer_utils.h"
using namespace onnx_transpose_optimization;
namespace onnxruntime {
static bool EPAwareHandleResize(HandlerArgs& args) {
// Whilst Resize is not technically layout sensitive, execution providers typically implement handling for only one
// layout. Due to that, only push a Transpose through a Resize once it is assigned and we know it's being handled
// by an EP that supports multiple layouts. Currently that's the CPU and XNNPACK EPs.
const auto ep_type = args.node.GetExecutionProviderType();
if (ep_type == kCpuExecutionProvider || ep_type == kXnnpackExecutionProvider) {
// allow NCHW <-> NHWC for now. not clear any other sort of transpose has a valid usage in a real model
int64_t rank_int = gsl::narrow_cast<int64_t>(args.perm.size());
if (rank_int == 4) {
static const std::vector<int64_t> nchw_to_nhwc_perm{0, 2, 3, 1};
static const std::vector<int64_t> nhwc_to_nchw_perm{0, 3, 1, 2};
if (args.perm == nchw_to_nhwc_perm || args.perm == nhwc_to_nchw_perm) {
return HandleResize(args);
}
}
}
return false;
}
constexpr HandlerInfo ep_aware_resize_handler = {&FirstInput, &EPAwareHandleResize};
static bool HandleQLinearConcat(HandlerArgs& args) {
return HandleSimpleNodeWithAxis(args);
}
@ -62,7 +85,7 @@ static bool HandleMaxPool(HandlerArgs& args) {
ORT_UNUSED_PARAMETER(args);
return false;
#else
if (args.node.GetExecutionProviderType() != "CPUExecutionProvider") {
if (args.node.GetExecutionProviderType() != kCpuExecutionProvider) {
return false;
}
@ -103,6 +126,7 @@ static bool HandleContribQuantizeDequantizeLinear(HandlerArgs& args) {
}
constexpr HandlerInfo max_pool_op_handler = {&FirstInput, &HandleMaxPool};
constexpr HandlerInfo node_1_inp_handler = {&FirstInput, &HandleSimpleNode};
constexpr HandlerInfo reduce_op_handler = {&FirstInput, &HandleReduceOps};
constexpr HandlerInfo contrib_quantize_dequantize_linear_handler = {&FirstInput,
@ -113,6 +137,7 @@ const HandlerMap& OrtExtendedHandlers() {
static const HandlerMap extended_handler_map = []() {
HandlerMap map = {
{"MaxPool", max_pool_op_handler},
{"Resize", ep_aware_resize_handler},
{"com.microsoft.QuantizeLinear", contrib_quantize_dequantize_linear_handler},
{"com.microsoft.DequantizeLinear", contrib_quantize_dequantize_linear_handler},
{"com.microsoft.QLinearAdd", q_linear_binary_op_handler},

1
onnxruntime/core/optimizer/transpose_optimization/ort_transpose_optimization.h
View file

@ -10,7 +10,6 @@ namespace onnxruntime {
/// <summary>
/// Get the extended handlers for ORT specific transpose optimization.
/// These include handlers for contrib ops, and where we have an NHWC version of a layout sensitive op.
/// Extends the handlers returned by OrtHandlers.
/// </summary>
/// <returns>HandlerMap</returns>
const onnx_transpose_optimization::HandlerMap& OrtExtendedHandlers();

14
onnxruntime/core/optimizer/transpose_optimizer.cc
View file

@ -18,10 +18,18 @@ namespace onnxruntime {
Status TransposeOptimizer::ApplyImpl(Graph& graph, bool& modified, int graph_level,
const logging::Logger& logger) const {
auto api_graph = MakeApiGraph(graph, cpu_allocator_, /*new_node_ep*/ nullptr);
OptimizeResult result;
OptimizeResult result = onnx_transpose_optimization::Optimize(*api_graph, "", /* default cost check*/ nullptr,
OrtExtendedHandlers());
if (ep_.empty()) {
// basic usage - no EP specific optimizations
auto api_graph = MakeApiGraph(graph, cpu_allocator_, /*new_node_ep*/ nullptr);
result = onnx_transpose_optimization::Optimize(*api_graph, "", /* default cost check*/ nullptr,
OrtExtendedHandlers());
} else {
// EP specific optimizations enabled. Currently only used for CPU EP.
auto api_graph = MakeApiGraph(graph, cpu_allocator_, /*new_node_ep*/ ep_.c_str());
result = onnx_transpose_optimization::Optimize(*api_graph, ep_, OrtEPCostCheck, OrtExtendedHandlers());
}
if (result.error_msg) {
// currently onnx_layout_transformation::Optimize only fails if we hit an unsupported opset.

8
onnxruntime/core/optimizer/transpose_optimizer.h
View file

@ -15,10 +15,14 @@ Push transposes through ops and eliminate them.
class TransposeOptimizer : public GraphTransformer {
private:
AllocatorPtr cpu_allocator_;
const std::string ep_;
public:
explicit TransposeOptimizer(AllocatorPtr cpu_allocator) noexcept
: GraphTransformer("TransposeOptimizer"), cpu_allocator_(std::move(cpu_allocator)) {}
explicit TransposeOptimizer(AllocatorPtr cpu_allocator,
const std::string& ep = {}) noexcept
: GraphTransformer(ep.empty() ? "TransposeOptimizer" : "TransposeOptimizer_" + ep),
cpu_allocator_(std::move(cpu_allocator)),
ep_{ep} {}
Status ApplyImpl(Graph& graph, bool& modified, int graph_level, const logging::Logger& logger) const override;

8
onnxruntime/core/providers/xnnpack/nn/resize.cc
View file

@ -331,7 +331,13 @@ ONNX_OPERATOR_VERSIONED_KERNEL_EX(Resize, kOnnxDomain, 13, 17, kXnnpackExecution
DataTypeImpl::GetTensorType<int8_t>()}),
Resize);
ONNX_OPERATOR_KERNEL_EX(Resize, kOnnxDomain, 18, kXnnpackExecutionProvider,
ONNX_OPERATOR_VERSIONED_KERNEL_EX(Resize, kOnnxDomain, 18, 18, kXnnpackExecutionProvider,
KernelDefBuilder().TypeConstraint("T1", {DataTypeImpl::GetTensorType<float>(),
DataTypeImpl::GetTensorType<uint8_t>(),
DataTypeImpl::GetTensorType<int8_t>()}),
Resize);
ONNX_OPERATOR_KERNEL_EX(Resize, kOnnxDomain, 19, kXnnpackExecutionProvider,
KernelDefBuilder().TypeConstraint("T1", {DataTypeImpl::GetTensorType<float>(),
DataTypeImpl::GetTensorType<uint8_t>(),
DataTypeImpl::GetTensorType<int8_t>()}),

7
onnxruntime/core/providers/xnnpack/xnnpack_execution_provider.cc
View file

@ -46,7 +46,8 @@ class ONNX_OPERATOR_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSInternalNHWC
class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kOnnxDomain, 10, 10, Resize);
class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kOnnxDomain, 11, 12, Resize);
class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kOnnxDomain, 13, 17, Resize);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kOnnxDomain, 18, Resize);
class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kOnnxDomain, 18, 18, Resize);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kOnnxDomain, 19, Resize);
class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 11, 11, MaxPool);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 12, MaxPool);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 11, AveragePool);
@ -84,7 +85,9 @@ std::unique_ptr<KernelRegistry> RegisterKernels() {
BuildKernelCreateInfo<
ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kOnnxDomain, 1, 12, Softmax)>,
BuildKernelCreateInfo<
ONNX_OPERATOR_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kOnnxDomain, 18, Resize)>,
ONNX_OPERATOR_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kOnnxDomain, 19, Resize)>,
BuildKernelCreateInfo<
ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kOnnxDomain, 18, 18, Resize)>,
BuildKernelCreateInfo<
ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kOnnxDomain, 13, 17, Resize)>,
BuildKernelCreateInfo<

4
onnxruntime/test/optimizer/qdq_transformer_test.cc
View file

@ -3085,12 +3085,12 @@ TEST(QDQTransformerTests, QDQPropagation_Per_Layer_No_Propagation) {
check_graph,
TransformerLevel::Default,
TransformerLevel::Level1,
18); // disable TransposeOptimizer for simplicity
18);
TransformerTester(build_test_case,
check_graph,
TransformerLevel::Default,
TransformerLevel::Level1,
19); // disable TransposeOptimizer for simplicity
19);
};
test_case({1, 13, 13, 23}, {0, 2, 3, 1}, false /*use_contrib_qdq*/);

96
onnxruntime/test/optimizer/transpose_optimizer_test.cc
View file

@ -320,20 +320,6 @@ TEST(TransposeOptimizerTests, TestPadNonconst) {
/*opset_version*/ {11, 18});
}
// The CUDA Resize kernel assumes that the input is NCHW and
// Resize can't be supported in ORT builds with CUDA enabled.
// TODO: Enable this once the CUDA Resize kernel is implemented
// "generically" (i.e.) aligning with the generic nature of the
// ONNX spec.
// See https://github.com/microsoft/onnxruntime/pull/10824 for
// a similar fix applied to the CPU Resize kernel.
// Per tests included in #10824, the ROCM EP also generates
// incorrect results when this handler is used, so the Resize
// handler is not enabled even for those builds.
//
// The QNN EP requires the input to be NHWC, so the Resize handler is also not enabled
// for QNN builds.
#if !defined(USE_CUDA) && !defined(USE_ROCM) && !defined(USE_QNN)
TEST(TransposeOptimizerTests, TestResize) {
auto build_test_case_1 = [&](ModelTestBuilder& builder) {
auto* input0_arg = MakeInput<float>(builder, {{4, -1, 2, -1}}, {4, 6, 2, 10}, 0.0, 1.0);
@ -362,7 +348,9 @@ TEST(TransposeOptimizerTests, TestResize) {
TransformerTester(build_test_case_1,
check_optimized_graph_1,
TransformerLevel::Default,
TransformerLevel::Level1,
// need the level 2 TransposeOptimizer as pushing a Transpose through a Resize requires it to be
// assigned to the CPU EP first
TransformerLevel::Level2,
/*opset_version*/ {10, 18});
}
@ -390,7 +378,9 @@ TEST(TransposeOptimizerTests, TestResizeOpset11) {
TransformerTester(build_test_case_1,
check_optimized_graph_1,
TransformerLevel::Default,
TransformerLevel::Level1,
// need the level 2 TransposeOptimizer as pushing a Transpose through a Resize requires it to be
// assigned to the CPU EP first
TransformerLevel::Level2,
/*opset_version*/ {11, 18});
}
@ -418,7 +408,9 @@ TEST(TransposeOptimizerTests, TestResizeOpset15) {
TransformerTester(build_test_case_1,
check_optimized_graph_1,
TransformerLevel::Default,
TransformerLevel::Level1,
// need the level 2 TransposeOptimizer as pushing a Transpose through a Resize requires it to be
// assigned to the CPU EP first
TransformerLevel::Level2,
/*opset_version*/ {15, 18});
}
@ -448,7 +440,9 @@ TEST(TransposeOptimizerTests, TestResizeSizeRoi) {
TransformerTester(build_test_case_1,
check_optimized_graph_1,
TransformerLevel::Default,
TransformerLevel::Level1,
// need the level 2 TransposeOptimizer as pushing a Transpose through a Resize requires it to be
// assigned to the CPU EP first
TransformerLevel::Level2,
/*opset_version*/ {15, 18});
}
@ -482,7 +476,9 @@ TEST(TransposeOptimizerTests, TestResizeRoiScalesZeroRank0) {
TransformerTester(build_test_case_1,
check_optimized_graph_1,
TransformerLevel::Default,
TransformerLevel::Level1,
// need the level 2 TransposeOptimizer as pushing a Transpose through a Resize requires it to be
// assigned to the CPU EP first
TransformerLevel::Level2,
{12, 18});
}
@ -511,7 +507,9 @@ TEST(TransposeOptimizerTests, TestResizeNonconst) {
TransformerTester(build_test_case_1,
check_optimized_graph_1,
TransformerLevel::Default,
TransformerLevel::Level1,
// need the level 2 TransposeOptimizer as pushing a Transpose through a Resize requires it to be
// assigned to the CPU EP first
TransformerLevel::Level2,
/*opset_version*/ {11, 18});
}
@ -540,11 +538,12 @@ TEST(TransposeOptimizerTests, TestResizeNonconstOpset13) {
TransformerTester(build_test_case_1,
check_optimized_graph_1,
TransformerLevel::Default,
TransformerLevel::Level1,
// need the level 2 TransposeOptimizer as pushing a Transpose through a Resize requires it to be
// assigned to the CPU EP first
TransformerLevel::Level2,
/*opset_version*/ {13, 18});
}
#endif
TEST(TransposeOptimizerTests, TestAdd) {
auto build_test_case_1 = [&](ModelTestBuilder& builder) {
auto* input0_arg = builder.MakeInput<float>({4, 6, 10}, 0.0, 1.0);
@ -4454,12 +4453,13 @@ TEST(TransposeOptimizerTests, RegressionTest_GitHubIssue12151) {
testing::ContainerEq(fetches[0].Get<Tensor>().DataAsSpan<float>()));
}
// These tests uses internal testing EP with static kernels which requires a full build,
// and the NHWC Conv which requires contrib ops
#if !defined(ORT_MINIMAL_BUILD) && !defined(DISABLE_CONTRIB_OPS)
// Test a Transpose node followed by a Reshape that is logically equivalent to an Transpose can be merged.
// The test graph was extracted from a model we were trying to use with the QNN EP.
TEST(TransposeOptimizerTests, QnnTransposeReshape) {
// test uses internal testing EP with static kernels which requires a full build,
// and the NHWC Conv with requires contrib ops
#if !defined(ORT_MINIMAL_BUILD) && !defined(DISABLE_CONTRIB_OPS)
Status status;
auto model_uri = ORT_TSTR("testdata/layout_transform_reshape.onnx");
@ -4509,13 +4509,9 @@ TEST(TransposeOptimizerTests, QnnTransposeReshape) {
EXPECT_TRUE(inputs[1]->Exists());
}
}
#endif
}
TEST(TransposeOptimizerTests, QnnTransposeReshapeQDQ) {
// test uses internal testing EP with static kernels which requires a full build,
// and the NHWC Conv with requires contrib ops
#if !defined(ORT_MINIMAL_BUILD) && !defined(DISABLE_CONTRIB_OPS)
Status status;
auto model_uri = ORT_TSTR("testdata/layout_transform_reshape.qdq.onnx");
@ -4552,9 +4548,49 @@ TEST(TransposeOptimizerTests, QnnTransposeReshapeQDQ) {
EXPECT_TRUE(node.GetExecutionProviderType() == expected_ep) << node.OpType() << " node named '" << node.Name()
<< "' was not assigned to the internal testing EP.";
}
#endif
}
// Validate handling for EP with layout specific Resize that prefers NHWC
TEST(TransposeOptimizerTests, QnnResizeOpset11) {
Status status;
auto model_uri = ORT_TSTR("testdata/nhwc_resize_scales_opset11.onnx");
SessionOptions so;
// Uncomment to debug
// ASSERT_STATUS_OK(so.config_options.AddConfigEntry(kDebugLayoutTransformation, "1"));
using InternalTestingEP = onnxruntime::internal_testing_ep::InternalTestingExecutionProvider;
// set the test EP to support all ops in the model so that the layout transform applies to all nodes
const std::unordered_set<std::string> empty_set;
auto internal_testing_ep = std::make_unique<InternalTestingEP>(empty_set, empty_set, DataLayout::NHWC);
internal_testing_ep->EnableStaticKernels().TakeAllNodes();
InferenceSessionWrapper session{so, GetEnvironment()};
ASSERT_STATUS_OK(session.RegisterExecutionProvider(std::move(internal_testing_ep)));
ASSERT_STATUS_OK(session.Load(model_uri));
ASSERT_STATUS_OK(session.Initialize());
const auto& graph = session.GetGraph();
// all nodes should be assigned to the internal testing EP, which also means they should be in NHWC layout
std::string expected_ep(onnxruntime::utils::kInternalTestingExecutionProvider);
for (const auto& node : graph.Nodes()) {
EXPECT_TRUE(node.GetExecutionProviderType() == expected_ep) << node.OpType() << " node named '" << node.Name()
<< "' was not assigned to the internal testing EP.";
if (node.OpType() == "Resize") {
EXPECT_EQ(node.Domain(), kMSInternalNHWCDomain) << "Resize was not converted to NHWC layout";
}
}
std::map<std::string, int> op_to_count = CountOpsInGraph(graph);
ASSERT_EQ(op_to_count["Transpose"], 2) << "Resize should have been wrapped in 2 Transpose nodes to convert to NHWC";
// And the post-Resize Transpose should have been pushed all the way to the end
GraphViewer viewer(graph);
EXPECT_EQ(graph.GetNode(viewer.GetNodesInTopologicalOrder().back())->OpType(), "Transpose");
}
#endif // !defined(ORT_MINIMAL_BUILD) && !defined(DISABLE_CONTRIB_OPS)
static void CheckSharedInitializerHandling(bool broadcast) {
auto model_uri = broadcast ? ORT_TSTR("testdata/transpose_optimizer_shared_initializers_broadcast.onnx")
: ORT_TSTR("testdata/transpose_optimizer_shared_initializers.onnx");

2
onnxruntime/test/providers/cpu/tensor/resize_op_test.cc
View file

@ -780,7 +780,7 @@ TEST(ResizeOpTest, ResizeOpLinearUpSampleTest_5DTrilinear_pytorch_half_pixel) {
}
TEST(ResizeOpTest, ResizeOpLinearScalesNoOpTest) {
// To test NNAPI EP, we need the sclaes/sizes to be in initializers
// To test NNAPI EP, we need the scales/sizes to be in initializers
auto run_test = [](bool scales_in_initializer) {
OpTester test("Resize", 13);
std::vector<float> roi{};