mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-25 19:48:11 +00:00
Update QDQ propagation transformer to insert QDQ nodes (#10487)
Update QDQ propagation transformer to insert new QDQ nodes instead of moving the existing one. This creates a more consistent `DQ -> op -> Q` pattern for other components to recognize. Upgrade this transformer to a basic level optimization as it yields a valid ONNX graph.
This commit is contained in:
Edward Chen
GitHub
parent
7691e7ed12
commit
3199074ac7
12 changed files with 657 additions and 302 deletions
|
|
@ -21,6 +21,8 @@ if (onnxruntime_MINIMAL_BUILD)
|
|||
"${ONNXRUNTIME_ROOT}/core/optimizer/qdq_transformer/qdq_util.cc"
|
||||
"${ONNXRUNTIME_ROOT}/core/optimizer/initializer.h"
|
||||
"${ONNXRUNTIME_ROOT}/core/optimizer/initializer.cc"
|
||||
"${ONNXRUNTIME_ROOT}/core/optimizer/selectors_actions/helpers.h"
|
||||
"${ONNXRUNTIME_ROOT}/core/optimizer/selectors_actions/helpers.cc"
|
||||
"${ONNXRUNTIME_ROOT}/core/optimizer/utils.h"
|
||||
"${ONNXRUNTIME_ROOT}/core/optimizer/utils.cc"
|
||||
)
|
||||
|
|
|
|||
128
onnxruntime/core/graph/extended_graph_edge.h
Normal file
128
onnxruntime/core/graph/extended_graph_edge.h
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <optional>
|
||||
#include <string>
|
||||
|
||||
#include "core/graph/basic_types.h"
|
||||
#include "core/graph/graph.h"
|
||||
#include "core/graph/graph_utils.h"
|
||||
|
||||
namespace onnxruntime::graph_utils {
|
||||
|
||||
/**
|
||||
* Represents an edge between two graph nodes, a graph input or initializer -> node connection, or a node -> graph
|
||||
* output connection.
|
||||
* Similar to graph_utils::GraphEdge.
|
||||
*/
|
||||
struct ExtendedGraphEdge {
|
||||
enum class End { Source,
|
||||
Destination };
|
||||
|
||||
/** Information about a source or destination node of the extended edge. */
|
||||
struct NodeInfo {
|
||||
/** Node index in the graph. */
|
||||
NodeIndex node_idx;
|
||||
/** Node input or output def index. */
|
||||
int arg_idx;
|
||||
};
|
||||
|
||||
/** Source node info. If empty, the source is a graph input or initializer. */
|
||||
std::optional<NodeInfo> src;
|
||||
/** Destination node info. If empty, the destination is a graph output. */
|
||||
std::optional<NodeInfo> dst;
|
||||
/** Edge/connection NodeArg name. */
|
||||
std::string arg_name;
|
||||
|
||||
/** Whether this extended edge has a graph input or initializer as the source. */
|
||||
bool HasGraphInputOrInitializer() const noexcept { return !src.has_value(); }
|
||||
|
||||
/** Whether this extended edge has a graph output as the destination. */
|
||||
bool HasGraphOutput() const noexcept { return !dst.has_value(); }
|
||||
|
||||
/** Gets the NodeInfo at the specified extended edge end. */
|
||||
const std::optional<NodeInfo>& GetNodeInfoAtEnd(End end) const noexcept {
|
||||
return end == End::Source ? src : dst;
|
||||
}
|
||||
|
||||
/** Gets the graph node at the specified end if it exists, otherwise nullptr. */
|
||||
const Node* GetNodeAtEnd(const Graph& graph, End end) const {
|
||||
if (const auto& node_info = GetNodeInfoAtEnd(end); node_info.has_value()) {
|
||||
const Node* node = graph.GetNode(node_info->node_idx);
|
||||
ORT_ENFORCE(node != nullptr, "Invalid node index ", node_info->node_idx);
|
||||
return node;
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
/** Gets the mutable graph node at the specified end if it exists, otherwise nullptr. */
|
||||
Node* GetMutableNodeAtEnd(Graph& graph, End end) const {
|
||||
if (const auto& node_info = GetNodeInfoAtEnd(end); node_info.has_value()) {
|
||||
Node* node = graph.GetNode(node_info->node_idx);
|
||||
ORT_ENFORCE(node != nullptr, "Invalid node index ", node_info->node_idx);
|
||||
return node;
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates an extended graph edge from a valid graph_utils::GraphEdge.
|
||||
* To be valid, `graph_edge` should represent an actual edge between two nodes in the graph.
|
||||
* Validity is assumed.
|
||||
*/
|
||||
static ExtendedGraphEdge CreateFromValidGraphEdge(const graph_utils::GraphEdge& graph_edge) {
|
||||
return ExtendedGraphEdge{
|
||||
NodeInfo{graph_edge.src_node, graph_edge.src_arg_index},
|
||||
NodeInfo{graph_edge.dst_node, graph_edge.dst_arg_index},
|
||||
graph_edge.arg_name};
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempts to create an extended graph edge from a graph input or initializer to a node.
|
||||
* Returns nullopt if there is not actually a graph input or initializer at the specified node's input index.
|
||||
*/
|
||||
static std::optional<ExtendedGraphEdge> TryCreateFromInputOrInitializerToNode(
|
||||
const Graph& graph, const Node& node, int node_input_def_idx) {
|
||||
const auto node_inputs = node.InputDefs();
|
||||
ORT_ENFORCE(node_input_def_idx >= 0 &&
|
||||
static_cast<size_t>(node_input_def_idx) < node_inputs.size());
|
||||
|
||||
const auto* node_input = node_inputs[node_input_def_idx];
|
||||
if (!graph.IsInputsIncludingInitializers(node_input)) {
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
return ExtendedGraphEdge{
|
||||
std::nullopt,
|
||||
NodeInfo{node.Index(), node_input_def_idx},
|
||||
node_input->Name()};
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempts to create an extended graph edge from a node to a graph output.
|
||||
* Returns nullopt if there is not actually a graph output at the specified node's output index.
|
||||
*/
|
||||
static std::optional<ExtendedGraphEdge> TryCreateFromNodeToOutput(
|
||||
const Graph& graph, const Node& node, int node_output_def_idx) {
|
||||
const auto node_outputs = node.OutputDefs();
|
||||
ORT_ENFORCE(node_output_def_idx >= 0 &&
|
||||
static_cast<size_t>(node_output_def_idx) < node_outputs.size());
|
||||
|
||||
const auto* node_output = node_outputs[node_output_def_idx];
|
||||
if (!graph.IsOutput(node_output)) {
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
return ExtendedGraphEdge{
|
||||
NodeInfo{node.Index(), node_output_def_idx},
|
||||
std::nullopt,
|
||||
node_output->Name()};
|
||||
}
|
||||
|
||||
// there is also the case where a graph input or initializer is an output
|
||||
// if that's useful to represent, add another creation function
|
||||
};
|
||||
|
||||
} // namespace onnxruntime::graph_utils
|
||||
|
|
@ -182,6 +182,11 @@ std::vector<std::unique_ptr<GraphTransformer>> GenerateTransformers(
|
|||
session_options.free_dimension_overrides));
|
||||
auto cpu_allocator = cpu_execution_provider.GetAllocator(0, OrtMemTypeDefault);
|
||||
transformers.emplace_back(std::make_unique<TransposeOptimizer>(std::move(cpu_allocator)));
|
||||
|
||||
if (!disable_quant_qdq) {
|
||||
transformers.emplace_back(std::make_unique<QDQPropagationTransformer>());
|
||||
}
|
||||
|
||||
} break;
|
||||
|
||||
case TransformerLevel::Level2: {
|
||||
|
|
@ -203,7 +208,6 @@ std::vector<std::unique_ptr<GraphTransformer>> GenerateTransformers(
|
|||
if (!QDQIsInt8Allowed()) {
|
||||
transformers.emplace_back(std::make_unique<QDQS8ToU8Transformer>(cpu_ep));
|
||||
}
|
||||
transformers.emplace_back(std::make_unique<QDQPropagationTransformer>(cpu_ep));
|
||||
transformers.emplace_back(std::make_unique<QDQSelectorActionTransformer>());
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,246 +1,304 @@
|
|||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
#include "qdq_propagation.h"
|
||||
#include "core/optimizer/qdq_transformer/qdq_propagation.h"
|
||||
|
||||
#include <deque>
|
||||
#include <optional>
|
||||
|
||||
#include "core/graph/extended_graph_edge.h"
|
||||
#include "core/graph/graph_utils.h"
|
||||
#include "core/optimizer/initializer.h"
|
||||
#include "core/optimizer/qdq_transformer/qdq_util.h"
|
||||
#include "core/optimizer/utils.h"
|
||||
|
||||
namespace onnxruntime {
|
||||
using onnxruntime::graph_utils::ExtendedGraphEdge;
|
||||
|
||||
static bool CanNodePropagate(const Node& node) {
|
||||
namespace onnxruntime {
|
||||
namespace {
|
||||
bool CanNodePropagate(const Node& node) {
|
||||
return graph_utils::IsSupportedOptypeVersionAndDomain(node, "MaxPool", {12}) ||
|
||||
graph_utils::IsSupportedOptypeVersionAndDomain(node, "Reshape", {5, 13, 14}) ||
|
||||
graph_utils::IsSupportedOptypeVersionAndDomain(node, "Transpose", {1, 13});
|
||||
}
|
||||
|
||||
static bool TryCancelOutDQQPair(Graph& graph, Node& dq_node, Node& q_node) {
|
||||
auto get_const_initializer = [&graph](const std::string& initializer_name) {
|
||||
return graph.GetConstantInitializer(initializer_name, true);
|
||||
// convert this: src_node -> dst_node
|
||||
// to this: src_node -> Q -> DQ -> dst_node
|
||||
// assumptions:
|
||||
// 1. insertion_edge is valid - node indexes refer to valid nodes, arg name refers to a valid NodeArg, and it
|
||||
// corresponds to an actual graph relationship
|
||||
// 2. scale_initializer_nodearg and zp_initializer_nodearg_ptr (if not null) are constant initializers
|
||||
Status InsertQDQPair(Graph& graph, const ExtendedGraphEdge& insertion_edge,
|
||||
NodeArg& scale_initializer_nodearg, NodeArg* zp_initializer_nodearg_ptr,
|
||||
const logging::Logger& logger) {
|
||||
auto* src_node = insertion_edge.GetMutableNodeAtEnd(graph, ExtendedGraphEdge::End::Source);
|
||||
auto* dst_node = insertion_edge.GetMutableNodeAtEnd(graph, ExtendedGraphEdge::End::Destination);
|
||||
|
||||
ORT_ENFORCE(src_node || dst_node, "At least one graph node must be specified in the propagation edge.");
|
||||
|
||||
const auto& base_name = insertion_edge.arg_name;
|
||||
auto& base_node_arg = *graph.GetNodeArg(base_name);
|
||||
|
||||
LOGS(logger, VERBOSE) << "Inserting Q/DQ pair between "
|
||||
<< (src_node ? MakeString("node (\"", src_node->Name(), "\", index: ", src_node->Index(), ")")
|
||||
: "input")
|
||||
<< " and "
|
||||
<< (dst_node ? MakeString("node (\"", dst_node->Name(), "\", index: ", dst_node->Index(), ")")
|
||||
: "output")
|
||||
<< " at NodeArg \"" << base_name << "\".";
|
||||
|
||||
// set up new NodeArgs
|
||||
auto& pre_q_nodearg = insertion_edge.HasGraphInputOrInitializer()
|
||||
? base_node_arg
|
||||
: graph.GetOrCreateNodeArg(graph.GenerateNodeArgName(base_name + "_pre_q"),
|
||||
nullptr);
|
||||
|
||||
auto& q_to_dq_nodearg = graph.GetOrCreateNodeArg(graph.GenerateNodeArgName(base_name + "_q_to_dq"),
|
||||
nullptr);
|
||||
|
||||
auto& post_dq_nodearg = insertion_edge.HasGraphOutput()
|
||||
? base_node_arg
|
||||
: graph.GetOrCreateNodeArg(graph.GenerateNodeArgName(base_name + "_post_dq"),
|
||||
nullptr);
|
||||
|
||||
// set up new Nodes
|
||||
auto make_q_or_dq_inputs = [](NodeArg& data, NodeArg& scale, NodeArg* zero_point) {
|
||||
return zero_point ? std::vector<NodeArg*>{&data, &scale, zero_point}
|
||||
: std::vector<NodeArg*>{&data, &scale};
|
||||
};
|
||||
|
||||
if (!QDQ::IsQDQPairSupported(q_node, dq_node, get_const_initializer, graph.ModelPath())) {
|
||||
return false;
|
||||
auto& q_node = graph.AddNode(graph.GenerateNodeName(base_name + "_q"),
|
||||
QDQ::QOpName,
|
||||
"Inserted by QDQPropagationTransformer",
|
||||
// inputs
|
||||
make_q_or_dq_inputs(pre_q_nodearg, scale_initializer_nodearg,
|
||||
zp_initializer_nodearg_ptr),
|
||||
// outputs
|
||||
{&q_to_dq_nodearg});
|
||||
|
||||
ORT_RETURN_IF_NOT(graph.SetOpSchemaFromRegistryForNode(q_node), "Failed to set op schema for added Q node.");
|
||||
|
||||
auto& dq_node = graph.AddNode(graph.GenerateNodeName(base_name + "_dq"),
|
||||
QDQ::DQOpName,
|
||||
"Inserted by QDQPropagationTransformer",
|
||||
// inputs
|
||||
make_q_or_dq_inputs(q_to_dq_nodearg, scale_initializer_nodearg,
|
||||
zp_initializer_nodearg_ptr),
|
||||
// outputs
|
||||
{&post_dq_nodearg});
|
||||
|
||||
ORT_RETURN_IF_NOT(graph.SetOpSchemaFromRegistryForNode(dq_node), "Failed to set op schema for added DQ node.");
|
||||
|
||||
// set up edges
|
||||
if (src_node && dst_node) {
|
||||
graph.RemoveEdge(src_node->Index(), dst_node->Index(),
|
||||
insertion_edge.src->arg_idx, insertion_edge.dst->arg_idx);
|
||||
}
|
||||
|
||||
// check if dq_node has only one output edge and,
|
||||
// dq_node and q_node output are not graph outputs
|
||||
if (!optimizer_utils::CheckOutputEdges(graph, dq_node, 1) ||
|
||||
graph.NodeProducesGraphOutput(q_node)) {
|
||||
return false;
|
||||
if (src_node) {
|
||||
src_node->MutableOutputDefs()[insertion_edge.src->arg_idx] = &pre_q_nodearg;
|
||||
graph.AddEdge(src_node->Index(), q_node.Index(), insertion_edge.src->arg_idx, 0);
|
||||
}
|
||||
|
||||
// remove edge between parent of DQ to DQ
|
||||
std::pair<NodeIndex, int> input_edge_info{0, -1};
|
||||
auto* dq_input_edge_0 = graph_utils::GetInputEdge(dq_node, 0);
|
||||
if (dq_input_edge_0) {
|
||||
input_edge_info.first = dq_input_edge_0->GetNode().Index();
|
||||
input_edge_info.second = dq_input_edge_0->GetSrcArgIndex();
|
||||
graph.RemoveEdge(dq_input_edge_0->GetNode().Index(), dq_node.Index(),
|
||||
dq_input_edge_0->GetSrcArgIndex(), dq_input_edge_0->GetDstArgIndex());
|
||||
graph.AddEdge(q_node.Index(), dq_node.Index(), 0, 0);
|
||||
|
||||
if (dst_node) {
|
||||
dst_node->MutableInputDefs()[insertion_edge.dst->arg_idx] = &post_dq_nodearg;
|
||||
graph.AddEdge(dq_node.Index(), dst_node->Index(), 0, insertion_edge.dst->arg_idx);
|
||||
}
|
||||
|
||||
graph_utils::RemoveNodeOutputEdges(graph, dq_node); // Remove DQ node output edges
|
||||
|
||||
auto output_edges = graph_utils::GraphEdge::GetNodeOutputEdges(q_node, 0);
|
||||
graph_utils::RemoveNodeOutputEdges(graph, q_node); // Remove Q node output edges
|
||||
for (auto& output_edge : output_edges) {
|
||||
// set input NodeArg of Q's children to the 1st input of DQ
|
||||
graph.GetNode(output_edge.dst_node)->MutableInputDefs()[output_edge.dst_arg_index] =
|
||||
dq_node.MutableInputDefs()[0];
|
||||
|
||||
// add edge between parent of DQ to children of Q
|
||||
if (input_edge_info.second != -1) {
|
||||
graph.AddEdge(input_edge_info.first, output_edge.dst_node,
|
||||
input_edge_info.second, output_edge.dst_arg_index);
|
||||
}
|
||||
}
|
||||
|
||||
graph.RemoveNode(dq_node.Index());
|
||||
graph.RemoveNode(q_node.Index());
|
||||
return true;
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
// A helper function that swap the relationship of 2 nodes.
|
||||
// @param up_node The original parent node
|
||||
// @param down_node The original child node
|
||||
// after calling the function, the up_node will become the child of down_node
|
||||
// Assumptions of this function are:
|
||||
// 1. up_node only has one Edge that points to down_node and its output is not graph output.
|
||||
// 2. NodeArg slots of the edge between up_node and down_node are (0, 0)
|
||||
static void SwapAdjacentNodes(Graph& graph, Node& up_node, Node& down_node) {
|
||||
ORT_ENFORCE(optimizer_utils::CheckOutputEdges(graph, up_node, 1),
|
||||
"up_node should have only one Edge that points to down_node and its output is not graph output");
|
||||
std::optional<ExtendedGraphEdge> GetPreviousEdge(const Graph& graph, const Node& node) {
|
||||
// for now we can just consider the first input (index 0)
|
||||
|
||||
auto edge_it = up_node.OutputEdgesBegin();
|
||||
ORT_ENFORCE(edge_it->GetDstArgIndex() == 0 &&
|
||||
edge_it->GetSrcArgIndex() == 0 &&
|
||||
edge_it->GetNode().Index() == down_node.Index(),
|
||||
"up_node should be parent of down_node and NodeArg slots of the edge between up_node and down_node should be (0, 0).");
|
||||
const auto input_edges = graph_utils::GraphEdge::GetNodeInputEdges(node);
|
||||
const auto input_edge_it = std::find_if(
|
||||
input_edges.begin(), input_edges.end(),
|
||||
[](const graph_utils::GraphEdge& edge) { return edge.dst_arg_index == 0; });
|
||||
|
||||
// ************** Remove edges **************
|
||||
// Remove the edge between parent of up_node and up_node, and keep the info of parent of up_node
|
||||
std::pair<NodeIndex, int> up_node_input_edge_info{0, -1};
|
||||
auto* up_node_input_edge_0 = graph_utils::GetInputEdge(up_node, 0);
|
||||
if (up_node_input_edge_0) {
|
||||
up_node_input_edge_info.first = up_node_input_edge_0->GetNode().Index();
|
||||
up_node_input_edge_info.second = up_node_input_edge_0->GetSrcArgIndex();
|
||||
graph.RemoveEdge(up_node_input_edge_0->GetNode().Index(),
|
||||
up_node.Index(),
|
||||
up_node_input_edge_0->GetSrcArgIndex(),
|
||||
up_node_input_edge_0->GetDstArgIndex());
|
||||
if (input_edge_it == input_edges.end()) {
|
||||
// maybe edge from input
|
||||
return ExtendedGraphEdge::TryCreateFromInputOrInitializerToNode(graph, node, 0);
|
||||
}
|
||||
|
||||
auto down_node_output_edges_info = graph_utils::GraphEdge::GetNodeOutputEdges(down_node);
|
||||
graph_utils::RemoveNodeOutputEdges(graph, up_node);
|
||||
graph_utils::RemoveNodeOutputEdges(graph, down_node);
|
||||
|
||||
// *********** Rebuild NodeArg ****************/
|
||||
down_node.MutableInputDefs()[0] = up_node.MutableInputDefs()[0];
|
||||
up_node.MutableOutputDefs()[0] = down_node.MutableOutputDefs()[0];
|
||||
|
||||
NodeArg* new_node_arg = &graph.GetOrCreateNodeArg(graph.GenerateNodeArgName("SwapAdjacentNodes"), nullptr);
|
||||
down_node.MutableOutputDefs()[0] = new_node_arg;
|
||||
up_node.MutableInputDefs()[0] = new_node_arg;
|
||||
|
||||
// *********** Rebuild Edges ****************/
|
||||
if (up_node_input_edge_info.second >= 0) {
|
||||
graph.AddEdge(up_node_input_edge_info.first, down_node.Index(), up_node_input_edge_info.second, 0);
|
||||
const auto& src_node = *graph.GetNode(input_edge_it->src_node);
|
||||
const auto src_node_output_edges =
|
||||
graph_utils::GraphEdge::GetNodeOutputEdges(src_node, input_edge_it->src_arg_index);
|
||||
if (!graph.IsOutput(src_node.OutputDefs()[input_edge_it->src_arg_index]) &&
|
||||
src_node_output_edges.size() == 1) {
|
||||
// single edge from previous node
|
||||
return ExtendedGraphEdge::CreateFromValidGraphEdge(*input_edge_it);
|
||||
}
|
||||
|
||||
graph.AddEdge(down_node.Index(), up_node.Index(), 0, 0);
|
||||
|
||||
for (auto output_edge_info : down_node_output_edges_info) {
|
||||
graph.AddEdge(up_node.Index(), output_edge_info.dst_node, 0, output_edge_info.dst_arg_index);
|
||||
}
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
bool QDQPropagationTransformer::PropagateDQForward(Graph& graph) const {
|
||||
bool is_modified = false;
|
||||
std::optional<ExtendedGraphEdge> GetPreviousPropagationEdge(const Graph& graph,
|
||||
const ExtendedGraphEdge& edge) {
|
||||
if (edge.HasGraphInputOrInitializer()) {
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
GraphViewer graph_viewer(graph);
|
||||
const auto& node_topology_list = graph_viewer.GetNodesInTopologicalOrder();
|
||||
const auto* src_node = edge.GetNodeAtEnd(graph, ExtendedGraphEdge::End::Source);
|
||||
ORT_ENFORCE(src_node != nullptr);
|
||||
|
||||
for (auto node_index : node_topology_list) {
|
||||
if (!CanNodePropagate(*src_node)) {
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
return GetPreviousEdge(graph, *src_node);
|
||||
}
|
||||
|
||||
std::optional<ExtendedGraphEdge> GetNextEdge(const Graph& graph, const Node& node) {
|
||||
// for now we can just consider the first output (index 0)
|
||||
|
||||
const auto output_edges = graph_utils::GraphEdge::GetNodeOutputEdges(node, 0);
|
||||
if (output_edges.empty()) {
|
||||
// maybe edge to output
|
||||
return ExtendedGraphEdge::TryCreateFromNodeToOutput(graph, node, 0);
|
||||
}
|
||||
|
||||
if (!graph.IsOutput(node.OutputDefs()[0]) && output_edges.size() == 1) {
|
||||
// single edge to next node
|
||||
return ExtendedGraphEdge::CreateFromValidGraphEdge(output_edges.front());
|
||||
}
|
||||
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
std::optional<ExtendedGraphEdge> GetNextPropagationEdge(const Graph& graph,
|
||||
const ExtendedGraphEdge& edge) {
|
||||
if (edge.HasGraphOutput()) {
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
const auto* dst_node = edge.GetNodeAtEnd(graph, ExtendedGraphEdge::End::Destination);
|
||||
ORT_ENFORCE(dst_node != nullptr);
|
||||
|
||||
if (!CanNodePropagate(*dst_node)) {
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
return GetNextEdge(graph, *dst_node);
|
||||
}
|
||||
|
||||
class GraphConstantInitializerGetter {
|
||||
const Graph& graph_;
|
||||
|
||||
public:
|
||||
GraphConstantInitializerGetter(const Graph& graph) : graph_{graph} {}
|
||||
const ONNX_NAMESPACE::TensorProto* operator()(const std::string& initializer_name) const {
|
||||
return graph_utils::GetConstantInitializer(graph_, initializer_name);
|
||||
}
|
||||
};
|
||||
|
||||
Status PropagateDQForward(Graph& graph, gsl::span<const NodeIndex> node_indices,
|
||||
const std::unordered_set<std::string>& compatible_eps,
|
||||
const logging::Logger& logger,
|
||||
bool& modified) {
|
||||
for (auto node_index : node_indices) {
|
||||
auto* dq_node_ptr = graph.GetNode(node_index);
|
||||
if (dq_node_ptr == nullptr)
|
||||
if (dq_node_ptr == nullptr) {
|
||||
continue; // node removed as part of an earlier fusion
|
||||
}
|
||||
|
||||
Node& dq_node = *dq_node_ptr;
|
||||
|
||||
if (!graph_utils::IsSupportedOptypeVersionAndDomain(dq_node, "DequantizeLinear", {10, 13}) ||
|
||||
!graph_utils::IsSupportedProvider(dq_node, GetCompatibleExecutionProviders()) ||
|
||||
if (!QDQ::MatchDQNode(dq_node) ||
|
||||
!graph_utils::IsSupportedProvider(dq_node, compatible_eps) ||
|
||||
!optimizer_utils::CheckOutputEdges(graph, dq_node, 1)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
std::vector<NodeArg*>& dq_input_defs = dq_node.MutableInputDefs();
|
||||
if (dq_input_defs.size() != QDQ::InputIndex::TOTAL_COUNT) {
|
||||
bool dq_zero_point_exists = false;
|
||||
if (!QDQ::QOrDQNodeHasConstantScalarScaleAndZeroPoint(dq_node, GraphConstantInitializerGetter{graph},
|
||||
dq_zero_point_exists)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!optimizer_utils::IsScalar(*dq_input_defs[QDQ::InputIndex::ZERO_POINT_ID]) ||
|
||||
!optimizer_utils::IsScalar(*dq_input_defs[QDQ::InputIndex::SCALE_ID])) {
|
||||
auto& dq_scale = *dq_node.MutableInputDefs()[QDQ::InputIndex::SCALE_ID];
|
||||
auto* dq_zero_point = dq_zero_point_exists
|
||||
? dq_node.MutableInputDefs()[QDQ::InputIndex::ZERO_POINT_ID]
|
||||
: nullptr;
|
||||
|
||||
const auto edge_after_dq = GetNextEdge(graph, dq_node);
|
||||
if (!edge_after_dq) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const ONNX_NAMESPACE::TensorProto* dq_zp_tensor_proto =
|
||||
graph_utils::GetConstantInitializer(graph, dq_input_defs[QDQ::InputIndex::ZERO_POINT_ID]->Name());
|
||||
const ONNX_NAMESPACE::TensorProto* dq_scale_tensor_proto =
|
||||
graph_utils::GetConstantInitializer(graph, dq_input_defs[QDQ::InputIndex::SCALE_ID]->Name());
|
||||
|
||||
if (nullptr == dq_zp_tensor_proto || nullptr == dq_scale_tensor_proto) {
|
||||
continue;
|
||||
}
|
||||
|
||||
do {
|
||||
Node& next_node = *graph.GetNode(dq_node.OutputNodesBegin()->Index());
|
||||
if (!CanNodePropagate(next_node)) {
|
||||
// Try canceling out DQ/Q pair
|
||||
if (graph_utils::IsSupportedOptypeVersionAndDomain(next_node, "QuantizeLinear", {10, 13}) &&
|
||||
graph_utils::IsSupportedProvider(next_node, GetCompatibleExecutionProviders()) &&
|
||||
TryCancelOutDQQPair(graph, dq_node, next_node)) {
|
||||
is_modified = true;
|
||||
}
|
||||
|
||||
for (auto curr_edge = GetNextPropagationEdge(graph, *edge_after_dq);
|
||||
curr_edge.has_value();
|
||||
curr_edge = GetNextPropagationEdge(graph, *curr_edge)) {
|
||||
if (const auto* dst_node = curr_edge->GetNodeAtEnd(graph, ExtendedGraphEdge::End::Destination);
|
||||
dst_node && QDQ::MatchQNode(*dst_node)) {
|
||||
break;
|
||||
}
|
||||
SwapAdjacentNodes(graph, dq_node, next_node);
|
||||
is_modified = true;
|
||||
} while (optimizer_utils::CheckOutputEdges(graph, dq_node, 1));
|
||||
|
||||
ORT_RETURN_IF_ERROR(InsertQDQPair(graph, *curr_edge, dq_scale, dq_zero_point, logger));
|
||||
modified = true;
|
||||
}
|
||||
}
|
||||
|
||||
return is_modified;
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
bool QDQPropagationTransformer::PropagateQBackward(Graph& graph) const {
|
||||
bool is_modified = false;
|
||||
|
||||
GraphViewer graph_viewer(graph);
|
||||
const auto& node_topology_list = graph_viewer.GetNodesInTopologicalOrder();
|
||||
|
||||
for (auto node_index : node_topology_list) {
|
||||
Status PropagateQBackward(Graph& graph, gsl::span<const NodeIndex> node_indices,
|
||||
const std::unordered_set<std::string>& compatible_eps,
|
||||
const logging::Logger& logger,
|
||||
bool& modified) {
|
||||
for (auto node_index : node_indices) {
|
||||
auto* q_node_ptr = graph.GetNode(node_index);
|
||||
if (q_node_ptr == nullptr)
|
||||
if (q_node_ptr == nullptr) {
|
||||
continue; // node removed as part of an earlier fusion
|
||||
}
|
||||
|
||||
Node& q_node = *q_node_ptr;
|
||||
|
||||
if (!graph_utils::IsSupportedOptypeVersionAndDomain(q_node, "QuantizeLinear", {10, 13}) ||
|
||||
!graph_utils::IsSupportedProvider(q_node, GetCompatibleExecutionProviders())) {
|
||||
if (!QDQ::MatchQNode(q_node) ||
|
||||
!graph_utils::IsSupportedProvider(q_node, compatible_eps)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
std::vector<NodeArg*>& q_input_defs = q_node.MutableInputDefs();
|
||||
if (q_input_defs.size() != QDQ::InputIndex::TOTAL_COUNT ||
|
||||
!optimizer_utils::IsScalar(*q_input_defs[QDQ::InputIndex::ZERO_POINT_ID]) ||
|
||||
!optimizer_utils::IsScalar(*q_input_defs[QDQ::InputIndex::SCALE_ID])) {
|
||||
bool q_zero_point_exists = false;
|
||||
if (!QDQ::QOrDQNodeHasConstantScalarScaleAndZeroPoint(q_node, GraphConstantInitializerGetter{graph},
|
||||
q_zero_point_exists)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const ONNX_NAMESPACE::TensorProto* q_zp_tensor_proto =
|
||||
graph_utils::GetConstantInitializer(graph, q_input_defs[QDQ::InputIndex::ZERO_POINT_ID]->Name());
|
||||
const ONNX_NAMESPACE::TensorProto* q_scale_tensor_proto =
|
||||
graph_utils::GetConstantInitializer(graph, q_input_defs[QDQ::InputIndex::SCALE_ID]->Name());
|
||||
auto& q_scale = *q_node.MutableInputDefs()[QDQ::InputIndex::SCALE_ID];
|
||||
auto* q_zero_point = q_zero_point_exists
|
||||
? q_node.MutableInputDefs()[QDQ::InputIndex::ZERO_POINT_ID]
|
||||
: nullptr;
|
||||
|
||||
if (nullptr == q_zp_tensor_proto || nullptr == q_scale_tensor_proto) {
|
||||
const auto edge_before_q = GetPreviousEdge(graph, q_node);
|
||||
if (!edge_before_q) {
|
||||
continue;
|
||||
}
|
||||
|
||||
do {
|
||||
if (q_node.InputNodesBegin() == q_node.InputNodesEnd()) {
|
||||
break;
|
||||
}
|
||||
Node& prev_node = *graph.GetNode(q_node.InputNodesBegin()->Index());
|
||||
if (!optimizer_utils::CheckOutputEdges(graph, prev_node, 1)) break;
|
||||
if (!CanNodePropagate(prev_node)) {
|
||||
// Try canceling out DQ/Q pair
|
||||
Node& dq_node = prev_node;
|
||||
if (graph_utils::IsSupportedOptypeVersionAndDomain(dq_node, "DequantizeLinear", {10, 13}) &&
|
||||
graph_utils::IsSupportedProvider(dq_node, GetCompatibleExecutionProviders()) &&
|
||||
TryCancelOutDQQPair(graph, dq_node, q_node)) {
|
||||
is_modified = true;
|
||||
}
|
||||
for (auto curr_edge = GetPreviousPropagationEdge(graph, *edge_before_q);
|
||||
curr_edge.has_value();
|
||||
curr_edge = GetPreviousPropagationEdge(graph, *curr_edge)) {
|
||||
if (auto* src_node = curr_edge->GetNodeAtEnd(graph, ExtendedGraphEdge::End::Source);
|
||||
src_node && QDQ::MatchDQNode(*src_node)) {
|
||||
break;
|
||||
}
|
||||
|
||||
SwapAdjacentNodes(graph, prev_node, q_node);
|
||||
is_modified = true;
|
||||
} while (true);
|
||||
ORT_RETURN_IF_ERROR(InsertQDQPair(graph, *curr_edge, q_scale, q_zero_point, logger));
|
||||
modified = true;
|
||||
}
|
||||
}
|
||||
|
||||
return is_modified;
|
||||
return Status::OK();
|
||||
}
|
||||
} // namespace
|
||||
|
||||
Status QDQPropagationTransformer::ApplyImpl(Graph& graph, bool& modified, int graph_level, const logging::Logger& logger) const {
|
||||
Status QDQPropagationTransformer::ApplyImpl(Graph& graph, bool& modified, int graph_level,
|
||||
const logging::Logger& logger) const {
|
||||
GraphViewer graph_viewer(graph);
|
||||
const auto& node_topology_list = graph_viewer.GetNodesInTopologicalOrder();
|
||||
const auto node_indices = gsl::make_span(graph_viewer.GetNodesInTopologicalOrder());
|
||||
|
||||
for (auto node_index : node_topology_list) {
|
||||
for (auto node_index : node_indices) {
|
||||
auto* node_ptr = graph.GetNode(node_index);
|
||||
if (node_ptr == nullptr)
|
||||
continue; // node removed as part of an earlier fusion
|
||||
|
|
@ -248,12 +306,11 @@ Status QDQPropagationTransformer::ApplyImpl(Graph& graph, bool& modified, int gr
|
|||
ORT_RETURN_IF_ERROR(Recurse(*node_ptr, modified, graph_level, logger));
|
||||
}
|
||||
|
||||
while (PropagateQBackward(graph) || PropagateDQForward(graph)) {
|
||||
}
|
||||
const auto& compatible_eps = GetCompatibleExecutionProviders();
|
||||
|
||||
modified = true;
|
||||
ORT_RETURN_IF_ERROR(PropagateQBackward(graph, node_indices, compatible_eps, logger, modified));
|
||||
ORT_RETURN_IF_ERROR(PropagateDQForward(graph, node_indices, compatible_eps, logger, modified));
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
} // namespace onnxruntime
|
||||
|
|
|
|||
|
|
@ -11,7 +11,8 @@ namespace onnxruntime {
|
|||
/**
|
||||
@Class QDQPropagationTransformer
|
||||
|
||||
Propagate Q backward, DQ forward and remove DQ/Q pair
|
||||
Propagate Q backward, DQ forward. This is done by inserting Q/DQ pairs that match the starting Q or DQ past each
|
||||
subsequent (previous for Q, next for DQ) op supporting propagation.
|
||||
*/
|
||||
class QDQPropagationTransformer : public GraphTransformer {
|
||||
public:
|
||||
|
|
@ -21,8 +22,6 @@ class QDQPropagationTransformer : public GraphTransformer {
|
|||
|
||||
private:
|
||||
Status ApplyImpl(Graph& graph, bool& modified, int graph_level, const logging::Logger& logger) const override;
|
||||
bool PropagateDQForward(Graph& graph) const;
|
||||
bool PropagateQBackward(Graph& graph) const;
|
||||
};
|
||||
|
||||
} // namespace onnxruntime
|
||||
|
|
|
|||
|
|
@ -1,16 +1,18 @@
|
|||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
#include "qdq_s8_to_u8.h"
|
||||
#include "core/optimizer/qdq_transformer/qdq_s8_to_u8.h"
|
||||
|
||||
#include "core/graph/graph_utils.h"
|
||||
#include "core/optimizer/initializer.h"
|
||||
#include "core/optimizer/qdq_transformer/qdq_util.h"
|
||||
#include "core/optimizer/utils.h"
|
||||
|
||||
namespace onnxruntime {
|
||||
|
||||
// Convert QuantizeLinear and DequantizeLinear pair with type int8_t to type uint8_t
|
||||
Status QDQS8ToU8Transformer::ApplyImpl(Graph& graph, bool& modified, int graph_level, const logging::Logger& logger) const {
|
||||
Status QDQS8ToU8Transformer::ApplyImpl(Graph& graph, bool& modified, int graph_level,
|
||||
const logging::Logger& logger) const {
|
||||
GraphViewer graph_viewer(graph);
|
||||
const auto& node_topology_list = graph_viewer.GetNodesInTopologicalOrder();
|
||||
|
||||
|
|
@ -23,14 +25,14 @@ Status QDQS8ToU8Transformer::ApplyImpl(Graph& graph, bool& modified, int graph_l
|
|||
Node& q_node = *q_node_ptr;
|
||||
ORT_RETURN_IF_ERROR(Recurse(q_node, modified, graph_level, logger));
|
||||
|
||||
if (!graph_utils::IsSupportedOptypeVersionAndDomain(q_node, "QuantizeLinear", {10, 13}) ||
|
||||
if (!QDQ::MatchQNode(q_node) ||
|
||||
!graph_utils::IsSupportedProvider(q_node, GetCompatibleExecutionProviders()) ||
|
||||
!optimizer_utils::CheckOutputEdges(graph, q_node, 1)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
Node& dq_node = *graph.GetNode(q_node.OutputNodesBegin()->Index());
|
||||
if (!graph_utils::IsSupportedOptypeVersionAndDomain(dq_node, "DequantizeLinear", {10, 13}) ||
|
||||
if (!QDQ::MatchDQNode(dq_node) ||
|
||||
!graph_utils::IsSupportedProvider(dq_node, GetCompatibleExecutionProviders())) {
|
||||
continue;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,21 +1,21 @@
|
|||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
#include "qdq_util.h"
|
||||
#include "core/optimizer/qdq_transformer/qdq_util.h"
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "core/common/common.h"
|
||||
#include "core/graph/graph.h"
|
||||
#include "core/graph/graph_utils.h"
|
||||
#include "core/optimizer/initializer.h"
|
||||
#include "core/optimizer/utils.h"
|
||||
|
||||
namespace onnxruntime {
|
||||
namespace QDQ {
|
||||
namespace onnxruntime::QDQ {
|
||||
|
||||
bool IsQDQPairSupported(
|
||||
const Node& q_node, const Node& dq_node,
|
||||
const std::function<const ONNX_NAMESPACE::TensorProto*(const std::string&)>& get_const_initializer,
|
||||
const GetConstantInitializerFn& get_const_initializer,
|
||||
const Path& model_path) {
|
||||
ConstPointerContainer<std::vector<NodeArg*>> dq_input_defs = dq_node.InputDefs();
|
||||
ConstPointerContainer<std::vector<NodeArg*>> q_input_defs = q_node.InputDefs();
|
||||
|
|
@ -58,31 +58,56 @@ bool IsQDQPairSupported(
|
|||
*q_scale.data<float>() == *dq_scale.data<float>();
|
||||
}
|
||||
|
||||
bool IsDQSupported(
|
||||
const Node& dq_node,
|
||||
const std::function<const ONNX_NAMESPACE::TensorProto*(const std::string&)>& get_const_initializer) {
|
||||
ConstPointerContainer<std::vector<NodeArg*>> dq_input_defs = dq_node.InputDefs();
|
||||
|
||||
// DQ contains optional input is not supported
|
||||
// non-scalar DQ scale and zero point needs are not supported
|
||||
if (dq_input_defs.size() != InputIndex::TOTAL_COUNT ||
|
||||
!optimizer_utils::IsScalar(*dq_input_defs[InputIndex::SCALE_ID]) ||
|
||||
!optimizer_utils::IsScalar(*dq_input_defs[InputIndex::ZERO_POINT_ID])) {
|
||||
bool IsDQSupported(const Node& dq_node, const GetConstantInitializerFn& get_const_initializer) {
|
||||
bool zero_point_exists = false;
|
||||
if (!QOrDQNodeHasConstantScalarScaleAndZeroPoint(dq_node, get_const_initializer, zero_point_exists)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// if DQ scale and zero point are not constant, return false
|
||||
const ONNX_NAMESPACE::TensorProto* dq_scale_tensor_proto =
|
||||
get_const_initializer(dq_input_defs[InputIndex::SCALE_ID]->Name());
|
||||
const ONNX_NAMESPACE::TensorProto* dq_zp_tensor_proto =
|
||||
get_const_initializer(dq_input_defs[InputIndex::ZERO_POINT_ID]->Name());
|
||||
if (nullptr == dq_zp_tensor_proto ||
|
||||
nullptr == dq_scale_tensor_proto) {
|
||||
if (!zero_point_exists) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace QDQ
|
||||
} // namespace onnxruntime
|
||||
bool QOrDQNodeHasConstantScalarScaleAndZeroPoint(
|
||||
const Node& q_or_dq_node,
|
||||
const GetConstantInitializerFn& get_const_initializer,
|
||||
bool& zero_point_exists) {
|
||||
auto q_or_dq_input_defs = q_or_dq_node.InputDefs();
|
||||
|
||||
ORT_ENFORCE(q_or_dq_input_defs.size() >= 2);
|
||||
|
||||
zero_point_exists = q_or_dq_input_defs.size() > 2 &&
|
||||
q_or_dq_input_defs[InputIndex::ZERO_POINT_ID]->Exists();
|
||||
|
||||
auto is_constant_scalar = [&](const NodeArg& node_arg) {
|
||||
return optimizer_utils::IsScalar(node_arg) && get_const_initializer(node_arg.Name()) != nullptr;
|
||||
};
|
||||
|
||||
if (!is_constant_scalar(*q_or_dq_input_defs[InputIndex::SCALE_ID])) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (zero_point_exists &&
|
||||
!is_constant_scalar(*q_or_dq_input_defs[InputIndex::ZERO_POINT_ID])) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
#if !defined(ORT_MINIMAL_BUILD)
|
||||
|
||||
bool MatchQNode(const Node& node) {
|
||||
return graph_utils::IsSupportedOptypeVersionAndDomain(node, QOpName, {10, 13});
|
||||
}
|
||||
|
||||
bool MatchDQNode(const Node& node) {
|
||||
return graph_utils::IsSupportedOptypeVersionAndDomain(node, DQOpName, {10, 13});
|
||||
}
|
||||
|
||||
#endif // !defined(ORT_MINIMAL_BUILD)
|
||||
|
||||
} // namespace onnxruntime::QDQ
|
||||
|
|
|
|||
|
|
@ -17,8 +17,8 @@ class Path;
|
|||
|
||||
namespace QDQ {
|
||||
|
||||
static constexpr const char* QOpName = "QuantizeLinear";
|
||||
static constexpr const char* DQOpName = "DequantizeLinear";
|
||||
constexpr const char* QOpName = "QuantizeLinear";
|
||||
constexpr const char* DQOpName = "DequantizeLinear";
|
||||
|
||||
enum InputIndex : int {
|
||||
INPUT_ID = 0,
|
||||
|
|
@ -27,21 +27,39 @@ enum InputIndex : int {
|
|||
TOTAL_COUNT = 3,
|
||||
};
|
||||
|
||||
// Check if Q/DQ pair is supported in the QDQ transformer. It requires:
|
||||
using GetConstantInitializerFn = std::function<const ONNX_NAMESPACE::TensorProto*(const std::string&)>;
|
||||
|
||||
// Check if Q/DQ pair is supported in extended level QDQ transformers. It requires:
|
||||
// 1. Q/DQ doesn't have optional input.
|
||||
// 2. scale and zero point is constant scalar
|
||||
// 3. Q and DQ have same scale and zero point
|
||||
bool IsQDQPairSupported(
|
||||
const Node& q_node, const Node& dq_node,
|
||||
const std::function<const ONNX_NAMESPACE::TensorProto*(const std::string&)>& get_const_initializer,
|
||||
const GetConstantInitializerFn& get_const_initializer,
|
||||
const Path& model_path);
|
||||
|
||||
// Check if DQ is supported in the QDQ transformer. It requires:
|
||||
// Check if DQ is supported in extended level QDQ transformers. It requires:
|
||||
// 1. DQ doesn't have optional input.
|
||||
// 2. scale and zero point is constant scalar
|
||||
bool IsDQSupported(
|
||||
const Node& dq_node,
|
||||
const std::function<const ONNX_NAMESPACE::TensorProto*(const std::string&)>& get_const_initializer);
|
||||
bool IsDQSupported(const Node& dq_node, const GetConstantInitializerFn& get_const_initializer);
|
||||
|
||||
// Check if Q or DQ node's scale and zero point inputs are constant scalars.
|
||||
// If the zero point input does not exist, it is assumed to have a default scalar value.
|
||||
// `zero_point_exists` will indicate if it does exist.
|
||||
bool QOrDQNodeHasConstantScalarScaleAndZeroPoint(
|
||||
const Node& q_or_dq_node,
|
||||
const GetConstantInitializerFn& get_const_initializer,
|
||||
bool& zero_point_exists);
|
||||
|
||||
#if !defined(ORT_MINIMAL_BUILD)
|
||||
|
||||
// Check Q node op type, version, and domain.
|
||||
bool MatchQNode(const Node& node);
|
||||
|
||||
// Check DQ node op type, version, and domain.
|
||||
bool MatchDQNode(const Node& node);
|
||||
|
||||
#endif // !defined(ORT_MINIMAL_BUILD)
|
||||
|
||||
} // namespace QDQ
|
||||
} // namespace onnxruntime
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
#include "test/optimizer/graph_transform_test_builder.h"
|
||||
|
||||
#include <functional>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
|
@ -9,10 +11,9 @@
|
|||
#include "core/session/inference_session.h"
|
||||
#include "test/compare_ortvalue.h"
|
||||
#include "test/test_environment.h"
|
||||
#include "test/util/include/asserts.h"
|
||||
#include "test/util/include/inference_session_wrapper.h"
|
||||
|
||||
#include "graph_transform_test_builder.h"
|
||||
|
||||
namespace onnxruntime {
|
||||
namespace test {
|
||||
|
||||
|
|
@ -34,39 +35,33 @@ void TransformerTester(const std::function<void(ModelTestBuilder& helper)>& buil
|
|||
ModelTestBuilder helper(graph);
|
||||
build_test_case(helper);
|
||||
helper.SetGraphOutputs();
|
||||
ASSERT_TRUE(model.MainGraph().Resolve().IsOK());
|
||||
ASSERT_STATUS_OK(model.MainGraph().Resolve());
|
||||
|
||||
// Serialize the model to a string.
|
||||
std::string model_data;
|
||||
model.ToProto().SerializeToString(&model_data);
|
||||
|
||||
auto run_model = [&](TransformerLevel level, std::vector<OrtValue>& fetches, std::unique_ptr<GraphTransformer> transformer = nullptr) {
|
||||
auto run_model = [&](TransformerLevel level, std::vector<OrtValue>& fetches,
|
||||
std::unique_ptr<GraphTransformer> transformer = nullptr) {
|
||||
SessionOptions session_options;
|
||||
session_options.graph_optimization_level = transformer ? baseline_level : level;
|
||||
#if 0 // enable to dump model for debugging
|
||||
session_options.optimized_model_filepath = L"model" + std::to_wstring(static_cast<int>(level)) + L".onnx";
|
||||
session_options.optimized_model_filepath =
|
||||
ToPathString("model" + std::to_string(static_cast<int>(level)) + ".onnx");
|
||||
#endif
|
||||
InferenceSessionWrapper session{session_options, GetEnvironment()};
|
||||
ASSERT_TRUE(session.Load(model_data.data(), static_cast<int>(model_data.size())).IsOK());
|
||||
ASSERT_STATUS_OK(session.Load(model_data.data(), static_cast<int>(model_data.size())));
|
||||
if (transformer) {
|
||||
ASSERT_TRUE(session.RegisterGraphTransformer(std::move(transformer), level).IsOK());
|
||||
ASSERT_STATUS_OK(session.RegisterGraphTransformer(std::move(transformer), level));
|
||||
}
|
||||
|
||||
auto status = session.Initialize();
|
||||
if (!status.IsOK()) {
|
||||
std::cout << "Model initialized failed with status message: " << status.ErrorMessage() << std::endl;
|
||||
}
|
||||
ASSERT_TRUE(status.IsOK());
|
||||
ASSERT_STATUS_OK(session.Initialize());
|
||||
|
||||
RunOptions run_options;
|
||||
status = session.Run(run_options,
|
||||
helper.feeds_,
|
||||
helper.output_names_,
|
||||
&fetches);
|
||||
if (!status.IsOK()) {
|
||||
std::cout << "Run failed with status message: " << status.ErrorMessage() << std::endl;
|
||||
}
|
||||
ASSERT_TRUE(status.IsOK());
|
||||
ASSERT_STATUS_OK(session.Run(run_options,
|
||||
helper.feeds_,
|
||||
helper.output_names_,
|
||||
&fetches));
|
||||
|
||||
if (level == target_level) {
|
||||
check_transformed_graph(session);
|
||||
|
|
@ -74,13 +69,13 @@ void TransformerTester(const std::function<void(ModelTestBuilder& helper)>& buil
|
|||
};
|
||||
|
||||
std::vector<OrtValue> baseline_fetches;
|
||||
run_model(baseline_level, baseline_fetches);
|
||||
ASSERT_NO_FATAL_FAILURE(run_model(baseline_level, baseline_fetches));
|
||||
|
||||
std::vector<OrtValue> target_fetches;
|
||||
run_model(target_level, target_fetches, std::move(transformer));
|
||||
ASSERT_NO_FATAL_FAILURE(run_model(target_level, target_fetches, std::move(transformer)));
|
||||
|
||||
size_t num_outputs = baseline_fetches.size();
|
||||
ASSERT_TRUE(num_outputs == target_fetches.size());
|
||||
ASSERT_EQ(num_outputs, target_fetches.size());
|
||||
|
||||
for (size_t i = 0; i < num_outputs; i++) {
|
||||
std::pair<COMPARE_RESULT, std::string> ret =
|
||||
|
|
|
|||
|
|
@ -172,11 +172,9 @@ class ModelTestBuilder {
|
|||
return AddNode("DequantizeLinear", input_args, {output_arg});
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
typename std::enable_if<IsTypeDequantLinearCompatible<T>::value, Node&>::type
|
||||
AddDequantizeLinearNode(NodeArg* input_arg,
|
||||
float input_scale,
|
||||
NodeArg* output_arg) {
|
||||
Node& AddDequantizeLinearNode(NodeArg* input_arg,
|
||||
float input_scale,
|
||||
NodeArg* output_arg) {
|
||||
std::vector<NodeArg*> input_args;
|
||||
input_args.push_back(input_arg);
|
||||
input_args.push_back(MakeScalarInitializer<float>(input_scale));
|
||||
|
|
|
|||
|
|
@ -34,7 +34,17 @@
|
|||
namespace onnxruntime {
|
||||
namespace test {
|
||||
|
||||
#ifndef DISABLE_CONTRIB_OPS
|
||||
static std::vector<std::string> GetNodeOpTypesInTopologicalOrder(const Graph& graph) {
|
||||
std::vector<std::string> op_types{};
|
||||
GraphViewer graph_viewer{graph};
|
||||
const auto& ordering = graph_viewer.GetNodesInTopologicalOrder();
|
||||
for (const auto node_idx : ordering) {
|
||||
op_types.push_back(graph.GetNode(node_idx)->OpType());
|
||||
}
|
||||
return op_types;
|
||||
}
|
||||
|
||||
#if !defined(DISABLE_CONTRIB_OPS)
|
||||
|
||||
template <typename InputType, typename WeightType, typename BiasType, typename OutputType>
|
||||
void QDQTransformerConvTests() {
|
||||
|
|
@ -198,12 +208,13 @@ TEST(QDQTransformerTests, ConvMaxPoolReshape_Int8) {
|
|||
};
|
||||
|
||||
auto check_graph = [&](InferenceSessionWrapper& session) {
|
||||
auto op_to_count = CountOpsInGraph(session.GetGraph());
|
||||
EXPECT_EQ(op_to_count["QLinearConv"], 1);
|
||||
EXPECT_EQ(op_to_count["MaxPool"], 1);
|
||||
EXPECT_EQ(op_to_count["Reshape"], 1);
|
||||
EXPECT_EQ(op_to_count["QuantizeLinear"], 1);
|
||||
EXPECT_EQ(op_to_count["DequantizeLinear"], 0);
|
||||
const std::vector<std::string> expected_op_types_in_order{
|
||||
"QuantizeLinear",
|
||||
"QLinearConv",
|
||||
"MaxPool",
|
||||
"Reshape"};
|
||||
const auto op_types_in_order = GetNodeOpTypesInTopologicalOrder(session.GetGraph());
|
||||
EXPECT_EQ(op_types_in_order, expected_op_types_in_order);
|
||||
};
|
||||
|
||||
TransformerTester(build_test_case, check_graph, TransformerLevel::Level1, TransformerLevel::Level2);
|
||||
|
|
@ -1673,27 +1684,115 @@ TEST(QDQTransformerTests, Concat) {
|
|||
test_case({{1, 6, 36}, {1, 6, 8}, {1, 6, 2}}, 2, false, false, true);
|
||||
}
|
||||
|
||||
TEST(QDQTransformerTests, QDQPropagation_QDQCancelOut) {
|
||||
auto test_case = [&](const std::vector<int64_t>& input_shape, size_t maxpool_dim, const std::vector<int64_t>& perms) {
|
||||
#endif // !defined(DISABLE_CONTRIB_OPS)
|
||||
|
||||
TEST(QDQTransformerTests, QDQPropagation_QBackward) {
|
||||
auto test_case = [&](const std::vector<int64_t>& input_shape,
|
||||
size_t maxpool_dim,
|
||||
const std::vector<int64_t>& perms,
|
||||
bool add_op_boundary,
|
||||
bool include_zp) {
|
||||
auto build_test_case = [&](ModelTestBuilder& builder) {
|
||||
auto* input_arg = builder.MakeInput<float>(input_shape, -1.f, 1.f);
|
||||
auto* output_arg = builder.MakeOutput();
|
||||
|
||||
// add QDQ
|
||||
auto* qdq_output = AddQDQNodePair<uint8_t>(builder, input_arg, .004f, 129);
|
||||
auto* transpose_input = add_op_boundary ? builder.MakeIntermediate() : input_arg;
|
||||
if (add_op_boundary) {
|
||||
// add Sign as boundary for QDQ propagation
|
||||
builder.AddNode("Sign", {input_arg}, {transpose_input});
|
||||
}
|
||||
|
||||
// add Transpose
|
||||
auto* transpose_output = builder.MakeIntermediate();
|
||||
Node& transpose_node = builder.AddNode("Transpose", {qdq_output}, {transpose_output});
|
||||
Node& transpose_node = builder.AddNode("Transpose", {transpose_input}, {transpose_output});
|
||||
transpose_node.AddAttribute("perm", perms);
|
||||
|
||||
// add Q
|
||||
auto* q_output = builder.MakeIntermediate();
|
||||
builder.AddQuantizeLinearNode<uint8_t>(transpose_output, .004f, 129, q_output);
|
||||
|
||||
// add MaxPool
|
||||
auto* maxpool_output = builder.MakeIntermediate();
|
||||
Node& pool_node = builder.AddNode("MaxPool", {q_output}, {maxpool_output});
|
||||
Node& pool_node = builder.AddNode("MaxPool", {transpose_output}, {maxpool_output});
|
||||
std::vector<int64_t> pads((maxpool_dim - 2) * 2, 1);
|
||||
pool_node.AddAttribute("pads", pads);
|
||||
std::vector<int64_t> kernel_shape(maxpool_dim - 2, 3);
|
||||
pool_node.AddAttribute("kernel_shape", kernel_shape);
|
||||
|
||||
// Reshape
|
||||
auto* reshape_output = builder.MakeIntermediate();
|
||||
auto* reshape_shape = builder.Make1DInitializer<int64_t>({-1, 0});
|
||||
builder.AddNode("Reshape", {maxpool_output, reshape_shape}, {reshape_output});
|
||||
|
||||
// add Q
|
||||
constexpr float qdq_scale = 0.004f;
|
||||
if (include_zp) {
|
||||
constexpr uint8_t qdq_zero_point = 129;
|
||||
builder.AddQuantizeLinearNode<uint8_t>(reshape_output, qdq_scale, qdq_zero_point, output_arg);
|
||||
} else {
|
||||
builder.AddQuantizeLinearNode(reshape_output, qdq_scale, output_arg);
|
||||
}
|
||||
};
|
||||
|
||||
auto check_graph = [&](InferenceSessionWrapper& session) {
|
||||
std::vector<std::string> expected_op_types_in_order{};
|
||||
if (add_op_boundary) {
|
||||
expected_op_types_in_order.push_back("Sign");
|
||||
}
|
||||
expected_op_types_in_order.insert(
|
||||
expected_op_types_in_order.end(),
|
||||
{"QuantizeLinear", "DequantizeLinear",
|
||||
"Transpose",
|
||||
"QuantizeLinear", "DequantizeLinear",
|
||||
"MaxPool",
|
||||
"QuantizeLinear", "DequantizeLinear",
|
||||
"Reshape",
|
||||
"QuantizeLinear"});
|
||||
|
||||
const auto op_types_in_order = GetNodeOpTypesInTopologicalOrder(session.GetGraph());
|
||||
EXPECT_EQ(op_types_in_order, expected_op_types_in_order);
|
||||
};
|
||||
|
||||
TransformerTester(build_test_case,
|
||||
check_graph,
|
||||
TransformerLevel::Default,
|
||||
TransformerLevel::Level1);
|
||||
};
|
||||
|
||||
// TODO re-enable tests after updating ONNX to get QuantizeLinear shape inference fix
|
||||
// https://github.com/onnx/onnx/pull/3806
|
||||
//test_case({1, 13, 13, 23}, 4, {0, 3, 1, 2}, false, false);
|
||||
test_case({1, 13, 13, 23}, 4, {0, 3, 1, 2}, false, true);
|
||||
//test_case({1, 13, 13, 23}, 4, {0, 3, 1, 2}, true, false);
|
||||
test_case({1, 13, 13, 23}, 4, {0, 3, 1, 2}, true, true);
|
||||
}
|
||||
|
||||
TEST(QDQTransformerTests, QDQPropagation_DQForward) {
|
||||
auto test_case = [&](const std::vector<int64_t>& input_shape,
|
||||
size_t maxpool_dim,
|
||||
const std::vector<int64_t>& perms,
|
||||
bool add_op_boundary,
|
||||
bool include_zp) {
|
||||
auto build_test_case = [&](ModelTestBuilder& builder) {
|
||||
auto* input_arg = builder.MakeInput<uint8_t>(input_shape,
|
||||
std::numeric_limits<uint8_t>::min(),
|
||||
std::numeric_limits<uint8_t>::max());
|
||||
auto* output_arg = builder.MakeOutput();
|
||||
|
||||
// add DQ
|
||||
constexpr float qdq_scale = 0.004f;
|
||||
auto* dq_output = builder.MakeIntermediate();
|
||||
if (include_zp) {
|
||||
constexpr uint8_t qdq_zero_point = 129;
|
||||
builder.AddDequantizeLinearNode<uint8_t>(input_arg, qdq_scale, qdq_zero_point, dq_output);
|
||||
} else {
|
||||
builder.AddDequantizeLinearNode(input_arg, qdq_scale, dq_output);
|
||||
}
|
||||
|
||||
// add Transpose
|
||||
auto* transpose_output = builder.MakeIntermediate();
|
||||
Node& transpose_node = builder.AddNode("Transpose", {dq_output}, {transpose_output});
|
||||
transpose_node.AddAttribute("perm", perms);
|
||||
|
||||
// add MaxPool
|
||||
auto* maxpool_output = builder.MakeIntermediate();
|
||||
Node& pool_node = builder.AddNode("MaxPool", {transpose_output}, {maxpool_output});
|
||||
std::vector<int64_t> pads((maxpool_dim - 2) * 2, 1);
|
||||
pool_node.AddAttribute("pads", pads);
|
||||
std::vector<int64_t> kernel_shape(maxpool_dim - 2, 3);
|
||||
|
|
@ -1701,28 +1800,47 @@ TEST(QDQTransformerTests, QDQPropagation_QDQCancelOut) {
|
|||
|
||||
// Reshape
|
||||
auto* reshape_shape = builder.Make1DInitializer<int64_t>({-1, 0});
|
||||
builder.AddNode("Reshape", {maxpool_output, reshape_shape}, {output_arg});
|
||||
auto* reshape_output = add_op_boundary ? builder.MakeIntermediate() : output_arg;
|
||||
builder.AddNode("Reshape", {maxpool_output, reshape_shape}, {reshape_output});
|
||||
|
||||
if (add_op_boundary) {
|
||||
// add Sign as boundary for QDQ propagation
|
||||
builder.AddNode("Sign", {reshape_output}, {output_arg});
|
||||
}
|
||||
};
|
||||
|
||||
auto check_graph = [&](InferenceSessionWrapper& session) {
|
||||
auto op_to_count = CountOpsInGraph(session.GetGraph());
|
||||
EXPECT_EQ(op_to_count["MaxPool"], 1);
|
||||
EXPECT_EQ(op_to_count["Reshape"], 1);
|
||||
EXPECT_EQ(op_to_count["Transpose"], 1);
|
||||
EXPECT_EQ(op_to_count["QuantizeLinear"], 1);
|
||||
EXPECT_EQ(op_to_count["DequantizeLinear"], 0);
|
||||
std::vector<std::string> expected_op_types_in_order{
|
||||
"DequantizeLinear",
|
||||
"Transpose",
|
||||
"QuantizeLinear", "DequantizeLinear",
|
||||
"MaxPool",
|
||||
"QuantizeLinear", "DequantizeLinear",
|
||||
"Reshape",
|
||||
"QuantizeLinear", "DequantizeLinear"};
|
||||
if (add_op_boundary) {
|
||||
expected_op_types_in_order.push_back("Sign");
|
||||
}
|
||||
|
||||
const auto op_types_in_order = GetNodeOpTypesInTopologicalOrder(session.GetGraph());
|
||||
EXPECT_EQ(op_types_in_order, expected_op_types_in_order);
|
||||
};
|
||||
|
||||
TransformerTester(build_test_case,
|
||||
check_graph,
|
||||
TransformerLevel::Level1,
|
||||
TransformerLevel::Level2);
|
||||
TransformerLevel::Default,
|
||||
TransformerLevel::Level1);
|
||||
};
|
||||
|
||||
test_case({1, 13, 13, 23}, 4, {0, 3, 1, 2});
|
||||
// TODO re-enable tests after updating ONNX to get QuantizeLinear shape inference fix
|
||||
// https://github.com/onnx/onnx/pull/3806
|
||||
//test_case({1, 13, 13, 23}, 4, {0, 3, 1, 2}, false, false);
|
||||
test_case({1, 13, 13, 23}, 4, {0, 3, 1, 2}, false, true);
|
||||
//test_case({1, 13, 13, 23}, 4, {0, 3, 1, 2}, true, false);
|
||||
test_case({1, 13, 13, 23}, 4, {0, 3, 1, 2}, true, true);
|
||||
}
|
||||
|
||||
TEST(QDQTransformerTests, QDQPropagation_QDQ_CancelOut_More) {
|
||||
TEST(QDQTransformerTests, QDQPropagation_StopAtOtherQDQ) {
|
||||
auto test_case = [&](const std::vector<int64_t>& input_shape, bool same_scale, bool same_zp) {
|
||||
auto build_test_case = [&](ModelTestBuilder& builder) {
|
||||
auto* input_arg = builder.MakeInput<float>(input_shape, -1.f, 1.f);
|
||||
|
|
@ -1741,16 +1859,18 @@ TEST(QDQTransformerTests, QDQPropagation_QDQ_CancelOut_More) {
|
|||
};
|
||||
|
||||
auto check_graph = [&](InferenceSessionWrapper& session) {
|
||||
auto op_to_count = CountOpsInGraph(session.GetGraph());
|
||||
EXPECT_EQ(op_to_count["Reshape"], 1);
|
||||
EXPECT_EQ(op_to_count["QuantizeLinear"], same_scale && same_zp ? 1 : 2);
|
||||
EXPECT_EQ(op_to_count["DequantizeLinear"], same_scale && same_zp ? 0 : 1);
|
||||
const std::vector<std::string> expected_op_types_in_order{
|
||||
"QuantizeLinear", "DequantizeLinear",
|
||||
"Reshape",
|
||||
"QuantizeLinear"};
|
||||
const auto op_types_in_order = GetNodeOpTypesInTopologicalOrder(session.GetGraph());
|
||||
EXPECT_EQ(op_types_in_order, expected_op_types_in_order);
|
||||
};
|
||||
|
||||
TransformerTester(build_test_case,
|
||||
check_graph,
|
||||
TransformerLevel::Level1,
|
||||
TransformerLevel::Level2);
|
||||
TransformerLevel::Default,
|
||||
TransformerLevel::Level1);
|
||||
};
|
||||
|
||||
test_case({1, 13, 13, 23}, false, false);
|
||||
|
|
@ -1775,16 +1895,18 @@ TEST(QDQTransformerTests, QDQPropagation_Q_No_Parent) {
|
|||
};
|
||||
|
||||
auto check_graph = [&](InferenceSessionWrapper& session) {
|
||||
GraphViewer graph_viewer(session.GetGraph());
|
||||
const auto& node_topology_list = graph_viewer.GetNodesInTopologicalOrder();
|
||||
EXPECT_EQ(graph_viewer.GetNode(node_topology_list[0])->OpType(), "QuantizeLinear");
|
||||
EXPECT_EQ(graph_viewer.GetNode(node_topology_list[1])->OpType(), "Transpose");
|
||||
const std::vector<std::string> expected_op_types_in_order{
|
||||
"QuantizeLinear", "DequantizeLinear",
|
||||
"Transpose",
|
||||
"QuantizeLinear"};
|
||||
const auto op_types_in_order = GetNodeOpTypesInTopologicalOrder(session.GetGraph());
|
||||
EXPECT_EQ(op_types_in_order, expected_op_types_in_order);
|
||||
};
|
||||
|
||||
TransformerTester(build_test_case,
|
||||
check_graph,
|
||||
TransformerLevel::Level1,
|
||||
TransformerLevel::Level2);
|
||||
TransformerLevel::Default,
|
||||
TransformerLevel::Level1);
|
||||
};
|
||||
|
||||
test_case({1, 13, 13, 23}, {0, 2, 3, 1});
|
||||
|
|
@ -1808,17 +1930,18 @@ TEST(QDQTransformerTests, QDQPropagation_DQ_No_Children) {
|
|||
};
|
||||
|
||||
auto check_graph = [&](InferenceSessionWrapper& session) {
|
||||
auto op_to_count = CountOpsInGraph(session.GetGraph());
|
||||
GraphViewer graph_viewer(session.GetGraph());
|
||||
const auto& node_topology_list = graph_viewer.GetNodesInTopologicalOrder();
|
||||
EXPECT_EQ(graph_viewer.GetNode(node_topology_list[0])->OpType(), "Transpose");
|
||||
EXPECT_EQ(graph_viewer.GetNode(node_topology_list[1])->OpType(), "DequantizeLinear");
|
||||
const std::vector<std::string> expected_op_types_in_order{
|
||||
"DequantizeLinear",
|
||||
"Transpose",
|
||||
"QuantizeLinear", "DequantizeLinear"};
|
||||
const auto op_types_in_order = GetNodeOpTypesInTopologicalOrder(session.GetGraph());
|
||||
EXPECT_EQ(op_types_in_order, expected_op_types_in_order);
|
||||
};
|
||||
|
||||
TransformerTester(build_test_case,
|
||||
check_graph,
|
||||
TransformerLevel::Level1,
|
||||
TransformerLevel::Level2);
|
||||
TransformerLevel::Default,
|
||||
TransformerLevel::Level1);
|
||||
};
|
||||
|
||||
test_case({1, 13, 13, 23}, {0, 2, 3, 1});
|
||||
|
|
@ -1844,17 +1967,17 @@ TEST(QDQTransformerTests, QDQPropagation_Per_Layer_No_Propagation) {
|
|||
};
|
||||
|
||||
auto check_graph = [&](InferenceSessionWrapper& session) {
|
||||
auto op_to_count = CountOpsInGraph(session.GetGraph());
|
||||
GraphViewer graph_viewer(session.GetGraph());
|
||||
const auto& node_topology_list = graph_viewer.GetNodesInTopologicalOrder();
|
||||
EXPECT_EQ(graph_viewer.GetNode(node_topology_list[0])->OpType(), "DequantizeLinear");
|
||||
EXPECT_EQ(graph_viewer.GetNode(node_topology_list[1])->OpType(), "Transpose");
|
||||
const std::vector<std::string> expected_op_types_in_order{
|
||||
"DequantizeLinear",
|
||||
"Transpose"};
|
||||
const auto op_types_in_order = GetNodeOpTypesInTopologicalOrder(session.GetGraph());
|
||||
EXPECT_EQ(op_types_in_order, expected_op_types_in_order);
|
||||
};
|
||||
|
||||
TransformerTester(build_test_case,
|
||||
check_graph,
|
||||
TransformerLevel::Level1,
|
||||
TransformerLevel::Level2);
|
||||
TransformerLevel::Default,
|
||||
TransformerLevel::Level1);
|
||||
};
|
||||
|
||||
test_case({1, 13, 13, 23}, {0, 2, 3, 1});
|
||||
|
|
@ -1877,22 +2000,22 @@ TEST(QDQTransformerTests, QDQPropagation_DQ_Q) {
|
|||
};
|
||||
|
||||
auto check_graph = [&](InferenceSessionWrapper& session) {
|
||||
auto op_to_count = CountOpsInGraph(session.GetGraph());
|
||||
EXPECT_EQ(op_to_count["QuantizeLinear"], 1);
|
||||
EXPECT_EQ(op_to_count["DequantizeLinear"], 1);
|
||||
const std::vector<std::string> expected_op_types_in_order{
|
||||
"DequantizeLinear",
|
||||
"QuantizeLinear"};
|
||||
const auto op_types_in_order = GetNodeOpTypesInTopologicalOrder(session.GetGraph());
|
||||
EXPECT_EQ(op_types_in_order, expected_op_types_in_order);
|
||||
};
|
||||
|
||||
TransformerTester(build_test_case,
|
||||
check_graph,
|
||||
TransformerLevel::Level1,
|
||||
TransformerLevel::Level2);
|
||||
TransformerLevel::Default,
|
||||
TransformerLevel::Level1);
|
||||
};
|
||||
|
||||
test_case({1, 13, 13, 23});
|
||||
}
|
||||
|
||||
#endif // DISABLE_CONTRIB_OPS
|
||||
|
||||
TEST(QDQTransformerTests, QDQ_Selector_Test) {
|
||||
const ORTCHAR_T* model_file_name = ORT_TSTR("testdata/transform/qdq_conv.onnx");
|
||||
|
||||
|
|
|
|||
|
|
@ -60,7 +60,7 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device
|
|||
} else if (provider_name == onnxruntime::kTensorrtExecutionProvider) {
|
||||
#ifdef USE_TENSORRT
|
||||
int device_id = 0;
|
||||
int trt_max_partition_iterations = 1000;
|
||||
int trt_max_partition_iterations = 1000;
|
||||
int trt_min_subgraph_size = 1;
|
||||
size_t trt_max_workspace_size = 1 << 30;
|
||||
bool trt_fp16_enable = false;
|
||||
|
|
@ -76,11 +76,11 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device
|
|||
std::string trt_engine_decryption_lib_path = "";
|
||||
bool trt_force_sequential_engine_build = false;
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#ifdef _MSC_VER
|
||||
std::string ov_string = ToUTF8String(performance_test_config.run_config.ep_runtime_config_string);
|
||||
#else
|
||||
#else
|
||||
std::string ov_string = performance_test_config.run_config.ep_runtime_config_string;
|
||||
#endif
|
||||
#endif
|
||||
std::istringstream ss(ov_string);
|
||||
std::string token;
|
||||
while (ss >> token) {
|
||||
|
|
@ -215,7 +215,7 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device
|
|||
tensorrt_options.has_user_compute_stream = 0;
|
||||
tensorrt_options.user_compute_stream = nullptr;
|
||||
tensorrt_options.trt_max_partition_iterations = trt_max_partition_iterations;
|
||||
tensorrt_options.trt_min_subgraph_size = trt_min_subgraph_size;
|
||||
tensorrt_options.trt_min_subgraph_size = trt_min_subgraph_size;
|
||||
tensorrt_options.trt_max_workspace_size = trt_max_workspace_size;
|
||||
tensorrt_options.trt_fp16_enable = trt_fp16_enable;
|
||||
tensorrt_options.trt_int8_enable = trt_int8_enable;
|
||||
|
|
@ -232,9 +232,9 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device
|
|||
session_options.AppendExecutionProvider_TensorRT_V2(tensorrt_options);
|
||||
|
||||
OrtCUDAProviderOptions cuda_options;
|
||||
cuda_options.device_id=device_id;
|
||||
cuda_options.cudnn_conv_algo_search=static_cast<OrtCudnnConvAlgoSearch>(performance_test_config.run_config.cudnn_conv_algo);
|
||||
cuda_options.do_copy_in_default_stream=!performance_test_config.run_config.do_cuda_copy_in_separate_stream;
|
||||
cuda_options.device_id = device_id;
|
||||
cuda_options.cudnn_conv_algo_search = static_cast<OrtCudnnConvAlgoSearch>(performance_test_config.run_config.cudnn_conv_algo);
|
||||
cuda_options.do_copy_in_default_stream = !performance_test_config.run_config.do_cuda_copy_in_separate_stream;
|
||||
// TODO: Support arena configuration for users of perf test
|
||||
session_options.AppendExecutionProvider_CUDA(cuda_options);
|
||||
#else
|
||||
|
|
@ -325,7 +325,11 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device
|
|||
} else if (provider_name == onnxruntime::kNnapiExecutionProvider) {
|
||||
#ifdef USE_NNAPI
|
||||
uint32_t nnapi_flags = 0;
|
||||
#ifdef _MSC_VER
|
||||
std::string ov_string = ToUTF8String(performance_test_config.run_config.ep_runtime_config_string);
|
||||
#else
|
||||
std::string ov_string = performance_test_config.run_config.ep_runtime_config_string;
|
||||
#endif
|
||||
std::istringstream ss(ov_string);
|
||||
std::string key;
|
||||
while (ss >> key) {
|
||||
|
|
|
|||
Loading…
Reference in a new issue