saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-30 20:18:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Enable QDQ quantization for DML EP (#18367)

Browse source 
### Description
This enables QDQ transforms with the DML EP
This commit is contained in:
raoanag 2024-01-02 18:06:05 -08:00 committed by Jeff Bloomfield
parent ee60e3af6c
commit 56fcea94e3
5 changed files with 67 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

60
onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_selector_action_transformer.cc
View file

@ -105,8 +105,8 @@ void UnaryOpQDQRules(SelectorActionRegistry& qdq_selector_action_registry) {
std::unique_ptr<Action> action = std::make_unique<QDQ::UnaryReplaceWithQLinear>(kMSDomain);
#if !defined(ORT_MINIMAL_BUILD)
// TODO: Enable 16-bit types in selector when unary QLinear* ops support 16-bit.
std::unique_ptr<NodeSelector> selector = std::make_unique<QDQ::UnarySelector>();
std::vector<const char*> providers = {kCpuExecutionProvider};
std::unique_ptr<NodeSelector> selector = std::make_unique<QDQ::UnarySelector>(providers);
qdq_selector_action_registry.RegisterSelectorAndAction(action_name,
{{"AveragePool", {}},
{"LeakyRelu", {}},
@ -123,20 +123,43 @@ void UnaryOpQDQRules(SelectorActionRegistry& qdq_selector_action_registry) {
void BinaryOpQDQRules(SelectorActionRegistry& qdq_selector_action_registry) {
// 4 nodes. 2 x DQ for inputs, target, Q
// Replace with internal QLinear version of operator. Delete all original nodes.
const std::string action_name{"2DQ"};
std::unique_ptr<Action> action = std::make_unique<QDQ::BinaryReplaceWithQLinear>(kMSDomain);
{
const std::string action_name{"2DQ"};
std::unique_ptr<Action> action = std::make_unique<QDQ::BinaryReplaceWithQLinear>(kMSDomain);
#if !defined(ORT_MINIMAL_BUILD)
// TODO: Enable 16-bit types in selector when binary QLinear* ops support 16-bit.
std::unique_ptr<NodeSelector> selector = std::make_unique<QDQ::BinarySelector>();
qdq_selector_action_registry.RegisterSelectorAndAction(action_name,
{{"Add", {}},
{"Mul", {}}},
std::move(selector),
std::move(action));
// TODO: Enable 16-bit types in selector when binary QLinear* ops support 16-bit.
std::vector<const char*> providers = {kCpuExecutionProvider};
std::unique_ptr<NodeSelector> selector = std::make_unique<QDQ::BinarySelector>(providers);
qdq_selector_action_registry.RegisterSelectorAndAction(action_name,
{{"Add", {}},
{"Mul", {}}},
std::move(selector),
std::move(action));
#else
qdq_selector_action_registry.RegisterAction(action_name, std::move(action));
qdq_selector_action_registry.RegisterAction(action_name, std::move(action));
#endif
}
#ifdef USE_DML
{
const std::string action_name{"2DQ_DML"};
std::unique_ptr<Action> action = std::make_unique<QDQ::BinaryReplaceWithQLinear>(kMSDomain);
#if !defined(ORT_MINIMAL_BUILD)
std::vector<const char*> providers = {kDmlExecutionProvider};
std::unique_ptr<NodeSelector> selector = std::make_unique<QDQ::BinarySelector>(providers);
qdq_selector_action_registry.RegisterSelectorAndAction(action_name,
{{"Add", {}}},
std::move(selector),
std::move(action));
#else
#error "ORT_MINIMAL_BUILD and USE_DML are not expected simultaneously. This would require RegisterAction to be called here."
#endif
}
#endif
}
@ -214,8 +237,8 @@ void GemmQDQRules(SelectorActionRegistry& qdq_selector_action_registry) {
std::unique_ptr<Action> action = std::make_unique<QDQ::GemmReplaceWithQuant>();
#if !defined(ORT_MINIMAL_BUILD)
// TODO: Enable 16-bit types in selector when QGemm supports 16-bit.
std::unique_ptr<NodeSelector> selector = std::make_unique<QDQ::GemmSelector>();
std::vector<const char*> providers = {kCpuExecutionProvider};
std::unique_ptr<NodeSelector> selector = std::make_unique<QDQ::GemmSelector>(providers);
qdq_selector_action_registry.RegisterSelectorAndAction(action_name,
{{"Gemm", {}}},
std::move(selector),
@ -235,8 +258,9 @@ void WhereQDQRules(SelectorActionRegistry& qdq_selector_action_registry) {
std::unique_ptr<Action> action = std::make_unique<QDQ::WhereReplaceWithQLinear>();
#if !defined(ORT_MINIMAL_BUILD)
// TODO: Enable 16-bit types in selector when QLinearWhere supports 16-bit.
std::unique_ptr<NodeSelector> selector = std::make_unique<QDQ::WhereSelector>();
std::vector<const char*> providers = {kCpuExecutionProvider};
std::unique_ptr<NodeSelector> selector = std::make_unique<QDQ::WhereSelector>(providers);
qdq_selector_action_registry.RegisterSelectorAndAction(action_name,
{{"Where", {}}},
std::move(selector),
@ -271,8 +295,8 @@ QDQSelectorActionTransformer::QDQSelectorActionTransformer(
"QDQSelectorActionTransformer",
CreateSelectorActionRegistry(is_int8_allowed),
apply_context,
// this transformer is only compatible with the CPU EP
{kCpuExecutionProvider}} {
// this transformer is only compatible with the CPU and DML EP
{kCpuExecutionProvider, kDmlExecutionProvider}} {
}
} // namespace onnxruntime

7
onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_selectors.cc
View file

@ -91,6 +91,13 @@ std::optional<NodeGroup> NodeGroupSelector::GetQDQSelection(const GraphViewer& g
}
std::optional<NodesToOptimizeIndices> BaseSelector::Select(const GraphViewer& graph_viewer, const Node& node) const {
const std::string_view node_ep = node.GetExecutionProviderType();
if (!compatible_providers_.empty() &&
std::find(compatible_providers_.begin(), compatible_providers_.end(), node_ep) == compatible_providers_.end()) {
return std::nullopt;
}
const auto qdq_group = node_group_selector_->GetQDQSelection(graph_viewer, node);
if (!qdq_group.has_value()) {
return std::nullopt;

26
onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_selectors.h
View file

@ -257,12 +257,15 @@ class BaseSelector : public NodeSelector {
// We std::move SelectorActionRegistry into the SelectorActionTransformer so this class needs to have a move ctor
BaseSelector(BaseSelector&& rhs) noexcept
: node_group_selector_{std::move(rhs.node_group_selector_)} {
: node_group_selector_{std::move(rhs.node_group_selector_)},
compatible_providers_{std::move(rhs.compatible_providers_)} {
}
protected:
BaseSelector(std::unique_ptr<NodeGroupSelector> node_group_selector)
: node_group_selector_{std::move(node_group_selector)} {}
BaseSelector(std::unique_ptr<NodeGroupSelector> node_group_selector, gsl::span<const char*> compatible_providers = {})
: node_group_selector_{std::move(node_group_selector)},
compatible_providers_(compatible_providers.begin(), compatible_providers.end()) {
}
// override if you need to adjust the values in NodesToOptimize.
// e.g. add entries for missing optional DQ inputs or set num_inputs to handle variadic inputs
@ -271,6 +274,7 @@ class BaseSelector : public NodeSelector {
private:
std::unique_ptr<NodeGroupSelector> node_group_selector_;
std::vector<std::string> compatible_providers_;
};
class DropQDQNodesSelector : public BaseSelector {
@ -287,14 +291,14 @@ class DropDQNodesSelector : public BaseSelector {
class UnarySelector : public BaseSelector {
public:
explicit UnarySelector(bool allow_16bit = false)
: BaseSelector(std::make_unique<UnaryNodeGroupSelector>(allow_16bit)) {}
explicit UnarySelector(gsl::span<const char*> compatible_providers = {}, bool allow_16bit = false)
: BaseSelector(std::make_unique<UnaryNodeGroupSelector>(allow_16bit), compatible_providers) {}
};
class BinarySelector : public BaseSelector {
public:
explicit BinarySelector(bool allow_16bit = false)
: BaseSelector(std::make_unique<BinaryNodeGroupSelector>(allow_16bit)) {}
explicit BinarySelector(gsl::span<const char*> compatible_providers = {}, bool allow_16bit = false)
: BaseSelector(std::make_unique<BinaryNodeGroupSelector>(allow_16bit), compatible_providers) {}
};
// Variadic DQ nodes -> node -> Q
@ -326,8 +330,8 @@ class ConvSelector : public BaseSelector {
class WhereSelector : public BaseSelector {
public:
explicit WhereSelector(bool allow_16bit = false)
: BaseSelector(std::make_unique<WhereNodeGroupSelector>(allow_16bit)) {}
explicit WhereSelector(gsl::span<const char*> compatible_providers = {}, bool allow_16bit = false)
: BaseSelector(std::make_unique<WhereNodeGroupSelector>(allow_16bit), compatible_providers) {}
};
// 2 DQ nodes for input -> node -> optional Q if QLinearMatMul, MatMulIntegerToFloat if not
@ -342,8 +346,8 @@ class MatMulSelector : public BaseSelector {
// Output: optional Q node for Y
class GemmSelector : public BaseSelector {
public:
explicit GemmSelector(bool allow_16bit = false)
: BaseSelector(std::make_unique<GemmNodeGroupSelector>(allow_16bit)) {}
explicit GemmSelector(gsl::span<const char*> compatible_providers = {}, bool allow_16bit = false)
: BaseSelector(std::make_unique<GemmNodeGroupSelector>(allow_16bit), compatible_providers) {}
void UpdateBuilder(NodesToOptimizeIndicesBuilder&) const override;
};

20
onnxruntime/core/session/inference_session.cc
View file

@ -633,26 +633,6 @@ common::Status InferenceSession::RegisterExecutionProvider(const std::shared_ptr
session_options_.enable_mem_pattern = false;
}
// Default this option to true when the DML EP is registered.
// This should be removed if QDQ is supported for DML through QDQSelectorActionTransformer and the DML EP does not
// rely on the constant folding pass for DequantizeLinear.
optional<std::string> disable_quant_qdq = session_options_.config_options.GetConfigEntry(kOrtSessionOptionsDisableQuantQDQ);
if (disable_quant_qdq == std::nullopt) {
LOGS(*session_logger_, INFO)
<< "QDQ quantization is not supported while using the DML Execution Provider. "
<< "So disabling it for this session since it uses the DML Execution Provider.";
auto st = session_options_.config_options.AddConfigEntry(kOrtSessionOptionsDisableQuantQDQ, "1");
if (!st.IsOK()) {
return st;
}
} else if (*disable_quant_qdq != "1") {
LOGS(*session_logger_, WARNING)
<< "QDQ quantization is not supported while using the DML Execution Provider. "
<< "It is enabled within session options which may result in lower performance.";
}
// Parallel execution mode does not support DML EP
if (session_options_.execution_mode != ExecutionMode::ORT_SEQUENTIAL) {
LOGS(*session_logger_, INFO)

3
tools/ci_build/build.py
View file

@ -1641,6 +1641,9 @@ def setup_dml_build(args, cmake_path, build_dir, configs):
]
run_subprocess(cmd_args)
if args.minimal_build is not None:
raise BuildError("use_dml and minimal_build may not both be set")
def setup_rocm_build(args):
rocm_home = None