saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-29 20:14:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

constant fold initializer for QD

Browse source
This commit is contained in:
Chi Lo 2025-01-14 15:11:43 -08:00
parent 04030f64be
commit bbb5862b9e
6 changed files with 76 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h
View file

@ -300,3 +300,9 @@ static const char* const kOrtSessionOptionsQDQMatMulNBitsAccuracyLevel = "sessio
// “Default”: OS determines the scheduling priority and processor performance to service this workload. [Default]
// “Efficient”: OS treats this workload is efficiency oriented with low scheduling priority and efficient processor performance.
static const char* const kOrtEpDynamicOptionsWorkloadType = "ep.dynamic.workload_type";
// Dequantize initializer using ORT ConstantFolding optimizer for dq node if initializer has specific(? TBD) data type.
// This feature is required by some NPU's.
// "0": disable. ORT doesn't constant fold the DQ node. [DEFAULT]
// "1": enable. ORT constant folds the DQ node.
static const char* const kOrtSessionOptionsDequantizeInitializerForDQNode = "session.dequantize_initializer_for_dq_node";

23
onnxruntime/core/optimizer/constant_folding.cc
View file

@ -18,11 +18,13 @@ namespace onnxruntime {
ConstantFolding::ConstantFolding(const IExecutionProvider& execution_provider,
bool skip_dequantize_linear,
bool dequantize_initializer_for_dequantize_linear,
const ConfigOptions& config_options,
const InlinedHashSet<std::string_view>& compatible_execution_providers,
const InlinedHashSet<std::string>& excluded_initializers) noexcept
: GraphTransformer("ConstantFolding", compatible_execution_providers),
skip_dequantize_linear_(skip_dequantize_linear),
dequantize_initializer_for_dequantize_linear_(dequantize_initializer_for_dequantize_linear),
config_options_(config_options),
excluded_initializers_(excluded_initializers),
execution_provider_(execution_provider) {
@ -219,7 +221,26 @@ Status ConstantFolding::ApplyImpl(Graph& graph, bool& modified, int graph_level,
}
}
if (!can_constant_fold_qdq_node_unit) {
bool can_constant_fold_dq_node = false;
// Another scenario where dequantizing initializer (ex: initializer -> DQ -> bias of X) can be constant folded is if:
// - the DQ node does not produce a graph output
// - The data type of initializer is INT32, UINT16 or INT16 (More? How can user specify the data type?)
// Note: Some NPU's only support FP32, FP16, INT8, UNIT8 or INT4 weights if consumed by Q/DQ nodes.
// - Does X need to be Gemm, Conv or LayerNormalization ?
if (!can_constant_fold_qdq_node_unit && dequantize_initializer_for_dequantize_linear_) {
if (!graph.NodeProducesGraphOutput(*node)) { // DQ does not produce graph output
const auto* input_def = node->InputDefs()[0]; // Get NodeArg of the initializer of the DequantizeLinear node;
auto data_type = input_def->TypeAsProto()->tensor_type().elem_type();
if (data_type == ONNX_NAMESPACE::TensorProto_DataType_INT32 ||
data_type == ONNX_NAMESPACE::TensorProto_DataType_INT16 ||
data_type == ONNX_NAMESPACE::TensorProto_DataType_UINT16) {
can_constant_fold_dq_node = true;
}
}
}
if (!can_constant_fold_qdq_node_unit && !can_constant_fold_dq_node) {
continue;
}
}

2
onnxruntime/core/optimizer/constant_folding.h
View file

@ -24,6 +24,7 @@ class ConstantFolding : public GraphTransformer {
*/
ConstantFolding(const IExecutionProvider& execution_provider,
bool skip_dequantize_linear,
bool dequantize_initializer_for_dequantize_linear,
const ConfigOptions& config_options,
const InlinedHashSet<std::string_view>& compatible_execution_providers = {},
const InlinedHashSet<std::string>& excluded_initializers = {}) noexcept;
@ -32,6 +33,7 @@ class ConstantFolding : public GraphTransformer {
Status ApplyImpl(Graph& graph, bool& modified, int graph_level, const logging::Logger& logger) const override;
bool skip_dequantize_linear_;
bool dequantize_initializer_for_dequantize_linear_;
const ConfigOptions& config_options_;
const InlinedHashSet<std::string> excluded_initializers_;
const IExecutionProvider& execution_provider_;

6
onnxruntime/core/optimizer/graph_transformer_utils.cc
View file

@ -197,6 +197,8 @@ InlinedVector<std::unique_ptr<GraphTransformer>> GenerateTransformers(
InlinedVector<std::unique_ptr<GraphTransformer>> transformers;
const bool disable_quant_qdq =
session_options.config_options.GetConfigOrDefault(kOrtSessionOptionsDisableQuantQDQ, "0") == "1";
const bool quantize_initializer_for_dq_node =
session_options.config_options.GetConfigOrDefault(kOrtSessionOptionsDequantizeInitializerForDQNode, "1") == "1";
#ifndef DISABLE_CONTRIB_OPS
const InlinedHashSet<std::string_view> cpu_ep = {onnxruntime::kCpuExecutionProvider};
const InlinedHashSet<std::string_view> cpu_acl_eps = {onnxruntime::kCpuExecutionProvider,
@ -233,7 +235,9 @@ InlinedVector<std::unique_ptr<GraphTransformer>> GenerateTransformers(
const InlinedHashSet<std::string_view> no_limit_empty_ep_list = {};
transformers.emplace_back(std::make_unique<ConstantSharing>(no_limit_empty_ep_list, excluded_initializers));
transformers.emplace_back(std::make_unique<CommonSubexpressionElimination>());
transformers.emplace_back(std::make_unique<ConstantFolding>(cpu_execution_provider, !disable_quant_qdq,
transformers.emplace_back(std::make_unique<ConstantFolding>(cpu_execution_provider,
!disable_quant_qdq,
quantize_initializer_for_dq_node,
session_options.config_options));
transformers.emplace_back(std::make_unique<MatMulAddFusion>());
transformers.emplace_back(std::make_unique<ReshapeFusion>());

5
onnxruntime/test/optimizer/cse_test.cc
View file

@ -284,7 +284,10 @@ TEST(CseTests, MergeConstants) {
TransformerLevel::Level1));
const ConfigOptions empty_config_options;
ASSERT_STATUS_OK(graph_transformation_mgr.Register(
std::make_unique<ConstantFolding>(*e.get(), false /*skip_dequantize_linear*/, empty_config_options),
std::make_unique<ConstantFolding>(*e.get(),
false /*skip_dequantize_linear*/,
false /*dequantize_initializer_for_dequantize_linear*/,
empty_config_options),
TransformerLevel::Level1));
ASSERT_STATUS_OK(graph_transformation_mgr.ApplyTransformers(graph, TransformerLevel::Level1,
DefaultLoggingManager().DefaultLogger()));

46
onnxruntime/test/optimizer/graph_transform_test.cc
View file

@ -588,7 +588,10 @@ TEST_F(GraphTransformationTests, ConstantFolding) {
onnxruntime::GraphTransformerManager graph_transformation_mgr{5};
const ConfigOptions empty_config_options;
ASSERT_STATUS_OK(graph_transformation_mgr.Register(
std::make_unique<ConstantFolding>(*e.get(), false /*skip_dequantize_linear*/, empty_config_options),
std::make_unique<ConstantFolding>(*e.get(),
false /*skip_dequantize_linear*/,
false /*dequantize_initializer_for_dequantize_linear*/,
empty_config_options),
TransformerLevel::Level1));
ASSERT_STATUS_OK(graph_transformation_mgr.ApplyTransformers(graph, TransformerLevel::Level1, *logger_));
@ -609,7 +612,10 @@ TEST_F(GraphTransformationTests, ConstantFoldingNodesOnDifferentEP) {
const ConfigOptions empty_config_options;
ASSERT_STATUS_OK(graph_transformation_mgr.Register(
std::make_unique<ConstantFolding>(*e.get(), false /*skip_dequantize_linear*/, empty_config_options),
std::make_unique<ConstantFolding>(*e.get(),
false /*skip_dequantize_linear*/,
false /*dequantize_initializer_for_dequantize_linear*/,
empty_config_options),
TransformerLevel::Level1));
// assign all nodes to CUDA. the constant folding should override this to perform the constant folding on cpu
@ -639,7 +645,10 @@ TEST_F(GraphTransformationTests, ConstantFoldingUnsupportedFloat16) {
onnxruntime::GraphTransformerManager graph_transformation_mgr{5};
const ConfigOptions empty_config_options;
ASSERT_STATUS_OK(graph_transformation_mgr.Register(
std::make_unique<ConstantFolding>(*e.get(), false /*skip_dequantize_linear*/, empty_config_options),
std::make_unique<ConstantFolding>(*e.get(),
false /*skip_dequantize_linear*/,
false /*dequantize_initializer_for_dequantize_linear*/,
empty_config_options),
TransformerLevel::Level1));
// assign all nodes to CUDA. the constant folding should try folding the node on the CPU and fail, thus leaving the
@ -723,7 +732,10 @@ TEST_F(GraphTransformationTests, ConstantFoldingSubgraph) {
onnxruntime::GraphTransformerManager graph_transformation_mgr{5};
const ConfigOptions empty_config_options;
ASSERT_STATUS_OK(graph_transformation_mgr.Register(
std::make_unique<ConstantFolding>(*e.get(), false /*skip_dequantize_linear*/, empty_config_options),
std::make_unique<ConstantFolding>(*e.get(),
false /*skip_dequantize_linear*/,
false /*dequantize_initializer_for_dequantize_linear*/,
empty_config_options),
TransformerLevel::Level1));
ASSERT_STATUS_OK(graph_transformation_mgr.ApplyTransformers(graph, TransformerLevel::Level1, *logger_));
@ -752,6 +764,7 @@ TEST_F(GraphTransformationTests, ConstantFoldingWithShapeToInitializer) {
ASSERT_STATUS_OK(graph_transformation_mgr.Register(
std::make_unique<ConstantFolding>(*e.get(),
false /*skip_dequantize_linear*/,
false /*dequantize_initializer_for_dequantize_linear*/,
empty_config_options,
compatible_eps,
excluded_initializers),
@ -781,7 +794,10 @@ TEST_F(GraphTransformationTests, ConstantFoldingWithScalarShapeToInitializer) {
std::unique_ptr<CPUExecutionProvider> e = std::make_unique<CPUExecutionProvider>(CPUExecutionProviderInfo());
ASSERT_STATUS_OK(graph_transformation_mgr.Register(
std::make_unique<ConstantFolding>(*e.get(), false /*skip_dequantize_linear*/, empty_config_options,
std::make_unique<ConstantFolding>(*e.get(),
false /*skip_dequantize_linear*/,
false /*dequantize_initializer_for_dequantize_linear*/,
empty_config_options,
compatible_eps),
TransformerLevel::Level1));
@ -810,7 +826,10 @@ TEST_F(GraphTransformationTests, ConstantFoldingForOpsWithMissingOptionalInputs)
std::unique_ptr<CPUExecutionProvider> e = std::make_unique<CPUExecutionProvider>(CPUExecutionProviderInfo());
ASSERT_STATUS_OK(graph_transformation_mgr.Register(
std::make_unique<ConstantFolding>(*e.get(), false /*skip_dequantize_linear*/, empty_config_options,
std::make_unique<ConstantFolding>(*e.get(),
false /*skip_dequantize_linear*/,
false /*dequantize_initializer_for_dequantize_linear*/,
empty_config_options,
compatible_eps),
TransformerLevel::Level1));
@ -988,7 +1007,10 @@ TEST_F(GraphTransformationTests, ConstantFolding_RemoveDanglingInputNodesToConst
onnxruntime::GraphTransformerManager graph_transformation_mgr{5};
const ConfigOptions empty_config_options;
ASSERT_STATUS_OK(graph_transformation_mgr.Register(
std::make_unique<ConstantFolding>(*e.get(), false /*skip_dequantize_linear*/, empty_config_options),
std::make_unique<ConstantFolding>(*e.get(),
false /*skip_dequantize_linear*/,
false /*dequantize_initializer_for_dequantize_linear*/,
empty_config_options),
TransformerLevel::Level1));
ASSERT_STATUS_OK(graph_transformation_mgr.ApplyTransformers(graph, TransformerLevel::Level1, *logger_));
@ -1012,7 +1034,10 @@ TEST_F(GraphTransformationTests, ConstantFoldingAShapeNodeDeepInTheGraph) {
std::unique_ptr<CPUExecutionProvider> e = std::make_unique<CPUExecutionProvider>(CPUExecutionProviderInfo());
ASSERT_STATUS_OK(graph_transformation_mgr.Register(
std::make_unique<ConstantFolding>(*e.get(), false /*skip_dequantize_linear*/, empty_config_options),
std::make_unique<ConstantFolding>(*e.get(),
false /*skip_dequantize_linear*/,
false /*dequantize_initializer_for_dequantize_linear*/,
empty_config_options),
TransformerLevel::Level1));
ASSERT_STATUS_OK(graph_transformation_mgr.ApplyTransformers(graph, TransformerLevel::Level1, *logger_));
@ -1041,7 +1066,10 @@ TEST_F(GraphTransformationTests, ConstantFoldingStringInitializer) {
std::unique_ptr<CPUExecutionProvider> e = std::make_unique<CPUExecutionProvider>(CPUExecutionProviderInfo());
ASSERT_STATUS_OK(graph_transformation_mgr.Register(
std::make_unique<ConstantFolding>(*e.get(), false /*skip_dequantize_linear*/, empty_config_options),
std::make_unique<ConstantFolding>(*e.get(),
false /*skip_dequantize_linear*/,
false /*dequantize_initializer_for_dequantize_linear*/,
empty_config_options),
TransformerLevel::Level1));
ASSERT_STATUS_OK(graph_transformation_mgr.ApplyTransformers(graph, TransformerLevel::Level1, *logger_));