diff --git a/onnxruntime/contrib_ops/cpu/quantization/qembed_layer_norm.cc b/onnxruntime/contrib_ops/cpu/quantization/qembed_layer_norm.cc index 2f684362e1..f9cb974cb8 100644 --- a/onnxruntime/contrib_ops/cpu/quantization/qembed_layer_norm.cc +++ b/onnxruntime/contrib_ops/cpu/quantization/qembed_layer_norm.cc @@ -198,7 +198,7 @@ Status CheckQuantizedInputs(OpKernelContext* context, bool* is_signed_inputs) { const Tensor* gamma_zero_point_tensor = context->Input(16); const Tensor* beta_zero_point_tensor = context->Input(17); - bool word_embedding_is_signed_inputs = word_embedding_scale_tensor->IsDataType(); + bool word_embedding_is_signed_inputs = word_embedding_zero_point_tensor->IsDataType(); bool has_segment_embedding = context->Input(1) != nullptr; if (!IsScalarOr1ElementVector(word_embedding_scale_tensor)) { diff --git a/onnxruntime/python/tools/quantization/quantize.py b/onnxruntime/python/tools/quantization/quantize.py index 955a74e525..a0a0b93522 100644 --- a/onnxruntime/python/tools/quantization/quantize.py +++ b/onnxruntime/python/tools/quantization/quantize.py @@ -141,7 +141,7 @@ def quantize_static(model_input, per_channel=False, reduce_range=False, activation_type=QuantType.QUInt8, - weight_type=QuantType.QUInt8, + weight_type=QuantType.QInt8, nodes_to_quantize=[], nodes_to_exclude=[], optimize_model=True, @@ -161,8 +161,8 @@ def quantize_static(model_input, :param op_types: operators to quantize :param per_channel: quantize weights per channel :param reduce_range: quantize weights with 7-bits. It may improve the accuracy for some models running on non-VNNI machine, especially for per-channel mode - :param activation_type: quantization data type of activation - :param weight_type: quantization data type of weight + :param activation_type: quantization data type of activation. Please refer to https://onnxruntime.ai/docs/performance/quantization.html for more details on data type selection + :param weight_type: quantization data type of weight. Please refer to https://onnxruntime.ai/docs/performance/quantization.html for more details on data type selection :param nodes_to_quantize: List of nodes names to quantize. When this list is not None only the nodes in this list are quantized. @@ -256,7 +256,7 @@ def quantize_dynamic(model_input: Path, per_channel=False, reduce_range=False, activation_type=QuantType.QUInt8, - weight_type=QuantType.QUInt8, + weight_type=QuantType.QInt8, nodes_to_quantize=[], nodes_to_exclude=[], optimize_model=True, @@ -270,8 +270,8 @@ def quantize_dynamic(model_input: Path, :param per_channel: quantize weights per channel :param reduce_range: quantize weights with 7-bits. It may improve the accuracy for some models running on non-VNNI machine, especially for per-channel mode :param nbits: number of bits to represent quantized data. Currently only supporting 8-bit types - :param activation_type: quantization data type of activation - :param weight_type: quantization data type of weight + :param activation_type: quantization data type of activation. Please refer to https://onnxruntime.ai/docs/performance/quantization.html for more details on data type selection + :param weight_type: quantization data type of weight. Please refer to https://onnxruntime.ai/docs/performance/quantization.html for more details on data type selection :param nodes_to_quantize: List of nodes names to quantize. When this list is not None only the nodes in this list are quantized. diff --git a/onnxruntime/test/contrib_ops/qembed_layer_norm_op_test.cc b/onnxruntime/test/contrib_ops/qembed_layer_norm_op_test.cc index 02b7c7f5f9..5c02cfda4a 100644 --- a/onnxruntime/test/contrib_ops/qembed_layer_norm_op_test.cc +++ b/onnxruntime/test/contrib_ops/qembed_layer_norm_op_test.cc @@ -12,6 +12,7 @@ namespace test { namespace { +template static void RunTest(const embedlayernorm::OpData& data, float accuracy_threshold = 0.25f) { ASSERT_TRUE(data.word_embedding_data.size() % data.hidden_size == 0); @@ -34,30 +35,30 @@ static void RunTest(const embedlayernorm::OpData& data, std::vector output_dims = {data.batch_size, data.sequence_size, data.hidden_size}; std::vector mask_index_dims = {data.batch_size}; - quantization::Params word_embedding_params; - std::vector word_embedding_data_quant = - QuantizeLinearTestVector(data.word_embedding_data, + quantization::Params word_embedding_params; + std::vector word_embedding_data_quant = + QuantizeLinearTestVector(data.word_embedding_data, word_embedding_params); - quantization::Params position_embedding_params; - std::vector position_embedding_data_quant = - QuantizeLinearTestVector(data.position_embedding_data, + quantization::Params position_embedding_params; + std::vector position_embedding_data_quant = + QuantizeLinearTestVector(data.position_embedding_data, position_embedding_params); - quantization::Params segment_embedding_params = {}; - std::vector segment_embedding_data_quant; + quantization::Params segment_embedding_params = {}; + std::vector segment_embedding_data_quant; if (data.has_segment) { - segment_embedding_data_quant = QuantizeLinearTestVector( + segment_embedding_data_quant = QuantizeLinearTestVector( data.segment_embedding_data, segment_embedding_params); } - quantization::Params gamma_params; - std::vector gamma_data_quant = - QuantizeLinearTestVector(data.gamma_data, gamma_params); + quantization::Params gamma_params; + std::vector gamma_data_quant = + QuantizeLinearTestVector(data.gamma_data, gamma_params); - quantization::Params beta_params; - std::vector beta_data_quant = - QuantizeLinearTestVector(data.beta_data, beta_params); + quantization::Params beta_params; + std::vector beta_data_quant = + QuantizeLinearTestVector(data.beta_data, beta_params); OpTester tester("QEmbedLayerNormalization", 1, onnxruntime::kMSDomain); @@ -70,27 +71,27 @@ static void RunTest(const embedlayernorm::OpData& data, } // Quantized initializer inputs: - tester.AddInput("word_embedding_data", + tester.AddInput("word_embedding_data", word_embedding_dims, word_embedding_data_quant, /*is_initializer=*/true); - tester.AddInput("position_embedding_data", + tester.AddInput("position_embedding_data", position_embedding_dims, position_embedding_data_quant, /*is_initializer=*/true); if (data.has_segment) { - tester.AddInput("segment_embedding_data", + tester.AddInput("segment_embedding_data", segment_embedding_dims, segment_embedding_data_quant, /*is_initializer=*/true); } else { - tester.AddOptionalInputEdge(); + tester.AddOptionalInputEdge(); } - tester.AddInput("gamma", + tester.AddInput("gamma", gamma_dims, gamma_data_quant, /*is_initializer=*/true); - tester.AddInput("beta", + tester.AddInput("beta", beta_dims, beta_data_quant, /*is_initializer=*/true); @@ -128,27 +129,27 @@ static void RunTest(const embedlayernorm::OpData& data, /*is_initializer=*/true); // Quantized zero points: - tester.AddInput("word_embedding_zero_point", + tester.AddInput("word_embedding_zero_point", /*dims=*/{}, {word_embedding_params.zero_point}, /*is_initializer=*/true); - tester.AddInput("position_embedding_zero_point", + tester.AddInput("position_embedding_zero_point", /*dims=*/{}, {position_embedding_params.zero_point}, /*is_initializer=*/true); if (data.has_segment) { - tester.AddInput("segment_embedding_zero_point", + tester.AddInput("segment_embedding_zero_point", /*dims=*/{}, {segment_embedding_params.zero_point}, /*is_initializer=*/true); } else { - tester.AddOptionalInputEdge(); + tester.AddOptionalInputEdge(); } - tester.AddInput("gamma_zero_point", + tester.AddInput("gamma_zero_point", /*dims=*/{}, {gamma_params.zero_point}, /*is_initializer=*/true); - tester.AddInput("beta_zero_point", + tester.AddInput("beta_zero_point", /*dims=*/{}, {beta_params.zero_point}, /*is_initializer=*/true); @@ -170,28 +171,34 @@ static void RunTest(const embedlayernorm::OpData& data, } // namespace TEST(QEmbedLayerNormTest, EmbedLayerNormBatch1) { - RunTest(embedlayernorm::EmbedLayerNormBatch1()); + RunTest(embedlayernorm::EmbedLayerNormBatch1()); + RunTest(embedlayernorm::EmbedLayerNormBatch1()); } TEST(QEmbedLayerNormTest, EmbedLayerNormBatch1_Float16) { - RunTest(embedlayernorm::EmbedLayerNormBatch1(), /*use_float16=*/true); + RunTest(embedlayernorm::EmbedLayerNormBatch1(), /*use_float16=*/true); + RunTest(embedlayernorm::EmbedLayerNormBatch1(), /*use_float16=*/true); } TEST(QEmbedLayerNormTest, EmbedLayerNormBatch2) { - RunTest(embedlayernorm::EmbedLayerNormBatch2()); + RunTest(embedlayernorm::EmbedLayerNormBatch2()); + RunTest(embedlayernorm::EmbedLayerNormBatch2()); } TEST(QEmbedLayerNormTest, EmbedLayerNormBatch2_NoMask) { - RunTest(embedlayernorm::EmbedLayerNormBatch2(/*has_mask=*/false)); + RunTest(embedlayernorm::EmbedLayerNormBatch2(/*has_mask=*/false)); + RunTest(embedlayernorm::EmbedLayerNormBatch2(/*has_mask=*/false)); } // BatchSize > HiddenSize to reproduce mask processing bug TEST(QEmbedLayerNormTest, EmbedLayerNormLargeBatchSmallHiddenSize) { - RunTest(embedlayernorm::EmbedLayerNormLargeBatchSmallHiddenSize()); + RunTest(embedlayernorm::EmbedLayerNormLargeBatchSmallHiddenSize()); + RunTest(embedlayernorm::EmbedLayerNormLargeBatchSmallHiddenSize()); } TEST(QEmbedLayerNormTest, EmbedLayerNormBatch_Distill) { - RunTest(embedlayernorm::EmbedLayerNormBatch_Distill()); + RunTest(embedlayernorm::EmbedLayerNormBatch_Distill()); + RunTest(embedlayernorm::EmbedLayerNormBatch_Distill()); } } // namespace test diff --git a/onnxruntime/test/python/quantization/test_qdq.py b/onnxruntime/test/python/quantization/test_qdq.py index 2aca1eacdb..6946280dbe 100644 --- a/onnxruntime/test/python/quantization/test_qdq.py +++ b/onnxruntime/test/python/quantization/test_qdq.py @@ -231,7 +231,7 @@ class TestQDQFormatConv(TestQDQFormat): onnx.save(model, output_model_path) - def verify_quantize_conv(self, has_bias, per_channel): + def verify_quantize_conv(self, has_bias, per_channel, is_weight_int8 = False): np.random.seed(1) model_fp32_path = 'conv_fp32.{}.{}.onnx'.format(has_bias, per_channel) model_int8_qdq_path = 'conv_quant_qdq.{}.{}.onnx'.format(has_bias, per_channel) @@ -247,7 +247,8 @@ class TestQDQFormatConv(TestQDQFormat): data_reader, quant_format=QuantFormat.QDQ, per_channel = per_channel, - reduce_range = per_channel + reduce_range = per_channel, + weight_type = QuantType.QInt8 if is_weight_int8 else QuantType.QUInt8 ) data_reader.rewind() qdq_nodes = {'Conv': 1, 'QuantizeLinear': 2, 'DequantizeLinear': 4 if has_bias else 3} @@ -260,7 +261,8 @@ class TestQDQFormatConv(TestQDQFormat): data_reader, quant_format=QuantFormat.QOperator, per_channel = per_channel, - reduce_range = per_channel + reduce_range = per_channel, + weight_type = QuantType.QInt8 if is_weight_int8 else QuantType.QUInt8 ) data_reader.rewind() qop_nodes = {'QLinearConv': 1, 'QuantizeLinear': 1, 'DequantizeLinear': 1} @@ -268,10 +270,14 @@ class TestQDQFormatConv(TestQDQFormat): check_model_correctness(self, model_fp32_path, model_int8_qop_path, data_reader.get_next()) def test_quantize_conv_without_bias(self): - self.verify_quantize_conv(False, False) # has_bias:False, per_channel:False - self.verify_quantize_conv(False, True) # has_bias:False, per_channel:True - self.verify_quantize_conv(True, False) # has_bias:True, per_channel:False - self.verify_quantize_conv(True, True) # has_bias:True, per_channel:True + # only test cases per_channel=True and reduce_range=True to avoid saturation on avx2 and avx512 for weight type int8 + self.verify_quantize_conv(False, True, True) # has_bias:False, per_channel:True, is_weight_int8:True + self.verify_quantize_conv(True, True, True) # has_bias:True, per_channel:True, is_weight_int8:True + + self.verify_quantize_conv(False, False, False) # has_bias:False, per_channel:False, is_weight_int8:False + self.verify_quantize_conv(True, False, False) # has_bias:True, per_channel:False, is_weight_int8:False + self.verify_quantize_conv(False, True, False) # has_bias:False, per_channel:True, is_weight_int8:False + self.verify_quantize_conv(True, True, False) # has_bias:True, per_channel:True, is_weight_int8:False class TestQDQFormatConvClip(TestQDQFormat): def construct_model_conv_clip(self, output_model_path, input_shape, weight_shape, output_shape): @@ -324,7 +330,7 @@ class TestQDQFormatConvClip(TestQDQFormat): onnx.save(model, output_model_path) - def verify(self, per_channel): + def verify(self, per_channel, is_weight_int8): np.random.seed(1) model_fp32_path = 'conv_clip_fp32.{}.onnx'.format(per_channel) model_int8_qdq_path = 'conv_clip_quant_qdq.{}.onnx'.format(per_channel) @@ -339,7 +345,8 @@ class TestQDQFormatConvClip(TestQDQFormat): data_reader, quant_format=QuantFormat.QDQ, per_channel = per_channel, - reduce_range = per_channel + reduce_range = per_channel, + weight_type = QuantType.QInt8 if is_weight_int8 else QuantType.QUInt8 ) data_reader.rewind() #topo sort check @@ -352,7 +359,8 @@ class TestQDQFormatConvClip(TestQDQFormat): data_reader, quant_format=QuantFormat.QOperator, per_channel = per_channel, - reduce_range = per_channel + reduce_range = per_channel, + weight_type = QuantType.QInt8 if is_weight_int8 else QuantType.QUInt8 ) data_reader.rewind() qop_nodes = {'QLinearConv': 1, 'QuantizeLinear': 1, 'DequantizeLinear': 1} @@ -360,8 +368,11 @@ class TestQDQFormatConvClip(TestQDQFormat): check_model_correctness(self, model_fp32_path, model_int8_qop_path, data_reader.get_next()) def test_quantize_conv_without_bias(self): - self.verify(False) # per_channel:False - #self.verify(True) # per_channel:True + # only test cases per_channel=True and reduce_range=True to avoid saturation on avx2 and avx512 for weight type int8 + self.verify(True, True) # per_channel:False, is_weight_int8:True + + self.verify(False, False) # per_channel:False, is_weight_int8:False + self.verify(True, False) # per_channel:True, is_weight_int8:False if __name__ == '__main__': unittest.main()