saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-20 19:12:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Set default quantization weight type to int8 (#10209)

Browse source 
* Set default quantization weight type to int8
This commit is contained in:
Yufeng Li 2022-01-11 14:24:57 -08:00 committed by GitHub
parent ce103ace93
commit cb9b0275b6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 70 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
onnxruntime/contrib_ops/cpu/quantization/qembed_layer_norm.cc
View file

@ -198,7 +198,7 @@ Status CheckQuantizedInputs(OpKernelContext* context, bool* is_signed_inputs) {
const Tensor* gamma_zero_point_tensor = context->Input<Tensor>(16);
const Tensor* beta_zero_point_tensor = context->Input<Tensor>(17);
bool word_embedding_is_signed_inputs = word_embedding_scale_tensor->IsDataType<int8_t>();
bool word_embedding_is_signed_inputs = word_embedding_zero_point_tensor->IsDataType<int8_t>();
bool has_segment_embedding = context->Input<Tensor>(1) != nullptr;
if (!IsScalarOr1ElementVector(word_embedding_scale_tensor)) {

12
onnxruntime/python/tools/quantization/quantize.py
View file

@ -141,7 +141,7 @@ def quantize_static(model_input,
per_channel=False,
reduce_range=False,
activation_type=QuantType.QUInt8,
weight_type=QuantType.QUInt8,
weight_type=QuantType.QInt8,
nodes_to_quantize=[],
nodes_to_exclude=[],
optimize_model=True,
@ -161,8 +161,8 @@ def quantize_static(model_input,
:param op_types: operators to quantize
:param per_channel: quantize weights per channel
:param reduce_range: quantize weights with 7-bits. It may improve the accuracy for some models running on non-VNNI machine, especially for per-channel mode
:param activation_type: quantization data type of activation
:param weight_type: quantization data type of weight
:param activation_type: quantization data type of activation. Please refer to https://onnxruntime.ai/docs/performance/quantization.html for more details on data type selection
:param weight_type: quantization data type of weight. Please refer to https://onnxruntime.ai/docs/performance/quantization.html for more details on data type selection
:param nodes_to_quantize:
List of nodes names to quantize. When this list is not None only the nodes in this list
are quantized.
@ -256,7 +256,7 @@ def quantize_dynamic(model_input: Path,
per_channel=False,
reduce_range=False,
activation_type=QuantType.QUInt8,
weight_type=QuantType.QUInt8,
weight_type=QuantType.QInt8,
nodes_to_quantize=[],
nodes_to_exclude=[],
optimize_model=True,
@ -270,8 +270,8 @@ def quantize_dynamic(model_input: Path,
:param per_channel: quantize weights per channel
:param reduce_range: quantize weights with 7-bits. It may improve the accuracy for some models running on non-VNNI machine, especially for per-channel mode
:param nbits: number of bits to represent quantized data. Currently only supporting 8-bit types
:param activation_type: quantization data type of activation
:param weight_type: quantization data type of weight
:param activation_type: quantization data type of activation. Please refer to https://onnxruntime.ai/docs/performance/quantization.html for more details on data type selection
:param weight_type: quantization data type of weight. Please refer to https://onnxruntime.ai/docs/performance/quantization.html for more details on data type selection
:param nodes_to_quantize:
List of nodes names to quantize. When this list is not None only the nodes in this list
are quantized.

73
onnxruntime/test/contrib_ops/qembed_layer_norm_op_test.cc
View file

@ -12,6 +12,7 @@ namespace test {
namespace {
template<typename WeightT>
static void RunTest(const embedlayernorm::OpData& data,
float accuracy_threshold = 0.25f) {
ASSERT_TRUE(data.word_embedding_data.size() % data.hidden_size == 0);
@ -34,30 +35,30 @@ static void RunTest(const embedlayernorm::OpData& data,
std::vector<int64_t> output_dims = {data.batch_size, data.sequence_size, data.hidden_size};
std::vector<int64_t> mask_index_dims = {data.batch_size};
quantization::Params<uint8_t> word_embedding_params;
std::vector<uint8_t> word_embedding_data_quant =
QuantizeLinearTestVector<uint8_t>(data.word_embedding_data,
quantization::Params<WeightT> word_embedding_params;
std::vector<WeightT> word_embedding_data_quant =
QuantizeLinearTestVector<WeightT>(data.word_embedding_data,
word_embedding_params);
quantization::Params<uint8_t> position_embedding_params;
std::vector<uint8_t> position_embedding_data_quant =
QuantizeLinearTestVector<uint8_t>(data.position_embedding_data,
quantization::Params<WeightT> position_embedding_params;
std::vector<WeightT> position_embedding_data_quant =
QuantizeLinearTestVector<WeightT>(data.position_embedding_data,
position_embedding_params);
quantization::Params<uint8_t> segment_embedding_params = {};
std::vector<uint8_t> segment_embedding_data_quant;
quantization::Params<WeightT> segment_embedding_params = {};
std::vector<WeightT> segment_embedding_data_quant;
if (data.has_segment) {
segment_embedding_data_quant = QuantizeLinearTestVector<uint8_t>(
segment_embedding_data_quant = QuantizeLinearTestVector<WeightT>(
data.segment_embedding_data, segment_embedding_params);
}
quantization::Params<uint8_t> gamma_params;
std::vector<uint8_t> gamma_data_quant =
QuantizeLinearTestVector<uint8_t>(data.gamma_data, gamma_params);
quantization::Params<WeightT> gamma_params;
std::vector<WeightT> gamma_data_quant =
QuantizeLinearTestVector<WeightT>(data.gamma_data, gamma_params);
quantization::Params<uint8_t> beta_params;
std::vector<uint8_t> beta_data_quant =
QuantizeLinearTestVector<uint8_t>(data.beta_data, beta_params);
quantization::Params<WeightT> beta_params;
std::vector<WeightT> beta_data_quant =
QuantizeLinearTestVector<WeightT>(data.beta_data, beta_params);
OpTester tester("QEmbedLayerNormalization", 1, onnxruntime::kMSDomain);
@ -70,27 +71,27 @@ static void RunTest(const embedlayernorm::OpData& data,
}
// Quantized initializer inputs:
tester.AddInput<uint8_t>("word_embedding_data",
tester.AddInput<WeightT>("word_embedding_data",
word_embedding_dims,
word_embedding_data_quant,
/*is_initializer=*/true);
tester.AddInput<uint8_t>("position_embedding_data",
tester.AddInput<WeightT>("position_embedding_data",
position_embedding_dims,
position_embedding_data_quant,
/*is_initializer=*/true);
if (data.has_segment) {
tester.AddInput<uint8_t>("segment_embedding_data",
tester.AddInput<WeightT>("segment_embedding_data",
segment_embedding_dims,
segment_embedding_data_quant,
/*is_initializer=*/true);
} else {
tester.AddOptionalInputEdge<uint8_t>();
tester.AddOptionalInputEdge<WeightT>();
}
tester.AddInput<uint8_t>("gamma",
tester.AddInput<WeightT>("gamma",
gamma_dims,
gamma_data_quant,
/*is_initializer=*/true);
tester.AddInput<uint8_t>("beta",
tester.AddInput<WeightT>("beta",
beta_dims,
beta_data_quant,
/*is_initializer=*/true);
@ -128,27 +129,27 @@ static void RunTest(const embedlayernorm::OpData& data,
/*is_initializer=*/true);
// Quantized zero points:
tester.AddInput<uint8_t>("word_embedding_zero_point",
tester.AddInput<WeightT>("word_embedding_zero_point",
/*dims=*/{},
{word_embedding_params.zero_point},
/*is_initializer=*/true);
tester.AddInput<uint8_t>("position_embedding_zero_point",
tester.AddInput<WeightT>("position_embedding_zero_point",
/*dims=*/{},
{position_embedding_params.zero_point},
/*is_initializer=*/true);
if (data.has_segment) {
tester.AddInput<uint8_t>("segment_embedding_zero_point",
tester.AddInput<WeightT>("segment_embedding_zero_point",
/*dims=*/{},
{segment_embedding_params.zero_point},
/*is_initializer=*/true);
} else {
tester.AddOptionalInputEdge<uint8_t>();
tester.AddOptionalInputEdge<WeightT>();
}
tester.AddInput<uint8_t>("gamma_zero_point",
tester.AddInput<WeightT>("gamma_zero_point",
/*dims=*/{},
{gamma_params.zero_point},
/*is_initializer=*/true);
tester.AddInput<uint8_t>("beta_zero_point",
tester.AddInput<WeightT>("beta_zero_point",
/*dims=*/{},
{beta_params.zero_point},
/*is_initializer=*/true);
@ -170,28 +171,34 @@ static void RunTest(const embedlayernorm::OpData& data,
} // namespace
TEST(QEmbedLayerNormTest, EmbedLayerNormBatch1) {
RunTest(embedlayernorm::EmbedLayerNormBatch1());
RunTest<int8_t>(embedlayernorm::EmbedLayerNormBatch1());
RunTest<uint8_t>(embedlayernorm::EmbedLayerNormBatch1());
}
TEST(QEmbedLayerNormTest, EmbedLayerNormBatch1_Float16) {
RunTest(embedlayernorm::EmbedLayerNormBatch1(), /*use_float16=*/true);
RunTest<int8_t>(embedlayernorm::EmbedLayerNormBatch1(), /*use_float16=*/true);
RunTest<uint8_t>(embedlayernorm::EmbedLayerNormBatch1(), /*use_float16=*/true);
}
TEST(QEmbedLayerNormTest, EmbedLayerNormBatch2) {
RunTest(embedlayernorm::EmbedLayerNormBatch2());
RunTest<int8_t>(embedlayernorm::EmbedLayerNormBatch2());
RunTest<uint8_t>(embedlayernorm::EmbedLayerNormBatch2());
}
TEST(QEmbedLayerNormTest, EmbedLayerNormBatch2_NoMask) {
RunTest(embedlayernorm::EmbedLayerNormBatch2(/*has_mask=*/false));
RunTest<int8_t>(embedlayernorm::EmbedLayerNormBatch2(/*has_mask=*/false));
RunTest<uint8_t>(embedlayernorm::EmbedLayerNormBatch2(/*has_mask=*/false));
}
// BatchSize > HiddenSize to reproduce mask processing bug
TEST(QEmbedLayerNormTest, EmbedLayerNormLargeBatchSmallHiddenSize) {
RunTest(embedlayernorm::EmbedLayerNormLargeBatchSmallHiddenSize());
RunTest<int8_t>(embedlayernorm::EmbedLayerNormLargeBatchSmallHiddenSize());
RunTest<uint8_t>(embedlayernorm::EmbedLayerNormLargeBatchSmallHiddenSize());
}
TEST(QEmbedLayerNormTest, EmbedLayerNormBatch_Distill) {
RunTest(embedlayernorm::EmbedLayerNormBatch_Distill());
RunTest<int8_t>(embedlayernorm::EmbedLayerNormBatch_Distill());
RunTest<uint8_t>(embedlayernorm::EmbedLayerNormBatch_Distill());
}
} // namespace test

35
onnxruntime/test/python/quantization/test_qdq.py
View file

@ -231,7 +231,7 @@ class TestQDQFormatConv(TestQDQFormat):
onnx.save(model, output_model_path)
def verify_quantize_conv(self, has_bias, per_channel):
def verify_quantize_conv(self, has_bias, per_channel, is_weight_int8 = False):
np.random.seed(1)
model_fp32_path = 'conv_fp32.{}.{}.onnx'.format(has_bias, per_channel)
model_int8_qdq_path = 'conv_quant_qdq.{}.{}.onnx'.format(has_bias, per_channel)
@ -247,7 +247,8 @@ class TestQDQFormatConv(TestQDQFormat):
data_reader,
quant_format=QuantFormat.QDQ,
per_channel = per_channel,
reduce_range = per_channel
reduce_range = per_channel,
weight_type = QuantType.QInt8 if is_weight_int8 else QuantType.QUInt8
)
data_reader.rewind()
qdq_nodes = {'Conv': 1, 'QuantizeLinear': 2, 'DequantizeLinear': 4 if has_bias else 3}
@ -260,7 +261,8 @@ class TestQDQFormatConv(TestQDQFormat):
data_reader,
quant_format=QuantFormat.QOperator,
per_channel = per_channel,
reduce_range = per_channel
reduce_range = per_channel,
weight_type = QuantType.QInt8 if is_weight_int8 else QuantType.QUInt8
)
data_reader.rewind()
qop_nodes = {'QLinearConv': 1, 'QuantizeLinear': 1, 'DequantizeLinear': 1}
@ -268,10 +270,14 @@ class TestQDQFormatConv(TestQDQFormat):
check_model_correctness(self, model_fp32_path, model_int8_qop_path, data_reader.get_next())
def test_quantize_conv_without_bias(self):
self.verify_quantize_conv(False, False) # has_bias:False, per_channel:False
self.verify_quantize_conv(False, True) # has_bias:False, per_channel:True
self.verify_quantize_conv(True, False) # has_bias:True, per_channel:False
self.verify_quantize_conv(True, True) # has_bias:True, per_channel:True
# only test cases per_channel=True and reduce_range=True to avoid saturation on avx2 and avx512 for weight type int8
self.verify_quantize_conv(False, True, True) # has_bias:False, per_channel:True, is_weight_int8:True
self.verify_quantize_conv(True, True, True) # has_bias:True, per_channel:True, is_weight_int8:True
self.verify_quantize_conv(False, False, False) # has_bias:False, per_channel:False, is_weight_int8:False
self.verify_quantize_conv(True, False, False) # has_bias:True, per_channel:False, is_weight_int8:False
self.verify_quantize_conv(False, True, False) # has_bias:False, per_channel:True, is_weight_int8:False
self.verify_quantize_conv(True, True, False) # has_bias:True, per_channel:True, is_weight_int8:False
class TestQDQFormatConvClip(TestQDQFormat):
def construct_model_conv_clip(self, output_model_path, input_shape, weight_shape, output_shape):
@ -324,7 +330,7 @@ class TestQDQFormatConvClip(TestQDQFormat):
onnx.save(model, output_model_path)
def verify(self, per_channel):
def verify(self, per_channel, is_weight_int8):
np.random.seed(1)
model_fp32_path = 'conv_clip_fp32.{}.onnx'.format(per_channel)
model_int8_qdq_path = 'conv_clip_quant_qdq.{}.onnx'.format(per_channel)
@ -339,7 +345,8 @@ class TestQDQFormatConvClip(TestQDQFormat):
data_reader,
quant_format=QuantFormat.QDQ,
per_channel = per_channel,
reduce_range = per_channel
reduce_range = per_channel,
weight_type = QuantType.QInt8 if is_weight_int8 else QuantType.QUInt8
)
data_reader.rewind()
#topo sort check
@ -352,7 +359,8 @@ class TestQDQFormatConvClip(TestQDQFormat):
data_reader,
quant_format=QuantFormat.QOperator,
per_channel = per_channel,
reduce_range = per_channel
reduce_range = per_channel,
weight_type = QuantType.QInt8 if is_weight_int8 else QuantType.QUInt8
)
data_reader.rewind()
qop_nodes = {'QLinearConv': 1, 'QuantizeLinear': 1, 'DequantizeLinear': 1}
@ -360,8 +368,11 @@ class TestQDQFormatConvClip(TestQDQFormat):
check_model_correctness(self, model_fp32_path, model_int8_qop_path, data_reader.get_next())
def test_quantize_conv_without_bias(self):
self.verify(False) # per_channel:False
#self.verify(True) # per_channel:True
# only test cases per_channel=True and reduce_range=True to avoid saturation on avx2 and avx512 for weight type int8
self.verify(True, True) # per_channel:False, is_weight_int8:True
self.verify(False, False) # per_channel:False, is_weight_int8:False
self.verify(True, False) # per_channel:True, is_weight_int8:False
if __name__ == '__main__':
unittest.main()