saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-23 19:32:23 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

[QLinearSoftmax]remove input_shape check in Ctor (#13489)

Browse source 
### Description
In some case, we can't get node's shape to do pre-process.


### Motivation and Context
<!-- - Why is this change required? What problem does it solve?
- If it fixes an open issue, please link to the issue here. -->
This commit is contained in:
JiCheng 2022-11-15 12:02:17 +08:00 committed by GitHub
parent ad31ac466b
commit 2490cf84c9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 144 additions and 121 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

39
onnxruntime/contrib_ops/cpu/quantization/qlinear_softmax.cc
View file

@ -70,9 +70,6 @@ QLinearSoftmax::QLinearSoftmax(const OpKernelInfo& info)
auto input_defs = node.InputDefs();
auto input_type = input_defs[0]->TypeAsProto()->tensor_type().elem_type();
is_signed_ = (input_type == ONNX_NAMESPACE::TensorProto_DataType_INT8);
const auto* x_shape = input_defs[0]->Shape();
ORT_ENFORCE(x_shape != nullptr && x_shape->dim_size() > 0, "input_shape of QLinearSoftmax must be existed");
int rank = x_shape->dim_size();
int64_t opset = -1;
Status status = info.GetAttr<int64_t>("opset", &opset);
@ -89,12 +86,15 @@ QLinearSoftmax::QLinearSoftmax(const OpKernelInfo& info)
axis_ = opset_ < OPSET13 ? 1 : -1;
}
axis_ = static_cast<int>(HandleNegativeAxis(axis_, int64_t(rank)));
auto input_shape = utils::GetTensorShapeFromTensorShapeProto(*x_shape);
int64_t reduce_size = opset_ < OPSET13 ? input_shape.SizeFromDimension(axis_) : input_shape[axis_];
// reduce_size could be negative if input-shape has a dynamic axis
if (reduce_size > 0) {
BuildLookupTableIfFixed(info, fixed_lookup_table_, onnxruntime::narrow<size_t>(reduce_size), is_signed_);
const auto* x_shape = input_defs[0]->Shape();
if (x_shape != nullptr && x_shape->dim_size() > 0) {
axis_ = static_cast<int>(HandleNegativeAxis(axis_, int64_t(x_shape->dim_size())));
auto input_shape = utils::GetTensorShapeFromTensorShapeProto(*x_shape);
int64_t reduce_size = opset_ < OPSET13 ? input_shape.SizeFromDimension(axis_) : input_shape[axis_];
// reduce_size could be negative if input-shape has a dynamic axis
if (reduce_size > 0) {
BuildLookupTableIfFixed(info, fixed_lookup_table_, reduce_size, is_signed_);
}
}
}
@ -102,21 +102,24 @@ QLinearSoftmax::QLinearSoftmax(const OpKernelInfo& info)
Status QLinearSoftmax::Compute(OpKernelContext* ctx) const {
const auto* X = ctx->Input<Tensor>(0);
const auto& X_shape = X->Shape();
auto* Y = ctx->Output(0, X_shape);
// edge case. one or more dims with value of 0. nothing to do
if (X_shape.Size() == 0) {
return Status::OK();
}
auto axis = static_cast<int>(HandleNegativeAxis(axis_, int64_t(X_shape.NumDimensions())));
auto* Y = ctx->Output(0, X_shape);
concurrency::ThreadPool* thread_pool = ctx->GetOperatorThreadPool();
const size_t D = opset_ < OPSET13 ? onnxruntime::narrow<size_t>(X_shape.SizeFromDimension(onnxruntime::narrow<size_t>(axis_))) : onnxruntime::narrow<size_t>(X_shape[onnxruntime::narrow<size_t>(axis_)]);
const size_t D = opset_ < OPSET13 ? X_shape.SizeFromDimension(axis) : X_shape[axis];
EXP_OUT_DTYPE tmp_lookup_table[256];
gsl::span<const EXP_OUT_DTYPE> lookup_table = GetLookupTable(ctx, tmp_lookup_table, D);
if (opset_ < OPSET13) {
return ComputeInternal(ctx, *X, *Y, lookup_table, axis_, thread_pool);
return ComputeInternal(ctx, *X, *Y, lookup_table, axis, thread_pool);
} else {
return ComputeImplOpset13(ctx, *X, *Y, lookup_table, thread_pool);
return ComputeImplOpset13(ctx, *X, *Y, lookup_table, axis, thread_pool);
}
}
@ -291,12 +294,12 @@ Status QLinearSoftmax::ComputeInternal(OpKernelContext* context, const Tensor& i
// opset-13 and above
Status QLinearSoftmax::ComputeImplOpset13(OpKernelContext* context,
const Tensor& input, Tensor& output,
gsl::span<const EXP_OUT_DTYPE> lookup_table,
gsl::span<const EXP_OUT_DTYPE> lookup_table, int axis,
concurrency::ThreadPool* thread_pool) const {
const auto& X_shape = input.Shape();
size_t rank = X_shape.NumDimensions();
bool is_transpose_required = (size_t(axis_) != (rank - 1));
bool is_transpose_required = (size_t(axis) != (rank - 1));
Tensor transposed_input;
Tensor intermediate_output; // output that the softmax implementation will write into while using transposed input
std::vector<size_t> permutation(rank);
@ -307,8 +310,8 @@ Status QLinearSoftmax::ComputeImplOpset13(OpKernelContext* context,
std::iota(std::begin(permutation), std::end(permutation), 0);
// swap the innermost dim with the dim corresponding to axis
permutation[axis_] = rank - 1;
permutation[rank - 1] = axis_;
permutation[axis] = rank - 1;
permutation[rank - 1] = axis;
std::vector<int64_t> transposed_input_dims(rank);
std::transform(permutation.cbegin(), permutation.cend(),
transposed_input_dims.begin(), [&X_shape](size_t e) { return X_shape[e]; });

2
onnxruntime/contrib_ops/cpu/quantization/qlinear_softmax.h
View file

@ -26,7 +26,7 @@ class QLinearSoftmax final : public OpKernel {
int axis, concurrency::ThreadPool* thread_pool) const;
Status ComputeImplOpset13(OpKernelContext* context, const Tensor& input, Tensor& output,
gsl::span<const EXP_OUT_DTYPE> lookup_table,
gsl::span<const EXP_OUT_DTYPE> lookup_table, int axis,
concurrency::ThreadPool* thread_pool) const;
private:

224
onnxruntime/test/contrib_ops/qlinear_lookup_table_test.cc
View file

@ -134,139 +134,159 @@ TEST(QLinearLookupTableBasedOperatorTests, QLinearSigmoid_UInt8_0_Y_ZP) {
\see then followed by the [DOC](https://pytorch.org/docs/stable/quantization.html)
*/
TEST(QLinearLookupTableBasedOperatorTests, QLinearSoftmax_UInt8_v12) {
OpTester test("QLinearSoftmax", 1, onnxruntime::kMSDomain);
test.AddAttribute<int64_t>("axis", -2);
test.AddAttribute<int64_t>("opset", 12);
float X_scale = 0.166099221f;
//
uint8_t X_zero_point = 128;
float Y_scale = 1.0f / 256.0f;
uint8_t Y_zero_point = 0;
//
auto run_test = [](bool add_shape_to_input) {
OpTester test("QLinearSoftmax", 1, onnxruntime::kMSDomain);
test.AddAttribute<int64_t>("axis", -2);
test.AddAttribute<int64_t>("opset", 12);
float X_scale = 0.166099221f;
//
uint8_t X_zero_point = 128;
float Y_scale = 1.0f / 256.0f;
uint8_t Y_zero_point = 0;
//
std::vector<int64_t> dims = {2, 4, 5};
auto x_in = std::vector<uint8_t>{50, 67, 58, 68, 46, 69, 77, 91, 62, 74, 67, 72, 71, 70, 83, 88, 75, 54, 74, 88};
auto y_out = std::vector<uint8_t> { 0, 2, 0, 2, 0, 2, 8, 86, 1, 5, 2, 4, 3, 3, 23, 52, 6, 0, 5, 52 };
for (int64_t i = 1; i < dims[0]; i++) {
for (int64_t j = 0; j < dims[1] * dims[2]; j++) {
x_in.push_back(x_in[j]);
y_out.push_back(y_out[j]);
std::vector<int64_t> dims = {2, 4, 5};
auto x_in = std::vector<uint8_t>{50, 67, 58, 68, 46, 69, 77, 91, 62, 74, 67, 72, 71, 70, 83, 88, 75, 54, 74, 88};
auto y_out = std::vector<uint8_t>{0, 2, 0, 2, 0, 2, 8, 86, 1, 5, 2, 4, 3, 3, 23, 52, 6, 0, 5, 52};
for (int64_t i = 1; i < dims[0]; i++) {
for (int64_t j = 0; j < dims[1] * dims[2]; j++) {
x_in.push_back(x_in[j]);
y_out.push_back(y_out[j]);
}
}
}
test.AddInput<uint8_t>("X", dims, x_in);
test.AddInput<float>("X_scale", {}, {X_scale});
test.AddInput<uint8_t>("X_zero_point", {}, {X_zero_point});
test.AddInput<float>("Y_scale", {}, {Y_scale});
test.AddInput<uint8_t>("Y_zero_point", {}, {Y_zero_point});
test.AddOutput<uint8_t>("Y", dims, y_out);
auto origin_round_mode = std::fegetround();
std::fesetround(FE_TONEAREST);
test.Run();
std::fesetround(origin_round_mode);
test.AddShapeToTensorData(add_shape_to_input);
test.AddInput<uint8_t>("X", dims, x_in);
test.AddInput<float>("X_scale", {}, {X_scale});
test.AddInput<uint8_t>("X_zero_point", {}, {X_zero_point});
test.AddInput<float>("Y_scale", {}, {Y_scale});
test.AddInput<uint8_t>("Y_zero_point", {}, {Y_zero_point});
test.AddOutput<uint8_t>("Y", dims, y_out);
auto origin_round_mode = std::fegetround();
std::fesetround(FE_TONEAREST);
test.Run();
std::fesetround(origin_round_mode);
};
run_test(true);
run_test(false);
}
TEST(QLinearLookupTableBasedOperatorTests, QLinearSoftmax_UInt8_v13) {
OpTester test("QLinearSoftmax", 1, onnxruntime::kMSDomain);
test.AddAttribute<int64_t>("axis", -2);
test.AddAttribute<int64_t>("opset", 13);
float X_scale = 0.0304f;
//
uint8_t X_zero_point = 128;
float Y_scale = 0.0059f;
uint8_t Y_zero_point = 0;
//
auto run_test = [](bool add_shape_to_input) {
OpTester test("QLinearSoftmax", 1, onnxruntime::kMSDomain);
test.AddAttribute<int64_t>("axis", -2);
test.AddAttribute<int64_t>("opset", 13);
float X_scale = 0.0304f;
//
uint8_t X_zero_point = 128;
float Y_scale = 0.0059f;
uint8_t Y_zero_point = 0;
//
std::vector<int64_t> dims = {4, 4, 4};
auto x_in = std::vector<uint8_t> {
62, 50, 71, 37, 68, 88, 64, 51, 59, 95, 41, 54, 55, 20, 77, 32, 92,
std::vector<int64_t> dims = {4, 4, 4};
auto x_in = std::vector<uint8_t>{
62, 50, 71, 37, 68, 88, 64, 51, 59, 95, 41, 54, 55, 20, 77, 32, 92,
63, 43, 13, 76, 82, 53, 43, 60, 18, 73, 74, 22, 89, 44, 106, 17,
95, 27, 35, 47, 57, 0, 78, 97, 66, 56, 28, 127, 33, 106, 71, 119,
64, 16, 0, 16, 79, 27, 89, 110, 126, 88, 90, 67, 11, 4, 90};
auto y_out = std::vector<uint8_t> {
43, 20, 50, 33, 52, 63, 40, 51, 39, 78,
auto y_out = std::vector<uint8_t>{
43, 20, 50, 33, 52, 63, 40, 51, 39, 78,
20, 56, 35, 8, 59, 29, 80, 32, 29, 6, 49, 57, 39, 16, 30, 8, 72, 40,
10, 71, 30, 107, 4, 90, 11, 20, 10, 28, 5, 74, 45, 37, 27, 16, 111, 14,
125, 59, 84, 18, 14, 4, 4, 28, 20, 54, 64, 119, 126, 56, 17, 4, 10, 56};
test.AddInput<uint8_t>("X", dims, x_in);
test.AddInput<float>("X_scale", {}, {X_scale});
test.AddInput<uint8_t>("X_zero_point", {}, {X_zero_point});
test.AddInput<float>("Y_scale", {}, {Y_scale});
test.AddInput<uint8_t>("Y_zero_point", {}, {Y_zero_point});
test.AddOutput<uint8_t>("Y", dims, y_out);
auto origin_round_mode = std::fegetround();
std::fesetround(FE_TONEAREST);
test.Run();
std::fesetround(origin_round_mode);
test.AddShapeToTensorData(add_shape_to_input);
test.AddInput<uint8_t>("X", dims, x_in);
test.AddInput<float>("X_scale", {}, {X_scale});
test.AddInput<uint8_t>("X_zero_point", {}, {X_zero_point});
test.AddInput<float>("Y_scale", {}, {Y_scale});
test.AddInput<uint8_t>("Y_zero_point", {}, {Y_zero_point});
test.AddOutput<uint8_t>("Y", dims, y_out);
auto origin_round_mode = std::fegetround();
std::fesetround(FE_TONEAREST);
test.Run();
std::fesetround(origin_round_mode);
};
run_test(true);
run_test(false);
}
TEST(QLinearLookupTableBasedOperatorTests, QLinearSoftmax_Int8_v13) {
OpTester test("QLinearSoftmax", 1, onnxruntime::kMSDomain);
test.AddAttribute<int64_t>("axis", -2);
test.AddAttribute<int64_t>("opset", 13);
float X_scale = 0.0304F;
//
int8_t X_zero_point = 0;
float Y_scale = 0.0059F;
int8_t Y_zero_point = -128;
//
auto run_test = [](bool add_shape_to_input) {
OpTester test("QLinearSoftmax", 1, onnxruntime::kMSDomain);
test.AddAttribute<int64_t>("axis", -2);
test.AddAttribute<int64_t>("opset", 13);
float X_scale = 0.0304F;
//
int8_t X_zero_point = 0;
float Y_scale = 0.0059F;
int8_t Y_zero_point = -128;
//
std::vector<int64_t> dims = {4, 4, 4};
auto x_in = std::vector<int8_t> {
-4, -16, 5, -29, 2, 22, -2, -15, -7, 29, -25, -12, -11, -46, 11, -34, 26,
std::vector<int64_t> dims = {4, 4, 4};
auto x_in = std::vector<int8_t>{
-4, -16, 5, -29, 2, 22, -2, -15, -7, 29, -25, -12, -11, -46, 11, -34, 26,
-3, -23, -53, 10, 16, -13, -23, -6, -48, 7, 8, -44, 23, -22, 40, -49, 29, -39, -31, -19, -9,
-72, 12, 31, 0, -10, -38, 61, -33, 40, 5, 53, -2, -50, -66, -50, 13, -39, 23, 44, 60, 22, 24,
1, -55, -62, 24};
auto y_out = std::vector<int8_t> {
-85, -108, -78, -95, -76, -65, -88, -77, -89, -50, -108, -72, -93,
auto y_out = std::vector<int8_t>{
-85, -108, -78, -95, -76, -65, -88, -77, -89, -50, -108, -72, -93,
-120, -69, -99, -48, -96, -99, -122, -79, -71, -89, -112, -98, -120, -56, -88, -118, -57, -98,
-21, -124, -38, -117, -108, -118, -100, -124, -54, -83, -91, -100, -112, -17, -114, -2, -69, -44,
-110, -114, -124, -124, -100, -108, -74, -64, -9, -2, -72, -111, -124, -118, -72};
test.AddInput<int8_t>("X", dims, x_in);
test.AddInput<float>("X_scale", {}, {X_scale});
test.AddInput<int8_t>("X_zero_point", {}, {X_zero_point});
test.AddInput<float>("Y_scale", {}, {Y_scale});
test.AddInput<int8_t>("Y_zero_point", {}, {Y_zero_point});
test.AddOutput<int8_t>("Y", dims, y_out);
auto origin_round_mode = std::fegetround();
std::fesetround(FE_TONEAREST);
test.Run();
std::fesetround(origin_round_mode);
test.AddShapeToTensorData(add_shape_to_input);
test.AddInput<int8_t>("X", dims, x_in);
test.AddInput<float>("X_scale", {}, {X_scale});
test.AddInput<int8_t>("X_zero_point", {}, {X_zero_point});
test.AddInput<float>("Y_scale", {}, {Y_scale});
test.AddInput<int8_t>("Y_zero_point", {}, {Y_zero_point});
test.AddOutput<int8_t>("Y", dims, y_out);
auto origin_round_mode = std::fegetround();
std::fesetround(FE_TONEAREST);
test.Run();
std::fesetround(origin_round_mode);
};
run_test(true);
run_test(false);
}
TEST(QLinearLookupTableBasedOperatorTests, QLinearSoftmax_Int8_v12) {
OpTester test("QLinearSoftmax", 1, onnxruntime::kMSDomain);
test.AddAttribute<int64_t>("axis", -2);
test.AddAttribute<int64_t>("opset", 12);
float X_scale = 0.166099221f;
//
int8_t X_zero_point = 0;
float Y_scale = 1.0f / 128.0f;
int8_t Y_zero_point = 0;
//
auto run_test = [](bool add_shape_to_input) {
OpTester test("QLinearSoftmax", 1, onnxruntime::kMSDomain);
test.AddAttribute<int64_t>("axis", -2);
test.AddAttribute<int64_t>("opset", 12);
float X_scale = 0.166099221f;
//
int8_t X_zero_point = 0;
float Y_scale = 1.0f / 128.0f;
int8_t Y_zero_point = 0;
//
std::vector<int64_t> dims = {2, 4, 5};
auto x_in = std::vector<int8_t>{-28, -4, -4, -7, 3, -26, 4, -16, 23, 14, -7, 26, -8, 19, -16, -13, 7, 17, 27, 5};
auto y_out = std::vector<int8_t>{0, 0, 0, 0, 1, 0, 1, 0, 22, 5, 0, 35, 0, 11, 0, 0, 2, 8, 42, 1};
for (int64_t i = 1; i < dims[0]; i++) {
for (int64_t j = 0; j < dims[1] * dims[2]; j++) {
x_in.push_back(x_in[j]);
y_out.push_back(y_out[j]);
std::vector<int64_t> dims = {2, 4, 5};
auto x_in = std::vector<int8_t>{-28, -4, -4, -7, 3, -26, 4, -16, 23, 14, -7, 26, -8, 19, -16, -13, 7, 17, 27, 5};
auto y_out = std::vector<int8_t>{0, 0, 0, 0, 1, 0, 1, 0, 22, 5, 0, 35, 0, 11, 0, 0, 2, 8, 42, 1};
for (int64_t i = 1; i < dims[0]; i++) {
for (int64_t j = 0; j < dims[1] * dims[2]; j++) {
x_in.push_back(x_in[j]);
y_out.push_back(y_out[j]);
}
}
}
test.AddInput<int8_t>("X", dims, x_in);
test.AddInput<float>("X_scale", {}, {X_scale});
test.AddInput<int8_t>("X_zero_point", {}, {X_zero_point});
test.AddInput<float>("Y_scale", {}, {Y_scale});
test.AddInput<int8_t>("Y_zero_point", {}, {Y_zero_point});
test.AddOutput<int8_t>("Y", dims, y_out);
auto origin_round_mode = std::fegetround();
std::fesetround(FE_TONEAREST);
test.Run();
std::fesetround(origin_round_mode);
test.AddShapeToTensorData(add_shape_to_input);
test.AddInput<int8_t>("X", dims, x_in);
test.AddInput<float>("X_scale", {}, {X_scale});
test.AddInput<int8_t>("X_zero_point", {}, {X_zero_point});
test.AddInput<float>("Y_scale", {}, {Y_scale});
test.AddInput<int8_t>("Y_zero_point", {}, {Y_zero_point});
test.AddOutput<int8_t>("Y", dims, y_out);
auto origin_round_mode = std::fegetround();
std::fesetround(FE_TONEAREST);
test.Run();
std::fesetround(origin_round_mode);
};
run_test(true);
run_test(false);
}
} // namespace test