saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-07 17:15:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

using copy to when scale/zp are same in qlinear concat. (#7761)

Browse source 
Fix typo during op construction which not treat zp as constant and affect perf.
This commit is contained in:
Zhang Lei 2021-05-19 16:43:22 -07:00 committed by GitHub
parent 374acf1423
commit 38c1c6d2da
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 61 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

83
onnxruntime/contrib_ops/cpu/qlinear_concat.cc
View file

@ -10,6 +10,25 @@
namespace onnxruntime {
namespace contrib {
constexpr int LOOKUP_TABLE_IS_FIXED = 1;
constexpr int LOOKUP_TABLE_IS_COPY = 2;
static inline bool has_same_scale(const Tensor* tensor_x_scale, const Tensor* tensor_y_scale) {
return *(tensor_x_scale->Data<float>()) == *(tensor_y_scale->Data<float>());
}
static inline bool has_same_zero_point(bool is_signed, const Tensor* tensor_x_zero_point, const Tensor* tensor_y_zero_point) {
if (is_signed) {
const int8_t X_zero_point = (tensor_x_zero_point == nullptr) ? static_cast<int8_t>(0) : *(tensor_x_zero_point->template Data<int8_t>());
const int8_t Y_zero_point = (tensor_y_zero_point == nullptr) ? static_cast<int8_t>(0) : *(tensor_y_zero_point->template Data<int8_t>());
return X_zero_point == Y_zero_point;
} else {
const uint8_t X_zero_point = (tensor_x_zero_point == nullptr) ? static_cast<uint8_t>(0) : *(tensor_x_zero_point->template Data<uint8_t>());
const uint8_t Y_zero_point = (tensor_y_zero_point == nullptr) ? static_cast<uint8_t>(0) : *(tensor_y_zero_point->template Data<uint8_t>());
return X_zero_point == Y_zero_point;
}
}
QLinearConcat::QLinearConcat(const OpKernelInfo& info) : OpKernel(info), ConcatBase(info) {
size_t input_def_count = info.node().InputDefs().size();
ORT_ENFORCE(input_def_count >= 8 && (input_def_count - 2) % 3 == 0,
@ -17,6 +36,7 @@ QLinearConcat::QLinearConcat(const OpKernelInfo& info) : OpKernel(info), ConcatB
size_t input_count = (input_def_count - 2) / 3;
fixed_lookup_tables_.resize(input_count);
fixed_table_attrs_.resize(input_count, 0);
const Tensor* tensor_y_scale = nullptr;
const Tensor* tensor_y_zero_point = nullptr;
@ -34,7 +54,7 @@ QLinearConcat::QLinearConcat(const OpKernelInfo& info) : OpKernel(info), ConcatB
const Tensor* tensor_x_scale = nullptr;
const Tensor* tensor_x_zero_point = nullptr;
bool get_x_scale = info.TryGetConstantInput(static_cast<int>(def_index) + 1, &tensor_x_scale);
bool get_x_zero_point = !info.TryGetConstantInput(static_cast<int>(def_index) + 2, &tensor_x_zero_point);
bool get_x_zero_point = info.TryGetConstantInput(static_cast<int>(def_index) + 2, &tensor_x_zero_point);
if (!get_x_scale || !get_x_zero_point) {
continue; // try to optimize next one
}
@ -43,15 +63,20 @@ QLinearConcat::QLinearConcat(const OpKernelInfo& info) : OpKernel(info), ConcatB
"Wrong input type encountered for zero point input def @", def_index + 2);
size_t input_idx = (def_index - 2) / 3;
fixed_lookup_tables_[input_idx].resize(256);
if (is_signed_int8) {
QlinearBuildLookupTable<int8_t>(
fixed_lookup_tables_[input_idx].data(), tensor_x_scale, tensor_x_zero_point,
tensor_y_scale, tensor_y_zero_point, identity_float);
fixed_table_attrs_[input_idx] |= LOOKUP_TABLE_IS_FIXED;
if (has_same_scale(tensor_x_scale, tensor_y_scale) && has_same_zero_point(is_signed_int8, tensor_x_zero_point, tensor_y_zero_point)) {
fixed_table_attrs_[input_idx] |= LOOKUP_TABLE_IS_COPY;
} else {
QlinearBuildLookupTable<uint8_t>(
fixed_lookup_tables_[input_idx].data(), tensor_x_scale, tensor_x_zero_point,
tensor_y_scale, tensor_y_zero_point, identity_float);
fixed_lookup_tables_[input_idx].resize(256);
if (is_signed_int8) {
QlinearBuildLookupTable<int8_t>(
fixed_lookup_tables_[input_idx].data(), tensor_x_scale, tensor_x_zero_point,
tensor_y_scale, tensor_y_zero_point, identity_float);
} else {
QlinearBuildLookupTable<uint8_t>(
fixed_lookup_tables_[input_idx].data(), tensor_x_scale, tensor_x_zero_point,
tensor_y_scale, tensor_y_zero_point, identity_float);
}
}
}
}
@ -70,6 +95,7 @@ Status QLinearConcat::Compute(OpKernelContext* ctx) const {
// Hold pointers to the input tensors to be used in the PrepareForCompute() step
auto input_count = input_count_x3 / 3;
std::vector<std::vector<uint8_t>> dynamic_lookup_tables(input_count);
std::vector<int> dynamic_table_attrs(input_count, 0);
std::vector<const Tensor*> input_tensors(input_count);
for (auto input_index = 0; input_index < input_count; ++input_index) {
@ -77,24 +103,28 @@ Status QLinearConcat::Compute(OpKernelContext* ctx) const {
input_tensors[input_index] = ctx->Input<Tensor>(static_cast<int>(tuple_start));
// Build lookup table for non-const parameters (scale + zero point)
if (fixed_lookup_tables_[input_index].size() == 0) {
if (!(fixed_table_attrs_[input_index] & LOOKUP_TABLE_IS_FIXED)) {
// Check tensor type first
const Tensor* tensor_x_scale = ctx->Input<Tensor>(tuple_start + 1);
const Tensor* tensor_x_zero_point = ctx->Input<Tensor>(tuple_start + 2);
ORT_ENFORCE(tensor_x_scale->IsDataType<float>(), "Input scale is not float for quantized input @", tuple_start + 1);
ORT_ENFORCE(tensor_x_zero_point->GetElementType() == tensor_y_zero_point->GetElementType(),
"Wrong input type encountered for zero point of quantized input @", tuple_start + 2);
"Wrong input type encountered for zero point of quantized input @", tuple_start + 2);
dynamic_lookup_tables[input_index].resize(256);
if (is_signed_int8) {
QlinearBuildLookupTable<int8_t>(
dynamic_lookup_tables[input_index].data(), tensor_x_scale, tensor_x_zero_point,
tensor_y_scale, tensor_y_zero_point, identity_float);
if (has_same_scale(tensor_x_scale, tensor_y_scale) && has_same_zero_point(is_signed_int8, tensor_x_zero_point, tensor_y_zero_point)) {
dynamic_table_attrs[input_index] |= LOOKUP_TABLE_IS_COPY;
} else {
QlinearBuildLookupTable<uint8_t>(
dynamic_lookup_tables[input_index].data(), tensor_x_scale, tensor_x_zero_point,
tensor_y_scale, tensor_y_zero_point, identity_float);
dynamic_lookup_tables[input_index].resize(256);
if (is_signed_int8) {
QlinearBuildLookupTable<int8_t>(
dynamic_lookup_tables[input_index].data(), tensor_x_scale, tensor_x_zero_point,
tensor_y_scale, tensor_y_zero_point, identity_float);
} else {
QlinearBuildLookupTable<uint8_t>(
dynamic_lookup_tables[input_index].data(), tensor_x_scale, tensor_x_zero_point,
tensor_y_scale, tensor_y_zero_point, identity_float);
}
}
}
}
@ -115,15 +145,22 @@ Status QLinearConcat::Compute(OpKernelContext* ctx) const {
if (prep.num_elements == 0)
continue;
const uint8_t* table = (fixed_lookup_tables_[input_index].size() > 0)
? fixed_lookup_tables_[input_index].data()
: dynamic_lookup_tables[input_index].data();
const uint8_t* table = (fixed_table_attrs_[input_index] & LOOKUP_TABLE_IS_FIXED)
? fixed_lookup_tables_[input_index].data()
: dynamic_lookup_tables[input_index].data();
const bool is_copy = (fixed_table_attrs_[input_index] & LOOKUP_TABLE_IS_FIXED)
? bool(fixed_table_attrs_[input_index] & LOOKUP_TABLE_IS_COPY)
: bool(dynamic_table_attrs[input_index] & LOOKUP_TABLE_IS_COPY);
auto input_axis_pitch = prep.axis_pitch;
const uint8_t* input = static_cast<const uint8_t*>(prep.tensor->DataRaw());
uint8_t* output = static_cast<uint8_t*>(p.output_tensor->MutableDataRaw()) + initial_output_offset;
for (int64_t cur_in_offset = 0; cur_in_offset < prep.num_elements; cur_in_offset += input_axis_pitch) {
QLinearLookupTableTransform(input + cur_in_offset, table, output, input_axis_pitch);
if (is_copy) {
memcpy(output, input + cur_in_offset, input_axis_pitch);
} else {
QLinearLookupTableTransform(input + cur_in_offset, table, output, input_axis_pitch);
}
output += p.output_axis_pitch;
}

1
onnxruntime/contrib_ops/cpu/qlinear_concat.h
View file

@ -19,6 +19,7 @@ class QLinearConcat final : public OpKernel, public ConcatBase {
private:
std::vector<std::vector<uint8_t>> fixed_lookup_tables_;
std::vector<int> fixed_table_attrs_; // is_static or not, is_copy or not
};
} // namespace contrib