mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-11 17:48:34 +00:00
Rework initializer.cc to eliminate code duplication (#11131)
Rework initializer.cc to eliminate code duplication and add type enforcement. Address review comments. Add literal operators for MLFloat16 abd BFloat16 and tests.
This commit is contained in:
Dmitri Smirnov
GitHub
parent
bcc62e0cbf
commit
12c687f594
7 changed files with 479 additions and 850 deletions
|
|
@ -7,6 +7,10 @@
|
|||
#include "cuda_bf16.h"
|
||||
#endif
|
||||
|
||||
#if !defined(__CUDACC__) && !defined(__HIPCC__)
|
||||
#include <gsl/gsl>
|
||||
#endif
|
||||
|
||||
#include "core/common/common.h"
|
||||
|
||||
namespace onnxruntime {
|
||||
|
|
@ -19,10 +23,10 @@ namespace onnxruntime {
|
|||
|
||||
// MLFloat16
|
||||
struct MLFloat16 {
|
||||
uint16_t val;
|
||||
uint16_t val{0};
|
||||
|
||||
MLFloat16() : val(0) {}
|
||||
explicit MLFloat16(uint16_t x) : val(x) {}
|
||||
MLFloat16() = default;
|
||||
explicit constexpr MLFloat16(uint16_t x) : val(x) {}
|
||||
explicit MLFloat16(float f);
|
||||
|
||||
float ToFloat() const;
|
||||
|
|
@ -45,7 +49,7 @@ struct BFloat16 {
|
|||
|
||||
struct FromBitsT {};
|
||||
static constexpr ORT_HOST_DEVICE FromBitsT FromBits() { return FromBitsT(); }
|
||||
constexpr ORT_HOST_DEVICE BFloat16(unsigned short bits, FromBitsT) : val(bits){};
|
||||
constexpr ORT_HOST_DEVICE BFloat16(unsigned short bits, FromBitsT) : val(bits) {}
|
||||
|
||||
inline ORT_HOST_DEVICE BFloat16(float v) {
|
||||
#if defined(CUDA_VERSION) && CUDA_VERSION >= 11000 && defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
|
||||
|
|
@ -109,6 +113,33 @@ struct BFloat16 {
|
|||
#endif
|
||||
};
|
||||
|
||||
inline bool operator==(const BFloat16& left, const BFloat16& right) { return left.val == right.val; }
|
||||
inline bool operator!=(const BFloat16& left, const BFloat16& right) { return left.val != right.val; }
|
||||
inline bool operator<(const BFloat16& left, const BFloat16& right) { return left.val < right.val; }
|
||||
|
||||
|
||||
// User defined suffixes to make it easier to declare
|
||||
// initializers with MLFloat16 and BFloat16 from unsigned short
|
||||
// E.g 10_f16 or 10_b16
|
||||
#if !defined(__CUDACC__) && !defined(__HIPCC__)
|
||||
inline MLFloat16 operator"" _f16(unsigned long long int v) {
|
||||
return MLFloat16(gsl::narrow<uint16_t>(v));
|
||||
}
|
||||
|
||||
inline MLFloat16 operator"" _fp16(long double v) {
|
||||
return MLFloat16(static_cast<float>(v));
|
||||
}
|
||||
|
||||
inline BFloat16 operator"" _b16(unsigned long long int v) {
|
||||
return BFloat16(gsl::narrow<uint16_t>(v), BFloat16::FromBits());
|
||||
}
|
||||
|
||||
inline BFloat16 operator"" _bfp16(long double v) {
|
||||
return BFloat16(static_cast<float>(v));
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
inline void BFloat16ToFloat(const BFloat16* blf, float* flt, size_t size) {
|
||||
auto src = blf;
|
||||
auto d = flt;
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
#if !defined(ORT_MINIMAL_BUILD)
|
||||
#include "core/optimizer/initializer.h"
|
||||
|
||||
#include "gsl/gsl"
|
||||
|
|
@ -11,49 +10,314 @@
|
|||
#include "core/framework/tensor_external_data_info.h"
|
||||
#include "core/platform/env.h"
|
||||
|
||||
#include <functional>
|
||||
|
||||
namespace onnxruntime {
|
||||
Status Initializer::ReadExternalRawData(
|
||||
const ONNX_NAMESPACE::TensorProto& tensor_proto, const Path& model_path, std::vector<char>& raw_data) {
|
||||
ORT_RETURN_IF_NOT(
|
||||
tensor_proto.data_type() != ONNX_NAMESPACE::TensorProto_DataType_UNDEFINED &&
|
||||
tensor_proto.data_type() != ONNX_NAMESPACE::TensorProto_DataType_STRING,
|
||||
"External data type must not be UNDEFINED or STRING.");
|
||||
|
||||
ORT_RETURN_IF(
|
||||
model_path.IsEmpty(),
|
||||
"model_path must not be empty. Ensure that a path is provided when the model is created or loaded.");
|
||||
|
||||
std::unique_ptr<ExternalDataInfo> external_data{};
|
||||
ORT_RETURN_IF_ERROR(ExternalDataInfo::Create(tensor_proto.external_data(), external_data));
|
||||
|
||||
size_t actual_tensor_data_length;
|
||||
ORT_RETURN_IF_ERROR(utils::GetSizeInBytesFromTensorProto<0>(
|
||||
tensor_proto, &actual_tensor_data_length));
|
||||
const size_t external_data_length = external_data->GetLength();
|
||||
|
||||
ORT_RETURN_IF_NOT(
|
||||
external_data_length == 0 ||
|
||||
external_data_length == actual_tensor_data_length,
|
||||
"TensorProto external data size mismatch. ",
|
||||
"Computed size: ", actual_tensor_data_length,
|
||||
", external_data.length: ", external_data_length);
|
||||
|
||||
Path external_data_relative_path{};
|
||||
ORT_RETURN_IF_ERROR(Path::Parse(
|
||||
external_data->GetRelPath(), external_data_relative_path));
|
||||
|
||||
std::vector<char> buffer(actual_tensor_data_length);
|
||||
|
||||
ORT_RETURN_IF_ERROR(Env::Default().ReadFileIntoBuffer(
|
||||
(model_path.ParentPath() / external_data_relative_path).ToPathString().c_str(),
|
||||
external_data->GetOffset(),
|
||||
actual_tensor_data_length,
|
||||
gsl::make_span(buffer)));
|
||||
|
||||
raw_data = std::move(buffer);
|
||||
|
||||
return Status::OK();
|
||||
Initializer::Initializer(ONNX_NAMESPACE::TensorProto_DataType data_type,
|
||||
std::string_view name,
|
||||
gsl::span<const int64_t> dims)
|
||||
: name_(name),
|
||||
data_(DataTypeImpl::TensorTypeFromONNXEnum(data_type)->GetElementType(), dims, std::make_shared<CPUAllocator>()) {
|
||||
if (!data_.IsDataTypeString()) {
|
||||
memset(data_.MutableDataRaw(), 0, data_.SizeInBytes());
|
||||
}
|
||||
}
|
||||
} // namespace onnxruntime
|
||||
|
||||
#endif // !(ORT_MINIMAL_BUILD)
|
||||
Initializer::Initializer(const ONNX_NAMESPACE::TensorProto& tensor_proto, const Path& model_path) {
|
||||
ORT_ENFORCE(utils::HasDataType(tensor_proto), "Initializer must have a datatype");
|
||||
if (utils::HasExternalData(tensor_proto)) {
|
||||
ORT_ENFORCE(!model_path.IsEmpty(),
|
||||
"model_path must not be empty. Ensure that a path is provided when the model is created or loaded.");
|
||||
}
|
||||
|
||||
auto proto_data_type = tensor_proto.data_type();
|
||||
if (utils::HasName(tensor_proto)) {
|
||||
name_ = tensor_proto.name();
|
||||
}
|
||||
|
||||
auto proto_dims = utils::GetTensorShapeFromTensorProto(tensor_proto);
|
||||
TensorShape proto_shape(proto_dims);
|
||||
|
||||
// This must be pre-allocated
|
||||
Tensor w(DataTypeImpl::TensorTypeFromONNXEnum(proto_data_type)->GetElementType(), proto_shape, std::make_shared<CPUAllocator>());
|
||||
ORT_THROW_IF_ERROR(utils::TensorProtoToTensor(Env::Default(), model_path.ToPathString().c_str(), tensor_proto, w));
|
||||
data_ = std::move(w);
|
||||
}
|
||||
|
||||
namespace {
|
||||
template <typename T>
|
||||
struct ToFp16;
|
||||
|
||||
template <>
|
||||
struct ToFp16<MLFloat16> {
|
||||
uint16_t operator()(const MLFloat16& fl) const {
|
||||
return fl.val;
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct ToFp16<float> {
|
||||
uint16_t operator()(float f) const {
|
||||
return MLFloat16(f).val;
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct ToFp16<double> {
|
||||
uint16_t operator()(double d) const {
|
||||
// The same code as in Eigen. We assume the loss of precision will occur
|
||||
// hence static_cast
|
||||
return MLFloat16(static_cast<float>(d)).val;
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct TensorToProtoFP16 {
|
||||
void operator()(const Tensor& data, ONNX_NAMESPACE::TensorProto& proto) const {
|
||||
ToFp16<T> to_fp16;
|
||||
auto span = data.DataAsSpan<T>();
|
||||
for (const auto& v : span) {
|
||||
proto.add_int32_data(to_fp16(v));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct ToBFloat16;
|
||||
|
||||
template <>
|
||||
struct ToBFloat16<BFloat16> {
|
||||
uint16_t operator()(const BFloat16& bf) const {
|
||||
return bf.val;
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct ToBFloat16<float> {
|
||||
uint16_t operator()(float f) const {
|
||||
return BFloat16(f).val;
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct ToBFloat16<double> {
|
||||
uint16_t operator()(double d) const {
|
||||
// The same code as in Eigen. We assume the loss of precision will occur
|
||||
// hence static_cast
|
||||
return BFloat16(static_cast<float>(d)).val;
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct TensorToProtoBFloat16 {
|
||||
void operator()(const Tensor& data, ONNX_NAMESPACE::TensorProto& proto) const {
|
||||
ToBFloat16<T> to_bfloat16;
|
||||
auto span = data.DataAsSpan<T>();
|
||||
for (const auto& v : span) {
|
||||
proto.add_int32_data(to_bfloat16(v));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
inline void SetNameDims(const std::string& name,
|
||||
gsl::span<const int64_t> dims,
|
||||
ONNX_NAMESPACE::TensorProto_DataType dt,
|
||||
ONNX_NAMESPACE::TensorProto& tensor_proto) {
|
||||
tensor_proto.set_name(name);
|
||||
tensor_proto.set_data_type(dt);
|
||||
|
||||
for (auto d : dims) {
|
||||
tensor_proto.add_dims(d);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
ONNX_NAMESPACE::TensorProto Initializer::ToFP16(const std::string& name) const {
|
||||
ONNX_NAMESPACE::TensorProto tensor_proto;
|
||||
SetNameDims(name, data_.Shape().GetDims(), ONNX_NAMESPACE::TensorProto_DataType_FLOAT16, tensor_proto);
|
||||
utils::MLTypeCallDispatcher<MLFloat16, float, double> t_disp(data_.GetElementType());
|
||||
t_disp.Invoke<TensorToProtoFP16>(data_, tensor_proto);
|
||||
return tensor_proto;
|
||||
}
|
||||
|
||||
ONNX_NAMESPACE::TensorProto Initializer::ToBFloat16(const std::string& name) const {
|
||||
ONNX_NAMESPACE::TensorProto tensor_proto;
|
||||
SetNameDims(name, data_.Shape().GetDims(), ONNX_NAMESPACE::TensorProto_DataType_BFLOAT16, tensor_proto);
|
||||
utils::MLTypeCallDispatcher<BFloat16, float, double> t_disp(data_.GetElementType());
|
||||
t_disp.Invoke<TensorToProtoBFloat16>(data_, tensor_proto);
|
||||
return tensor_proto;
|
||||
}
|
||||
|
||||
namespace {
|
||||
|
||||
// std::identity c++20
|
||||
template <typename T>
|
||||
struct ToNumeric {
|
||||
using type = T;
|
||||
constexpr const T& operator()(const T& v) const {
|
||||
return v;
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct ToNumeric<MLFloat16> {
|
||||
using type = float;
|
||||
float operator()(const MLFloat16& v) const {
|
||||
return v.ToFloat();
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct ToNumeric<BFloat16> {
|
||||
using type = float;
|
||||
float operator()(const BFloat16& v) const {
|
||||
return v.ToFloat();
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T, typename Op>
|
||||
struct OpElementWise {
|
||||
void Invoke(Tensor& lhs, const Tensor& rhs) const {
|
||||
Op op;
|
||||
ToNumeric<T> to_numeric;
|
||||
auto dst_span = lhs.MutableDataAsSpan<T>();
|
||||
auto src_span = rhs.DataAsSpan<T>();
|
||||
for (size_t i = 0, limit = dst_span.size(); i < limit; ++i) {
|
||||
dst_span[i] = T(op(to_numeric(dst_span[i]), to_numeric(src_span[i])));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct ScalarAdd {
|
||||
void operator()(Tensor& tensor, float v) const {
|
||||
ToNumeric<T> to_numeric;
|
||||
auto span = tensor.MutableDataAsSpan<T>();
|
||||
for (auto& dst : span) {
|
||||
dst = T(to_numeric(dst) + v);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct Sqrt {
|
||||
void operator()(Tensor& tensor) const {
|
||||
ToNumeric<T> to_numeric;
|
||||
auto span = tensor.MutableDataAsSpan<T>();
|
||||
for (auto& dst : span) {
|
||||
auto v = to_numeric(dst);
|
||||
dst = T(std::sqrt(v));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct ElementWiseAdd : OpElementWise<T, std::plus<typename ToNumeric<T>::type>> {
|
||||
void operator()(Tensor& lhs, const Tensor& rhs) const {
|
||||
this->Invoke(lhs, rhs);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct ElementWiseSub : OpElementWise<T, std::minus<typename ToNumeric<T>::type>> {
|
||||
void operator()(Tensor& lhs, const Tensor& rhs) const {
|
||||
this->Invoke(lhs, rhs);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct ElementWiseMul : OpElementWise<T, std::multiplies<typename ToNumeric<T>::type>> {
|
||||
void operator()(Tensor& lhs, const Tensor& rhs) const {
|
||||
this->Invoke(lhs, rhs);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct ElementWiseDiv : OpElementWise<T, std::divides<typename ToNumeric<T>::type>> {
|
||||
void operator()(Tensor& lhs, const Tensor& rhs) const {
|
||||
this->Invoke(lhs, rhs);
|
||||
}
|
||||
};
|
||||
} // namespace
|
||||
|
||||
Initializer& Initializer::add(float value) {
|
||||
utils::MLTypeCallDispatcher<MLFloat16, BFloat16, float, double> t_disp(data_.GetElementType());
|
||||
t_disp.Invoke<ScalarAdd>(data_, value);
|
||||
return *this;
|
||||
}
|
||||
|
||||
Initializer& Initializer::add(const Initializer& other) {
|
||||
ORT_ENFORCE(data_type() == other.data_type(), "Expecting the same data type");
|
||||
ORT_ENFORCE(size() == other.size(), "Expecting the same size");
|
||||
utils::MLTypeCallDispatcher<MLFloat16, BFloat16, float, double, int32_t, int64_t> t_disp(data_.GetElementType());
|
||||
t_disp.Invoke<ElementWiseAdd>(data_, other.data_);
|
||||
return *this;
|
||||
}
|
||||
|
||||
Initializer& Initializer::sub(const Initializer& other) {
|
||||
ORT_ENFORCE(data_type() == other.data_type(), "Expecting the same data type");
|
||||
ORT_ENFORCE(size() == other.size(), "Expecting the same size");
|
||||
utils::MLTypeCallDispatcher<MLFloat16, BFloat16, float, double, int32_t, int64_t> t_disp(data_.GetElementType());
|
||||
t_disp.Invoke<ElementWiseSub>(data_, other.data_);
|
||||
return *this;
|
||||
}
|
||||
|
||||
Initializer& Initializer::mul(const Initializer& other) {
|
||||
ORT_ENFORCE(data_type() == other.data_type(), "Expecting the same data type");
|
||||
ORT_ENFORCE(size() == other.size(), "Expecting the same size");
|
||||
utils::MLTypeCallDispatcher<MLFloat16, BFloat16, float, double, int32_t, int64_t> t_disp(data_.GetElementType());
|
||||
t_disp.Invoke<ElementWiseMul>(data_, other.data_);
|
||||
return *this;
|
||||
}
|
||||
|
||||
Initializer& Initializer::div(const Initializer& other) {
|
||||
ORT_ENFORCE(data_type() == other.data_type(), "Expecting the same data type");
|
||||
ORT_ENFORCE(size() == other.size(), "Expecting the same size");
|
||||
utils::MLTypeCallDispatcher<MLFloat16, BFloat16, float, double, int32_t, int64_t> t_disp(data_.GetElementType());
|
||||
t_disp.Invoke<ElementWiseDiv>(data_, other.data_);
|
||||
return *this;
|
||||
}
|
||||
|
||||
Initializer& Initializer::sqrt() {
|
||||
utils::MLTypeCallDispatcher<MLFloat16, BFloat16, float, double> t_disp(data_.GetElementType());
|
||||
t_disp.Invoke<Sqrt>(data_);
|
||||
return *this;
|
||||
}
|
||||
|
||||
namespace {
|
||||
template <typename T>
|
||||
struct ScaleByAxis {
|
||||
void operator()(Tensor& data, const Tensor& scalers, const int64_t block_size, const int64_t num_blocks) const {
|
||||
ToNumeric<T> to_numeric;
|
||||
const auto scaler_size = scalers.Shape().Size();
|
||||
T* dst = data.MutableData<T>();
|
||||
const T* scalers_data = scalers.Data<T>();
|
||||
if (scaler_size == 1) {
|
||||
const auto numeric_scaler = to_numeric(scalers_data[0]);
|
||||
for (int64_t block_offset = 0, limit = block_size * num_blocks; block_offset < limit; ++block_offset) {
|
||||
dst[block_offset] = T(to_numeric(dst[block_offset]) * numeric_scaler);
|
||||
}
|
||||
} else
|
||||
for (int64_t block_offset = 0, i = 0; i < num_blocks; i++) {
|
||||
const auto numeric_scaler = to_numeric(scalers_data[i]);
|
||||
for (int64_t j = 0; j < block_size; ++j, ++block_offset) {
|
||||
dst[block_offset] = T(to_numeric(dst[block_offset]) * numeric_scaler);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
void Initializer::scale_by_axis(const Initializer& scalers, int axis) {
|
||||
ORT_ENFORCE(axis >= 0, "Axis must be non-negative");
|
||||
const int64_t block_size = data_.Shape().SizeFromDimension(gsl::narrow_cast<size_t>(axis));
|
||||
const int64_t num_blocks = size() / block_size;
|
||||
ORT_ENFORCE(scalers.size() == 1 || scalers.size() == num_blocks, "Invalid other(scalers) size");
|
||||
utils::MLTypeCallDispatcher<MLFloat16, BFloat16, float, double, int32_t, int64_t> t_disp(data_.GetElementType());
|
||||
t_disp.Invoke<ScaleByAxis>(data_, scalers.data_, block_size, num_blocks);
|
||||
}
|
||||
|
||||
} // namespace onnxruntime
|
||||
|
|
|
|||
|
|
@ -4,11 +4,15 @@
|
|||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <functional>
|
||||
#include <vector>
|
||||
#include <cmath>
|
||||
|
||||
#include "core/common/common.h"
|
||||
#include "core/common/path.h"
|
||||
#include "core/framework/allocator.h"
|
||||
#include "core/optimizer/graph_transformer.h"
|
||||
#include "core/framework/tensor_shape.h"
|
||||
#include "core/framework/tensorprotoutils.h"
|
||||
#include "core/graph/onnx_protobuf.h"
|
||||
#include "core/util/math.h"
|
||||
|
|
@ -19,804 +23,68 @@ class Initializer final {
|
|||
public:
|
||||
// Construct an initializer with the provided name and data type, with all values initialized to 0
|
||||
Initializer(ONNX_NAMESPACE::TensorProto_DataType data_type,
|
||||
const std::string& name,
|
||||
const std::vector<int64_t>& dims) : dims_(dims), size_(0) {
|
||||
data_type_ = data_type;
|
||||
name_ = name;
|
||||
size_ = std::accumulate(dims_.begin(), dims_.end(), int64_t(1), std::multiplies<int64_t>{});
|
||||
std::string_view name,
|
||||
gsl::span<const int64_t> dims);
|
||||
|
||||
switch (data_type_) {
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_FLOAT16: {
|
||||
float16_data_.assign(static_cast<size_t>(size_), math::floatToHalf(0.f));
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_BFLOAT16: {
|
||||
// Reuse float16 field
|
||||
float16_data_.assign(static_cast<size_t>(size_), BFloat16(0.f).val);
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: {
|
||||
float_data_.assign(static_cast<size_t>(size_), 0.0f);
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_DOUBLE: {
|
||||
double_data_.assign(static_cast<size_t>(size_), 0.0);
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_INT8: {
|
||||
int8_data_.assign(static_cast<size_t>(size_), 0);
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_UINT8: {
|
||||
uint8_data_.assign(static_cast<size_t>(size_), 0);
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_INT32: {
|
||||
int32_data_.assign(static_cast<size_t>(size_), 0);
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_INT64: {
|
||||
int64_data_.assign(static_cast<size_t>(size_), 0);
|
||||
break;
|
||||
}
|
||||
default:
|
||||
ORT_THROW("data type ", data_type_, "is not supported.");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Initializer(const ONNX_NAMESPACE::TensorProto& tensor_proto, const Path& model_path) {
|
||||
data_type_ = tensor_proto.data_type();
|
||||
if (utils::HasName(tensor_proto)) {
|
||||
name_ = tensor_proto.name();
|
||||
}
|
||||
dims_.reserve(tensor_proto.dims_size());
|
||||
for (int i = 0; i < tensor_proto.dims_size(); i++) {
|
||||
dims_.push_back(tensor_proto.dims(i));
|
||||
}
|
||||
|
||||
size_ = std::accumulate(dims_.begin(), dims_.end(), static_cast<int64_t>(1), std::multiplies<int64_t>{});
|
||||
|
||||
if (tensor_proto.data_location() != ONNX_NAMESPACE::TensorProto_DataLocation_EXTERNAL) {
|
||||
if (utils::HasRawData(tensor_proto)) {
|
||||
raw_data_.assign(tensor_proto.raw_data().begin(), tensor_proto.raw_data().end());
|
||||
} else {
|
||||
switch (data_type_) {
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_FLOAT16:
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_BFLOAT16: {
|
||||
int64_t size = tensor_proto.int32_data_size();
|
||||
ORT_ENFORCE(size_ == size, "size is different");
|
||||
for (int i = 0; i < size_; i++) {
|
||||
float16_data_.push_back(static_cast<uint16_t>(tensor_proto.int32_data(i)));
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: {
|
||||
int64_t size = tensor_proto.float_data_size();
|
||||
ORT_ENFORCE(size_ == size, "size is different");
|
||||
for (int i = 0; i < size_; i++) {
|
||||
float_data_.push_back(tensor_proto.float_data(i));
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_DOUBLE: {
|
||||
int64_t size = tensor_proto.double_data_size();
|
||||
ORT_ENFORCE(size_ == size, "size is different");
|
||||
for (int i = 0; i < size_; i++) {
|
||||
double_data_.push_back(tensor_proto.double_data(i));
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_INT8: {
|
||||
int64_t size = tensor_proto.int32_data_size();
|
||||
ORT_ENFORCE(size_ == size, "size is different");
|
||||
for (int i = 0; i < size_; i++) {
|
||||
int8_data_.push_back(static_cast<int8_t>(tensor_proto.int32_data(i)));
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_UINT8: {
|
||||
int64_t size = tensor_proto.int32_data_size();
|
||||
ORT_ENFORCE(size_ == size, "size is different");
|
||||
for (int i = 0; i < size_; i++) {
|
||||
uint8_data_.push_back(static_cast<uint8_t>(tensor_proto.int32_data(i)));
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_INT32: {
|
||||
int64_t size = tensor_proto.int32_data_size();
|
||||
ORT_ENFORCE(size_ == size, "size is different");
|
||||
for (int i = 0; i < size_; i++) {
|
||||
int32_data_.push_back(tensor_proto.int32_data(i));
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_INT64: {
|
||||
int64_t size = tensor_proto.int64_data_size();
|
||||
ORT_ENFORCE(size_ == size, "size is different");
|
||||
for (int i = 0; i < size_; i++) {
|
||||
int64_data_.push_back(tensor_proto.int64_data(i));
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
ORT_NOT_IMPLEMENTED(__FUNCTION__, "unsupported data type: ", data_type_);
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else { // tensor_proto.data_location() == ONNX_NAMESPACE::TensorProto_DataLocation_EXTERNAL
|
||||
#if !defined(ORT_MINIMAL_BUILD)
|
||||
const auto status = ReadExternalRawData(tensor_proto, model_path, raw_data_);
|
||||
ORT_ENFORCE(status.IsOK(), "ReadExternalRawData() failed: ", status.ErrorMessage());
|
||||
#else
|
||||
ORT_UNUSED_PARAMETER(model_path);
|
||||
ORT_THROW("External data is not supported in an ORT formal model.");
|
||||
#endif
|
||||
}
|
||||
}
|
||||
Initializer(const ONNX_NAMESPACE::TensorProto& tensor_proto,
|
||||
const Path& model_path);
|
||||
|
||||
~Initializer() = default;
|
||||
|
||||
void ToProto(ONNX_NAMESPACE::TensorProto& tensor_proto) {
|
||||
tensor_proto.clear_name();
|
||||
if (!name_.empty()) {
|
||||
tensor_proto.set_name(name_);
|
||||
}
|
||||
|
||||
tensor_proto.clear_dims();
|
||||
for (auto d : dims_) {
|
||||
tensor_proto.add_dims(d);
|
||||
}
|
||||
|
||||
tensor_proto.clear_data_type();
|
||||
tensor_proto.set_data_type(data_type_);
|
||||
|
||||
if (!raw_data_.empty()) {
|
||||
tensor_proto.clear_raw_data();
|
||||
tensor_proto.set_raw_data(raw_data_.data(), raw_data_.size());
|
||||
} else {
|
||||
switch (data_type_) {
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_FLOAT16:
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_BFLOAT16: {
|
||||
tensor_proto.clear_int32_data();
|
||||
for (int i = 0; i < size_; i++) {
|
||||
tensor_proto.add_int32_data(float16_data_[i]);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: {
|
||||
tensor_proto.clear_float_data();
|
||||
for (int i = 0; i < size_; i++) {
|
||||
tensor_proto.add_float_data(float_data_[i]);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_DOUBLE: {
|
||||
tensor_proto.clear_double_data();
|
||||
for (int i = 0; i < size_; i++) {
|
||||
tensor_proto.add_double_data(double_data_[i]);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_INT8: {
|
||||
tensor_proto.clear_int32_data();
|
||||
for (int i = 0; i < size_; i++) {
|
||||
tensor_proto.add_int32_data(int8_data_[i]);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_UINT8: {
|
||||
tensor_proto.clear_int32_data();
|
||||
for (int i = 0; i < size_; i++) {
|
||||
tensor_proto.add_int32_data(uint8_data_[i]);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_INT32: {
|
||||
tensor_proto.clear_int32_data();
|
||||
for (int i = 0; i < size_; i++) {
|
||||
tensor_proto.add_int32_data(int32_data_[i]);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_INT64: {
|
||||
tensor_proto.clear_int64_data();
|
||||
for (int i = 0; i < size_; i++) {
|
||||
tensor_proto.add_int64_data(int64_data_[i]);
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
ORT_NOT_IMPLEMENTED(__FUNCTION__, "data type is not supported");
|
||||
break;
|
||||
}
|
||||
}
|
||||
void ToProto(ONNX_NAMESPACE::TensorProto& tensor_proto) const {
|
||||
tensor_proto = utils::TensorToTensorProto(data_, name_);
|
||||
}
|
||||
|
||||
ONNX_NAMESPACE::TensorProto ToFP16(const std::string name) {
|
||||
ONNX_NAMESPACE::TensorProto tensor_proto;
|
||||
tensor_proto.set_name(name);
|
||||
ONNX_NAMESPACE::TensorProto ToFP16(const std::string& name) const;
|
||||
|
||||
for (auto d : dims_) {
|
||||
tensor_proto.add_dims(d);
|
||||
}
|
||||
|
||||
tensor_proto.set_data_type(ONNX_NAMESPACE::TensorProto_DataType_FLOAT16);
|
||||
|
||||
switch (data_type_) {
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_FLOAT16: {
|
||||
uint16_t* dst = data<uint16_t>();
|
||||
for (int i = 0; i < size_; i++) {
|
||||
tensor_proto.add_int32_data(dst[i]);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: {
|
||||
float* dst = data<float>();
|
||||
for (int i = 0; i < size_; i++) {
|
||||
tensor_proto.add_int32_data(math::floatToHalf(dst[i]));
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_DOUBLE: {
|
||||
double* dst = data<double>();
|
||||
for (int i = 0; i < size_; i++) {
|
||||
tensor_proto.add_int32_data(math::doubleToHalf(dst[i]));
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
ORT_NOT_IMPLEMENTED(__FUNCTION__, "data type is not supported");
|
||||
break;
|
||||
}
|
||||
return tensor_proto;
|
||||
}
|
||||
|
||||
ONNX_NAMESPACE::TensorProto ToBFloat16(const std::string name) {
|
||||
ONNX_NAMESPACE::TensorProto tensor_proto;
|
||||
tensor_proto.set_name(name);
|
||||
|
||||
for (auto d : dims_) {
|
||||
tensor_proto.add_dims(d);
|
||||
}
|
||||
|
||||
tensor_proto.set_data_type(ONNX_NAMESPACE::TensorProto_DataType_BFLOAT16);
|
||||
|
||||
switch (data_type_) {
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: {
|
||||
float* dst = data<float>();
|
||||
for (int i = 0; i < size_; i++) {
|
||||
tensor_proto.add_int32_data(BFloat16(dst[i]).val);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_DOUBLE: {
|
||||
double* dst = data<double>();
|
||||
for (int i = 0; i < size_; i++) {
|
||||
tensor_proto.add_int32_data(math::doubleToHalf(dst[i]));
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_BFLOAT16: {
|
||||
uint16_t* dst = data<uint16_t>();
|
||||
for (int i = 0; i < size_; i++) {
|
||||
tensor_proto.add_int32_data(dst[i]);
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
ORT_NOT_IMPLEMENTED(__FUNCTION__, "data type is not supported");
|
||||
break;
|
||||
}
|
||||
return tensor_proto;
|
||||
}
|
||||
ONNX_NAMESPACE::TensorProto ToBFloat16(const std::string& name) const;
|
||||
|
||||
int data_type() const {
|
||||
return data_type_;
|
||||
return data_.GetElementType();
|
||||
}
|
||||
|
||||
int& data_type() {
|
||||
return data_type_;
|
||||
}
|
||||
|
||||
const std::string& name() {
|
||||
std::string_view name() const {
|
||||
return name_;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
T* data() {
|
||||
if (!raw_data_.empty()) {
|
||||
return reinterpret_cast<T*>(raw_data_.data());
|
||||
}
|
||||
switch (data_type_) {
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_FLOAT16:
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_BFLOAT16: {
|
||||
return reinterpret_cast<T*>(float16_data_.data());
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: {
|
||||
return reinterpret_cast<T*>(float_data_.data());
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_DOUBLE: {
|
||||
return reinterpret_cast<T*>(double_data_.data());
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_INT8: {
|
||||
return reinterpret_cast<T*>(int8_data_.data());
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_UINT8: {
|
||||
return reinterpret_cast<T*>(uint8_data_.data());
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_INT32: {
|
||||
return reinterpret_cast<T*>(int32_data_.data());
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_INT64: {
|
||||
return reinterpret_cast<T*>(int64_data_.data());
|
||||
break;
|
||||
}
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
return nullptr;
|
||||
return data_.MutableData<T>();
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
const T* data() const {
|
||||
if (!raw_data_.empty()) {
|
||||
return reinterpret_cast<const T*>(raw_data_.data());
|
||||
}
|
||||
switch (data_type_) {
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_FLOAT16:
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_BFLOAT16: {
|
||||
return reinterpret_cast<const T*>(float16_data_.data());
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: {
|
||||
return reinterpret_cast<const T*>(float_data_.data());
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_DOUBLE: {
|
||||
return reinterpret_cast<const T*>(double_data_.data());
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_INT8: {
|
||||
return reinterpret_cast<const T*>(int8_data_.data());
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_UINT8: {
|
||||
return reinterpret_cast<const T*>(uint8_data_.data());
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_INT32: {
|
||||
return reinterpret_cast<const T*>(int32_data_.data());
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_INT64: {
|
||||
return reinterpret_cast<const T*>(int64_data_.data());
|
||||
break;
|
||||
}
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
return nullptr;
|
||||
return data_.Data<T>();
|
||||
}
|
||||
|
||||
const std::vector<int64_t>& dims() const {
|
||||
return dims_;
|
||||
const int8_t* raw_data() const {
|
||||
return reinterpret_cast<const int8_t*>(data_.DataRaw());
|
||||
}
|
||||
|
||||
const std::vector<int64_t>& dims() {
|
||||
return dims_;
|
||||
gsl::span<const int64_t> dims() const {
|
||||
return data_.Shape().GetDims();
|
||||
}
|
||||
|
||||
int64_t size() const { return size_; }
|
||||
int64_t size() const { return data_.Shape().Size(); }
|
||||
|
||||
Initializer& add(float value) {
|
||||
int64_t n = size();
|
||||
switch (data_type_) {
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_FLOAT16: {
|
||||
uint16_t* dst = data<uint16_t>();
|
||||
for (int i = 0; i < n; i++) {
|
||||
dst[i] = math::floatToHalf(math::halfToFloat(dst[i]) + value);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_BFLOAT16: {
|
||||
uint16_t* dst = data<uint16_t>();
|
||||
for (int i = 0; i < n; i++) {
|
||||
dst[i] = BFloat16((reinterpret_cast<BFloat16*>(dst + i))->ToFloat() + value).val;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: {
|
||||
float* dst = data<float>();
|
||||
for (int i = 0; i < n; i++) {
|
||||
dst[i] += value;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_DOUBLE: {
|
||||
double* dst = data<double>();
|
||||
for (int i = 0; i < n; i++) {
|
||||
dst[i] += value;
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
break;
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
Initializer& add(float value);
|
||||
|
||||
Initializer& add(const Initializer& other) {
|
||||
int64_t n = size();
|
||||
switch (data_type_) {
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_FLOAT16: {
|
||||
uint16_t* dst = data<uint16_t>();
|
||||
const uint16_t* src = other.data<uint16_t>();
|
||||
for (int i = 0; i < n; i++) {
|
||||
dst[i] = math::floatToHalf(math::halfToFloat(dst[i]) + math::halfToFloat(src[i]));
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_BFLOAT16: {
|
||||
uint16_t* dst = data<uint16_t>();
|
||||
const uint16_t* src = other.data<uint16_t>();
|
||||
for (int i = 0; i < n; i++) {
|
||||
dst[i] = BFloat16((reinterpret_cast<BFloat16*>(dst + i))->ToFloat() + (reinterpret_cast<const BFloat16*>(src + i))->ToFloat()).val;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: {
|
||||
float* dst = data<float>();
|
||||
const float* src = other.data<float>();
|
||||
for (int i = 0; i < n; i++) {
|
||||
dst[i] += src[i];
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_DOUBLE: {
|
||||
double* dst = data<double>();
|
||||
const double* src = other.data<double>();
|
||||
for (int i = 0; i < n; i++) {
|
||||
dst[i] += src[i];
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_INT32: {
|
||||
int32_t* dst = data<int32_t>();
|
||||
const int32_t* src = other.data<int32_t>();
|
||||
for (int i = 0; i < n; i++) {
|
||||
dst[i] += src[i];
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_INT64: {
|
||||
int64_t* dst = data<int64_t>();
|
||||
const int64_t* src = other.data<int64_t>();
|
||||
for (int i = 0; i < n; i++) {
|
||||
dst[i] += src[i];
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
break;
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
Initializer& sub(const Initializer& other) {
|
||||
int64_t n = size();
|
||||
switch (data_type_) {
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_FLOAT16: {
|
||||
uint16_t* dst = data<uint16_t>();
|
||||
const uint16_t* src = other.data<uint16_t>();
|
||||
for (int i = 0; i < n; i++) {
|
||||
dst[i] = math::floatToHalf(math::halfToFloat(dst[i]) - math::halfToFloat(src[i]));
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_BFLOAT16: {
|
||||
uint16_t* dst = data<uint16_t>();
|
||||
const uint16_t* src = other.data<uint16_t>();
|
||||
for (int i = 0; i < n; i++) {
|
||||
dst[i] = BFloat16((reinterpret_cast<BFloat16*>(dst + i))->ToFloat() - (reinterpret_cast<const BFloat16*>(src + i))->ToFloat()).val;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: {
|
||||
float* dst = data<float>();
|
||||
const float* src = other.data<float>();
|
||||
for (int i = 0; i < n; i++) {
|
||||
dst[i] -= src[i];
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_DOUBLE: {
|
||||
double* dst = data<double>();
|
||||
const double* src = other.data<double>();
|
||||
for (int i = 0; i < n; i++) {
|
||||
dst[i] -= src[i];
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_INT32: {
|
||||
int32_t* dst = data<int32_t>();
|
||||
const int32_t* src = other.data<int32_t>();
|
||||
for (int i = 0; i < n; i++) {
|
||||
dst[i] -= src[i];
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_INT64: {
|
||||
int64_t* dst = data<int64_t>();
|
||||
const int64_t* src = other.data<int64_t>();
|
||||
for (int i = 0; i < n; i++) {
|
||||
dst[i] -= src[i];
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
break;
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
Initializer& add(const Initializer& other);
|
||||
|
||||
Initializer& mul(const Initializer& other) {
|
||||
int64_t n = size();
|
||||
switch (data_type_) {
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_FLOAT16: {
|
||||
uint16_t* dst = data<uint16_t>();
|
||||
const uint16_t* src = other.data<uint16_t>();
|
||||
for (int i = 0; i < n; i++) {
|
||||
dst[i] = math::floatToHalf(math::halfToFloat(dst[i]) * math::halfToFloat(src[i]));
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_BFLOAT16: {
|
||||
uint16_t* dst = data<uint16_t>();
|
||||
const uint16_t* src = other.data<uint16_t>();
|
||||
for (int i = 0; i < n; i++) {
|
||||
dst[i] = BFloat16((reinterpret_cast<BFloat16*>(dst + i))->ToFloat() * (reinterpret_cast<const BFloat16*>(src + i))->ToFloat()).val;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: {
|
||||
float* dst = data<float>();
|
||||
const float* src = other.data<float>();
|
||||
for (int i = 0; i < n; i++) {
|
||||
dst[i] *= src[i];
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_DOUBLE: {
|
||||
double* dst = data<double>();
|
||||
const double* src = other.data<double>();
|
||||
for (int i = 0; i < n; i++) {
|
||||
dst[i] *= src[i];
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_INT32: {
|
||||
int32_t* dst = data<int32_t>();
|
||||
const int32_t* src = other.data<int32_t>();
|
||||
for (int i = 0; i < n; i++) {
|
||||
dst[i] *= src[i];
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_INT64: {
|
||||
int64_t* dst = data<int64_t>();
|
||||
const int64_t* src = other.data<int64_t>();
|
||||
for (int i = 0; i < n; i++) {
|
||||
dst[i] *= src[i];
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
break;
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
Initializer& div(const Initializer& other) {
|
||||
int64_t n = size();
|
||||
switch (data_type_) {
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_FLOAT16: {
|
||||
uint16_t* dst = data<uint16_t>();
|
||||
const uint16_t* src = other.data<uint16_t>();
|
||||
for (int i = 0; i < n; i++) {
|
||||
dst[i] = math::floatToHalf(math::halfToFloat(dst[i]) / math::halfToFloat(src[i]));
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_BFLOAT16: {
|
||||
uint16_t* dst = data<uint16_t>();
|
||||
const uint16_t* src = other.data<uint16_t>();
|
||||
for (int i = 0; i < n; i++) {
|
||||
dst[i] = BFloat16((reinterpret_cast<BFloat16*>(dst + i))->ToFloat() / (reinterpret_cast<const BFloat16*>(src + i))->ToFloat()).val;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: {
|
||||
float* dst = data<float>();
|
||||
const float* src = other.data<float>();
|
||||
for (int i = 0; i < n; i++) {
|
||||
dst[i] /= src[i];
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_DOUBLE: {
|
||||
double* dst = data<double>();
|
||||
const double* src = other.data<double>();
|
||||
for (int i = 0; i < n; i++) {
|
||||
dst[i] /= src[i];
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_INT32: {
|
||||
int32_t* dst = data<int32_t>();
|
||||
const int32_t* src = other.data<int32_t>();
|
||||
for (int i = 0; i < n; i++) {
|
||||
dst[i] /= src[i];
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_INT64: {
|
||||
int64_t* dst = data<int64_t>();
|
||||
const int64_t* src = other.data<int64_t>();
|
||||
for (int i = 0; i < n; i++) {
|
||||
dst[i] /= src[i];
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
break;
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
Initializer& sub(const Initializer& other);
|
||||
|
||||
Initializer& mul(const Initializer& other);
|
||||
|
||||
Initializer& sqrt() {
|
||||
int64_t n = size();
|
||||
switch (data_type_) {
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_FLOAT16: {
|
||||
uint16_t* dst = data<uint16_t>();
|
||||
for (int i = 0; i < n; i++) {
|
||||
dst[i] = math::floatToHalf(std::sqrt(math::halfToFloat(dst[i])));
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_BFLOAT16: {
|
||||
uint16_t* dst = data<uint16_t>();
|
||||
for (int i = 0; i < n; i++) {
|
||||
dst[i] = BFloat16(std::sqrt((reinterpret_cast<BFloat16*>(dst + i))->ToFloat())).val;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: {
|
||||
float* dst = data<float>();
|
||||
for (int i = 0; i < n; i++) {
|
||||
dst[i] = std::sqrt(dst[i]);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_DOUBLE: {
|
||||
double* dst = data<double>();
|
||||
for (int i = 0; i < n; i++) {
|
||||
dst[i] = std::sqrt(dst[i]);
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
break;
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
Initializer& div(const Initializer& other);
|
||||
|
||||
inline void scale_by_axis(const Initializer& other, int axis) {
|
||||
int64_t num = 1;
|
||||
for (size_t k = axis; k < dims_.size(); k++) {
|
||||
num *= dims_[k];
|
||||
}
|
||||
Initializer& sqrt();
|
||||
|
||||
int64_t n = size() / num;
|
||||
switch (data_type_) {
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_FLOAT16: {
|
||||
uint16_t* dst = data<uint16_t>();
|
||||
const uint16_t* src = other.data<uint16_t>();
|
||||
for (int i = 0; i < n; i++) {
|
||||
int index = other.size() == 1 ? 0 : i;
|
||||
for (int64_t j = 0; j < num; j++) {
|
||||
auto k = i * num + j;
|
||||
dst[k] = math::floatToHalf(math::halfToFloat(dst[k]) * math::halfToFloat(src[index]));
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_BFLOAT16: {
|
||||
uint16_t* dst = data<uint16_t>();
|
||||
const uint16_t* src = other.data<uint16_t>();
|
||||
for (int i = 0; i < n; i++) {
|
||||
int index = other.size() == 1 ? 0 : i;
|
||||
for (int64_t j = 0; j < num; j++) {
|
||||
auto k = i * num + j;
|
||||
dst[k] = BFloat16((reinterpret_cast<BFloat16*>(dst + k))->ToFloat() * (reinterpret_cast<const BFloat16*>(src + index))->ToFloat()).val;
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: {
|
||||
float* dst = data<float>();
|
||||
const float* src = other.data<float>();
|
||||
for (int i = 0; i < n; i++) {
|
||||
int index = other.size() == 1 ? 0 : i;
|
||||
for (int64_t j = 0; j < num; j++) {
|
||||
dst[i * num + j] *= src[index];
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_DOUBLE: {
|
||||
double* dst = data<double>();
|
||||
const double* src = other.data<double>();
|
||||
for (int i = 0; i < n; i++) {
|
||||
int index = other.size() == 1 ? 0 : i;
|
||||
for (int64_t j = 0; j < num; j++) {
|
||||
dst[i * num + j] *= src[index];
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_INT32: {
|
||||
int32_t* dst = data<int32_t>();
|
||||
const int32_t* src = other.data<int32_t>();
|
||||
for (int i = 0; i < n; i++) {
|
||||
int index = other.size() == 1 ? 0 : i;
|
||||
for (int64_t j = 0; j < num; j++) {
|
||||
dst[i * num + j] *= src[index];
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_INT64: {
|
||||
int64_t* dst = data<int64_t>();
|
||||
const int64_t* src = other.data<int64_t>();
|
||||
for (int i = 0; i < n; i++) {
|
||||
int index = other.size() == 1 ? 0 : i;
|
||||
for (int64_t j = 0; j < num; j++) {
|
||||
dst[i * num + j] *= src[index];
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
void scale_by_axis(const Initializer& other, int axis);
|
||||
|
||||
private:
|
||||
#if !defined(ORT_MINIMAL_BUILD)
|
||||
static Status ReadExternalRawData(
|
||||
const ONNX_NAMESPACE::TensorProto& tensor_proto, const Path& model_path, std::vector<char>& raw_data);
|
||||
#endif
|
||||
|
||||
int data_type_;
|
||||
std::string name_;
|
||||
std::vector<int64_t> dims_;
|
||||
int64_t size_;
|
||||
|
||||
std::vector<char> raw_data_;
|
||||
std::vector<float> float_data_;
|
||||
std::vector<uint16_t> float16_data_;
|
||||
std::vector<double> double_data_;
|
||||
std::vector<int8_t> int8_data_;
|
||||
std::vector<uint8_t> uint8_data_;
|
||||
std::vector<int32_t> int32_data_;
|
||||
std::vector<int64_t> int64_data_;
|
||||
Tensor data_;
|
||||
};
|
||||
|
||||
} // namespace onnxruntime
|
||||
|
|
|
|||
|
|
@ -54,7 +54,7 @@ bool IsQDQPairSupported(
|
|||
Initializer dq_scale(*dq_scale_tensor_proto, model_path);
|
||||
|
||||
return q_zp.data_type() == dq_zp.data_type() &&
|
||||
*q_zp.data<int8_t>() == *dq_zp.data<int8_t>() &&
|
||||
*q_zp.raw_data() == *dq_zp.raw_data() &&
|
||||
*q_scale.data<float>() == *dq_scale.data<float>();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@
|
|||
#include "core/common/inlined_containers.h"
|
||||
#include "core/framework/data_types.h"
|
||||
#include "core/framework/data_types_internal.h"
|
||||
#include "core/framework/float16.h"
|
||||
#include "core/graph/onnx_protobuf.h"
|
||||
#include "gtest/gtest.h"
|
||||
|
||||
|
|
@ -666,7 +667,7 @@ TEST_F(DataTypeTest, DataUtilsTest) {
|
|||
}
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
template <typename T>
|
||||
using Calc = CalculateInlinedVectorDefaultInlinedElements<T>;
|
||||
|
||||
template <typename... Types>
|
||||
|
|
@ -682,11 +683,34 @@ struct TypeMinimunInlinedElements {
|
|||
};
|
||||
|
||||
TEST(InlinedVectorTests, TestDefaultInlinedCapacity) {
|
||||
|
||||
// We want to test all the type here
|
||||
TypeMinimunInlinedElements<int8_t, int16_t, int32_t, int64_t, std::string> sizes;
|
||||
sizes.print(std::cout);
|
||||
|
||||
}
|
||||
|
||||
TEST(TypeLiterals, Tests) {
|
||||
{
|
||||
// uint16_t test
|
||||
MLFloat16 mlfloat{static_cast<uint16_t>(16)};
|
||||
auto mlfloat_literal = 16_f16;
|
||||
ASSERT_EQ(mlfloat, mlfloat_literal);
|
||||
|
||||
BFloat16 bfloat{static_cast<uint16_t>(16), BFloat16::FromBits()};
|
||||
auto bfloat_literal = 16_b16;
|
||||
ASSERT_EQ(bfloat, bfloat_literal);
|
||||
}
|
||||
|
||||
{
|
||||
// float
|
||||
MLFloat16 mlfloat{17.0f};
|
||||
auto mlfloat_literal = 17.0_fp16;
|
||||
ASSERT_EQ(mlfloat, mlfloat_literal);
|
||||
|
||||
BFloat16 bfloat{17.0f};
|
||||
auto bfloat_literal = 17.0_bfp16;
|
||||
ASSERT_EQ(bfloat, bfloat_literal);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace test
|
||||
} // namespace onnxruntime
|
||||
|
|
|
|||
|
|
@ -2075,13 +2075,13 @@ TEST_F(GraphTransformationTests, ReluClip11Fusion) {
|
|||
|
||||
// add initializer for min_input_1 so it's constant
|
||||
TensorProto const_min_1;
|
||||
Initializer i1(TensorProto_DataType_FLOAT16, "min_input_1", {1});
|
||||
Initializer i1(TensorProto_DataType_FLOAT16, "min_input_1", AsSpan<int64_t>({1}));
|
||||
i1.data<MLFloat16>()->val = math::floatToHalf(-1.f);
|
||||
i1.ToProto(const_min_1);
|
||||
graph.AddInitializedTensor(const_min_1);
|
||||
|
||||
TensorProto const_min_2;
|
||||
Initializer i2(TensorProto_DataType_FLOAT, "min_input_2", {1});
|
||||
Initializer i2(TensorProto_DataType_FLOAT, "min_input_2", AsSpan<int64_t>({1}));
|
||||
*i2.data<float>() = 1.f;
|
||||
i2.ToProto(const_min_2);
|
||||
graph.AddInitializedTensor(const_min_2);
|
||||
|
|
|
|||
|
|
@ -108,28 +108,54 @@ TEST(OptimizerInitializerTest, LoadExternalData) {
|
|||
}
|
||||
|
||||
template <typename T>
|
||||
ONNX_NAMESPACE::TensorProto_DataType GetTensorProtoDataType();
|
||||
constexpr ONNX_NAMESPACE::TensorProto_DataType GetTensorProtoDataType();
|
||||
|
||||
#define CppTypeToTensorProto_DataType(CppType, TP_DataType) \
|
||||
template <> \
|
||||
ONNX_NAMESPACE::TensorProto_DataType GetTensorProtoDataType<CppType>() { \
|
||||
return ONNX_NAMESPACE::TP_DataType; \
|
||||
#define CppTypeToTensorProto_DataType(CppType, TP_DataType) \
|
||||
template <> \
|
||||
constexpr ONNX_NAMESPACE::TensorProto_DataType GetTensorProtoDataType<CppType>() { \
|
||||
return ONNX_NAMESPACE::TP_DataType; \
|
||||
}
|
||||
|
||||
CppTypeToTensorProto_DataType(int8_t, TensorProto_DataType_INT8)
|
||||
CppTypeToTensorProto_DataType(uint8_t, TensorProto_DataType_UINT8)
|
||||
CppTypeToTensorProto_DataType(int32_t, TensorProto_DataType_INT32)
|
||||
CppTypeToTensorProto_DataType(int64_t, TensorProto_DataType_INT64)
|
||||
CppTypeToTensorProto_DataType(uint16_t, TensorProto_DataType_FLOAT16)
|
||||
CppTypeToTensorProto_DataType(float, TensorProto_DataType_FLOAT)
|
||||
CppTypeToTensorProto_DataType(double, TensorProto_DataType_DOUBLE)
|
||||
CppTypeToTensorProto_DataType(int8_t, TensorProto_DataType_INT8);
|
||||
CppTypeToTensorProto_DataType(uint8_t, TensorProto_DataType_UINT8);
|
||||
CppTypeToTensorProto_DataType(int32_t, TensorProto_DataType_INT32);
|
||||
CppTypeToTensorProto_DataType(int64_t, TensorProto_DataType_INT64);
|
||||
CppTypeToTensorProto_DataType(MLFloat16, TensorProto_DataType_FLOAT16);
|
||||
CppTypeToTensorProto_DataType(BFloat16, TensorProto_DataType_BFLOAT16);
|
||||
CppTypeToTensorProto_DataType(float, TensorProto_DataType_FLOAT);
|
||||
CppTypeToTensorProto_DataType(double, TensorProto_DataType_DOUBLE);
|
||||
|
||||
template <typename T>
|
||||
void TestInitializerRawData() {
|
||||
std::vector<T> GetInitializerData() {
|
||||
std::vector<T> data{
|
||||
0, 1, 2, 3,
|
||||
4, 5, 6, 7,
|
||||
8, 9, 10, 11};
|
||||
return data;
|
||||
}
|
||||
|
||||
template <>
|
||||
std::vector<MLFloat16> GetInitializerData<MLFloat16>() {
|
||||
std::vector<MLFloat16> data{
|
||||
0_f16, 1_f16, 2_f16, 3_f16,
|
||||
4_f16, 5_f16, 6_f16, 7_f16,
|
||||
8_f16, 9_f16, 10_f16, 11_f16};
|
||||
return data;
|
||||
}
|
||||
|
||||
template <>
|
||||
std::vector<BFloat16> GetInitializerData<BFloat16>() {
|
||||
std::vector<BFloat16> data{
|
||||
0_b16, 1_b16, 2_b16, 3_b16,
|
||||
4_b16, 5_b16, 6_b16, 7_b16,
|
||||
8_b16, 9_b16, 10_b16, 11_b16};
|
||||
return data;
|
||||
}
|
||||
|
||||
|
||||
template <typename T>
|
||||
void TestInitializerRawData() {
|
||||
std::vector<T> data = GetInitializerData<T>();
|
||||
|
||||
ONNX_NAMESPACE::TensorProto tensor_proto;
|
||||
tensor_proto.set_data_type(GetTensorProtoDataType<T>());
|
||||
|
|
@ -138,7 +164,7 @@ void TestInitializerRawData() {
|
|||
tensor_proto.add_dims(4);
|
||||
tensor_proto.set_raw_data(data.data(), data.size() * sizeof(T));
|
||||
|
||||
Initializer init(tensor_proto, Path());
|
||||
const Initializer init(tensor_proto, Path());
|
||||
|
||||
for (size_t idx = 0; idx < data.size(); idx++) {
|
||||
EXPECT_EQ(data[idx], init.data<T>()[idx]);
|
||||
|
|
@ -150,36 +176,51 @@ TEST(OptimizerInitializerTest, RawData) {
|
|||
TestInitializerRawData<uint8_t>();
|
||||
TestInitializerRawData<int32_t>();
|
||||
TestInitializerRawData<int64_t>();
|
||||
TestInitializerRawData<uint16_t>();
|
||||
TestInitializerRawData<MLFloat16>();
|
||||
TestInitializerRawData<BFloat16>();
|
||||
TestInitializerRawData<float>();
|
||||
TestInitializerRawData<double>();
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void TestInitializerDataField() {
|
||||
std::vector<T> data{
|
||||
0, 1, 2, 3,
|
||||
4, 5, 6, 7,
|
||||
8, 9, 10, 11};
|
||||
void AddData(const std::vector<T>& data, size_t idx, ONNX_NAMESPACE::TensorProto& tensor_proto) {
|
||||
tensor_proto.add_int32_data(data[idx]);
|
||||
}
|
||||
|
||||
template <>
|
||||
void AddData<MLFloat16>(const std::vector<MLFloat16>& data, size_t idx, ONNX_NAMESPACE::TensorProto& tensor_proto) {
|
||||
tensor_proto.add_int32_data(data[idx].val);
|
||||
}
|
||||
|
||||
template <>
|
||||
void AddData<BFloat16>(const std::vector<BFloat16>& data, size_t idx, ONNX_NAMESPACE::TensorProto& tensor_proto) {
|
||||
tensor_proto.add_int32_data(data[idx].val);
|
||||
}
|
||||
|
||||
|
||||
template <typename T>
|
||||
void TestInitializerDataField() {
|
||||
constexpr auto dt = GetTensorProtoDataType<T>();
|
||||
static_assert((dt == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT8 ||
|
||||
dt == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_UINT8 ||
|
||||
dt == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT32 ||
|
||||
dt == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT16 ||
|
||||
dt == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_BFLOAT16),
|
||||
"tensor type is not supported");
|
||||
|
||||
const std::vector<T> data = GetInitializerData<T>();
|
||||
|
||||
auto dt = GetTensorProtoDataType<T>();
|
||||
ONNX_NAMESPACE::TensorProto tensor_proto;
|
||||
tensor_proto.set_data_type(GetTensorProtoDataType<T>());
|
||||
tensor_proto.set_name("OptimizerInitializerTest_DataField");
|
||||
tensor_proto.add_dims(3);
|
||||
tensor_proto.add_dims(4);
|
||||
|
||||
for (size_t idx = 0; idx < data.size(); idx++) {
|
||||
if (dt == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT8 ||
|
||||
dt == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_UINT8 ||
|
||||
dt == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT32 ||
|
||||
dt == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT16) {
|
||||
tensor_proto.add_int32_data(data[idx]);
|
||||
} else {
|
||||
ORT_NOT_IMPLEMENTED("tensor type ", GetTensorProtoDataType<T>(), " is not supported");
|
||||
}
|
||||
AddData<T>(data, idx, tensor_proto);
|
||||
}
|
||||
|
||||
Initializer init(tensor_proto, Path());
|
||||
const Initializer init(tensor_proto, Path());
|
||||
|
||||
for (size_t idx = 0; idx < data.size(); idx++) {
|
||||
EXPECT_EQ(data[idx], init.data<T>()[idx]);
|
||||
|
|
@ -203,7 +244,7 @@ void TestInitializerDataField() {
|
|||
tensor_proto.add_##type##_data(data[idx]); \
|
||||
} \
|
||||
\
|
||||
Initializer init(tensor_proto, Path()); \
|
||||
const Initializer init(tensor_proto, Path()); \
|
||||
\
|
||||
for (size_t idx = 0; idx < data.size(); idx++) { \
|
||||
EXPECT_EQ(data[idx], init.data<type>()[idx]); \
|
||||
|
|
@ -211,16 +252,17 @@ void TestInitializerDataField() {
|
|||
}
|
||||
|
||||
typedef int64_t int64;
|
||||
TestInitializerDataFieldSpecialized(float)
|
||||
TestInitializerDataFieldSpecialized(double)
|
||||
TestInitializerDataFieldSpecialized(int64)
|
||||
TestInitializerDataFieldSpecialized(float);
|
||||
TestInitializerDataFieldSpecialized(double);
|
||||
TestInitializerDataFieldSpecialized(int64);
|
||||
|
||||
TEST(OptimizerInitializerTest, DataField) {
|
||||
TestInitializerDataField<int8_t>();
|
||||
TestInitializerDataField<uint8_t>();
|
||||
TestInitializerDataField<int32_t>();
|
||||
TestInitializerDataField<int64_t>();
|
||||
TestInitializerDataField<uint16_t>();
|
||||
TestInitializerDataField<MLFloat16>();
|
||||
TestInitializerDataField<BFloat16>();
|
||||
TestInitializerDataField<float>();
|
||||
TestInitializerDataField<double>();
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue