saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-29 20:14:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Make SparseTensor infrastructure optional (#8802)

Browse source 
Add cmake parameter and #ifdefs to allow for disabling sparse tensor support. This comes with a significant binary size cost so we want to be able to exclude it in a minimal build.
This commit is contained in:
Rachel Guo 2021-08-27 00:12:26 -07:00 committed by GitHub
parent cb59f46e04
commit 1886f1a737
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
50 changed files with 595 additions and 69 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
cmake/CMakeLists.txt
View file

@ -107,6 +107,7 @@ option(onnxruntime_USE_ROCM "Build with AMD GPU support" OFF)
# Options related to reducing the binary size produced by the build
option(onnxruntime_DISABLE_CONTRIB_OPS "Disable contrib ops" OFF)
option(onnxruntime_DISABLE_ML_OPS "Disable traditional ML ops" OFF)
option(onnxruntime_DISABLE_SPARSE_TENSORS "Disable sparse tensors data types" OFF)
cmake_dependent_option(onnxruntime_DISABLE_RTTI "Disable RTTI" ON "NOT onnxruntime_ENABLE_PYTHON" OFF)
# For now onnxruntime_DISABLE_EXCEPTIONS will only work with onnxruntime_MINIMAL_BUILD, more changes (ONNX, non-CPU EP, ...) are required to run this standalone
option(onnxruntime_DISABLE_EXCEPTIONS "Disable exception handling. Requires onnxruntime_MINIMAL_BUILD currently." OFF)
@ -758,6 +759,10 @@ if (onnxruntime_DISABLE_ML_OPS)
add_definitions(-DDISABLE_ML_OPS)
endif()
if (onnxruntime_DISABLE_SPARSE_TENSORS)
add_compile_definitions(DISABLE_SPARSE_TENSORS)
endif()
if (onnxruntime_USE_CUDA AND "${onnxruntime_CUDNN_HOME}" STREQUAL "")
message(FATAL_ERROR "onnxruntime_CUDNN_HOME required for onnxruntime_USE_CUDA")
endif()

16
include/onnxruntime/core/framework/data_types.h
View file

@ -53,7 +53,9 @@ using VectorInt64 = std::vector<int64_t>;
class DataTypeImpl;
class TensorTypeBase;
#if !defined(DISABLE_SPARSE_TENSORS)
class SparseTensorTypeBase;
#endif
class SequenceTensorTypeBase;
class NonTensorTypeBase;
class PrimitiveDataTypeBase;
@ -114,10 +116,12 @@ class DataTypeImpl {
return nullptr;
}
#if !defined(DISABLE_SPARSE_TENSORS)
// Returns this if this is of sparse-tensor-type and null otherwise
virtual const SparseTensorTypeBase* AsSparseTensorType() const {
return nullptr;
}
#endif
virtual const NonTensorTypeBase* AsNonTensorTypeBase() const {
return nullptr;
@ -140,9 +144,11 @@ class DataTypeImpl {
template <typename elemT>
static MLDataType GetSequenceTensorType();
#if !defined(DISABLE_SPARSE_TENSORS)
// Return the MLDataType for a concrete sparse tensor type.
template <typename elemT>
static MLDataType GetSparseTensorType();
#endif
/**
* Convert an ONNX TypeProto to onnxruntime DataTypeImpl.
@ -154,8 +160,10 @@ class DataTypeImpl {
static MLDataType TypeFromProto(const ONNX_NAMESPACE::TypeProto& proto);
static const TensorTypeBase* TensorTypeFromONNXEnum(int type);
static const SparseTensorTypeBase* SparseTensorTypeFromONNXEnum(int type);
static const NonTensorTypeBase* SequenceTensorTypeFromONNXEnum(int type);
#if !defined(DISABLE_SPARSE_TENSORS)
static const SparseTensorTypeBase* SparseTensorTypeFromONNXEnum(int type);
#endif
static const char* ToString(MLDataType type);
static std::vector<std::string> ToString(const std::vector<MLDataType>& types);
@ -282,6 +290,7 @@ struct IsTensorContainedType : public IsAnyOf<T, float, uint8_t, int8_t, uint16_
double, uint32_t, uint64_t, BFloat16> {
};
#if !defined(DISABLE_SPARSE_TENSORS)
/// Use "IsSparseTensorContainedType<T>::value" to test if a type T
/// is permitted as the element-type of a sparse-tensor.
@ -290,6 +299,7 @@ struct IsSparseTensorContainedType : public IsAnyOf<T, float, uint8_t, int8_t, u
int32_t, int64_t, std::string, bool, MLFloat16,
double, uint32_t, uint64_t, BFloat16> {
};
#endif
/// This template's Get() returns a corresponding MLDataType
/// It dispatches the call to either GetTensorType<>() or
@ -441,6 +451,7 @@ class TensorType : public TensorTypeBase {
}
};
#if !defined(DISABLE_SPARSE_TENSORS)
/// Common base-class for all sparse-tensors (with different element types).
class SparseTensorTypeBase : public DataTypeImpl {
public:
@ -500,6 +511,7 @@ class SparseTensorType : public SparseTensorTypeBase {
TensorElementTypeSetter<elemT>::SetSparseTensorElementType(mutable_type_proto());
}
};
#endif // !defined(DISABLE_SPARSE_TENSORS)
/**
* \brief Provide a specialization for your C++ Non-tensor type
@ -850,6 +862,7 @@ class PrimitiveDataType : public PrimitiveDataTypeBase {
return TensorType<ELEM_TYPE>::Type(); \
}
#if !defined(DISABLE_SPARSE_TENSORS)
#define ORT_REGISTER_SPARSE_TENSOR_TYPE(ELEM_TYPE) \
template <> \
MLDataType SparseTensorType<ELEM_TYPE>::Type() { \
@ -860,6 +873,7 @@ class PrimitiveDataType : public PrimitiveDataTypeBase {
MLDataType DataTypeImpl::GetSparseTensorType<ELEM_TYPE>() { \
return SparseTensorType<ELEM_TYPE>::Type(); \
}
#endif
#if !defined(DISABLE_ML_OPS)
#define ORT_REGISTER_MAP(TYPE) \

15
include/onnxruntime/core/framework/op_kernel.h
View file

@ -110,8 +110,8 @@ class OpKernel {
// @param used_shared_buffers: Boolean flag set by the kernel implementation indicating
// that the provided weight has been used by the kernel.
virtual Status UseSharedPrePackedBuffers(std::vector<BufferUniquePtr>& /*prepacked_buffers*/,
int /*input_idx*/,
/*out*/ bool& used_shared_buffers) {
int /*input_idx*/,
/*out*/ bool& used_shared_buffers) {
used_shared_buffers = false;
return Status::OK();
}
@ -327,12 +327,14 @@ struct BuildKernelDefConstraintsImpl {
}
};
template<typename... Types>
#if !defined(DISABLE_SPARSE_TENSORS)
template <typename... Types>
struct BuildKernelDefSparseConstraintsImpl {
std::vector<MLDataType> operator()() const {
return {DataTypeImpl::GetSparseTensorType<Types>()...};
}
};
#endif
// Use within macro definitions to create a custom vector of constraints.
// Example: #define REG_KERNEL(OP, VERSION, KERNEL_CLASS, Type, ...)
@ -342,10 +344,12 @@ inline std::vector<MLDataType> BuildKernelDefConstraints() {
return BuildKernelDefConstraintsImpl<Types...>{}();
}
#if !defined(DISABLE_SPARSE_TENSORS)
template <typename... Types>
inline std::vector<MLDataType> BuildKernelDefSparseConstraints() {
return BuildKernelDefSparseConstraintsImpl<Types...>{}();
}
#endif
// version of BuildKernelDefConstraints() which takes a type list
template <typename L>
@ -353,11 +357,12 @@ inline std::vector<MLDataType> BuildKernelDefConstraintsFromTypeList() {
return boost::mp11::mp_apply<BuildKernelDefConstraintsImpl, L>{}();
}
template<typename L>
#if !defined(DISABLE_SPARSE_TENSORS)
template <typename L>
inline std::vector<MLDataType> BuildKernelDefSparseConstraintsFromTypeList() {
return boost::mp11::mp_apply<BuildKernelDefSparseConstraintsImpl, L>{}();
}
#endif
} // namespace onnxruntime

4
include/onnxruntime/core/framework/op_kernel_context.h
View file

@ -69,11 +69,13 @@ class OpKernelContext {
return *output_ptr;
}
#if !defined(DISABLE_SPARSE_TENSORS)
// Fetch a sparse-tensor output corresponding to the specified index.
// shape must specify the shape of the underlying dense-tensor.
// Memory allocation for the output may happen when this method is invoked,
// unless static optimization pre-allocates it.
SparseTensor* OutputSparse(int index, const TensorShape& shape);
#endif
// Retrieve indexed shape obtained from memory planning before actual
// computation. If the indexed shape cannot be inferred, this function returns
@ -210,11 +212,13 @@ inline Tensor* OpKernelContext::Output<Tensor>(int index) {
return p_ml_value->GetMutable<Tensor>();
}
#if !defined(DISABLE_SPARSE_TENSORS)
template <>
inline SparseTensor* OpKernelContext::Output<SparseTensor>(int index) {
OrtValue* p_ml_value = GetOutputMLValue(index);
ORT_ENFORCE(p_ml_value, "Please fetch output sparse tensor with specified shape.");
return p_ml_value->GetMutable<SparseTensor>();
}
#endif
} // namespace onnxruntime

12
include/onnxruntime/core/framework/ort_value.h
View file

@ -13,12 +13,13 @@
#include "core/framework/TensorSeq.h"
namespace onnxruntime {
#if !defined(DISABLE_SPARSE_TENSORS)
class SparseTensor;
#endif
} // namespace onnxruntime
#endif
/**
Represents both tensors and non-tensors.
*/
@ -65,8 +66,12 @@ struct OrtValue {
return (type_ != nullptr && type_->IsTensorSequenceType());
}
bool IsSparseTensor() const noexcept {
bool IsSparseTensor() const {
#if !defined(DISABLE_SPARSE_TENSORS)
return (type_ != nullptr && type_->IsSparseTensorType());
#else
ORT_THROW("Sparse tensor is not supported in this build.");
#endif
}
onnxruntime::MLDataType Type() const {
@ -115,6 +120,7 @@ inline onnxruntime::TensorSeq* OrtValue::GetMutable<onnxruntime::TensorSeq>() {
return static_cast<onnxruntime::TensorSeq*>(data_.get());
}
#if !defined(DISABLE_SPARSE_TENSORS)
template <>
inline const onnxruntime::SparseTensor& OrtValue::Get<onnxruntime::SparseTensor>() const {
ORT_ENFORCE(IsSparseTensor(), "Trying to get a SparseTensor, but got: ", onnxruntime::DataTypeImpl::ToString(type_));
@ -126,4 +132,4 @@ inline onnxruntime::SparseTensor* OrtValue::GetMutable<onnxruntime::SparseTensor
ORT_ENFORCE(IsSparseTensor(), "Trying to get a SparseTensor, but got: ", onnxruntime::DataTypeImpl::ToString(type_));
return static_cast<onnxruntime::SparseTensor*>(data_.get());
}
#endif

4
include/onnxruntime/core/framework/sparse_tensor.h
View file

@ -1,6 +1,8 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#if !defined(DISABLE_SPARSE_TENSORS)
#pragma once
#include "core/framework/data_types.h"
@ -528,3 +530,5 @@ class SparseTensor final {
};
} // namespace onnxruntime
#endif

4
include/onnxruntime/core/graph/graph.h
View file

@ -331,7 +331,9 @@ class Node {
ADD_ATTR_INTERFACES(std::string)
ADD_ATTR_INTERFACES(ONNX_NAMESPACE::TensorProto)
ADD_ATTR_INTERFACES(ONNX_NAMESPACE::GraphProto)
#if !defined(DISABLE_SPARSE_TENSORS)
ADD_ATTR_INTERFACES(ONNX_NAMESPACE::SparseTensorProto)
#endif
ADD_ATTR_INTERFACES(ONNX_NAMESPACE::TypeProto)
/** Gets the Node's attributes. */
@ -625,11 +627,13 @@ class Graph {
/** Check if a given name is an initializer tensor's name in this graph. */
bool IsInitializedTensor(const std::string& name) const;
#if !defined(DISABLE_SPARSE_TENSORS)
/** Check if a given name is a sparse initializer's name in the model
* we currently convert sparse_initializer field in the model into dense Tensor instances.
* However, we sometimes want to check if this initializer was stored as sparse in the model.
*/
bool IsSparseInitializer(const std::string& name) const;
#endif
/** Gets an initializer tensor with the provided name.
@param[out] value Set to the TensorProto* if the initializer is found, or nullptr if not.

13
include/onnxruntime/core/session/onnxruntime_cxx_api.h
View file

@ -449,7 +449,7 @@ struct Value : Base<OrtValue> {
static Value CreateTensor(const OrtMemoryInfo* info, void* p_data, size_t p_data_byte_count, const int64_t* shape, size_t shape_len,
ONNXTensorElementDataType type);
#if !defined(DISABLE_SPARSE_TENSORS)
/// <summary>
/// This is a simple forwarding method to the other overload that helps deducing
/// data type enum value from the type of the buffer.
@ -515,10 +515,13 @@ struct Value : Base<OrtValue> {
/// <param name="indices_data">user allocated buffer with indices or nullptr for fully spare tensors</param>
void UseBlockSparseIndices(const Shape& indices_shape, int32_t* indices_data);
#endif // !defined(DISABLE_SPARSE_TENSORS)
template <typename T>
static Value CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len);
static Value CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len, ONNXTensorElementDataType type);
#if !defined(DISABLE_SPARSE_TENSORS)
/// <summary>
/// This is a simple forwarding method the below CreateSparseTensor.
/// This helps to specify data type enum in terms of C++ data type.
@ -622,6 +625,8 @@ struct Value : Base<OrtValue> {
template <typename T>
const T* GetSparseTensorIndicesData(OrtSparseIndicesFormat indices_format, size_t& num_indices) const;
#endif // !defined(DISABLE_SPARSE_TENSORS)
static Value CreateMap(Value& keys, Value& values);
static Value CreateSequence(std::vector<Value>& values);
@ -638,11 +643,13 @@ struct Value : Base<OrtValue> {
bool IsTensor() const;
#if !defined(DISABLE_SPARSE_TENSORS)
/// <summary>
/// Returns true if the OrtValue contains a sparse tensor
/// </summary>
/// <returns></returns>
bool IsSparseTensor() const;
#endif
size_t GetCount() const; // If a non tensor, returns 2 for map and N for sequence, where N is the number of elements
Value GetValue(int index, OrtAllocator* allocator) const;
@ -660,7 +667,7 @@ struct Value : Base<OrtValue> {
/// into a supplied buffer. Use GetStringTensorDataLength() to find out the length of the buffer to allocate.
/// The user must also allocate offsets buffer with the number of entries equal to that of the contained
/// strings.
///
///
/// Strings are always assumed to be on CPU, no X-device copy.
/// </summary>
/// <param name="buffer">user allocated buffer</param>
@ -677,6 +684,7 @@ struct Value : Base<OrtValue> {
template <typename T>
const T* GetTensorData() const;
#if !defined(DISABLE_SPARSE_TENSORS)
/// <summary>
/// The API returns a pointer to an internal buffer of the sparse tensor
/// containing non-zero values. The API merely does casting. Make sure you
@ -687,6 +695,7 @@ struct Value : Base<OrtValue> {
/// <returns>a pointer to the internal values buffer. Do not free this pointer.</returns>
template <typename T>
const T* GetSparseTensorValues() const;
#endif
template <typename T>
T& At(const std::vector<int64_t>& location);

8
include/onnxruntime/core/session/onnxruntime_cxx_inline.h
View file

@ -755,6 +755,7 @@ inline Value Value::CreateTensor(const OrtMemoryInfo* info, void* p_data, size_t
return Value{out};
}
#if !defined(DISABLE_SPARSE_TENSORS)
template <typename T>
inline Value Value::CreateSparseTensor(const OrtMemoryInfo* info, T* p_data, const Shape& dense_shape,
const Shape& values_shape) {
@ -830,6 +831,7 @@ inline const T* Value::GetSparseTensorIndicesData(OrtSparseIndicesFormat indices
ThrowOnError(GetApi().GetSparseTensorIndices(p_, indices_format, &num_indices, &out));
return reinterpret_cast<const T*>(out);
}
#endif // !defined(DISABLE_SPARSE_TENSORS)
template <typename T>
inline Value Value::CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len) {
@ -842,6 +844,7 @@ inline Value Value::CreateTensor(OrtAllocator* allocator, const int64_t* shape,
return Value{out};
}
#if !defined(DISABLE_SPARSE_TENSORS)
template <typename T>
inline Value Value::CreateSparseTensor(OrtAllocator* allocator, const Shape& dense_shape) {
return CreateSparseTensor(allocator, dense_shape, TypeToTensorType<T>::type);
@ -853,6 +856,7 @@ inline Value Value::CreateSparseTensor(OrtAllocator* allocator, const Shape& den
ThrowOnError(GetApi().CreateSparseTensorAsOrtValue(allocator, dense_shape.shape, dense_shape.shape_len, type, &out));
return Value{out};
}
#endif // !defined(DISABLE_SPARSE_TENSORS)
inline Value Value::CreateMap(Value& keys, Value& values) {
OrtValue* out;
@ -886,11 +890,13 @@ inline bool Value::IsTensor() const {
return out != 0;
}
#if !defined(DISABLE_SPARSE_TENSORS)
inline bool Value::IsSparseTensor() const {
int out;
ThrowOnError(GetApi().IsSparseTensor(p_, &out));
return out != 0;
}
#endif
inline size_t Value::GetCount() const {
size_t out;
@ -946,12 +952,14 @@ const T* Value::GetTensorData() const {
return out;
}
#if !defined(DISABLE_SPARSE_TENSORS)
template <typename T>
inline const T* Value::GetSparseTensorValues() const {
const void* out;
ThrowOnError(GetApi().GetSparseTensorValues(p_, &out));
return reinterpret_cast<const T*>(out);
}
#endif // !defined(DISABLE_SPARSE_TENSORS)
template <typename T>
inline T& Value::At(const std::vector<int64_t>& location) {

4
onnxruntime/contrib_ops/cpu/cpu_contrib_kernels.cc
View file

@ -23,7 +23,9 @@ class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, WordC
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, GatherND);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, TransposeMatMul); // backward compatibility
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, FusedMatMul);
#if !defined(DISABLE_SPARSE_TENSORS)
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, SparseToDenseMatMul);
#endif
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, MurmurHash3);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, MaxpoolWithMask);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, Pad);
@ -189,7 +191,9 @@ Status RegisterCpuContribKernels(KernelRegistry& kernel_registry) {
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, Range)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, WordConvEmbedding)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, GatherND)>,
#if !defined(DISABLE_SPARSE_TENSORS)
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, SparseToDenseMatMul)>,
#endif
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, MurmurHash3)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, TransposeMatMul)>, // backward compatibility
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, FusedMatMul)>,

4
onnxruntime/contrib_ops/cpu/math/sparse_dense_matmul.cc
View file

@ -1,6 +1,8 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#if !defined(DISABLE_SPARSE_TENSORS)
#include "core/framework/sparse_tensor.h"
#include "core/providers/cpu/math/gemm_matmul_common.h"
#include "core/providers/cpu/math/matmul_helper.h"
@ -208,3 +210,5 @@ Status SparseToDenseMatMul::Compute(OpKernelContext* ctx) const {
} // namespace contrib
} // namespace onnxruntime
#endif //!defined(DISABLE_SPARSE_TENSORS)

2
onnxruntime/core/framework/data_transfer.cc
View file

@ -23,12 +23,14 @@ common::Status IDataTransfer::CopyTensors(const std::vector<IDataTransfer::SrcDs
return Status::OK();
}
#if !defined(DISABLE_SPARSE_TENSORS)
common::Status IDataTransfer::CopySparseTensors(const std::vector<SparseSrcDstPair>& src_dst_pairs) const {
for (const auto& pair : src_dst_pairs) {
ORT_RETURN_IF_ERROR(pair.src.get().Copy(*this, pair.dst, pair.exec_queue_id));
}
return Status::OK();
}
#endif
bool CPUDataTransfer::CanCopy(const OrtDevice& src_device, const OrtDevice& dst_device) const {
return src_device.Type() == OrtDevice::CPU && dst_device.Type() == OrtDevice::CPU;

4
onnxruntime/core/framework/data_transfer.h
View file

@ -11,8 +11,10 @@ struct OrtDevice;
namespace onnxruntime {
#ifndef SHARED_PROVIDER
class Tensor;
#if !defined(DISABLE_SPARSE_TENSORS)
class SparseTensor;
#endif
#endif
namespace common {
class Status;
}
@ -36,6 +38,7 @@ class IDataTransfer {
// batched copy. default implementation copies each entry sequentially, and returns on first failure.
virtual common::Status CopyTensors(const std::vector<SrcDstPair>& src_dst_pairs) const;
#if !defined(DISABLE_SPARSE_TENSORS)
struct SparseSrcDstPair {
std::reference_wrapper<const SparseTensor> src;
std::reference_wrapper<SparseTensor> dst;
@ -43,6 +46,7 @@ class IDataTransfer {
};
virtual common::Status CopySparseTensors(const std::vector<SparseSrcDstPair>& src_dst_pairs) const;
#endif
};
class CPUDataTransfer : public IDataTransfer {

6
onnxruntime/core/framework/data_transfer_manager.cc
View file

@ -31,9 +31,11 @@ Status DataTransferManager::CopyTensor(const Tensor& src, Tensor& dst) const {
return CopyTensor(src, dst, 0);
}
#if !defined(DISABLE_SPARSE_TENSORS)
common::Status DataTransferManager::CopySparseTensor(const SparseTensor& src, SparseTensor& dst) const {
return CopySparseTensor(src, dst, 0);
}
#endif
Status DataTransferManager::CopyTensor(const Tensor& src, Tensor& dst, int exec_queue_id) const {
if (src.Shape().Size() != dst.Shape().Size()) {
@ -56,6 +58,7 @@ Status DataTransferManager::CopyTensor(const Tensor& src, Tensor& dst, int exec_
dst.Location().device.ToString());
}
#if !defined(DISABLE_SPARSE_TENSORS)
Status DataTransferManager::CopySparseTensor(const SparseTensor& src, SparseTensor& dst, int exec_queue_id) const {
if (src.DenseShape().Size() != dst.DenseShape().Size()) {
return Status(ONNXRUNTIME, FAIL, "Tensor size mismatch");
@ -76,6 +79,7 @@ Status DataTransferManager::CopySparseTensor(const SparseTensor& src, SparseTens
" to ",
dst.Location().device.ToString());
}
#endif
common::Status DataTransferManager::CopyTensors(const std::vector<IDataTransfer::SrcDstPair>& src_dst_pairs) const {
if (src_dst_pairs.empty())
@ -128,6 +132,7 @@ common::Status DataTransferManager::CopyTensors(const std::vector<IDataTransfer:
return Status::OK();
}
#if !defined(DISABLE_SPARSE_TENSORS)
common::Status DataTransferManager::CopySparseTensors(const std::vector<IDataTransfer::SparseSrcDstPair>& src_dst_pairs) const {
if (src_dst_pairs.empty())
return Status::OK();
@ -178,5 +183,6 @@ common::Status DataTransferManager::CopySparseTensors(const std::vector<IDataTra
return Status::OK();
}
#endif
} // namespace onnxruntime

2
onnxruntime/core/framework/data_transfer_manager.h
View file

@ -23,9 +23,11 @@ class DataTransferManager {
common::Status CopyTensor(const Tensor& src, Tensor& dst) const;
common::Status CopyTensor(const Tensor& src, Tensor& dst, int exec_queue_id) const;
common::Status CopyTensors(const std::vector<IDataTransfer::SrcDstPair>& src_dst_pairs) const;
#if !defined(DISABLE_SPARSE_TENSORS)
common::Status CopySparseTensor(const SparseTensor& src, SparseTensor& dst) const;
common::Status CopySparseTensor(const SparseTensor& src, SparseTensor& dst, int exec_queue_id) const;
common::Status CopySparseTensors(const std::vector<IDataTransfer::SparseSrcDstPair>& src_dst_pairs) const;
#endif
private:
ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(DataTransferManager);

28
onnxruntime/core/framework/data_types.cc
View file

@ -46,11 +46,13 @@ MLDataType DataTypeImpl::GetType<Tensor>() {
namespace onnxruntime {
#if !defined(DISABLE_SPARSE_TENSORS)
// Return the MLDataType used for a generic SparseTensor
template <>
MLDataType DataTypeImpl::GetType<SparseTensor>() {
return SparseTensorTypeBase::Type();
}
#endif
template <>
MLDataType DataTypeImpl::GetType<TensorSeq>() {
@ -69,9 +71,12 @@ struct TensorElementTypeSetter<T> {
static void SetTensorElementType(ONNX_NAMESPACE::TypeProto& proto) {
proto.mutable_tensor_type()->set_elem_type(utils::ToTensorProtoElementType<T>());
}
#if !defined(DISABLE_SPARSE_TENSORS)
static void SetSparseTensorElementType(ONNX_NAMESPACE::TypeProto& proto) {
proto.mutable_sparse_tensor_type()->set_elem_type(utils::ToTensorProtoElementType<T>());
}
#endif
#if !defined(DISABLE_ML_OPS)
static void SetMapKeyType(ONNX_NAMESPACE::TypeProto& proto) {
@ -136,8 +141,10 @@ void AssignOpaqueDomainName(const char* domain, const char* name,
bool IsCompatible(const ONNX_NAMESPACE::TypeProto_Tensor& tensor_proto,
const ONNX_NAMESPACE::TypeProto_Tensor& type_proto);
#if !defined(DISABLE_SPARSE_TENSORS)
bool IsCompatible(const ONNX_NAMESPACE::TypeProto_SparseTensor& tensor_proto,
const ONNX_NAMESPACE::TypeProto_SparseTensor& type_proto);
#endif
#if !defined(DISABLE_ML_OPS)
bool IsCompatible(const ONNX_NAMESPACE::TypeProto_Map& map_proto,
@ -180,9 +187,11 @@ bool IsCompatible(const ONNX_NAMESPACE::TypeProto_Map& map_proto,
case TypeProto::ValueCase::kOpaqueType:
result = IsCompatible(lhs.value_type().opaque_type(), rhs.value_type().opaque_type());
break;
#if !defined(DISABLE_SPARSE_TENSORS)
case TypeProto::ValueCase::kSparseTensorType:
result = IsCompatible(lhs.value_type().sparse_tensor_type(), rhs.value_type().sparse_tensor_type());
break;
#endif
default:
ORT_ENFORCE(false);
break;
@ -215,9 +224,11 @@ bool IsCompatible(const ONNX_NAMESPACE::TypeProto_Sequence& sequence_proto,
case TypeProto::ValueCase::kOpaqueType:
result = IsCompatible(lhs.elem_type().opaque_type(), rhs.elem_type().opaque_type());
break;
#if !defined(DISABLE_SPARSE_TENSORS)
case TypeProto::ValueCase::kSparseTensorType:
result = IsCompatible(lhs.elem_type().sparse_tensor_type(), rhs.elem_type().sparse_tensor_type());
break;
#endif
default:
ORT_ENFORCE(false);
break;
@ -227,6 +238,7 @@ bool IsCompatible(const ONNX_NAMESPACE::TypeProto_Sequence& sequence_proto,
}
return result;
}
bool IsCompatible(const ONNX_NAMESPACE::TypeProto_Opaque& opaque_proto,
const ONNX_NAMESPACE::TypeProto_Opaque& type_proto) {
const auto& lhs = opaque_proto;
@ -245,11 +257,12 @@ bool IsCompatible(const ONNX_NAMESPACE::TypeProto_Opaque& opaque_proto,
return !((lhs_name != rhs_name) ||
(lhs_name && rhs_name && lhs.name() != rhs.name()));
}
#if !defined(DISABLE_SPARSE_TENSORS)
bool IsCompatible(const ONNX_NAMESPACE::TypeProto_SparseTensor& tensor_proto,
const ONNX_NAMESPACE::TypeProto_SparseTensor& type_proto) {
return type_proto.elem_type() == tensor_proto.elem_type();
}
#endif
void RegisterAllProtos(const std::function<void(MLDataType)>& /*reg_fn*/);
@ -368,6 +381,8 @@ MLDataType TensorTypeBase::Type() {
return &tensor_base;
}
#if !defined(DISABLE_SPARSE_TENSORS)
/// SparseTensor
struct SparseTensorTypeBase::Impl : public data_types_internal::TypeProtoImpl {
@ -413,6 +428,7 @@ MLDataType SparseTensorTypeBase::Type() {
static SparseTensorTypeBase sparse_tensor_base;
return &sparse_tensor_base;
}
#endif // !defined(DISABLE_SPARSE_TENSORS)
///// SequenceTensorTypeBase
@ -544,6 +560,7 @@ ORT_REGISTER_TENSOR_TYPE(uint64_t);
ORT_REGISTER_TENSOR_TYPE(MLFloat16);
ORT_REGISTER_TENSOR_TYPE(BFloat16);
#if !defined(DISABLE_SPARSE_TENSORS)
ORT_REGISTER_SPARSE_TENSOR_TYPE(int32_t);
ORT_REGISTER_SPARSE_TENSOR_TYPE(float);
ORT_REGISTER_SPARSE_TENSOR_TYPE(bool);
@ -558,6 +575,7 @@ ORT_REGISTER_SPARSE_TENSOR_TYPE(uint32_t);
ORT_REGISTER_SPARSE_TENSOR_TYPE(uint64_t);
ORT_REGISTER_SPARSE_TENSOR_TYPE(MLFloat16);
ORT_REGISTER_SPARSE_TENSOR_TYPE(BFloat16);
#endif
#if !defined(DISABLE_ML_OPS)
ORT_REGISTER_MAP(MapStringToString);
@ -603,11 +621,13 @@ ORT_REGISTER_SEQ(VectorMapInt64ToFloat);
reg_fn(mltype); \
}
#if !defined(DISABLE_SPARSE_TENSORS)
#define REGISTER_SPARSE_TENSOR_PROTO(TYPE, reg_fn) \
{ \
MLDataType mltype = DataTypeImpl::GetSparseTensorType<TYPE>(); \
reg_fn(mltype); \
}
#endif
#define REGISTER_ONNX_PROTO(TYPE, reg_fn) \
{ \
@ -633,6 +653,7 @@ void RegisterAllProtos(const std::function<void(MLDataType)>& reg_fn) {
REGISTER_TENSOR_PROTO(MLFloat16, reg_fn);
REGISTER_TENSOR_PROTO(BFloat16, reg_fn);
#if !defined(DISABLE_SPARSE_TENSORS)
REGISTER_SPARSE_TENSOR_PROTO(int32_t, reg_fn);
REGISTER_SPARSE_TENSOR_PROTO(float, reg_fn);
REGISTER_SPARSE_TENSOR_PROTO(bool, reg_fn);
@ -647,6 +668,7 @@ void RegisterAllProtos(const std::function<void(MLDataType)>& reg_fn) {
REGISTER_SPARSE_TENSOR_PROTO(uint64_t, reg_fn);
REGISTER_SPARSE_TENSOR_PROTO(MLFloat16, reg_fn);
REGISTER_SPARSE_TENSOR_PROTO(BFloat16, reg_fn);
#endif
#if !defined(DISABLE_ML_OPS)
REGISTER_ONNX_PROTO(MapStringToString, reg_fn);
@ -819,6 +841,7 @@ const NonTensorTypeBase* DataTypeImpl::SequenceTensorTypeFromONNXEnum(int type)
}
}
#if !defined(DISABLE_SPARSE_TENSORS)
const SparseTensorTypeBase* DataTypeImpl::SparseTensorTypeFromONNXEnum(int type) {
switch (type) {
case TensorProto_DataType_FLOAT:
@ -830,7 +853,7 @@ const SparseTensorTypeBase* DataTypeImpl::SparseTensorTypeFromONNXEnum(int type)
case TensorProto_DataType_DOUBLE:
return reinterpret_cast<const SparseTensorTypeBase*>(DataTypeImpl::GetSparseTensorType<double>());
case TensorProto_DataType_STRING:
return reinterpret_cast<const SparseTensorTypeBase*>(DataTypeImpl::GetSparseTensorType<std::string>());
return reinterpret_cast<const SparseTensorTypeBase*>(DataTypeImpl::GetSparseTensorType<std::string>());
case TensorProto_DataType_UINT8:
return reinterpret_cast<const SparseTensorTypeBase*>(DataTypeImpl::GetSparseTensorType<uint8_t>());
case TensorProto_DataType_UINT16:
@ -853,6 +876,7 @@ const SparseTensorTypeBase* DataTypeImpl::SparseTensorTypeFromONNXEnum(int type)
ORT_NOT_IMPLEMENTED("sparse tensor type ", type, " is not supported");
}
}
#endif
MLDataType DataTypeImpl::TypeFromProto(const ONNX_NAMESPACE::TypeProto& proto) {
const auto& registry = data_types_internal::DataTypeRegistry::instance();

24
onnxruntime/core/framework/execution_frame.cc
View file

@ -153,10 +153,12 @@ Status IExecutionFrame::GetOrCreateNodeOutputMLValue(const int output_index, int
"OrtValue shape verification failed. Current shape:", tensor.Shape(),
" Requested shape:", shape ? shape->ToString() : "null");
} else if (p_ort_value->IsSparseTensor()) {
#if !defined(DISABLE_SPARSE_TENSORS)
const SparseTensor& sp_tensor = p_ort_value->Get<SparseTensor>();
ORT_ENFORCE(shape && sp_tensor.DenseShape() == *shape,
"OrtValue shape verification failed. Current shape:", sp_tensor.DenseShape(),
" Requested shape:", shape ? shape->ToString() : "null");
#endif
}
} else {
// shape is nullptr for traditional ML output values
@ -252,10 +254,11 @@ void IExecutionFrame::Init(const std::vector<int>& feed_mlvalue_idxs, const std:
if (IsOutput(ort_value_index)) {
std::string name;
ORT_THROW_IF_ERROR(ort_value_idx_map_.GetName(ort_value_index, name));
const bool is_sparse_initializer = is_initializer_sparse_func(name);
const Tensor& src = entry.second.Get<Tensor>(); // all initializers in ONNX are tensors
OrtValue& dest = all_values_[ort_value_index];
#if !defined(DISABLE_SPARSE_TENSORS)
const bool is_sparse_initializer = is_initializer_sparse_func(name);
if (is_sparse_initializer) {
if (!dest.IsAllocated()) {
auto p_tensor = std::make_unique<SparseTensor>();
@ -270,6 +273,7 @@ void IExecutionFrame::Init(const std::vector<int>& feed_mlvalue_idxs, const std:
cpu_allocator, allocator, has_linear_coo_index,
*dest.GetMutable<SparseTensor>()));
} else {
#endif // !defined(DISABLE_SPARSE_TENSORS)
if (!dest.IsAllocated()) {
// NOTE: This doesn't need to support ExecutionFrame custom allocators as they only come into play
// for a subgraph with an output of unknown shape that needs to be accumulated by the control flow node.
@ -278,7 +282,9 @@ void IExecutionFrame::Init(const std::vector<int>& feed_mlvalue_idxs, const std:
Tensor::InitOrtValue(src.DataType(), src.Shape(), std::move(allocator), dest);
}
ORT_THROW_IF_ERROR(CopyTensor(src, *dest.GetMutable<Tensor>()));
#if !defined(DISABLE_SPARSE_TENSORS)
}
#endif
} else {
all_values_[ort_value_index] = entry.second;
}
@ -327,6 +333,7 @@ ExecutionFrame::ExecutionFrame(const std::vector<int>& feed_mlvalue_idxs, const
planner_(nullptr) {
Init(
feed_mlvalue_idxs, feeds, session_state.GetInitializedTensors(),
#if !defined(DISABLE_SPARSE_TENSORS)
[&session_state](const std::string& name) -> bool {
int idx = -1;
if (session_state.GetOrtValueNameIdxMap().GetIdx(name, idx).IsOK()) {
@ -334,7 +341,13 @@ ExecutionFrame::ExecutionFrame(const std::vector<int>& feed_mlvalue_idxs, const
}
return false;
},
#else
[&](const std::string& /*name*/) -> bool {
return false;
},
#endif
fetches);
#if !defined(ORT_MINIMAL_BUILD) && defined(ORT_MEMORY_PROFILE)
MemoryInfo::IncreaseIteration();
#endif
@ -355,7 +368,7 @@ ExecutionFrame::ExecutionFrame(const std::vector<int>& feed_mlvalue_idxs, const
// and we have execution plan generated, try to setup
// memory pattern optimization.
if (session_state.GetEnableMemoryPattern() && session_state.GetExecutionPlan()) {
std::vector<std::reference_wrapper<const TensorShape>> input_shapes;
std::vector<std::reference_wrapper<const TensorShape> > input_shapes;
bool all_tensors = true;
// Reserve mem to avoid re-allocation.
input_shapes.reserve(feeds.size());
@ -614,6 +627,7 @@ static Status AllocateTensorSequence(OrtValue& ort_value) {
return Status::OK();
}
#if !defined(DISABLE_SPARSE_TENSORS)
static Status AllocateSparseTensor(OrtValue& mlvalue, const DataTypeImpl& ml_type, AllocatorPtr allocator,
const TensorShape& shape, bool create_fence,
const SessionState& session_state) {
@ -629,6 +643,7 @@ static Status AllocateSparseTensor(OrtValue& mlvalue, const DataTypeImpl& ml_typ
return Status::OK();
}
#endif
// This method is not thread safe!
Status ExecutionFrame::AllocateAsPerAllocationPlan(OrtValue& ort_value, int ort_value_index, const TensorShape* shape) {
@ -705,8 +720,13 @@ Status ExecutionFrame::AllocateAsPerAllocationPlan(OrtValue& ort_value, int ort_
return Status::OK();
} else if (ml_type->IsSparseTensorType()) {
#if !defined(DISABLE_SPARSE_TENSORS)
return AllocateSparseTensor(ort_value, *ml_type, GetAllocator(alloc_info),
*shape, per_alloc_plan.create_fence_if_async, session_state_);
#else
// Model load should have failed so this should be unreachable
ORT_THROW("SparseTensor is not supported in this build.");
#endif
} else if (ml_type->IsTensorSequenceType()) {
return AllocateTensorSequence(ort_value);
} else {

8
onnxruntime/core/framework/memcpy.cc
View file

@ -25,7 +25,9 @@ Status Memcpy::Compute(OpKernelContext* ctx) const {
" Input shape:", X->Shape(), " Output shape:", Y->Shape(),
" X data:", X->DataRaw(), " Y data:", Y->DataRaw());
}
} else if (input_type_0->IsSparseTensorType()) {
}
#if !defined(DISABLE_SPARSE_TENSORS)
else if (input_type_0->IsSparseTensorType()) {
const auto* X = ctx->Input<SparseTensor>(0);
SparseTensor* Y = ctx->OutputSparse(0, X->DenseShape());
retval = X->Copy(Info().GetDataTransferManager(), Info().GetKernelDef().ExecQueueId(), *Y);
@ -35,7 +37,9 @@ Status Memcpy::Compute(OpKernelContext* ctx) const {
" to ", Node().OutputDefs()[0]->Name(),
" Input shape:", X->DenseShape(), " Output shape:", Y->DenseShape());
}
} else {
}
#endif
else {
ORT_NOT_IMPLEMENTED("Input type not supported: ", DataTypeImpl::ToString(input_type_0));
}

18
onnxruntime/core/framework/onnxruntime_typeinfo.cc
View file

@ -20,7 +20,9 @@
using onnxruntime::BFloat16;
using onnxruntime::DataTypeImpl;
using onnxruntime::MLFloat16;
#if !defined(DISABLE_SPARSE_TENSORS)
using onnxruntime::SparseTensor;
#endif
using onnxruntime::Tensor;
using onnxruntime::TensorShape;
@ -119,6 +121,7 @@ OrtStatus* OrtTypeInfo::FromOrtValue(const OrtValue& value, OrtTypeInfo** out) {
}
if (type->IsSparseTensorType()) {
#if !defined(DISABLE_SPARSE_TENSORS)
OrtTensorTypeAndShapeInfo* info = nullptr;
const SparseTensor& tensor = value.Get<onnxruntime::SparseTensor>();
const auto* tensor_data_type = tensor.DataType();
@ -128,6 +131,9 @@ OrtStatus* OrtTypeInfo::FromOrtValue(const OrtValue& value, OrtTypeInfo** out) {
}
*out = new OrtTypeInfo(ONNX_TYPE_SPARSETENSOR, info);
return nullptr;
#else
return OrtApis::CreateStatus(ORT_FAIL, "SparseTensor is not supported in this build.");
#endif
}
if (type->IsTensorSequenceType()) {
@ -157,17 +163,21 @@ OrtStatus* OrtTypeInfo::FromOrtValue(const OrtValue& value, OrtTypeInfo** out) {
*out = new OrtTypeInfo(ONNX_TYPE_OPAQUE);
return nullptr;
}
#if !defined(DISABLE_ML_OPS)
case on::TypeProto::kMapType: {
return OrtTypeInfo::FromTypeProto(type_proto, out);
}
#endif
case on::TypeProto::kSequenceType: {
return OrtTypeInfo::FromTypeProto(type_proto, out);
}
// Real Tensor support
case on::TypeProto::kTensorType:
#if !defined(DISABLE_SPARSE_TENSORS)
case on::TypeProto::kSparseTensorType: {
return OrtApis::CreateStatus(ORT_FAIL, "Tensor types should have been handled already");
}
#endif
default:
// NOT_IMPLEMENTED
break;
@ -220,7 +230,9 @@ OrtStatus* OrtTypeInfo::FromTypeProto(const ONNX_NAMESPACE::TypeProto* input, Or
case on::TypeProto::kSparseTensorType: {
ONNXType ten_type = ONNX_TYPE_UNKNOWN;
const on::TypeProto_Tensor* tensor_type = nullptr;
#if !defined(DISABLE_SPARSE_TENSORS)
const on::TypeProto_SparseTensor* sparse_type = nullptr;
#endif
const on::TensorShapeProto* sp = nullptr;
if (value_case == on::TypeProto::kTensorType) {
tensor_type = &input->tensor_type();
@ -229,11 +241,13 @@ OrtStatus* OrtTypeInfo::FromTypeProto(const ONNX_NAMESPACE::TypeProto* input, Or
sp = &tensor_type->shape();
}
} else if (value_case == on::TypeProto::kSparseTensorType) {
#if !defined(DISABLE_SPARSE_TENSORS)
sparse_type = &input->sparse_tensor_type();
ten_type = ONNX_TYPE_SPARSETENSOR;
if (onnxruntime::utils::HasShape(*sparse_type)) {
sp = &sparse_type->shape();
}
#endif
}
OrtStatus* st = nullptr;
@ -312,6 +326,7 @@ OrtStatus* OrtTypeInfo::Clone(OrtTypeInfo** out) {
switch (type) {
case ONNX_TYPE_TENSOR:
case ONNX_TYPE_SPARSETENSOR: {
#if !defined(DISABLE_SPARSE_TENSORS)
OrtTensorTypeAndShapeInfo* clone;
if (auto status = data->Clone(&clone)) {
return status;
@ -319,6 +334,9 @@ OrtStatus* OrtTypeInfo::Clone(OrtTypeInfo** out) {
*out = new OrtTypeInfo(type, clone);
(*out)->denotation = denotation;
return nullptr;
#else
return OrtApis::CreateStatus(ORT_FAIL, "SparseTensor is not supported in this build.");
#endif
}
case ONNX_TYPE_SEQUENCE: {
OrtSequenceTypeInfo* clone;

2
onnxruntime/core/framework/op_kernel.cc
View file

@ -49,10 +49,12 @@ Tensor* OpKernelContext::Output(int index, const std::initializer_list<int64_t>&
return Output(index, TensorShape(shape));
}
#if !defined(DISABLE_SPARSE_TENSORS)
SparseTensor* OpKernelContext::OutputSparse(int index, const TensorShape& shape) {
auto p_ml_value = OutputMLValue(index, shape);
return p_ml_value ? p_ml_value->GetMutable<SparseTensor>() : nullptr;
}
#endif
bool OpKernelContext::TryGetInferredInputShape(int index, TensorShape& shape) const {
return execution_frame_->TryGetInferredShape(GetInputArgIndex(index), shape);

4
onnxruntime/core/framework/session_state.cc
View file

@ -201,9 +201,11 @@ Status SessionState::AddInitializedTensor(int ort_value_index, const OrtValue& o
constant_initialized_tensors_.insert({ort_value_index, ort_value});
}
#if !defined(DISABLE_SPARSE_TENSORS)
if (sparse) {
sparse_initialized_tensors_.insert(ort_value_index);
}
#endif
return Status::OK();
}
@ -214,9 +216,11 @@ const std::unordered_map<int, OrtValue>& SessionState::GetConstantInitializedTen
return constant_initialized_tensors_;
}
#if !defined(DISABLE_SPARSE_TENSORS)
bool SessionState::IsSparseInitializer(int ort_value_index) const {
return sparse_initialized_tensors_.count(ort_value_index) > 0;
}
#endif
#ifdef ENABLE_TRAINING
Status SessionState::GetInitializedTensors(

4
onnxruntime/core/framework/session_state.h
View file

@ -164,7 +164,9 @@ class SessionState {
*/
const std::unordered_map<int, OrtValue>& GetConstantInitializedTensors() const;
#if !defined(DISABLE_SPARSE_TENSORS)
bool IsSparseInitializer(int ort_value_index) const;
#endif
#ifdef ENABLE_TRAINING
/**
@ -436,11 +438,13 @@ class SessionState {
// subset of initialized_tensors_ that are constant and cannot be overridden at runtime
std::unordered_map<int, OrtValue> constant_initialized_tensors_;
#if !defined(DISABLE_SPARSE_TENSORS)
// This is an auxiliary lookup to check if the OrtValue was actually a sparse tensor
// this is needed because we currently convert all sparse initializer into dense Tensors
// if and when we actually place SparseTensor instances (we should) into OrtValues, we
// will not need this structure.
std::unordered_set<int> sparse_initialized_tensors_;
#endif
// This data structure is for uninitializing string tensors and
// munmap memory region and close file descriptor

4
onnxruntime/core/framework/session_state_utils.cc
View file

@ -253,8 +253,12 @@ common::Status SaveInitializedTensors(
// any outer scope value is shadowed by a local value and can't override it.
// due to that check_outer_scope is false
const bool constant = graph.IsConstantInitializer(name, /* check_outer_scope */ false);
#if !defined(DISABLE_SPARSE_TENSORS)
const bool sparse = graph.GetGraph().IsSparseInitializer(name);
ORT_RETURN_IF_ERROR(save_tensor_func(ort_value_index, ort_value, deleter, constant, sparse));
#else
ORT_RETURN_IF_ERROR(save_tensor_func(ort_value_index, ort_value, deleter, constant, false));
#endif
VLOGS(logger, 1) << "Added weight with name : " << name << " with index: " << ort_value_index;
}

4
onnxruntime/core/framework/sparse_tensor.cc
View file

@ -1,6 +1,8 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#if !defined(DISABLE_SPARSE_TENSORS)
#include "core/framework/data_types.h"
#include "core/framework/sparse_tensor.h"
#include "core/framework/data_transfer_manager.h"
@ -588,3 +590,5 @@ Status SparseTensor::Copy(const IDataTransfer& data_transfer, SparseTensor& dst_
}
} // namespace onnxruntime
#endif // !defined(DISABLE_SPARSE_TENSORS)

15
onnxruntime/core/framework/sparse_utils.cc
View file

@ -1,6 +1,8 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#if !defined(DISABLE_SPARSE_TENSORS)
#include "core/framework/sparse_utils.h"
#include "core/common/status.h"
#include "core/framework/tensor.h"
@ -308,8 +310,8 @@ Status SparseCooToDenseTensor(const DataTransferManager& data_manager, const Spa
const int64_t* indices = nullptr;
const auto num_values = src.Values().Shape().Size();
const auto num_indices = src.AsCoo().Indices().Shape().Size();
ORT_RETURN_IF_NOT((num_values == num_indices || 2 * num_values == num_indices),
"Expecting indices to be equal the number of values or be twice as many");
ORT_RETURN_IF_NOT((num_values == num_indices || 2 * num_values == num_indices),
"Expecting indices to be equal the number of values or be twice as many");
SparseTensor src_cpu;
if (src.Location().device.Type() != OrtDevice::CPU) {
@ -400,13 +402,12 @@ void ScanAndRecordCoo(gsl::span<const T> src_span,
}
}
++index;
}
}
}
}
Status DenseTensorToSparseCoo(const DataTransferManager& data_manager, const Tensor& src,
const AllocatorPtr& cpu_allocator,
const AllocatorPtr& dst_allocator, bool linear_index, SparseTensor& dst) {
const IDataTransfer* data_transfer = data_manager.GetDataTransfer(cpu_allocator->Info().device,
dst_allocator->Info().device);
ORT_RETURN_IF_NOT(data_transfer != nullptr, "Unable to find a data transfer for copying from device type: ",
@ -509,4 +510,6 @@ Status DenseTensorToSparseCoo(const DataTransferManager& data_manager, const Ten
}
} // namespace sparse_utils
} // namespace onnxruntime
} // namespace onnxruntime
#endif // !defined(DISABLE_SPARSE_TENSORS)

6
onnxruntime/core/framework/sparse_utils.h
View file

@ -1,6 +1,8 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#if !defined(DISABLE_SPARSE_TENSORS)
#pragma once
#include "core/framework/allocator.h"
@ -77,7 +79,7 @@ Status SparseCsrToDenseTensor(const DataTransferManager& data_manager, const Spa
/// <returns>Status instance</returns>
Status SparseCooToDenseTensor(const DataTransferManager& data_manager, const SparseTensor& src, const AllocatorPtr& cpu_allocator,
const AllocatorPtr& dst_allocator, Tensor& dst);
#endif //ORT_MINIMAL_BUILD
#endif //!ORT_MINIMAL_BUILD
/// <summary>
/// Convert Dense Tensor to COO format.
@ -97,3 +99,5 @@ Status DenseTensorToSparseCoo(const DataTransferManager& data_manager, const Ten
} // namespace sparse_utils
} // namespace onnxruntime
#endif //!defined(DISABLE_SPARSE_TENSORS)

18
onnxruntime/core/framework/tensor_type_and_shape.cc
View file

@ -20,7 +20,9 @@
using onnxruntime::BFloat16;
using onnxruntime::DataTypeImpl;
using onnxruntime::MLFloat16;
#if !defined(DISABLE_SPARSE_TENSORS)
using onnxruntime::SparseTensor;
#endif
using onnxruntime::Tensor;
ORT_API_STATUS_IMPL(OrtApis::CreateTensorTypeAndShapeInfo, _Outptr_ OrtTensorTypeAndShapeInfo** out) {
@ -214,9 +216,11 @@ ORT_API_STATUS_IMPL(OrtApis::GetTensorTypeAndShape, _In_ const OrtValue* v, _Out
shape = &tensor.Shape();
data_type = tensor.DataType();
} else {
#if !defined(DISABLE_SPARSE_TENSORS)
const SparseTensor& tensor = v->Get<onnxruntime::SparseTensor>();
shape = &tensor.DenseShape();
data_type = tensor.DataType();
#endif
}
return GetTensorShapeAndType(*shape, *data_type, out);
} else {
@ -228,12 +232,17 @@ ORT_API_STATUS_IMPL(OrtApis::GetTensorTypeAndShape, _In_ const OrtValue* v, _Out
ORT_API_STATUS_IMPL(OrtApis::GetSparseTensorValuesTypeAndShape, _In_ const OrtValue* v,
_Outptr_ OrtTensorTypeAndShapeInfo** out) {
API_IMPL_BEGIN
#if !defined(DISABLE_SPARSE_TENSORS)
const auto& sparse_tensor = SparseTensor::GetSparseTensorFromOrtValue(*v);
const auto& values = sparse_tensor.Values();
return GetTensorShapeAndType(values.Shape(), *values.DataType(), out);
#else
return OrtApis::CreateStatus(ORT_FAIL, "SparseTensor is not supported in this build.");
#endif
API_IMPL_END
}
#if !defined(DISABLE_SPARSE_TENSORS)
namespace {
const Tensor& GetIndicesTensor(const OrtValue& v, OrtSparseIndicesFormat indices_format) {
const auto& sparse_tensor = SparseTensor::GetSparseTensorFromOrtValue(v);
@ -257,22 +266,31 @@ const Tensor& GetIndicesTensor(const OrtValue& v, OrtSparseIndicesFormat indices
return *indices_tensor;
}
} // namespace
#endif // !defined(DISABLE_SPARSE_TENSORS)
ORT_API_STATUS_IMPL(OrtApis::GetSparseTensorIndicesTypeShape, _In_ const OrtValue* v,
OrtSparseIndicesFormat indices_format, _Outptr_ OrtTensorTypeAndShapeInfo** out) {
API_IMPL_BEGIN
#if !defined(DISABLE_SPARSE_TENSORS)
const Tensor& indices_tensor = GetIndicesTensor(*v, indices_format);
return GetTensorShapeAndType(indices_tensor.Shape(), *indices_tensor.DataType(), out);
#else
return OrtApis::CreateStatus(ORT_FAIL, "SparseTensor is not supported in this build.");
#endif
API_IMPL_END
}
ORT_API_STATUS_IMPL(OrtApis::GetSparseTensorIndices, _In_ const OrtValue* v,
enum OrtSparseIndicesFormat indices_format, _Out_ size_t* num_indices, _Outptr_ const void** indices) {
API_IMPL_BEGIN
#if !defined(DISABLE_SPARSE_TENSORS)
const Tensor& indices_tensor = GetIndicesTensor(*v, indices_format);
*num_indices = gsl::narrow<size_t>(indices_tensor.Shape().Size());
*indices = indices_tensor.DataRaw();
return nullptr;
#else
return OrtApis::CreateStatus(ORT_FAIL, "SparseTensor is not supported in this build.");
#endif
API_IMPL_END
}

9
onnxruntime/core/framework/tensorprotoutils.cc
View file

@ -823,11 +823,13 @@ common::Status ConstantNodeProtoToTensorProto(const ONNX_NAMESPACE::NodeProto& n
*tensor.mutable_string_data() = constant_attribute.strings();
break;
}
#if !defined(DISABLE_SPARSE_TENSORS)
case AttributeProto_AttributeType_SPARSE_TENSOR: {
auto& s = constant_attribute.sparse_tensor();
ORT_RETURN_IF_ERROR(SparseTensorProtoToDenseTensorProto(s, model_path, tensor));
break;
}
#endif
default:
ORT_THROW("Unsupported attribute value type of ", constant_attribute.type(),
" in 'Constant' node '", node.name(), "'");
@ -839,6 +841,7 @@ common::Status ConstantNodeProtoToTensorProto(const ONNX_NAMESPACE::NodeProto& n
return Status::OK();
}
#if !defined(DISABLE_SPARSE_TENSORS)
template <typename T>
static Status CopySparseData(size_t n_sparse_elements,
const ONNX_NAMESPACE::TensorProto& indices,
@ -902,14 +905,16 @@ static Status CopySparseData(size_t n_sparse_elements,
return status;
}
#endif // !defined(DISABLE_SPARSE_TENSORS)
namespace conversion_internal {
#if !defined(DISABLE_SPARSE_TENSORS)
struct UnsupportedSparseDataType {
void operator()(int32_t dt_type, Status& status) const {
status = ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Unsupported sparse tensor data type of ", dt_type);
}
};
#endif
template <typename T>
struct GetElementSize {
Status operator()(size_t& element_size) const {
@ -922,6 +927,7 @@ using SupportedConversionTypeList = onnxruntime::TypeList<float, double, MLFloat
int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t, int64_t, uint64_t>;
} // namespace conversion_internal
#if !defined(DISABLE_SPARSE_TENSORS)
common::Status SparseTensorProtoToDenseTensorProto(const ONNX_NAMESPACE::SparseTensorProto& sparse,
const Path& model_path,
ONNX_NAMESPACE::TensorProto& dense) {
@ -1150,6 +1156,7 @@ common::Status DenseTensorToSparseTensorProto(const ONNX_NAMESPACE::TensorProto&
}
#endif // !ORT_MINIMAL_BUILD
#endif // !defined(DISABLE_SPARSE_TENSORS)
template common::Status GetSizeInBytesFromTensorProto<kAllocAlignment>(const ONNX_NAMESPACE::TensorProto& tensor_proto,
size_t* out);

32
onnxruntime/core/framework/tensorprotoutils.h
View file

@ -85,6 +85,7 @@ common::Status ConstantNodeProtoToTensorProto(const ONNX_NAMESPACE::NodeProto& n
const Path& model_path,
ONNX_NAMESPACE::TensorProto& tensor);
#if !defined(DISABLE_SPARSE_TENSORS)
// Convert a SparseTensorProto to a dense TensorProto
// If the SparseTensorProto contains external data then it loads the data and converts to dense tensor proto
// The resulting TensorProto will contain the data as raw data.
@ -102,6 +103,7 @@ common::Status DenseTensorToSparseTensorProto(const ONNX_NAMESPACE::TensorProto&
const Path& model_path,
ONNX_NAMESPACE::SparseTensorProto& sparse);
#endif // !ORT_MINIMAL_BUILD
#endif // !defined(DISABLE_SPARSE_TENSORS)
#endif
inline bool HasDimValue(const ONNX_NAMESPACE::TensorShapeProto_Dimension& dim) {
@ -125,6 +127,7 @@ inline bool HasShape(const ONNX_NAMESPACE::TypeProto_Tensor& ten_proto) {
return ten_proto.has_shape();
}
#if !defined(DISABLE_SPARSE_TENSORS)
inline bool HasSparseTensorType(const ONNX_NAMESPACE::TypeProto& type_proto) {
return type_proto.value_case() == ONNX_NAMESPACE::TypeProto::kSparseTensorType;
}
@ -137,28 +140,41 @@ inline bool HasShape(const ONNX_NAMESPACE::TypeProto_SparseTensor& ten_proto) {
inline bool HasElemType(const ONNX_NAMESPACE::TypeProto_SparseTensor& ten_proto) {
return ten_proto.elem_type() != ONNX_NAMESPACE::TensorProto::UNDEFINED;
}
inline bool HasShape(const ONNX_NAMESPACE::TypeProto& type_proto) {
if (HasTensorType(type_proto) && HasShape(type_proto.tensor_type())) {
return true;
}
return HasSparseTensorType(type_proto) && HasShape(type_proto.sparse_tensor_type());
}
#endif // !defined(DISABLE_SPARSE_TENSORS)
inline bool HasElementType(const ONNX_NAMESPACE::TypeProto& type_proto) {
if (HasTensorType(type_proto) && HasElemType(type_proto.tensor_type())) {
return true;
}
return HasSparseTensorType(type_proto) && HasElemType(type_proto.sparse_tensor_type());
#if !defined(DISABLE_SPARSE_TENSORS)
if (HasSparseTensorType(type_proto) && HasElemType(type_proto.sparse_tensor_type())) {
return true;
}
#endif // !defined(DISABLE_SPARSE_TENSORS)
return false;
}
inline bool HasShape(const ONNX_NAMESPACE::TypeProto& type_proto) {
if (HasTensorType(type_proto) && HasShape(type_proto.tensor_type())) {
return true;
}
#if !defined(DISABLE_SPARSE_TENSORS)
if (HasSparseTensorType(type_proto) && HasShape(type_proto.sparse_tensor_type())) {
return true;
}
#endif
return false;
}
inline const ONNX_NAMESPACE::TensorShapeProto& GetShape(const ONNX_NAMESPACE::TypeProto& type_proto) {
if (HasTensorType(type_proto) && HasShape(type_proto.tensor_type())) {
return type_proto.tensor_type().shape();
}
#if !defined(DISABLE_SPARSE_TENSORS)
if (HasSparseTensorType(type_proto) && HasShape(type_proto.sparse_tensor_type())) {
return type_proto.sparse_tensor_type().shape();
}
#endif
ORT_THROW("TypeProto must have shape for this to run");
}

40
onnxruntime/core/framework/utils.cc
View file

@ -139,8 +139,10 @@ static common::Status AllocateHelper(const AllocatorPtr& allocator,
source_tensor.Shape(),
allocator, target_mlvalue);
} else if (source_mlvalue.IsSparseTensor()) {
#if !defined(DISABLE_SPARSE_TENSORS)
const SparseTensor& source_tensor = source_mlvalue.Get<SparseTensor>();
SparseTensor::InitOrtValue(source_tensor.DataType(), source_tensor.DenseShape(), allocator, target_mlvalue);
#endif
} else if (source_mlvalue.IsTensorSequence()) {
const TensorSeq& source_tensor_seq = source_mlvalue.Get<TensorSeq>();
auto target_tensor_seq = std::make_unique<TensorSeq>(source_tensor_seq.DataType());
@ -182,8 +184,13 @@ static Status BatchOrCopyMLValue(const SessionState& session_state,
const MLValueCopyInfo& copy_info,
const OrtValue& source_mlvalue,
OrtValue& target_mlvalue,
#if !defined(DISABLE_SPARSE_TENSORS)
std::vector<IDataTransfer::SrcDstPair>* copy_tensor_pairs = nullptr,
std::vector<IDataTransfer::SparseSrcDstPair>* copy_sparse_pairs = nullptr) {
std::vector<IDataTransfer::SparseSrcDstPair>* copy_sparse_pairs = nullptr)
#else
std::vector<IDataTransfer::SrcDstPair>* copy_tensor_pairs = nullptr)
#endif
{
// same device so direct copy
if (copy_info.source_device == copy_info.target_device) {
target_mlvalue = source_mlvalue;
@ -206,6 +213,7 @@ static Status BatchOrCopyMLValue(const SessionState& session_state,
ORT_RETURN_IF_ERROR(session_state.GetDataTransferMgr().CopyTensor(source_tensor, *p_output_tensor));
}
} else if (source_mlvalue.IsSparseTensor()) {
#if !defined(DISABLE_SPARSE_TENSORS)
const auto& source_tensor = source_mlvalue.Get<SparseTensor>();
SparseTensor* p_output_tensor = target_mlvalue.GetMutable<SparseTensor>();
if (copy_sparse_pairs != nullptr) {
@ -213,6 +221,7 @@ static Status BatchOrCopyMLValue(const SessionState& session_state,
} else {
ORT_RETURN_IF_ERROR(session_state.GetDataTransferMgr().CopySparseTensor(source_tensor, *p_output_tensor));
}
#endif
} else if (source_mlvalue.IsTensorSequence()) {
const TensorSeq& source_tensor_seq = source_mlvalue.Get<TensorSeq>();
TensorSeq& target_tensor_seq = const_cast<TensorSeq&>(target_mlvalue.Get<TensorSeq>());
@ -242,7 +251,7 @@ static Status BatchOrCopyMLValue(const SessionState& session_state,
}
return Status::OK();
}
} // namespace utils
static bool HaveCpuExecutionProvidersOnly(const ExecutionProviders& execution_providers) {
for (const auto& execution_provider : execution_providers) {
@ -448,7 +457,9 @@ static void FinalizeFeedFetchCopyInfo(FeedsFetchesManager& feeds_fetches_manager
if (feed.IsTensor()) {
feed_locations[i] = feed.Get<Tensor>().Location().device;
} else if (feed.IsSparseTensor()) {
#if !defined(DISABLE_SPARSE_TENSORS)
feed_locations[i] = feed.Get<SparseTensor>().Location().device;
#endif
}
}
@ -461,7 +472,9 @@ static void FinalizeFeedFetchCopyInfo(FeedsFetchesManager& feeds_fetches_manager
if (fetch.IsTensor()) {
fetch_alloc_info[i] = &fetch.Get<Tensor>().Location();
} else if (fetch.IsSparseTensor()) {
#if !defined(DISABLE_SPARSE_TENSORS)
fetch_alloc_info[i] = &fetch.Get<SparseTensor>().Location();
#endif
}
}
}
@ -478,20 +491,29 @@ static common::Status CopyInputsAcrossDevices(const SessionState& session_state,
new_feeds.resize(num_feeds);
std::vector<IDataTransfer::SrcDstPair> batched_data_transfers;
#if !defined(DISABLE_SPARSE_TENSORS)
std::vector<IDataTransfer::SparseSrcDstPair> batched_sparse_data_transfers;
#endif
for (size_t idx = 0; idx < num_feeds; ++idx) {
#if !defined(DISABLE_SPARSE_TENSORS)
ORT_RETURN_IF_ERROR(BatchOrCopyMLValue(session_state, copy_info[idx], orig_feeds[idx], new_feeds[idx],
&batched_data_transfers, &batched_sparse_data_transfers));
#else
ORT_RETURN_IF_ERROR(BatchOrCopyMLValue(session_state, copy_info[idx], orig_feeds[idx], new_feeds[idx],
&batched_data_transfers));
#endif
}
if (!batched_data_transfers.empty()) {
ORT_RETURN_IF_ERROR(session_state.GetDataTransferMgr().CopyTensors(batched_data_transfers));
}
#if !defined(DISABLE_SPARSE_TENSORS)
if (!batched_sparse_data_transfers.empty()) {
ORT_RETURN_IF_ERROR(session_state.GetDataTransferMgr().CopySparseTensors(batched_sparse_data_transfers));
}
#endif
return Status::OK();
}
@ -506,9 +528,13 @@ common::Status CopyOneInputAcrossDevices(const SessionState& session_state, cons
MLValueCopyInfo copy_info;
ORT_RETURN_IF_ERROR(CalculateStaticCopyInfoForFeed(session_state, input_name, copy_info));
#if !defined(DISABLE_SPARSE_TENSORS)
copy_info.source_device = (orig_mlvalue.IsTensor())
? orig_mlvalue.Get<Tensor>().Location().device
: orig_mlvalue.Get<SparseTensor>().Location().device;
#else
copy_info.source_device = orig_mlvalue.Get<Tensor>().Location().device;
#endif
return BatchOrCopyMLValue(session_state, copy_info, orig_mlvalue, new_mlvalue);
}
@ -521,20 +547,29 @@ static common::Status CopyOutputsAcrossDevices(const SessionState& session_state
user_fetches.resize(num_outputs);
std::vector<IDataTransfer::SrcDstPair> batched_data_transfers;
#if !defined(DISABLE_SPARSE_TENSORS)
std::vector<IDataTransfer::SparseSrcDstPair> batched_sparse_data_transfers;
#endif
for (size_t idx = 0; idx < num_outputs; ++idx) {
#if !defined(DISABLE_SPARSE_TENSORS)
ORT_RETURN_IF_ERROR(BatchOrCopyMLValue(session_state, copy_info[idx], fetches[idx], user_fetches[idx],
&batched_data_transfers, &batched_sparse_data_transfers));
#else
ORT_RETURN_IF_ERROR(BatchOrCopyMLValue(session_state, copy_info[idx], fetches[idx], user_fetches[idx],
&batched_data_transfers));
#endif
}
if (!batched_data_transfers.empty()) {
ORT_RETURN_IF_ERROR(session_state.GetDataTransferMgr().CopyTensors(batched_data_transfers));
}
#if !defined(DISABLE_SPARSE_TENSORS)
if (!batched_sparse_data_transfers.empty()) {
ORT_RETURN_IF_ERROR(session_state.GetDataTransferMgr().CopySparseTensors(batched_sparse_data_transfers));
}
#endif
return Status::OK();
}
@ -633,7 +668,6 @@ common::Status ExecutePartialGraph(const SessionState& session_state, FeedsFetch
const std::vector<OrtValue>& feeds, std::vector<OrtValue>& fetches,
const logging::Logger& logger, PartialGraphExecutionState& state,
const OrtValueCachePtr& cache) {
// finalize the copy info using the provided feeds and fetches. will update device_copy_checks in the background
FinalizeFeedFetchCopyInfo(feeds_fetches_manager, feeds, fetches);
PartialExecutor executor{state, cache};

102
onnxruntime/core/graph/graph.cc
View file

@ -69,20 +69,30 @@ static bool UsingLatestOnnxOpset(const DomainToVersionMap& opset_versions) {
static Status MergeShapeInfo(const std::string& output_name,
const TypeProto& source, TypeProto& target,
bool strict, const logging::Logger& logger) {
#if !defined(DISABLE_SPARSE_TENSORS)
if (!(utils::HasTensorType(source) && utils::HasTensorType(target)) &&
!(utils::HasSparseTensorType(source) && utils::HasSparseTensorType(target))) {
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL,
"Source and target must both be either tensors or sparse tensors");
}
#else
if (!(utils::HasTensorType(source) && utils::HasTensorType(target))) {
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL,
"Source and target must both be tensors");
}
#endif
auto status = Status::OK();
ORT_TRY {
if (utils::HasTensorType(source)) {
ONNX_NAMESPACE::mergeInShapeInfo(source.tensor_type(), *target.mutable_tensor_type());
} else {
}
#if !defined(DISABLE_SPARSE_TENSORS)
else {
ONNX_NAMESPACE::mergeInShapeInfo(source.sparse_tensor_type(), *target.mutable_sparse_tensor_type());
}
#endif
}
ORT_CATCH(const ONNX_NAMESPACE::InferenceError& ex) {
// if this model was not created with the latest onnx version, allow the shape inferencing failure (strict == false).
@ -97,9 +107,12 @@ static Status MergeShapeInfo(const std::string& output_name,
<< ". Falling back to lenient merge.";
if (utils::HasTensorType(source)) {
ONNX_NAMESPACE::UnionShapeInfo(utils::GetShape(source), *target.mutable_tensor_type());
} else {
}
#if !defined(DISABLE_SPARSE_TENSORS)
else {
ONNX_NAMESPACE::UnionShapeInfo(utils::GetShape(source), *target.mutable_sparse_tensor_type());
}
#endif
} else {
ORT_UNUSED_PARAMETER(logger);
ORT_UNUSED_PARAMETER(strict);
@ -205,12 +218,14 @@ const TensorShapeProto* NodeArg::Shape() const {
}
return nullptr;
}
#if !defined(DISABLE_SPARSE_TENSORS)
case TypeProto::kSparseTensorType: {
if (utils::HasShape(type->sparse_tensor_type())) {
return &(type->sparse_tensor_type().shape());
}
return nullptr;
}
#endif
case TypeProto::kSequenceType:
case TypeProto::kMapType:
case TypeProto::kOpaqueType:
@ -226,11 +241,13 @@ bool NodeArg::HasTensorOrScalarShape() const {
const auto type_case = type->value_case();
switch (type_case) {
case TypeProto::kTensorType:
#if !defined(DISABLE_SPARSE_TENSORS)
case TypeProto::kSparseTensorType:
// Standard tensor has a valid shape field while
// scalar's shape is empty. Thus, we don't need to
// check shape here.
return true;
#endif
case TypeProto::kSequenceType:
case TypeProto::kMapType:
case TypeProto::kOpaqueType:
@ -247,9 +264,11 @@ void NodeArg::SetShape(const TensorShapeProto& shape) {
case TypeProto::kTensorType:
*(node_arg_info_.mutable_type()->mutable_tensor_type()->mutable_shape()) = shape;
break;
#if !defined(DISABLE_SPARSE_TENSORS)
case TypeProto::kSparseTensorType:
*(node_arg_info_.mutable_type()->mutable_sparse_tensor_type()->mutable_shape()) = shape;
break;
#endif
case TypeProto::kSequenceType:
case TypeProto::kMapType:
case TypeProto::kOpaqueType:
@ -265,9 +284,11 @@ void NodeArg::ClearShape() {
case TypeProto::kTensorType:
node_arg_info_.mutable_type()->mutable_tensor_type()->clear_shape();
break;
#if !defined(DISABLE_SPARSE_TENSORS)
case TypeProto::kSparseTensorType:
node_arg_info_.mutable_type()->mutable_sparse_tensor_type()->clear_shape();
break;
#endif
case TypeProto::kSequenceType:
case TypeProto::kMapType:
case TypeProto::kOpaqueType:
@ -328,6 +349,8 @@ common::Status NodeArg::UpdateTypeAndShape(const ONNX_NAMESPACE::TypeProto& inpu
break;
}
#if !defined(DISABLE_SPARSE_TENSORS)
case TypeProto::kSparseTensorType: {
const auto& input_tensor_type = input_type.sparse_tensor_type();
const auto input_tensor_elem_type = input_tensor_type.elem_type();
@ -357,12 +380,16 @@ common::Status NodeArg::UpdateTypeAndShape(const ONNX_NAMESPACE::TypeProto& inpu
*current_type.mutable_sparse_tensor_type() = input_tensor_type;
}
}
} break;
break;
}
#endif
case TypeProto::kSequenceType:
case TypeProto::kMapType:
case TypeProto::kOptionalType:
case TypeProto::kOpaqueType:
case TypeProto::VALUE_NOT_SET:
default:
break;
}
@ -812,15 +839,17 @@ ADD_BASIC_ATTR_IMPL(float, AttributeProto_AttributeType::AttributeProto_Attribut
ADD_BASIC_ATTR_IMPL(int64_t, AttributeProto_AttributeType::AttributeProto_AttributeType_INT, i)
ADD_BASIC_ATTR_IMPL(std::string, AttributeProto_AttributeType::AttributeProto_AttributeType_STRING, s)
ADD_ATTR_IMPL(TensorProto, AttributeProto_AttributeType::AttributeProto_AttributeType_TENSOR, t)
ADD_ATTR_IMPL(SparseTensorProto, AttributeProto_AttributeType::AttributeProto_AttributeType_SPARSE_TENSOR, sparse_tensor)
ADD_ATTR_IMPL(TypeProto, AttributeProto_AttributeType::AttributeProto_AttributeType_TYPE_PROTO, tp)
ADD_LIST_ATTR_IMPL(float, AttributeProto_AttributeType::AttributeProto_AttributeType_FLOATS, floats)
ADD_LIST_ATTR_IMPL(int64_t, AttributeProto_AttributeType::AttributeProto_AttributeType_INTS, ints)
ADD_LIST_ATTR_IMPL(std::string, AttributeProto_AttributeType::AttributeProto_AttributeType_STRINGS, strings)
ADD_LIST_ATTR_IMPL(TensorProto, AttributeProto_AttributeType::AttributeProto_AttributeType_TENSORS, tensors)
ADD_LIST_ATTR_IMPL(GraphProto, AttributeProto_AttributeType::AttributeProto_AttributeType_GRAPHS, graphs)
ADD_LIST_ATTR_IMPL(SparseTensorProto, AttributeProto_AttributeType::AttributeProto_AttributeType_SPARSE_TENSORS, sparse_tensors)
ADD_LIST_ATTR_IMPL(TypeProto, AttributeProto_AttributeType::AttributeProto_AttributeType_TYPE_PROTOS, type_protos)
#if !defined(DISABLE_SPARSE_TENSORS)
ADD_ATTR_IMPL(SparseTensorProto, AttributeProto_AttributeType::AttributeProto_AttributeType_SPARSE_TENSOR, sparse_tensor)
ADD_LIST_ATTR_IMPL(SparseTensorProto, AttributeProto_AttributeType::AttributeProto_AttributeType_SPARSE_TENSORS, sparse_tensors)
#endif
#if !defined(ORT_MINIMAL_BUILD)
bool Node::ClearAttribute(const std::string& attr_name) {
@ -998,10 +1027,12 @@ Graph::Graph(const Model& owning_model,
const gsl::not_null<TensorProto*> tensor{graph_proto_->add_initializer()};
auto status = utils::ConstantNodeProtoToTensorProto(node, model_path, *tensor);
ORT_ENFORCE(status.IsOK(), status.ToString());
#if !defined(DISABLE_SPARSE_TENSORS)
if (node.attribute(0).type() == AttributeProto_AttributeType_SPARSE_TENSOR) {
auto p = sparse_tensor_names_.emplace(tensor->name());
ORT_ENFORCE(p.second, "Duplicate constant node sparse initializer name: '", tensor->name(), "' Model is invalid.");
}
#endif
}
// Remove constant nodes as they're replaced with initializers above.
@ -1013,6 +1044,7 @@ Graph::Graph(const Model& owning_model,
}),
graph_mutable_nodes->end());
#if !defined(DISABLE_SPARSE_TENSORS)
// For now we convert sparse_intializer to dense tensors
// since there are currently no supported ops that consume sparse
// initializers directly. We remove them from graph_proto. We will reconstitute them
@ -1034,6 +1066,7 @@ Graph::Graph(const Model& owning_model,
delete graph_proto_->mutable_sparse_initializer()->ReleaseCleared();
}
}
#endif
// Collect all node arg name, type, shape information in the graph.
// type/shape information will be assigned to each node arg when going
@ -1780,10 +1813,12 @@ bool FullyDefinedType(const TypeProto& type_proto) {
auto& tensor_type = type_proto.tensor_type();
return utils::HasElemType(tensor_type);
}
#if !defined(DISABLE_SPARSE_TENSORS)
case TypeProto::kSparseTensorType: {
auto& tensor_type = type_proto.sparse_tensor_type();
return utils::HasElemType(tensor_type);
}
#endif
case TypeProto::kSequenceType: {
auto& seq_type = type_proto.sequence_type();
return utils::HasElemType(seq_type) && FullyDefinedType(seq_type.elem_type());
@ -2224,9 +2259,12 @@ Status Graph::InferAndVerifyTypeMatch(Node& node, const OpSchema& op, const Reso
TypeProto merge_target;
if (utils::HasTensorType(onnx_inferred_type)) {
*merge_target.mutable_tensor_type()->mutable_shape() = *output_def->Shape();
} else if (utils::HasSparseTensorType(onnx_inferred_type)) {
}
#if !defined(DISABLE_SPARSE_TENSORS)
else if (utils::HasSparseTensorType(onnx_inferred_type)) {
*merge_target.mutable_sparse_tensor_type()->mutable_shape() = *output_def->Shape();
}
#endif
auto status = MergeShapeInfo(output_def->Name(), onnx_inferred_type, merge_target, using_latest_onnx_opset_, logger_);
if (!status.IsOK()) {
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Node:", node_name, " ", status.ErrorMessage());
@ -2674,9 +2712,11 @@ bool Graph::IsInitializedTensor(const std::string& name) const {
return name_to_initial_tensor_.count(name) > 0;
}
#if !defined(DISABLE_SPARSE_TENSORS)
bool Graph::IsSparseInitializer(const std::string& name) const {
return sparse_tensor_names_.count(name) > 0;
}
#endif
void Graph::RemoveInitializedTensor(const std::string& tensor_name) {
bool found = false;
@ -2684,10 +2724,14 @@ void Graph::RemoveInitializedTensor(const std::string& tensor_name) {
found = iter != name_to_initial_tensor_.end();
if (found) {
name_to_initial_tensor_.erase(iter);
#if !defined(DISABLE_SPARSE_TENSORS)
sparse_tensor_names_.erase(tensor_name);
#endif
SetGraphResolveNeeded();
} else {
#if !defined(DISABLE_SPARSE_TENSORS)
ORT_ENFORCE(sparse_tensor_names_.count(tensor_name) == 0, "sparse_tensor_names_ not in sync with name_to_initial_tensor_");
#endif
}
auto& mutable_initializers = *(graph_proto_->mutable_initializer());
@ -2753,7 +2797,9 @@ bool Graph::GetInitializedTensor(const std::string& tensor_name, const TensorPro
void Graph::CleanAllInitializedTensors() noexcept {
name_to_initial_tensor_.clear();
#if !defined(DISABLE_SPARSE_TENSORS)
sparse_tensor_names_.clear();
#endif
// Clearing RepeatedPtrFields does not free objects' memory. The memory is retained
// and can be reused. Need to explicitly release the cleared objects and free the
@ -2902,13 +2948,19 @@ common::Status Graph::SaveToOrtFormat(flatbuffers::FlatBufferBuilder& builder,
auto inputs = SaveInputsOutputsToOrtFormat(builder, graph_inputs_including_initializers_);
auto outputs = SaveInputsOutputsToOrtFormat(builder, graph_outputs_);
#if !defined(DISABLE_SPARSE_TENSORS)
std::vector<flatbuffers::Offset<fbs::SparseTensor>> sparse_initializers_data;
sparse_initializers_data.reserve(sparse_tensor_names_.size());
#endif
const auto sparse_end = sparse_tensor_names_.end();
std::vector<flatbuffers::Offset<fbs::Tensor>> initializers_data;
#if !defined(DISABLE_SPARSE_TENSORS)
assert(sparse_tensor_names_.size() <= name_to_initial_tensor_.size());
initializers_data.reserve(name_to_initial_tensor_.size() - sparse_tensor_names_.size());
#else
initializers_data.reserve(name_to_initial_tensor_.size());
#endif
const auto& model_path = ModelPath();
for (const auto& pair : name_to_initial_tensor_) {
@ -2917,7 +2969,9 @@ common::Status Graph::SaveToOrtFormat(flatbuffers::FlatBufferBuilder& builder,
ORT_RETURN_IF_ERROR(
experimental::utils::SaveInitializerOrtFormat(builder, *pair.second, model_path, fbs_tensor));
initializers_data.push_back(fbs_tensor);
} else {
}
#if !defined(DISABLE_SPARSE_TENSORS)
else {
SparseTensorProto sparse_initializer;
ORT_RETURN_IF_ERROR(utils::DenseTensorToSparseTensorProto(*pair.second, model_path, sparse_initializer));
flatbuffers::Offset<fbs::SparseTensor> fbs_sparse_tensor;
@ -2925,9 +2979,12 @@ common::Status Graph::SaveToOrtFormat(flatbuffers::FlatBufferBuilder& builder,
experimental::utils::SaveSparseInitializerOrtFormat(builder, sparse_initializer, model_path, fbs_sparse_tensor));
sparse_initializers_data.push_back(fbs_sparse_tensor);
}
#endif
}
auto initializers = builder.CreateVector(initializers_data);
#if !defined(DISABLE_SPARSE_TENSORS)
auto sparse_initializers = builder.CreateVector(sparse_initializers_data);
#endif
auto initializers = builder.CreateVector(initializers_data);
std::vector<flatbuffers::Offset<fbs::ValueInfo>> node_args_data;
node_args_data.reserve(node_args_.size());
@ -2961,7 +3018,9 @@ common::Status Graph::SaveToOrtFormat(flatbuffers::FlatBufferBuilder& builder,
gb.add_node_edges(node_edges);
gb.add_inputs(inputs);
gb.add_outputs(outputs);
#if !defined(DISABLE_SPARSE_TENSORS)
gb.add_sparse_initializers(sparse_initializers);
#endif
fbs_graph = gb.Finish();
return Status::OK();
}
@ -3090,17 +3149,24 @@ const ONNX_NAMESPACE::GraphProto& Graph::ToGraphProto() {
}
ONNX_NAMESPACE::GraphProto Graph::ToGraphProto() const {
#if !defined(DISABLE_SPARSE_TENSORS)
if (!GraphProtoSyncNeeded() && sparse_tensor_names_.empty()) {
return *graph_proto_;
}
#else
if (!GraphProtoSyncNeeded()) {
return *graph_proto_;
}
#endif
GraphProto result;
ToGraphProtoInternal(result);
// Path of the owning model
// This is used for constructing full path for external data
// if it exists
const auto& model_path = ModelPath();
#if !defined(DISABLE_SPARSE_TENSORS)
const auto& model_path = ModelPath();
// We want to make sure that sparse initializers do not appear
// as dense duplicates within the initializers list.
if (!sparse_tensor_names_.empty()) {
@ -3118,6 +3184,9 @@ ONNX_NAMESPACE::GraphProto Graph::ToGraphProto() const {
} else {
*result.mutable_initializer() = graph_proto_->initializer();
}
#else
*result.mutable_initializer() = graph_proto_->initializer();
#endif
return result;
}
@ -3126,21 +3195,26 @@ ONNX_NAMESPACE::GraphProto Graph::ToGraphProtoWithExternalInitializers(const std
size_t initializer_size_threshold) const {
GraphProto result;
ToGraphProtoInternal(result);
const auto& model_path = ModelPath();
std::ofstream external_stream(external_file_name, std::ofstream::out | std::ofstream::binary);
ORT_ENFORCE(external_stream.is_open());
int64_t external_offset = 0;
// Add the initializers to the result graph.
#if !defined(DISABLE_SPARSE_TENSORS)
const auto& model_path = ModelPath();
const auto sparse_end = sparse_tensor_names_.end();
#endif
for (const auto& initializer : graph_proto_->initializer()) {
#if !defined(DISABLE_SPARSE_TENSORS)
if (sparse_end != sparse_tensor_names_.find(initializer.name())) {
// Sparse tensors are added to the ONNX file.
auto& sparse_initializer = *result.add_sparse_initializer();
auto status = utils::DenseTensorToSparseTensorProto(initializer, model_path, sparse_initializer);
ORT_ENFORCE(status.IsOK(), "Failed to convert dense initializer to sparse");
} else {
#endif
// Dense tensors larger than the threshold are added to the external file.
TensorProto* output_proto = result.add_initializer();
@ -3175,7 +3249,9 @@ ONNX_NAMESPACE::GraphProto Graph::ToGraphProtoWithExternalInitializers(const std
output_proto->set_doc_string(initializer.doc_string());
external_offset += tensor_bytes_size;
#if !defined(DISABLE_SPARSE_TENSORS)
}
#endif
}
return result;
@ -3882,9 +3958,13 @@ common::Status Graph::LoadFromOrtFormat(const onnxruntime::experimental::fbs::Gr
// Initializers
auto fbs_initializers = fbs_graph.initializers();
#if !defined(DISABLE_SPARSE_TENSORS)
auto fbs_sparse_initializers = fbs_graph.sparse_initializers();
flatbuffers::uoffset_t map_size = (fbs_initializers != nullptr ? fbs_initializers->size() : 0U) +
(fbs_sparse_initializers != nullptr ? fbs_sparse_initializers->size() : 0U);
#else
flatbuffers::uoffset_t map_size = (fbs_initializers != nullptr ? fbs_initializers->size() : 0U);
#endif
if (map_size > 0) {
name_to_initial_tensor_.reserve(map_size);
@ -3905,6 +3985,7 @@ common::Status Graph::LoadFromOrtFormat(const onnxruntime::experimental::fbs::Gr
}
}
#if !defined(DISABLE_SPARSE_TENSORS)
if (fbs_sparse_initializers) {
sparse_tensor_names_.reserve(fbs_sparse_initializers->size());
const auto& model_path = ModelPath();
@ -3925,6 +4006,7 @@ common::Status Graph::LoadFromOrtFormat(const onnxruntime::experimental::fbs::Gr
sparse_tensor_names_.emplace(initializer.name());
}
}
#endif
// NodeArgs
auto fbs_node_args = fbs_graph.node_args();

8
onnxruntime/core/optimizer/constant_folding.cc
View file

@ -94,9 +94,11 @@ Status ConstantFolding::ApplyImpl(Graph& graph, bool& modified, int graph_level,
GraphViewer graph_viewer(graph);
auto& order = graph_viewer.GetNodesInTopologicalOrder();
#if !defined(DISABLE_SPARSE_TENSORS)
std::function<bool(const std::string&)> is_sparse_initializer_check = [&graph](const std::string& name) -> bool {
return graph.IsSparseInitializer(name);
};
#endif
for (NodeIndex i : order) {
auto* node = graph.GetNode(i);
@ -146,9 +148,15 @@ Status ConstantFolding::ApplyImpl(Graph& graph, bool& modified, int graph_level,
continue;
}
#if !defined(DISABLE_SPARSE_TENSORS)
// Create execution frame for executing constant nodes.
OptimizerExecutionFrame::Info info({node}, constant_inputs, graph.ModelPath(), execution_provider_,
is_sparse_initializer_check);
#else
// Create execution frame for executing constant nodes.
OptimizerExecutionFrame::Info info({node}, constant_inputs, graph.ModelPath(), execution_provider_,
[](std::string const&) { return false; });
#endif
std::vector<int> fetch_mlvalue_idxs;
for (const auto* node_out : node->OutputDefs()) {

4
onnxruntime/core/optimizer/optimizer_execution_frame.cc
View file

@ -145,9 +145,13 @@ Status OptimizerExecutionFrame::CreateNodeOutputMLValueImpl(OrtValue& ort_value,
return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT,
"Tried to allocate without valid type information, ort_value index=" + std::to_string(ort_value_idx));
if (ml_type->IsSparseTensorType()) {
#if !defined(DISABLE_SPARSE_TENSORS)
auto element_type = ml_type->AsSparseTensorType()->GetElementType();
SparseTensor::InitOrtValue(element_type, *shape, info_.GetAllocator(), ort_value);
return Status::OK();
#else
return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Sparse tensor is not supported in this build");
#endif
}
if (ml_type->IsTensorSequenceType()) {

13
onnxruntime/core/providers/shared_library/provider_bridge_provider.cc
View file

@ -104,8 +104,10 @@ AllocatorPtr AllocatorManager::GetAllocator(int id, OrtMemType mem_type) const {
template <>
MLDataType DataTypeImpl::GetType<Tensor>() { return Provider_GetHost()->DataTypeImpl__GetType_Tensor(); }
#if !defined(DISABLE_SPARSE_TENSORS)
template <>
MLDataType DataTypeImpl::GetType<SparseTensor>() { return Provider_GetHost()->DataTypeImpl__GetType_SparseTensor(); }
#endif
template <>
MLDataType DataTypeImpl::GetType<TensorSeq>() { return Provider_GetHost()->DataTypeImpl__GetType_TensorSeq(); }
MLDataType DataTypeImpl::GetTypeFromOnnxType(int onnx_type) { return Provider_GetHost()->DataTypeImpl__GetTypeFromOnnxType(onnx_type); }
@ -164,6 +166,7 @@ MLDataType DataTypeImpl::GetTensorType<BFloat16>() { return Provider_GetHost()->
template <>
MLDataType DataTypeImpl::GetTensorType<MLFloat16>() { return Provider_GetHost()->DataTypeImpl__GetTensorType_MLFloat16(); }
#if !defined(DISABLE_SPARSE_TENSORS)
template <>
MLDataType DataTypeImpl::GetSparseTensorType<bool>() { return Provider_GetHost()->DataTypeImpl__GetSparseTensorType_bool(); }
template <>
@ -192,6 +195,7 @@ template <>
MLDataType DataTypeImpl::GetSparseTensorType<BFloat16>() { return Provider_GetHost()->DataTypeImpl__GetSparseTensorType_BFloat16(); }
template <>
MLDataType DataTypeImpl::GetSparseTensorType<MLFloat16>() { return Provider_GetHost()->DataTypeImpl__GetSparseTensorType_MLFloat16(); }
#endif
Status IDataTransfer::CopyTensor(const Tensor& src, Tensor& dst) const {
return g_host->IDataTransfer__CopyTensor(this, src, dst);
@ -200,10 +204,11 @@ Status IDataTransfer::CopyTensor(const Tensor& src, Tensor& dst) const {
Status IDataTransfer::CopyTensors(const std::vector<SrcDstPair>& src_dst_pairs) const {
return g_host->IDataTransfer__CopyTensors(this, src_dst_pairs);
}
#if !defined(DISABLE_SPARSE_TENSORS)
Status IDataTransfer::CopySparseTensors(const std::vector<SparseSrcDstPair>& src_dst_pairs) const {
return g_host->IDataTransfer__CopySparseTensors(this, src_dst_pairs);
}
#endif
const Node& OpKernel::Node() const { return g_host->OpKernel__Node(this); }
@ -373,6 +378,7 @@ float halfToFloat(uint16_t h) { return g_host->math__halfToFloat(h); }
} // namespace math
namespace sparse_utils {
#if !defined(DISABLE_SPARSE_TENSORS)
#if !defined(ORT_MINIMAL_BUILD)
Status DenseTensorToSparseCsr(const DataTransferManager& data_manager, const Tensor& src, const AllocatorPtr& cpu_allocator,
const AllocatorPtr& dst_allocator, SparseTensor& dst) {
@ -388,11 +394,14 @@ Status SparseCooToDenseTensor(const DataTransferManager& data_manager, const Spa
const AllocatorPtr& dst_allocator, Tensor& dst) {
return g_host->sparse_utils__SparseCooToDenseTensor(data_manager, src, cpu_allocator, dst_allocator, dst);
}
#endif // ORT_MINIMAL_BUILD
#endif // !ORT_MINIMAL_BUILD
Status DenseTensorToSparseCoo(const DataTransferManager& data_manager, const Tensor& src, const AllocatorPtr& cpu_allocator,
const AllocatorPtr& dst_allocator, bool linear_indexs, SparseTensor& dst) {
return g_host->sparse_utils__DenseTensorToSparseCoo(data_manager, src, cpu_allocator, dst_allocator, linear_indexs, dst);
}
#endif // !defined(DISABLE_SPARSE_TENSORS)
} // namespace sparse_utils
float MLFloat16::ToFloat() const {

26
onnxruntime/core/providers/shared_library/provider_interfaces.h
View file

@ -172,6 +172,7 @@ struct ProviderHost {
virtual float math__halfToFloat(uint16_t h) = 0;
// sparse_utils
#if !defined(DISABLE_SPARSE_TENSORS)
#if !defined(ORT_MINIMAL_BUILD)
virtual Status sparse_utils__DenseTensorToSparseCsr(const DataTransferManager& data_manager, const Tensor& src, const AllocatorPtr& cpu_allocator,
const AllocatorPtr& dst_allocator, SparseTensor& dst) = 0;
@ -180,9 +181,10 @@ struct ProviderHost {
virtual Status sparse_utils__SparseCooToDenseTensor(const DataTransferManager& data_manager, const SparseTensor& src, const AllocatorPtr& cpu_allocator,
const AllocatorPtr& dst_allocator, Tensor& dst) = 0;
#endif // ORT_MINIMAL_BUILD
#endif // !ORT_MINIMAL_BUILD
virtual Status sparse_utils__DenseTensorToSparseCoo(const DataTransferManager& data_manager, const Tensor& src, const AllocatorPtr& cpu_allocator,
const AllocatorPtr& dst_allocator, bool linear_indexs, SparseTensor& dst) = 0;
#endif // !defined(DISABLE_SPARSE_TENSORS)
// IAllocator
virtual bool IAllocator__CalcMemSizeForArrayWithAlignment(size_t nmemb, size_t size, size_t alignment, size_t* out) = 0;
@ -239,18 +241,22 @@ struct ProviderHost {
virtual ONNX_NAMESPACE::TensorShapeProto* TypeProto_Tensor__mutable_shape(ONNX_NAMESPACE::TypeProto_Tensor* p) = 0;
virtual int32_t TypeProto_Tensor__elem_type(const ONNX_NAMESPACE::TypeProto_Tensor* p) = 0;
#if !defined(DISABLE_SPARSE_TENSORS)
// TypeProto_SparseTensor
virtual bool TypeProto_SparseTensor__has_shape(const ONNX_NAMESPACE::TypeProto_SparseTensor* p) = 0;
virtual const ONNX_NAMESPACE::TensorShapeProto& TypeProto_SparseTensor__shape(const ONNX_NAMESPACE::TypeProto_SparseTensor* p) = 0;
virtual ONNX_NAMESPACE::TensorShapeProto* TypeProto_SparseTensor__mutable_shape(ONNX_NAMESPACE::TypeProto_SparseTensor* p) = 0;
virtual int32_t TypeProto_SparseTensor__elem_type(const ONNX_NAMESPACE::TypeProto_SparseTensor* p) = 0;
#endif
// TypeProto
virtual const ONNX_NAMESPACE::TypeProto_Tensor& TypeProto__tensor_type(const ONNX_NAMESPACE::TypeProto* p) = 0;
virtual ONNX_NAMESPACE::TypeProto_Tensor* TypeProto__mutable_tensor_type(ONNX_NAMESPACE::TypeProto* p) = 0;
#if !defined(DISABLE_SPARSE_TENSORS)
virtual const ONNX_NAMESPACE::TypeProto_SparseTensor& TypeProto__sparse_tensor_type(const ONNX_NAMESPACE::TypeProto* p) = 0;
virtual ONNX_NAMESPACE::TypeProto_SparseTensor* TypeProto__mutable_sparse_tensor_type(ONNX_NAMESPACE::TypeProto* p) = 0;
#endif
virtual int TypeProto__value_case(const ONNX_NAMESPACE::TypeProto* p) = 0;
// AttributeProto
@ -362,15 +368,19 @@ struct ProviderHost {
// DataTransferManager
virtual Status DataTransferManager__CopyTensor(const DataTransferManager* p, const Tensor& src, Tensor& dst, int exec_queue_id) = 0;
virtual Status DataTransferManager__CopyTensor(const DataTransferManager* p, const Tensor& src, Tensor& dst) = 0;
#if !defined(DISABLE_SPARSE_TENSORS)
virtual Status DataTransferManager__CopySparseTensor(const DataTransferManager* p, const SparseTensor& src, SparseTensor& dst) = 0;
virtual Status DataTransferManager__CopySparseTensor(const DataTransferManager* p, const SparseTensor& src, SparseTensor& dst, int exec_queue_id) = 0;
virtual Status DataTransferManager__CopySparseTensors(const DataTransferManager* p, const std::vector<IDataTransfer::SparseSrcDstPair>& src_dst_pairs) = 0;
#endif
virtual const IDataTransfer* DataTransferManager__GetDataTransfer(const DataTransferManager* p, const OrtDevice& src_device, const OrtDevice& dst_device) = 0;
// IDataTransfer
virtual Status IDataTransfer__CopyTensor(const IDataTransfer* p, const Tensor& src, Tensor& dst) = 0;
virtual Status IDataTransfer__CopyTensors(const IDataTransfer* p, const std::vector<IDataTransfer::SrcDstPair>& src_dst_pairs) = 0;
#if !defined(DISABLE_SPARSE_TENSORS)
virtual Status IDataTransfer__CopySparseTensors(const IDataTransfer* p, const std::vector<IDataTransfer::SparseSrcDstPair>& src_dst_pairs) = 0;
#endif
// IndexedSubGraph_MetaDef
virtual std::unique_ptr<IndexedSubGraph_MetaDef> IndexedSubGraph_MetaDef__construct() = 0;
@ -436,7 +446,9 @@ struct ProviderHost {
// DataTypeImpl
virtual MLDataType DataTypeImpl__GetType_Tensor() = 0;
#if !defined(DISABLE_SPARSE_TENSORS)
virtual MLDataType DataTypeImpl__GetType_SparseTensor() = 0;
#endif
virtual MLDataType DataTypeImpl__GetType_TensorSeq() = 0;
virtual MLDataType DataTypeImpl__GetTypeFromOnnxType(int) = 0;
virtual MLDataType DataTypeImpl__GetType_bool() = 0;
@ -467,6 +479,7 @@ struct ProviderHost {
virtual MLDataType DataTypeImpl__GetTensorType_BFloat16() = 0;
virtual MLDataType DataTypeImpl__GetTensorType_MLFloat16() = 0;
#if !defined(DISABLE_SPARSE_TENSORS)
virtual MLDataType DataTypeImpl__GetSparseTensorType_bool() = 0;
virtual MLDataType DataTypeImpl__GetSparseTensorType_int8() = 0;
virtual MLDataType DataTypeImpl__GetSparseTensorType_uint8() = 0;
@ -481,11 +494,14 @@ struct ProviderHost {
virtual MLDataType DataTypeImpl__GetSparseTensorType_string() = 0;
virtual MLDataType DataTypeImpl__GetSparseTensorType_BFloat16() = 0;
virtual MLDataType DataTypeImpl__GetSparseTensorType_MLFloat16() = 0;
#endif
virtual const char* DataTypeImpl__ToString(MLDataType type) = 0;
virtual bool DataTypeImpl__IsTensorType(const DataTypeImpl* p) = 0;
virtual bool DataTypeImpl__IsTensorSequenceType(const DataTypeImpl* p) = 0;
#if !defined(DISABLE_SPARSE_TENSORS)
virtual bool DataTypeImpl__IsSparseTensorType(const DataTypeImpl* p) = 0;
#endif
virtual DeleteFunc DataTypeImpl__GetDeleteFunc(const DataTypeImpl* p) = 0;
virtual const std::vector<MLDataType>& DataTypeImpl__AllFixedSizeTensorTypes() = 0;
virtual const std::vector<MLDataType>& DataTypeImpl__AllTensorTypes() = 0;
@ -612,13 +628,17 @@ struct ProviderHost {
// OpKernelContext
virtual const Tensor* OpKernelContext__Input_Tensor(const OpKernelContext* p, int index) = 0;
#if !defined(DISABLE_SPARSE_TENSORS)
virtual const SparseTensor* OpKernelContext__Input_SparseTensor(const OpKernelContext* p, int index) = 0;
#endif
virtual const TensorSeq* OpKernelContext__Input_TensorSeq(const OpKernelContext* p, int index) = 0;
virtual const Tensor& OpKernelContext__RequiredInput_Tensor(const OpKernelContext* p, int index) = 0;
virtual Tensor* OpKernelContext__Output_Tensor(OpKernelContext* p, int index) = 0;
virtual TensorSeq* OpKernelContext__Output_TensorSeq(OpKernelContext* p, int index) = 0;
virtual Tensor* OpKernelContext__Output(OpKernelContext* p, int index, const TensorShape& shape) = 0;
#if !defined(DISABLE_SPARSE_TENSORS)
virtual SparseTensor* OpKernelContext__OutputSparse(OpKernelContext* p, int index, const TensorShape& shape) = 0;
#endif
virtual Tensor& OpKernelContext__RequiredOutput(OpKernelContext* p, int index, const TensorShape& shape) = 0;
virtual MLDataType OpKernelContext__InputType(const OpKernelContext* p, int index) = 0;
virtual int OpKernelContext__InputCount(const OpKernelContext* p) = 0;
@ -718,9 +738,11 @@ struct ProviderHost {
virtual int32_t Tensor__GetElementType(const Tensor* p) = 0;
virtual MLDataType Tensor__DataType(const Tensor* p) = 0;
#if !defined(DISABLE_SPARSE_TENSORS)
// SparseTensor
virtual const TensorShape& SparseTensor__DenseShape(const SparseTensor*) = 0;
virtual Status SparseTensor__Copy(const SparseTensor*, const DataTransferManager&, int, SparseTensor&) = 0;
#endif
// TensorSeq
virtual MLDataType TensorSeq__DataType(const TensorSeq* p) noexcept = 0;
@ -756,7 +778,7 @@ struct ProviderHost {
#endif
#endif
virtual ProviderHostCPU& GetProviderHostCPU()=0;
virtual ProviderHostCPU& GetProviderHostCPU() = 0;
};
} // namespace onnxruntime

26
onnxruntime/core/providers/shared_library/provider_wrappedtypes.h
View file

@ -40,7 +40,7 @@ struct Capture final {
void operator=(const Capture&) = delete;
};
} // namespace logging
}
} // namespace onnxruntime
namespace ONNX_NAMESPACE {
@ -80,13 +80,17 @@ struct AttributeProto final {
static constexpr AttributeType STRING = AttributeProto_AttributeType_STRING;
static constexpr AttributeType TENSOR = AttributeProto_AttributeType_TENSOR;
static constexpr AttributeType GRAPH = AttributeProto_AttributeType_GRAPH;
#if !defined(DISABLE_SPARSE_TENSORS)
static constexpr AttributeType SPARSE_TENSOR = AttributeProto_AttributeType_SPARSE_TENSOR;
#endif
static constexpr AttributeType FLOATS = AttributeProto_AttributeType_FLOATS;
static constexpr AttributeType INTS = AttributeProto_AttributeType_INTS;
static constexpr AttributeType STRINGS = AttributeProto_AttributeType_STRINGS;
static constexpr AttributeType TENSORS = AttributeProto_AttributeType_TENSORS;
static constexpr AttributeType GRAPHS = AttributeProto_AttributeType_GRAPHS;
#if !defined(DISABLE_SPARSE_TENSORS)
static constexpr AttributeType SPARSE_TENSORS = AttributeProto_AttributeType_SPARSE_TENSORS;
#endif
AttributeProto() = delete;
AttributeProto(const AttributeProto&) = delete;
@ -210,6 +214,7 @@ struct TypeProto_Tensor final {
PROVIDER_DISALLOW_ALL(TypeProto_Tensor)
};
#if !defined(DISABLE_SPARSE_TENSORS)
struct TypeProto_SparseTensor final {
bool has_shape() const { return g_host->TypeProto_SparseTensor__has_shape(this); }
const TensorShapeProto& shape() const { return g_host->TypeProto_SparseTensor__shape(this); }
@ -218,13 +223,16 @@ struct TypeProto_SparseTensor final {
PROVIDER_DISALLOW_ALL(TypeProto_SparseTensor)
};
#endif
struct TypeProto final {
const TypeProto_Tensor& tensor_type() const { return g_host->TypeProto__tensor_type(this); }
TypeProto_Tensor* mutable_tensor_type() { return g_host->TypeProto__mutable_tensor_type(this); }
#if !defined(DISABLE_SPARSE_TENSORS)
const TypeProto_SparseTensor& sparse_tensor_type() const { return g_host->TypeProto__sparse_tensor_type(this); }
TypeProto_SparseTensor* mutable_sparse_tensor_type() { return g_host->TypeProto__mutable_sparse_tensor_type(this); }
#endif
enum ValueCase {
kTensorType = 1,
@ -291,9 +299,11 @@ struct ComputeCapability final {
struct DataTransferManager final {
Status CopyTensor(const Tensor& src, Tensor& dst, int exec_queue_id) const { return g_host->DataTransferManager__CopyTensor(this, src, dst, exec_queue_id); }
Status CopyTensor(const Tensor& src, Tensor& dst) const { return g_host->DataTransferManager__CopyTensor(this, src, dst); }
#if !defined(DISABLE_SPARSE_TENSORS)
Status CopySparseTensor(const SparseTensor& src, SparseTensor& dst) const { return g_host->DataTransferManager__CopySparseTensor(this, src, dst); }
Status CopySparseTensor(const SparseTensor& src, SparseTensor& dst, int exec_queue_id) const { return g_host->DataTransferManager__CopySparseTensor(this, src, dst, exec_queue_id); }
Status CopySparseTensors(const std::vector<IDataTransfer::SparseSrcDstPair>& src_dst_pairs) const { return g_host->DataTransferManager__CopySparseTensors(this, src_dst_pairs); }
#endif
const IDataTransfer* GetDataTransfer(const OrtDevice& src_device, const OrtDevice& dst_device) const { return g_host->DataTransferManager__GetDataTransfer(this, src_device, dst_device); }
PROVIDER_DISALLOW_ALL(DataTransferManager)
@ -466,14 +476,18 @@ class DataTypeImpl final {
static MLDataType GetType();
template <typename elemT>
static MLDataType GetTensorType();
#if !defined(DISABLE_SPARSE_TENSORS)
template <typename elemT>
static MLDataType GetSparseTensorType();
#endif
static MLDataType GetTypeFromOnnxType(int);
bool IsTensorType() const { return g_host->DataTypeImpl__IsTensorType(this); }
bool IsTensorSequenceType() const { return g_host->DataTypeImpl__IsTensorSequenceType(this); }
#if !defined(DISABLE_SPARSE_TENSORS)
bool IsSparseTensorType() const { return g_host->DataTypeImpl__IsSparseTensorType(this); }
#endif
DeleteFunc GetDeleteFunc() const { return g_host->DataTypeImpl__GetDeleteFunc(this); }
static const std::vector<MLDataType>& AllFixedSizeTensorTypes() { return g_host->DataTypeImpl__AllFixedSizeTensorTypes(); }
@ -680,7 +694,9 @@ struct OpKernelContext final {
T* Output(int index);
Tensor* Output(int index, const TensorShape& shape) { return g_host->OpKernelContext__Output(this, index, shape); }
SparseTensor* OutputSparse(int index, const TensorShape& shape) { return g_host->OpKernelContext__OutputSparse(this, index, shape); }
#if !defined(DISABLE_SPARSE_TENSORS)
SparseTensor* OutputSparse(int index, const TensorShape& shape) { return g_host->OpKernelContext__OutputSparse(this, index, shape); }
#endif
int OutputCount() const { return g_host->OpKernelContext__OutputCount(this); }
Status GetTempSpaceAllocator(AllocatorPtr* output) const { return g_host->OpKernelContext__GetTempSpaceAllocator(this, output); }
@ -698,10 +714,12 @@ inline const Tensor* OpKernelContext::Input<Tensor>(int index) const {
return g_host->OpKernelContext__Input_Tensor(this, index);
}
#if !defined(DISABLE_SPARSE_TENSORS)
template <>
inline const SparseTensor* OpKernelContext::Input<SparseTensor>(int index) const {
return g_host->OpKernelContext__Input_SparseTensor(this, index);
}
#endif
template <>
inline const TensorSeq* OpKernelContext::Input<TensorSeq>(int index) const {
@ -919,10 +937,12 @@ template <>
inline const MLFloat16* Tensor::Data<MLFloat16>() const { return g_host->Tensor__Data_MLFloat16(this); }
// SparseTensor
#if !defined(DISABLE_SPARSE_TENSORS)
struct SparseTensor final {
const TensorShape& DenseShape() const noexcept { return g_host->SparseTensor__DenseShape(this); }
Status Copy(const DataTransferManager& dtm, int exec_q_id, SparseTensor& dst) const { return g_host->SparseTensor__Copy(this, dtm, exec_q_id, dst); }
};
#endif
//TensorSeq
struct TensorSeq final {
@ -936,4 +956,4 @@ struct TensorSeq final {
template <>
inline gsl::span<const int64_t> Tensor::DataAsSpan() const { return g_host->Tensor__DataAsSpan_int64(this); }
}
} // namespace onnxruntime

6
onnxruntime/core/session/inference_session.cc
View file

@ -1587,6 +1587,7 @@ common::Status InferenceSession::ValidateInputs(const std::vector<std::string>&
ORT_RETURN_IF_ERROR_SESSIONID_(CheckShapes(feed_name, input_shape, expected_shape));
}
} else if (input_ml_value.IsSparseTensor()) {
#if !defined(DISABLE_SPARSE_TENSORS)
if (!expected_type->IsSparseTensorType()) {
return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input with name: ", feed_name,
" is not expected to be of type sparse tensor.");
@ -1601,6 +1602,11 @@ common::Status InferenceSession::ValidateInputs(const std::vector<std::string>&
const auto& input_shape = sparse_tensor.DenseShape();
ORT_RETURN_IF_ERROR_SESSIONID_(CheckShapes(feed_name, input_shape, expected_shape));
}
#else
return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input with name ", feed_name,
" is a sparse tensor, which is not supported in this build.");
#endif
} else if (input_ml_value.IsTensorSequence()) {
if (!expected_type->IsTensorSequenceType()) {
return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input with name: ", feed_name,

54
onnxruntime/core/session/onnxruntime_c_api.cc
View file

@ -231,6 +231,7 @@ ORT_API_STATUS_IMPL(OrtApis::CreateTensorAsOrtValue, _Inout_ OrtAllocator* alloc
ORT_API_STATUS_IMPL(OrtApis::CreateSparseTensorAsOrtValue, _Inout_ OrtAllocator* allocator, _In_ const int64_t* dense_shape,
size_t dense_shape_len, ONNXTensorElementDataType type, _Outptr_ OrtValue** out) {
API_IMPL_BEGIN
#if !defined(DISABLE_SPARSE_TENSORS)
auto sparse_tensor_type = DataTypeImpl::SparseTensorTypeFromONNXEnum(type);
auto element_type = sparse_tensor_type->GetElementType();
assert(element_type->AsPrimitiveDataType() != nullptr);
@ -245,10 +246,14 @@ ORT_API_STATUS_IMPL(OrtApis::CreateSparseTensorAsOrtValue, _Inout_ OrtAllocator*
SparseTensor::InitOrtValue(element_type, shape, std::move(alloc_ptr), *value);
*out = value.release();
return nullptr;
#else
return OrtApis::CreateStatus(ORT_FAIL, "SparseTensor is not supported in this build.");
#endif
API_IMPL_END
}
namespace {
#if !defined(DISABLE_SPARSE_TENSORS)
std::unique_ptr<IDataTransfer> GetDataTransfer(const OrtDevice& src_device, const OrtDevice& dst_device) {
if (src_device.Type() == OrtDevice::CPU && dst_device.Type() == OrtDevice::CPU) {
return std::make_unique<CPUDataTransfer>();
@ -284,12 +289,14 @@ union PtrConvert {
const char** strings;
};
#endif // !defined(DISABLE_SPARSE_TENSORS)
} // namespace
ORT_API_STATUS_IMPL(OrtApis::FillSparseTensorCoo, _Inout_ OrtValue* ort_value, _In_ const OrtMemoryInfo* data_mem_info,
_In_ const int64_t* values_shape, size_t values_shape_len, _In_ const void* values,
_In_ const int64_t* indices_data, size_t indices_num) {
API_IMPL_BEGIN
#if !defined(DISABLE_SPARSE_TENSORS)
TensorShape values_t_shape(values_shape, values_shape_len);
auto& sparse_tensor = ValidateFillInputArgs(ort_value, values_t_shape, data_mem_info);
@ -305,6 +312,9 @@ ORT_API_STATUS_IMPL(OrtApis::FillSparseTensorCoo, _Inout_ OrtValue* ort_value, _
values, indices_span));
}
return nullptr;
#else
return OrtApis::CreateStatus(ORT_FAIL, "SparseTensor is not supported in this build.");
#endif
API_IMPL_END
}
@ -313,6 +323,7 @@ ORT_API_STATUS_IMPL(OrtApis::FillSparseTensorCsr, _Inout_ OrtValue* ort_value, _
_In_ const int64_t* inner_indices_data, size_t inner_indices_num,
_In_ const int64_t* outer_indices_data, size_t outer_indices_num) {
API_IMPL_BEGIN
#if !defined(DISABLE_SPARSE_TENSORS)
TensorShape values_t_shape(values_shape, values_shape_len);
auto& sparse_tensor = ValidateFillInputArgs(ort_value, values_t_shape, data_mem_info);
auto values_size = gsl::narrow<size_t>(values_t_shape.Size());
@ -328,6 +339,9 @@ ORT_API_STATUS_IMPL(OrtApis::FillSparseTensorCsr, _Inout_ OrtValue* ort_value, _
values, inner_indices_span, outer_indices_span));
}
return nullptr;
#else
return OrtApis::CreateStatus(ORT_FAIL, "SparseTensor is not supported in this build.");
#endif
API_IMPL_END
}
@ -336,6 +350,7 @@ ORT_API_STATUS_IMPL(OrtApis::FillSparseTensorBlockSparse, _Inout_ OrtValue* ort_
_In_ const int64_t* indices_shape_data, size_t indices_shape_len,
_In_ const int32_t* indices_data) {
API_IMPL_BEGIN
#if !defined(DISABLE_SPARSE_TENSORS)
TensorShape values_t_shape(values_shape, values_shape_len);
auto& sparse_tensor = ValidateFillInputArgs(ort_value, values_t_shape, data_mem_info);
@ -354,6 +369,9 @@ ORT_API_STATUS_IMPL(OrtApis::FillSparseTensorBlockSparse, _Inout_ OrtValue* ort_
values, indices_t_shape, indices_data));
}
return nullptr;
#else
return OrtApis::CreateStatus(ORT_FAIL, "SparseTensor is not supported in this build.");
#endif
API_IMPL_END
}
@ -362,6 +380,7 @@ ORT_API_STATUS_IMPL(OrtApis::CreateSparseTensorWithValuesAsOrtValue, _In_ const
_In_ const int64_t* values_shape, size_t values_shape_len,
ONNXTensorElementDataType type, _Outptr_ OrtValue** out) {
API_IMPL_BEGIN
#if !defined(DISABLE_SPARSE_TENSORS)
auto sparse_tensor_type = DataTypeImpl::SparseTensorTypeFromONNXEnum(type);
auto element_type = sparse_tensor_type->GetElementType();
assert(element_type->AsPrimitiveDataType() != nullptr);
@ -380,11 +399,15 @@ ORT_API_STATUS_IMPL(OrtApis::CreateSparseTensorWithValuesAsOrtValue, _In_ const
SparseTensor::InitOrtValue(element_type, tensor_dense_shape, tensor_values_shape, p_data, *info, *value);
*out = value.release();
return nullptr;
#else
return OrtApis::CreateStatus(ORT_FAIL, "SparseTensor is not supported in this build.");
#endif
API_IMPL_END
}
ORT_API_STATUS_IMPL(OrtApis::UseCooIndices, _Inout_ OrtValue* ort_value, _Inout_ int64_t* indices_data, size_t indices_num) {
API_IMPL_BEGIN
#if !defined(DISABLE_SPARSE_TENSORS)
auto v = reinterpret_cast<::OrtValue*>(ort_value);
auto& sparse_tensor = SparseTensor::GetSparseTensorFromOrtValue(*v);
auto indices_span = (indices_num == 0 || indices_data == nullptr)
@ -393,6 +416,9 @@ ORT_API_STATUS_IMPL(OrtApis::UseCooIndices, _Inout_ OrtValue* ort_value, _Inout_
ORT_THROW_IF_ERROR(sparse_tensor.UseCooIndices(indices_span));
return nullptr;
#else
return OrtApis::CreateStatus(ORT_FAIL, "SparseTensor is not supported in this build.");
#endif
API_IMPL_END
}
@ -400,6 +426,7 @@ ORT_API_STATUS_IMPL(OrtApis::UseCsrIndices, _Inout_ OrtValue* ort_value,
_Inout_ int64_t* inner_data, size_t inner_num,
_Inout_ int64_t* outer_data, size_t outer_num) {
API_IMPL_BEGIN
#if !defined(DISABLE_SPARSE_TENSORS)
auto& sparse_tensor = SparseTensor::GetSparseTensorFromOrtValue(*ort_value);
auto inner_span = (inner_num == 0 || inner_data == nullptr)
? gsl::span<int64_t>()
@ -409,21 +436,29 @@ ORT_API_STATUS_IMPL(OrtApis::UseCsrIndices, _Inout_ OrtValue* ort_value,
: gsl::make_span(outer_data, outer_num);
ORT_THROW_IF_ERROR(sparse_tensor.UseCsrIndices(inner_span, outer_span));
return nullptr;
#else
return OrtApis::CreateStatus(ORT_FAIL, "SparseTensor is not supported in this build.");
#endif
API_IMPL_END
}
ORT_API_STATUS_IMPL(OrtApis::UseBlockSparseIndices, _Inout_ OrtValue* ort_value, const int64_t* indices_shape, size_t indices_shape_len,
_Inout_ int32_t* indices_data) {
API_IMPL_BEGIN
#if !defined(DISABLE_SPARSE_TENSORS)
auto& sparse_tensor = SparseTensor::GetSparseTensorFromOrtValue(*ort_value);
TensorShape ind_shape(indices_shape, indices_shape_len);
ORT_THROW_IF_ERROR(sparse_tensor.UseBlockSparseIndices(ind_shape, indices_data));
return nullptr;
#else
return OrtApis::CreateStatus(ORT_FAIL, "SparseTensor is not supported in this build.");
#endif
API_IMPL_END
}
ORT_API_STATUS_IMPL(OrtApis::GetSparseTensorFormat, _In_ const OrtValue* ort_value, _Out_ enum OrtSparseFormat* out) {
API_IMPL_BEGIN
#if !defined(DISABLE_SPARSE_TENSORS)
auto v = reinterpret_cast<const ::OrtValue*>(ort_value);
if (!v->IsAllocated()) {
return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, "the ort_value must contain a constructed tensor");
@ -431,11 +466,15 @@ ORT_API_STATUS_IMPL(OrtApis::GetSparseTensorFormat, _In_ const OrtValue* ort_val
const auto& sparse_tensor = v->Get<SparseTensor>();
*out = static_cast<OrtSparseFormat>(sparse_tensor.Format());
return nullptr;
#else
return OrtApis::CreateStatus(ORT_FAIL, "SparseTensor is not supported in this build.");
#endif
API_IMPL_END
}
ORT_API_STATUS_IMPL(OrtApis::GetSparseTensorValues, _In_ const OrtValue* ort_value, _Outptr_ const void** out) {
API_IMPL_BEGIN
#if !defined(DISABLE_SPARSE_TENSORS)
const auto& sparse_tensor = SparseTensor::GetSparseTensorFromOrtValue(*ort_value);
if (sparse_tensor.IsDataTypeString()) {
return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, "Use GetStringTensor*() API to retrieve strings");
@ -443,6 +482,9 @@ ORT_API_STATUS_IMPL(OrtApis::GetSparseTensorValues, _In_ const OrtValue* ort_val
const auto& values = sparse_tensor.Values();
*out = values.DataRaw();
return nullptr;
#else
return OrtApis::CreateStatus(ORT_FAIL, "SparseTensor is not supported in this build.");
#endif
API_IMPL_END
}
@ -883,9 +925,13 @@ ORT_API_STATUS_IMPL(OrtApis::IsTensor, _In_ const OrtValue* value, _Out_ int* ou
}
ORT_API_STATUS_IMPL(OrtApis::IsSparseTensor, _In_ const OrtValue* value, _Out_ int* out) {
#if !defined(DISABLE_SPARSE_TENSORS)
auto v = reinterpret_cast<const ::OrtValue*>(value);
*out = v->IsSparseTensor() ? 1 : 0;
return nullptr;
#else
return OrtApis::CreateStatus(ORT_FAIL, "SparseTensor is not supported in this build.");
#endif
}
ORT_API_STATUS_IMPL(OrtApis::GetTensorMutableData, _Inout_ OrtValue* value, _Outptr_ void** output) {
@ -943,7 +989,9 @@ OrtStatusPtr GetTensorStringSpan(const ::OrtValue& v, gsl::span<const std::strin
if (items >= 0) {
str_span = tensor.DataAsSpan<std::string>();
}
} else if (v.IsSparseTensor()) {
}
#if !defined(DISABLE_SPARSE_TENSORS)
else if (v.IsSparseTensor()) {
const auto& sparse_tensor = v.Get<SparseTensor>();
if (sparse_tensor.Format() == onnxruntime::SparseFormat::kUndefined) {
return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, "Sparse Tensor does not contain sparse data");
@ -952,7 +1000,9 @@ OrtStatusPtr GetTensorStringSpan(const ::OrtValue& v, gsl::span<const std::strin
if (items >= 0) {
str_span = sparse_tensor.Values().DataAsSpan<std::string>();
}
} else {
}
#endif
else {
return OrtApis::CreateStatus(ORT_NOT_IMPLEMENTED, "This API supports Tensors or SparseTensors");
}

24
onnxruntime/core/session/provider_bridge_ort.cc
View file

@ -198,6 +198,7 @@ struct ProviderHostImpl : ProviderHost {
float math__halfToFloat(uint16_t h) override { return math::halfToFloat(h); }
// sparse_utils
#if !defined(DISABLE_SPARSE_TENSORS)
#if !defined(ORT_MINIMAL_BUILD)
Status sparse_utils__DenseTensorToSparseCsr(const DataTransferManager& data_manager, const Tensor& src, const AllocatorPtr& cpu_allocator,
const AllocatorPtr& dst_allocator, SparseTensor& dst) override {
@ -213,12 +214,15 @@ struct ProviderHostImpl : ProviderHost {
const AllocatorPtr& dst_allocator, Tensor& dst) override {
return sparse_utils::SparseCooToDenseTensor(data_manager, src, cpu_allocator, dst_allocator, dst);
}
#endif // ORT_MINIMAL_BUILD
Status sparse_utils__DenseTensorToSparseCoo(const DataTransferManager& data_manager, const Tensor& src, const AllocatorPtr& cpu_allocator,
const AllocatorPtr& dst_allocator, bool linear_indexs, SparseTensor& dst) override {
return sparse_utils::DenseTensorToSparseCoo(data_manager, src, cpu_allocator, dst_allocator, linear_indexs, dst);
}
#endif // !defined(DISABLE_SPARSE_TENSORS)
// IAllocator (direct)
bool IAllocator__CalcMemSizeForArrayWithAlignment(size_t nmemb, size_t size, size_t alignment, size_t* out) override { return IAllocator::CalcMemSizeForArrayWithAlignment(nmemb, size, alignment, out); }
@ -290,6 +294,7 @@ struct ProviderHostImpl : ProviderHost {
int32_t TypeProto_Tensor__elem_type(const ONNX_NAMESPACE::TypeProto_Tensor* p) override { return p->elem_type(); }
//TypeProto_SparseTensor (wrapped)
#if !defined(DISABLE_SPARSE_TENSORS)
bool TypeProto_SparseTensor__has_shape(const ONNX_NAMESPACE::TypeProto_SparseTensor* p) override { return p->has_shape(); }
const ONNX_NAMESPACE::TensorShapeProto& TypeProto_SparseTensor__shape(const ONNX_NAMESPACE::TypeProto_SparseTensor* p) override {
return p->shape();
@ -300,17 +305,20 @@ struct ProviderHostImpl : ProviderHost {
int32_t TypeProto_SparseTensor__elem_type(const ONNX_NAMESPACE::TypeProto_SparseTensor* p) override {
return p->elem_type();
}
#endif
// TypeProto (wrapped)
const ONNX_NAMESPACE::TypeProto_Tensor& TypeProto__tensor_type(const ONNX_NAMESPACE::TypeProto* p) override { return p->tensor_type(); }
ONNX_NAMESPACE::TypeProto_Tensor* TypeProto__mutable_tensor_type(ONNX_NAMESPACE::TypeProto* p) override { return p->mutable_tensor_type(); }
int TypeProto__value_case(const ONNX_NAMESPACE::TypeProto* p) override { return p->value_case(); }
#if !defined(DISABLE_SPARSE_TENSORS)
const ONNX_NAMESPACE::TypeProto_SparseTensor& TypeProto__sparse_tensor_type(const ONNX_NAMESPACE::TypeProto* p) override {
return p->sparse_tensor_type();
}
ONNX_NAMESPACE::TypeProto_SparseTensor* TypeProto__mutable_sparse_tensor_type(ONNX_NAMESPACE::TypeProto* p) override {
return p->mutable_sparse_tensor_type();
}
#endif
// AttributeProto (wrapped)
std::unique_ptr<ONNX_NAMESPACE::AttributeProto> AttributeProto__construct() override { return std::make_unique<ONNX_NAMESPACE::AttributeProto>(); }
@ -427,17 +435,21 @@ struct ProviderHostImpl : ProviderHost {
// DataTransferManager (wrapped)
Status DataTransferManager__CopyTensor(const DataTransferManager* p, const Tensor& src, Tensor& dst, int exec_queue_id) override { return p->CopyTensor(src, dst, exec_queue_id); }
Status DataTransferManager__CopyTensor(const DataTransferManager* p, const Tensor& src, Tensor& dst) override { return p->CopyTensor(src, dst); }
#if !defined(DISABLE_SPARSE_TENSORS)
Status DataTransferManager__CopySparseTensor(const DataTransferManager* p, const SparseTensor& src, SparseTensor& dst) override { return p->CopySparseTensor(src, dst); }
Status DataTransferManager__CopySparseTensor(const DataTransferManager* p, const SparseTensor& src, SparseTensor& dst, int exec_queue_id) override { return p->CopySparseTensor(src, dst, exec_queue_id); }
Status DataTransferManager__CopySparseTensors(const DataTransferManager* p, const std::vector<IDataTransfer::SparseSrcDstPair>& src_dst_pairs) override { return p->CopySparseTensors(src_dst_pairs); };
#endif
const IDataTransfer* DataTransferManager__GetDataTransfer(const DataTransferManager* p, const OrtDevice& src_device, const OrtDevice& dst_device) override { return p->GetDataTransfer(src_device, dst_device); }
// IDataTransfer (direct)
Status IDataTransfer__CopyTensor(const IDataTransfer* p, const Tensor& src, Tensor& dst) override { return p->IDataTransfer::CopyTensor(src, dst); }
Status IDataTransfer__CopyTensors(const IDataTransfer* p, const std::vector<IDataTransfer::SrcDstPair>& src_dst_pairs) override { return p->IDataTransfer::CopyTensors(src_dst_pairs); }
#if !defined(DISABLE_SPARSE_TENSORS)
Status IDataTransfer__CopySparseTensors(const IDataTransfer* p, const std::vector<IDataTransfer::SparseSrcDstPair>& src_dst_pairs) override {
return p->CopySparseTensors(src_dst_pairs);
}
#endif
// IndexedSubGraph_MetaDef (wrapped)
std::unique_ptr<IndexedSubGraph_MetaDef> IndexedSubGraph_MetaDef__construct() override { return std::make_unique<IndexedSubGraph::MetaDef>(); }
@ -506,7 +518,9 @@ struct ProviderHostImpl : ProviderHost {
// DataTypeImpl (wrapped)
MLDataType DataTypeImpl__GetType_Tensor() override { return DataTypeImpl::GetType<Tensor>(); }
#if !defined(DISABLE_SPARSE_TENSORS)
MLDataType DataTypeImpl__GetType_SparseTensor() override { return DataTypeImpl::GetType<SparseTensor>(); }
#endif
MLDataType DataTypeImpl__GetType_TensorSeq() override { return DataTypeImpl::GetType<TensorSeq>(); }
MLDataType DataTypeImpl__GetTypeFromOnnxType(int onnx_type) override { return DataTypeImpl::TensorTypeFromONNXEnum(onnx_type)->GetElementType(); }
MLDataType DataTypeImpl__GetType_bool() override { return DataTypeImpl::GetType<bool>(); }
@ -537,6 +551,7 @@ struct ProviderHostImpl : ProviderHost {
MLDataType DataTypeImpl__GetTensorType_BFloat16() override { return DataTypeImpl::GetTensorType<BFloat16>(); }
MLDataType DataTypeImpl__GetTensorType_MLFloat16() override { return DataTypeImpl::GetTensorType<MLFloat16>(); }
#if !defined(DISABLE_SPARSE_TENSORS)
MLDataType DataTypeImpl__GetSparseTensorType_bool() override { return DataTypeImpl::GetSparseTensorType<bool>(); }
MLDataType DataTypeImpl__GetSparseTensorType_int8() override { return DataTypeImpl::GetSparseTensorType<int8_t>(); }
MLDataType DataTypeImpl__GetSparseTensorType_uint8() override { return DataTypeImpl::GetSparseTensorType<uint8_t>(); }
@ -551,11 +566,14 @@ struct ProviderHostImpl : ProviderHost {
MLDataType DataTypeImpl__GetSparseTensorType_string() override { return DataTypeImpl::GetSparseTensorType<std::string>(); }
MLDataType DataTypeImpl__GetSparseTensorType_BFloat16() override { return DataTypeImpl::GetSparseTensorType<BFloat16>(); }
MLDataType DataTypeImpl__GetSparseTensorType_MLFloat16() override { return DataTypeImpl::GetSparseTensorType<MLFloat16>(); }
#endif
const char* DataTypeImpl__ToString(MLDataType type) override { return DataTypeImpl::ToString(type); }
bool DataTypeImpl__IsTensorType(const DataTypeImpl* p) override { return p->IsTensorType(); }
bool DataTypeImpl__IsTensorSequenceType(const DataTypeImpl* p) override { return p->IsTensorSequenceType(); }
#if !defined(DISABLE_SPARSE_TENSORS)
bool DataTypeImpl__IsSparseTensorType(const DataTypeImpl* p) override { return p->IsSparseTensorType(); }
#endif
DeleteFunc DataTypeImpl__GetDeleteFunc(const DataTypeImpl* p) override { return p->GetDeleteFunc(); }
const std::vector<MLDataType>& DataTypeImpl__AllFixedSizeTensorTypes() override { return DataTypeImpl::AllFixedSizeTensorTypes(); }
const std::vector<MLDataType>& DataTypeImpl__AllTensorTypes() override { return DataTypeImpl::AllTensorTypes(); }
@ -694,14 +712,18 @@ struct ProviderHostImpl : ProviderHost {
// OpKernelContext (wrapped)
const Tensor* OpKernelContext__Input_Tensor(const OpKernelContext* p, int index) override { return p->Input<Tensor>(index); }
#if !defined(DISABLE_SPARSE_TENSORS)
const SparseTensor* OpKernelContext__Input_SparseTensor(const OpKernelContext* p, int index) override { return p->Input<SparseTensor>(index); }
#endif
const TensorSeq* OpKernelContext__Input_TensorSeq(const OpKernelContext* p, int index) override { return p->Input<TensorSeq>(index); }
const Tensor& OpKernelContext__RequiredInput_Tensor(const OpKernelContext* p, int index) override { return p->RequiredInput<Tensor>(index); }
MLDataType OpKernelContext__InputType(const OpKernelContext* p, int index) override { return p->InputType(index); }
Tensor* OpKernelContext__Output_Tensor(OpKernelContext* p, int index) override { return p->Output<Tensor>(index); }
TensorSeq* OpKernelContext__Output_TensorSeq(OpKernelContext* p, int index) override { return p->Output<TensorSeq>(index); }
Tensor* OpKernelContext__Output(OpKernelContext* p, int index, const TensorShape& shape) override { return p->Output(index, shape); }
#if !defined(DISABLE_SPARSE_TENSORS)
SparseTensor* OpKernelContext__OutputSparse(OpKernelContext* p, int index, const TensorShape& shape) override { return p->OutputSparse(index, shape); }
#endif
Tensor& OpKernelContext__RequiredOutput(OpKernelContext* p, int index, const TensorShape& shape) override { return p->RequiredOutput(index, shape); }
int OpKernelContext__InputCount(const OpKernelContext* p) override { return p->InputCount(); }
int OpKernelContext__OutputCount(const OpKernelContext* p) override { return p->OutputCount(); }
@ -806,8 +828,10 @@ struct ProviderHostImpl : ProviderHost {
MLDataType Tensor__DataType(const Tensor* p) override { return p->DataType(); }
// SparseTensor(wrapped)
#if !defined(DISABLE_SPARSE_TENSORS)
const TensorShape& SparseTensor__DenseShape(const SparseTensor* p) override { return p->DenseShape(); }
Status SparseTensor__Copy(const SparseTensor* p, const DataTransferManager& dtm, int exec_q_id, SparseTensor& dst) override { return p->Copy(dtm, exec_q_id, dst); }
#endif
// TensorSeq(wrapped)
MLDataType TensorSeq__DataType(const TensorSeq* p) noexcept override { return p->DataType(); }

3
onnxruntime/test/contrib_ops/math/matmul_sparse_test.cc
View file

@ -141,7 +141,7 @@ void resize(Index size, double reserveSizeFactor = 0) {
m_size = size;
}
*/
#if !defined(DISABLE_SPARSE_TENSORS)
#if !defined(__i386__) && !defined(_M_IX86) && !defined(__wasm__) && !defined(__ANDROID__)
TEST(SparseToDenseMatMul, TestCsr) {
constexpr int64_t rows = 9;
@ -383,6 +383,7 @@ TEST(SparseToDenseMatMul, TestCoo) {
tester.Run(OpTester::ExpectResult::kExpectSuccess);
}
}
#endif // !defined(DISABLE_SPARSE_TENSORS)
} // namespace test
} // namespace onnxruntime

8
onnxruntime/test/framework/execution_frame_test.cc
View file

@ -106,8 +106,8 @@ TEST_F(ExecutionFrameTest, TensorAllocationTest) {
}
TEST_F(ExecutionFrameTest, OutputShapeValidationTest) {
onnxruntime::Model model("test", false, ModelMetaData(), PathString(), IOnnxRuntimeOpSchemaRegistryList(),
{{kOnnxDomain, 12}}, {}, DefaultLoggingManager().DefaultLogger());
onnxruntime::Model model("test", false, ModelMetaData(), PathString(), IOnnxRuntimeOpSchemaRegistryList(),
{{kOnnxDomain, 12}}, {}, DefaultLoggingManager().DefaultLogger());
onnxruntime::Graph& graph = model.MainGraph();
TypeProto tensor_float;
tensor_float.mutable_tensor_type()->set_elem_type(TensorProto_DataType_FLOAT);
@ -145,7 +145,7 @@ TEST_F(ExecutionFrameTest, OutputShapeValidationTest) {
ASSERT_EQ(start_index, 0);
TensorShape actual_shape_same_as_input(std::vector<int64_t>{2, 3});
TensorShape actual_shape_diff_from_input(std::vector<int64_t>{2, 9});
OrtValue* p_ml_value = frame.GetMutableNodeInputOrOutputMLValue(0);
ASSERT_TRUE(p_ml_value != nullptr);
@ -467,6 +467,7 @@ TEST(ExecutionFrameTestInit, InitializerAsOutput) {
}
}
#if !defined(DISABLE_SPARSE_TENSORS)
TEST(ExecutionFrameTestInit, SparseInitializerAsOutput) {
const std::vector<int64_t> dense_shape{3, 3};
@ -508,6 +509,7 @@ TEST(ExecutionFrameTestInit, SparseInitializerAsOutput) {
EXPECT_THAT(coo_view.Indices().DataAsSpan<int64_t>(), ::testing::ContainerEq(gsl::make_span(expected_linear_indices)));
}
}
#endif // !defined(DISABLE_SPARSE_TENSORS)
} // namespace test
} // namespace onnxruntime

26
onnxruntime/test/framework/sparse_kernels_test.cc
View file

@ -129,6 +129,7 @@ This operator constructs a sparse tensor from three tensors that provide a COO
TensorShape shape(shape_tensor.Data<int64_t>(), shape_shape.Size());
#if !defined(DISABLE_SPARSE_TENSORS)
SparseTensor* output = ctx->OutputSparse(0, shape);
ORT_ENFORCE(output != nullptr);
const auto& dtm = Info().GetDataTransferManager();
@ -138,6 +139,7 @@ This operator constructs a sparse tensor from three tensors that provide a COO
static_cast<size_t>(val_shape.Size()),
values.DataRaw(),
indices.DataAsSpan<int64_t>()));
#endif
return Status::OK();
}
};
@ -147,8 +149,12 @@ This operator constructs a sparse tensor from three tensors that provide a COO
def.SetName(SparseFromCOO::OpName())
.TypeConstraint("values", DataTypeImpl::GetTensorType<int64_t>())
.TypeConstraint("indices", DataTypeImpl::GetTensorType<int64_t>())
#if !defined(DISABLE_SPARSE_TENSORS)
.TypeConstraint("shape", DataTypeImpl::GetTensorType<int64_t>())
.TypeConstraint("sparse_rep", DataTypeImpl::GetSparseTensorType<int64_t>());
#else
.TypeConstraint("shape", DataTypeImpl::GetTensorType<int64_t>());
#endif
return def;
}
};
@ -190,6 +196,7 @@ This operator applies the Abs op element-wise to the input sparse-tensor.
Status Compute(OpKernelContext* ctx) const override {
ORT_ENFORCE(ctx->InputCount() == 1, "Expecting 1 input");
#if !defined(DISABLE_SPARSE_TENSORS)
const SparseTensor* input = ctx->Input<SparseTensor>(0);
const auto* input_values = input->Values().Data<int64_t>();
const auto nnz = input->NumValues();
@ -211,6 +218,7 @@ This operator applies the Abs op element-wise to the input sparse-tensor.
// TODO: Extend allocation-planner to enable such sharing.
const auto& input_indices = input_coo_view.Indices();
memcpy(output_mutator.Indices().MutableData<int64_t>(), input_indices.Data<int64_t>(), input_indices.SizeInBytes());
#endif
return Status::OK();
}
};
@ -218,8 +226,10 @@ This operator applies the Abs op element-wise to the input sparse-tensor.
// A KernelDefBuilder for SparseAbs:
static KernelDefBuilder KernelDef() {
KernelDefBuilder def;
#if !defined(DISABLE_SPARSE_TENSORS)
def.SetName(OpName())
.TypeConstraint("T", DataTypeImpl::GetSparseTensorType<int64_t>());
#endif
return def;
}
};
@ -261,6 +271,8 @@ struct SparseToValues {
Status Compute(OpKernelContext* ctx) const override {
ORT_ENFORCE(ctx->InputCount() == 1, "Expecting a single SparseTensorSample input");
#if !defined(DISABLE_SPARSE_TENSORS)
const SparseTensor* sparse_input = ctx->Input<SparseTensor>(0);
const auto* values = sparse_input->Values().Data<int64_t>();
auto nnz = sparse_input->Values().Shape().Size();
@ -272,7 +284,7 @@ struct SparseToValues {
ORT_ENFORCE(output_data != nullptr);
memcpy(output_data, values, sparse_input->Values().SizeInBytes());
#endif
return Status::OK();
}
};
@ -280,9 +292,11 @@ struct SparseToValues {
// A KernelDefBuilder for SparseToValues
static KernelDefBuilder KernelDef() {
KernelDefBuilder def;
#if !defined(DISABLE_SPARSE_TENSORS)
def.SetName(OpName())
.TypeConstraint("sparse_rep", DataTypeImpl::GetSparseTensorType<int64_t>())
.TypeConstraint("values", DataTypeImpl::GetTensorType<int64_t>());
#endif
return def;
}
};
@ -347,12 +361,14 @@ class SparseTensorTests : public testing::Test {
EXPECT_TRUE(session_object.Initialize().IsOK());
}
#if !defined(DISABLE_SPARSE_TENSORS)
NodeArg* Sparse(const std::string& name) {
types.push_back(*DataTypeImpl::GetSparseTensorType<int64_t>()->GetTypeProto());
Graph& graph = model->MainGraph();
auto& arg = graph.GetOrCreateNodeArg(name, &types.back());
return &arg;
}
#endif
NodeArg* Dense(const std::string& name) {
types.push_back(*DataTypeImpl::GetTensorType<int64_t>()->GetTypeProto());
@ -431,6 +447,7 @@ class SparseTensorTests : public testing::Test {
}
};
#if !defined(DISABLE_SPARSE_TENSORS)
// Test ops SparseFromCOO, SparseAbs, and SparseToValues.
// Tests 1-dimensional int64 sparse tensor.
TEST_F(SparseTensorTests, Test1) {
@ -606,6 +623,7 @@ TEST(SparseCrcsFormatTests, Test1) {
csr_wrap.Outer().Data<int64_t>(),
outer_indices.size() * sizeof(int64_t)));
}
#endif // !defined(DISABLE_SPARSE_TENSORS)
// Code below depends on the values being size 4
template <typename T>
@ -928,6 +946,7 @@ TEST(SparseTensorConversionTests, TestConstantNodeConversion) {
}
/// Dense to Sparse conversion tests
#if !defined(DISABLE_SPARSE_TENSORS)
#if !defined(ORT_MINIMAL_BUILD)
template <typename T>
@ -1570,7 +1589,6 @@ TEST(SparseTensorConversionTests, CooConversion) {
ASSERT_TRUE(std::equal(expected_linear_indices.cbegin(), expected_linear_indices.cend(), indices.cbegin(), indices.cend()));
}
{
// test where both src and destination are on CPU. 2-D index
SparseTensor dst;
@ -1718,7 +1736,6 @@ TEST(SparseTensorConversionTests, BlockSparse) {
const std::string expected_strings[] = {
"1", "2", "3", "4", "5", "6", "7", "8"};
const TensorShape indices_shape{2, 2}; // two blocks by two coordinates
// (0, 0), (0,1)
std::vector<int32_t> blocksparse_indices = {
@ -1783,8 +1800,9 @@ TEST(SparseTensorConversionTests, BlockSparse) {
auto indices_span = indices.DataAsSpan<int32_t>();
ASSERT_TRUE(std::equal(blocksparse_indices.cbegin(), blocksparse_indices.cend(),
indices_span.cbegin(), indices_span.cend()));
}
}
#endif // !defined(DISABLE_SPARSE_TENSORS)
} // namespace test
} // namespace onnxruntime

9
onnxruntime/test/ir/graph_test.cc
View file

@ -144,8 +144,7 @@ static bool RegisterCustomSchemas() {
node.set_domain(kMSNchwcDomain);
}
}
return nodes;
}(),
return nodes; }(),
[]() {
std::vector<OperatorSetIdProto> operator_sets(2);
auto& onnx_opset = operator_sets[0];
@ -217,6 +216,7 @@ const std::vector<float> values = {13.f,
const std::vector<int64_t> indices = {9, 30, 50}; // Not to exceed 59
} // namespace sparse_details
#if !defined(DISABLE_SPARSE_TENSORS)
// To match a simple Add graph above
static void ConstructSparseTensor(const std::string& name,
SparseTensorProto& sparse_proto) {
@ -274,6 +274,7 @@ static void ValidateSparseTensorProto(const SparseTensorProto& proto) {
auto expected_shape = gsl::make_span(sparse_details::shape);
EXPECT_THAT(actual_shape, testing::ContainerEq(expected_shape));
}
#endif // !defined(DISABLE_SPARSE_TENSORS)
TEST_F(GraphTest, SimpleAddWithoutDomain) {
ModelProto m;
@ -1304,6 +1305,7 @@ TEST_F(GraphTest, UnusedInitializerIsIgnored) {
ASSERT_TRUE(graph.GetAllInitializedTensors().empty());
}
#if !defined(DISABLE_SPARSE_TENSORS)
TEST_F(GraphTest, UnusedSparseInitializerIsIgnored) {
std::string s1;
{
@ -1334,6 +1336,7 @@ TEST_F(GraphTest, UnusedSparseInitializerIsIgnored) {
auto& graph_proto = graph2.ToGraphProto();
ASSERT_TRUE(graph_proto.sparse_initializer().empty());
}
#endif // !defined(DISABLE_SPARSE_TENSORS)
TEST_F(GraphTest, GraphConstruction_CheckIsNotAcyclic) {
// A cyclic graph
@ -1798,6 +1801,7 @@ TEST_F(GraphTest, AddRemoveInitializerHandling) {
<< num_initializers << " remain.";
}
#if !defined(DISABLE_SPARSE_TENSORS)
TEST_F(GraphTest, SparseInitializerHandling) {
const char* const input_initializer_name = "x";
Model model("SparseInitializerHandling", false, *logger_);
@ -1848,6 +1852,7 @@ TEST_F(GraphTest, SparseInitializerHandling) {
ValidateSparseTensorProto(model_proto_get.graph().sparse_initializer().at(0));
}
}
#endif //!defined(DISABLE_SPARSE_TENSORS)
TEST_F(GraphTest, SetInputsAndSetOutputs_NewInputAndOutput) {
std::shared_ptr<Model> model;

7
onnxruntime/test/optimizer/optimizer_test.cc
View file

@ -66,12 +66,19 @@ TEST(OptimizerTest, Basic) {
std::unique_ptr<CPUExecutionProvider> cpu_execution_provider =
std::make_unique<CPUExecutionProvider>(CPUExecutionProviderInfo());
#if !defined(DISABLE_SPARSE_TENSORS)
OptimizerExecutionFrame::Info info(nodes, initialized_tensor_set,
graph.ModelPath(),
*cpu_execution_provider.get(),
[&graph](const std::string& name) -> bool {
return graph.IsSparseInitializer(name);
});
#else
OptimizerExecutionFrame::Info info(nodes, initialized_tensor_set,
graph.ModelPath(),
*cpu_execution_provider.get(),
[](std::string const& ) { return false; });
#endif //!defined(DISABLE_SPARSE_TENSORS)
std::vector<int> fetch_mlvalue_idxs{info.GetMLValueIndex("out")};
OptimizerExecutionFrame frame(info, fetch_mlvalue_idxs);

2
onnxruntime/test/providers/internal_testing/internal_testing_tests.cc
View file

@ -81,6 +81,7 @@ static void ExecuteMnist(InferenceSessionWrapper& session, bool custom_ep_enable
}
}
#if !defined(DISABLE_SPARSE_TENSORS)
#if !defined(ORT_MINIMAL_BUILD)
TEST(InternalTestingEP, TestSaveAndLoadOrtModel) {
const ORTCHAR_T* ort_model_path = ORT_TSTR("testdata/mnist.internal_testing_ep.test_output.ort");
@ -145,6 +146,7 @@ TEST(InternalTestingEP, PreventSaveOfModelWithCompiledOps) {
ASSERT_THAT(status.ErrorMessage(), ::testing::HasSubstr("Unable to serialize model as it contains compiled nodes"));
}
#endif // !defined(ORT_MINIMAL_BUILD)
#endif // !defined(DISABLE_SPARSE_TENSORS)
// test to validate a minimal build
TEST(InternalTestingEP, TestLoadOrtModel) {

3
onnxruntime/test/providers/provider_test_utils.cc
View file

@ -548,6 +548,7 @@ void OpTester::AddShapeToTensorData(NodeArg& node_arg, const std::vector<int64_t
}
}
#if !defined(DISABLE_SPARSE_TENSORS)
static std::unique_ptr<SparseTensor> MakeSparseTensor(MLDataType data_type, const std::vector<int64_t>& dims) {
TensorShape shape{dims};
auto allocator = test::AllocatorManager::Instance().GetAllocator(CPU);
@ -673,7 +674,7 @@ void OpTester::AddSparseCsrTensorStrings(std::vector<Data>& data,
NodeArg node_arg = MakeSparseNodeArg(dtype, name, dims, dim_params);
AddSparseTensorData(data, std::move(node_arg), std::move(p_tensor), CheckParams());
}
#endif // !defined(DISABLE_SPARSE_TENSORS)
void OpTester::AddInitializers(onnxruntime::Graph& graph) {
for (auto index : initializer_index_) {

12
onnxruntime/test/providers/provider_test_utils.h
View file

@ -149,6 +149,7 @@ struct TTensorType {
template <typename T>
const TTypeProto<T> TTensorType<T>::s_type_proto;
#if !defined(DISABLE_SPARSE_TENSORS)
struct TSparseTensorProto {
explicit TSparseTensorProto(int32_t dtype, const std::vector<int64_t>* shape = nullptr) {
proto.mutable_sparse_tensor_type()->set_elem_type(dtype);
@ -165,6 +166,7 @@ struct TSparseTensorProto {
}
ONNX_NAMESPACE::TypeProto proto;
};
#endif
// TypeProto for map<TKey, TVal>
template <typename TKey, typename TVal>
@ -295,6 +297,7 @@ class OpTester {
AddData(input_data_, name, dims, p_values, size, is_initializer, false, dim_params);
}
#if !defined(DISABLE_SPARSE_TENSORS)
// Useful to add boolean data
template <typename T>
void AddSparseCooInput(const char* name, const std::vector<int64_t>& dims,
@ -394,6 +397,7 @@ class OpTester {
gsl::make_span(outer_indices),
dim_params);
}
#endif
// Add other registered types, possibly experimental
template <typename T>
@ -475,7 +479,8 @@ class OpTester {
AddData(output_data_, name, dims, p_values, size, false,
sort_output, nullptr /* dim_params */, rel_error, abs_error);
}
#if !defined(DISABLE_SPARSE_TENSORS)
template <typename T>
void AddSparseCooOutput(const char* name, const std::vector<int64_t>& dims,
const std::initializer_list<T>& expected_values,
@ -571,6 +576,7 @@ class OpTester {
gsl::make_span(expected_inner_indices),
gsl::make_span(expected_outer_indices));
}
#endif
/*
* Use this API to add an output *edge* to the node/op being tested that shouldn't have any
@ -585,7 +591,7 @@ class OpTester {
output_data_.push_back(Data(NodeArg(name, &TTensorType<T>::s_type_proto.proto), OrtValue(), optional<float>(),
optional<float>()));
}
// Add other registered types, possibly experimental
template <typename T>
void AddOutput(const char* name, const T& val) {
@ -858,6 +864,7 @@ class OpTester {
void CopyDataToTensor(gsl::span<const gsl::byte> data, Tensor& dst);
#if !defined(DISABLE_SPARSE_TENSORS)
NodeArg MakeSparseNodeArg(int32_t dtype, const char* name,
const std::vector<int64_t>& dims,
const std::vector<std::string>* dim_params);
@ -899,6 +906,7 @@ class OpTester {
void AddSparseTensorData(std::vector<Data>& data, NodeArg node_arg,
std::unique_ptr<SparseTensor> p_tensor,
const CheckParams& check_params);
#endif
const char* domain_;
int opset_version_;

4
onnxruntime/test/shared_lib/test_inference.cc
View file

@ -176,10 +176,12 @@ static constexpr PATH_TYPE VARIED_INPUT_CUSTOM_OP_MODEL_URI_2 = TSTR("testdata/f
static constexpr PATH_TYPE OPTIONAL_INPUT_OUTPUT_CUSTOM_OP_MODEL_URI = TSTR("testdata/foo_bar_1.onnx");
static constexpr PATH_TYPE OPTIONAL_INPUT_OUTPUT_CUSTOM_OP_MODEL_URI_2 = TSTR("testdata/foo_bar_2.onnx");
static constexpr PATH_TYPE CUSTOM_OP_MODEL_WITH_ATTRIBUTES_URI = TSTR("testdata/foo_bar_3.onnx");
#if !defined(DISABLE_SPARSE_TENSORS)
static constexpr PATH_TYPE SPARSE_OUTPUT_MODEL_URI = TSTR("testdata/sparse_initializer_as_output.onnx");
#ifndef DISABLE_CONTRIB_OPS
static constexpr PATH_TYPE SPARSE_INPUT_MATMUL_MODEL_URI = TSTR("testdata/sparse_to_dense_matmul.onnx");
#endif
#endif // !defined(DISABLE_SPARSE_TENSORS)
#ifdef ENABLE_EXTENSION_CUSTOM_OPS
static constexpr PATH_TYPE ORT_CUSTOM_OPS_MODEL_URI = TSTR("testdata/custom_op_string_lower.onnx");
@ -244,6 +246,7 @@ INSTANTIATE_TEST_SUITE_P(CApiTestWithProviders,
CApiTestWithProvider,
::testing::Values(0, 1, 2, 3, 4));
#if !defined(DISABLE_SPARSE_TENSORS)
TEST(CApiTest, SparseOutputModel) {
std::vector<int64_t> dense_shape{3, 3};
std::vector<float> values{1.764052391052246, 0.40015721321105957, 0.978738009929657};
@ -358,6 +361,7 @@ TEST(CApiTest, SparseInputModel) {
ASSERT_TRUE(std::equal(Y_result.cbegin(), Y_result.cend(), result_span.cbegin(), result_span.cend()));
}
#endif // DISABLE_CONTRIB_OPS
#endif // !defined(DISABLE_SPARSE_TENSORS)
TEST(CApiTest, custom_op_handler) {
std::cout << "Running custom op inference" << std::endl;

4
onnxruntime/test/shared_lib/test_nontensor_types.cc
View file

@ -309,6 +309,7 @@ TEST(CApiTest, TypeInfoSequence) {
ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64);
}
#if !defined(DISABLE_SPARSE_TENSORS)
TEST(CApiTest, SparseTensorUsingAPI) {
Ort::MemoryInfo info("Cpu", OrtDeviceAllocator, 0, OrtMemTypeDefault);
@ -921,4 +922,5 @@ TEST(CApiTest, SparseTensorFillSparseFormatStringsAPI) {
ASSERT_TRUE(std::equal(blocksparse_indices.cbegin(), blocksparse_indices.cend(), ind_span.cbegin(), ind_span.cend()));
}
}
}
}
#endif // !defined(DISABLE_SPARSE_TENSORS)

4
onnxruntime/test/util/compare_ortvalue.cc
View file

@ -251,6 +251,7 @@ std::pair<COMPARE_RESULT, std::string> CompareSeqOfMapToFloat(const T& real_outp
return std::make_pair(COMPARE_RESULT::SUCCESS, "");
}
#if !defined(DISABLE_SPARSE_TENSORS)
std::pair<COMPARE_RESULT, std::string> CompareSparseTensors(const SparseTensor& actual, const SparseTensor& expected,
double per_sample_tolerance, double relative_per_sample_tolerance,
bool post_processing) {
@ -290,6 +291,7 @@ std::pair<COMPARE_RESULT, std::string> CompareSparseTensors(const SparseTensor&
return std::make_pair(COMPARE_RESULT::SUCCESS, "");
}
#endif // !defined(DISABLE_SPARSE_TENSORS)
// The expected_shape could contain unknown dimensions, but the real_shape cannot
bool AreShapesEqual(const std::vector<int64_t>& real_shape, const ::ONNX_NAMESPACE::TensorShapeProto& expected_shape) {
@ -353,12 +355,14 @@ std::pair<COMPARE_RESULT, std::string> CompareOrtValue(const OrtValue& o, const
return CompareTwoTensors(outvalue, expected_tensor, per_sample_tolerance, relative_per_sample_tolerance,
post_processing);
} else if (o.IsSparseTensor()) {
#if !defined(DISABLE_SPARSE_TENSORS)
TEST_RETURN_IF_NOT(expected_mlvalue.IsSparseTensor(), COMPARE_RESULT::TYPE_MISMATCH,
"SparseTensor is not expected as output");
TEST_RETURN_IF_ERROR(CompareSparseTensors(o.Get<SparseTensor>(), expected_mlvalue.Get<SparseTensor>(),
per_sample_tolerance, relative_per_sample_tolerance,
post_processing),
"while comaring sparse tensors");
#endif
return std::make_pair(COMPARE_RESULT::SUCCESS, "");
} else if (o.IsTensorSequence()) {
auto& expected_tensor_seq = expected_mlvalue.Get<TensorSeq>();