diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 83058da655..7505ee5ae8 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -107,6 +107,7 @@ option(onnxruntime_USE_ROCM "Build with AMD GPU support" OFF) # Options related to reducing the binary size produced by the build option(onnxruntime_DISABLE_CONTRIB_OPS "Disable contrib ops" OFF) option(onnxruntime_DISABLE_ML_OPS "Disable traditional ML ops" OFF) +option(onnxruntime_DISABLE_SPARSE_TENSORS "Disable sparse tensors data types" OFF) cmake_dependent_option(onnxruntime_DISABLE_RTTI "Disable RTTI" ON "NOT onnxruntime_ENABLE_PYTHON" OFF) # For now onnxruntime_DISABLE_EXCEPTIONS will only work with onnxruntime_MINIMAL_BUILD, more changes (ONNX, non-CPU EP, ...) are required to run this standalone option(onnxruntime_DISABLE_EXCEPTIONS "Disable exception handling. Requires onnxruntime_MINIMAL_BUILD currently." OFF) @@ -758,6 +759,10 @@ if (onnxruntime_DISABLE_ML_OPS) add_definitions(-DDISABLE_ML_OPS) endif() +if (onnxruntime_DISABLE_SPARSE_TENSORS) + add_compile_definitions(DISABLE_SPARSE_TENSORS) +endif() + if (onnxruntime_USE_CUDA AND "${onnxruntime_CUDNN_HOME}" STREQUAL "") message(FATAL_ERROR "onnxruntime_CUDNN_HOME required for onnxruntime_USE_CUDA") endif() diff --git a/include/onnxruntime/core/framework/data_types.h b/include/onnxruntime/core/framework/data_types.h index 5d90038eac..18bf756c05 100644 --- a/include/onnxruntime/core/framework/data_types.h +++ b/include/onnxruntime/core/framework/data_types.h @@ -53,7 +53,9 @@ using VectorInt64 = std::vector; class DataTypeImpl; class TensorTypeBase; +#if !defined(DISABLE_SPARSE_TENSORS) class SparseTensorTypeBase; +#endif class SequenceTensorTypeBase; class NonTensorTypeBase; class PrimitiveDataTypeBase; @@ -114,10 +116,12 @@ class DataTypeImpl { return nullptr; } +#if !defined(DISABLE_SPARSE_TENSORS) // Returns this if this is of sparse-tensor-type and null otherwise virtual const SparseTensorTypeBase* AsSparseTensorType() const { return nullptr; } +#endif virtual const NonTensorTypeBase* AsNonTensorTypeBase() const { return nullptr; @@ -140,9 +144,11 @@ class DataTypeImpl { template static MLDataType GetSequenceTensorType(); +#if !defined(DISABLE_SPARSE_TENSORS) // Return the MLDataType for a concrete sparse tensor type. template static MLDataType GetSparseTensorType(); +#endif /** * Convert an ONNX TypeProto to onnxruntime DataTypeImpl. @@ -154,8 +160,10 @@ class DataTypeImpl { static MLDataType TypeFromProto(const ONNX_NAMESPACE::TypeProto& proto); static const TensorTypeBase* TensorTypeFromONNXEnum(int type); - static const SparseTensorTypeBase* SparseTensorTypeFromONNXEnum(int type); static const NonTensorTypeBase* SequenceTensorTypeFromONNXEnum(int type); +#if !defined(DISABLE_SPARSE_TENSORS) + static const SparseTensorTypeBase* SparseTensorTypeFromONNXEnum(int type); +#endif static const char* ToString(MLDataType type); static std::vector ToString(const std::vector& types); @@ -282,6 +290,7 @@ struct IsTensorContainedType : public IsAnyOf { }; +#if !defined(DISABLE_SPARSE_TENSORS) /// Use "IsSparseTensorContainedType::value" to test if a type T /// is permitted as the element-type of a sparse-tensor. @@ -290,6 +299,7 @@ struct IsSparseTensorContainedType : public IsAnyOf { }; +#endif /// This template's Get() returns a corresponding MLDataType /// It dispatches the call to either GetTensorType<>() or @@ -441,6 +451,7 @@ class TensorType : public TensorTypeBase { } }; +#if !defined(DISABLE_SPARSE_TENSORS) /// Common base-class for all sparse-tensors (with different element types). class SparseTensorTypeBase : public DataTypeImpl { public: @@ -500,6 +511,7 @@ class SparseTensorType : public SparseTensorTypeBase { TensorElementTypeSetter::SetSparseTensorElementType(mutable_type_proto()); } }; +#endif // !defined(DISABLE_SPARSE_TENSORS) /** * \brief Provide a specialization for your C++ Non-tensor type @@ -850,6 +862,7 @@ class PrimitiveDataType : public PrimitiveDataTypeBase { return TensorType::Type(); \ } +#if !defined(DISABLE_SPARSE_TENSORS) #define ORT_REGISTER_SPARSE_TENSOR_TYPE(ELEM_TYPE) \ template <> \ MLDataType SparseTensorType::Type() { \ @@ -860,6 +873,7 @@ class PrimitiveDataType : public PrimitiveDataTypeBase { MLDataType DataTypeImpl::GetSparseTensorType() { \ return SparseTensorType::Type(); \ } +#endif #if !defined(DISABLE_ML_OPS) #define ORT_REGISTER_MAP(TYPE) \ diff --git a/include/onnxruntime/core/framework/op_kernel.h b/include/onnxruntime/core/framework/op_kernel.h index 4d615e87f8..fa89af8fdd 100644 --- a/include/onnxruntime/core/framework/op_kernel.h +++ b/include/onnxruntime/core/framework/op_kernel.h @@ -110,8 +110,8 @@ class OpKernel { // @param used_shared_buffers: Boolean flag set by the kernel implementation indicating // that the provided weight has been used by the kernel. virtual Status UseSharedPrePackedBuffers(std::vector& /*prepacked_buffers*/, - int /*input_idx*/, - /*out*/ bool& used_shared_buffers) { + int /*input_idx*/, + /*out*/ bool& used_shared_buffers) { used_shared_buffers = false; return Status::OK(); } @@ -327,12 +327,14 @@ struct BuildKernelDefConstraintsImpl { } }; -template +#if !defined(DISABLE_SPARSE_TENSORS) +template struct BuildKernelDefSparseConstraintsImpl { std::vector operator()() const { return {DataTypeImpl::GetSparseTensorType()...}; } }; +#endif // Use within macro definitions to create a custom vector of constraints. // Example: #define REG_KERNEL(OP, VERSION, KERNEL_CLASS, Type, ...) @@ -342,10 +344,12 @@ inline std::vector BuildKernelDefConstraints() { return BuildKernelDefConstraintsImpl{}(); } +#if !defined(DISABLE_SPARSE_TENSORS) template inline std::vector BuildKernelDefSparseConstraints() { return BuildKernelDefSparseConstraintsImpl{}(); } +#endif // version of BuildKernelDefConstraints() which takes a type list template @@ -353,11 +357,12 @@ inline std::vector BuildKernelDefConstraintsFromTypeList() { return boost::mp11::mp_apply{}(); } -template +#if !defined(DISABLE_SPARSE_TENSORS) +template inline std::vector BuildKernelDefSparseConstraintsFromTypeList() { return boost::mp11::mp_apply{}(); } - +#endif } // namespace onnxruntime diff --git a/include/onnxruntime/core/framework/op_kernel_context.h b/include/onnxruntime/core/framework/op_kernel_context.h index 0b425c1e42..fe98ff54c8 100644 --- a/include/onnxruntime/core/framework/op_kernel_context.h +++ b/include/onnxruntime/core/framework/op_kernel_context.h @@ -69,11 +69,13 @@ class OpKernelContext { return *output_ptr; } + #if !defined(DISABLE_SPARSE_TENSORS) // Fetch a sparse-tensor output corresponding to the specified index. // shape must specify the shape of the underlying dense-tensor. // Memory allocation for the output may happen when this method is invoked, // unless static optimization pre-allocates it. SparseTensor* OutputSparse(int index, const TensorShape& shape); + #endif // Retrieve indexed shape obtained from memory planning before actual // computation. If the indexed shape cannot be inferred, this function returns @@ -210,11 +212,13 @@ inline Tensor* OpKernelContext::Output(int index) { return p_ml_value->GetMutable(); } +#if !defined(DISABLE_SPARSE_TENSORS) template <> inline SparseTensor* OpKernelContext::Output(int index) { OrtValue* p_ml_value = GetOutputMLValue(index); ORT_ENFORCE(p_ml_value, "Please fetch output sparse tensor with specified shape."); return p_ml_value->GetMutable(); } +#endif } // namespace onnxruntime diff --git a/include/onnxruntime/core/framework/ort_value.h b/include/onnxruntime/core/framework/ort_value.h index 4c3db30d37..c97a4e3edf 100644 --- a/include/onnxruntime/core/framework/ort_value.h +++ b/include/onnxruntime/core/framework/ort_value.h @@ -13,12 +13,13 @@ #include "core/framework/TensorSeq.h" namespace onnxruntime { +#if !defined(DISABLE_SPARSE_TENSORS) class SparseTensor; +#endif } // namespace onnxruntime #endif - /** Represents both tensors and non-tensors. */ @@ -65,8 +66,12 @@ struct OrtValue { return (type_ != nullptr && type_->IsTensorSequenceType()); } - bool IsSparseTensor() const noexcept { + bool IsSparseTensor() const { +#if !defined(DISABLE_SPARSE_TENSORS) return (type_ != nullptr && type_->IsSparseTensorType()); +#else + ORT_THROW("Sparse tensor is not supported in this build."); +#endif } onnxruntime::MLDataType Type() const { @@ -115,6 +120,7 @@ inline onnxruntime::TensorSeq* OrtValue::GetMutable() { return static_cast(data_.get()); } +#if !defined(DISABLE_SPARSE_TENSORS) template <> inline const onnxruntime::SparseTensor& OrtValue::Get() const { ORT_ENFORCE(IsSparseTensor(), "Trying to get a SparseTensor, but got: ", onnxruntime::DataTypeImpl::ToString(type_)); @@ -126,4 +132,4 @@ inline onnxruntime::SparseTensor* OrtValue::GetMutable(data_.get()); } - +#endif diff --git a/include/onnxruntime/core/framework/sparse_tensor.h b/include/onnxruntime/core/framework/sparse_tensor.h index 6a0596882a..fa1ad27711 100644 --- a/include/onnxruntime/core/framework/sparse_tensor.h +++ b/include/onnxruntime/core/framework/sparse_tensor.h @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +#if !defined(DISABLE_SPARSE_TENSORS) + #pragma once #include "core/framework/data_types.h" @@ -528,3 +530,5 @@ class SparseTensor final { }; } // namespace onnxruntime + +#endif \ No newline at end of file diff --git a/include/onnxruntime/core/graph/graph.h b/include/onnxruntime/core/graph/graph.h index be49b2add7..722fa7f061 100644 --- a/include/onnxruntime/core/graph/graph.h +++ b/include/onnxruntime/core/graph/graph.h @@ -331,7 +331,9 @@ class Node { ADD_ATTR_INTERFACES(std::string) ADD_ATTR_INTERFACES(ONNX_NAMESPACE::TensorProto) ADD_ATTR_INTERFACES(ONNX_NAMESPACE::GraphProto) +#if !defined(DISABLE_SPARSE_TENSORS) ADD_ATTR_INTERFACES(ONNX_NAMESPACE::SparseTensorProto) +#endif ADD_ATTR_INTERFACES(ONNX_NAMESPACE::TypeProto) /** Gets the Node's attributes. */ @@ -625,11 +627,13 @@ class Graph { /** Check if a given name is an initializer tensor's name in this graph. */ bool IsInitializedTensor(const std::string& name) const; +#if !defined(DISABLE_SPARSE_TENSORS) /** Check if a given name is a sparse initializer's name in the model * we currently convert sparse_initializer field in the model into dense Tensor instances. * However, we sometimes want to check if this initializer was stored as sparse in the model. */ bool IsSparseInitializer(const std::string& name) const; +#endif /** Gets an initializer tensor with the provided name. @param[out] value Set to the TensorProto* if the initializer is found, or nullptr if not. diff --git a/include/onnxruntime/core/session/onnxruntime_cxx_api.h b/include/onnxruntime/core/session/onnxruntime_cxx_api.h index 0ae27590c6..f62630d943 100644 --- a/include/onnxruntime/core/session/onnxruntime_cxx_api.h +++ b/include/onnxruntime/core/session/onnxruntime_cxx_api.h @@ -449,7 +449,7 @@ struct Value : Base { static Value CreateTensor(const OrtMemoryInfo* info, void* p_data, size_t p_data_byte_count, const int64_t* shape, size_t shape_len, ONNXTensorElementDataType type); - +#if !defined(DISABLE_SPARSE_TENSORS) /// /// This is a simple forwarding method to the other overload that helps deducing /// data type enum value from the type of the buffer. @@ -515,10 +515,13 @@ struct Value : Base { /// user allocated buffer with indices or nullptr for fully spare tensors void UseBlockSparseIndices(const Shape& indices_shape, int32_t* indices_data); +#endif // !defined(DISABLE_SPARSE_TENSORS) + template static Value CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len); static Value CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len, ONNXTensorElementDataType type); +#if !defined(DISABLE_SPARSE_TENSORS) /// /// This is a simple forwarding method the below CreateSparseTensor. /// This helps to specify data type enum in terms of C++ data type. @@ -622,6 +625,8 @@ struct Value : Base { template const T* GetSparseTensorIndicesData(OrtSparseIndicesFormat indices_format, size_t& num_indices) const; +#endif // !defined(DISABLE_SPARSE_TENSORS) + static Value CreateMap(Value& keys, Value& values); static Value CreateSequence(std::vector& values); @@ -638,11 +643,13 @@ struct Value : Base { bool IsTensor() const; +#if !defined(DISABLE_SPARSE_TENSORS) /// /// Returns true if the OrtValue contains a sparse tensor /// /// bool IsSparseTensor() const; +#endif size_t GetCount() const; // If a non tensor, returns 2 for map and N for sequence, where N is the number of elements Value GetValue(int index, OrtAllocator* allocator) const; @@ -660,7 +667,7 @@ struct Value : Base { /// into a supplied buffer. Use GetStringTensorDataLength() to find out the length of the buffer to allocate. /// The user must also allocate offsets buffer with the number of entries equal to that of the contained /// strings. - /// + /// /// Strings are always assumed to be on CPU, no X-device copy. /// /// user allocated buffer @@ -677,6 +684,7 @@ struct Value : Base { template const T* GetTensorData() const; +#if !defined(DISABLE_SPARSE_TENSORS) /// /// The API returns a pointer to an internal buffer of the sparse tensor /// containing non-zero values. The API merely does casting. Make sure you @@ -687,6 +695,7 @@ struct Value : Base { /// a pointer to the internal values buffer. Do not free this pointer. template const T* GetSparseTensorValues() const; +#endif template T& At(const std::vector& location); diff --git a/include/onnxruntime/core/session/onnxruntime_cxx_inline.h b/include/onnxruntime/core/session/onnxruntime_cxx_inline.h index 684c8fbaa4..83eb60a39d 100644 --- a/include/onnxruntime/core/session/onnxruntime_cxx_inline.h +++ b/include/onnxruntime/core/session/onnxruntime_cxx_inline.h @@ -755,6 +755,7 @@ inline Value Value::CreateTensor(const OrtMemoryInfo* info, void* p_data, size_t return Value{out}; } +#if !defined(DISABLE_SPARSE_TENSORS) template inline Value Value::CreateSparseTensor(const OrtMemoryInfo* info, T* p_data, const Shape& dense_shape, const Shape& values_shape) { @@ -830,6 +831,7 @@ inline const T* Value::GetSparseTensorIndicesData(OrtSparseIndicesFormat indices ThrowOnError(GetApi().GetSparseTensorIndices(p_, indices_format, &num_indices, &out)); return reinterpret_cast(out); } +#endif // !defined(DISABLE_SPARSE_TENSORS) template inline Value Value::CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len) { @@ -842,6 +844,7 @@ inline Value Value::CreateTensor(OrtAllocator* allocator, const int64_t* shape, return Value{out}; } +#if !defined(DISABLE_SPARSE_TENSORS) template inline Value Value::CreateSparseTensor(OrtAllocator* allocator, const Shape& dense_shape) { return CreateSparseTensor(allocator, dense_shape, TypeToTensorType::type); @@ -853,6 +856,7 @@ inline Value Value::CreateSparseTensor(OrtAllocator* allocator, const Shape& den ThrowOnError(GetApi().CreateSparseTensorAsOrtValue(allocator, dense_shape.shape, dense_shape.shape_len, type, &out)); return Value{out}; } +#endif // !defined(DISABLE_SPARSE_TENSORS) inline Value Value::CreateMap(Value& keys, Value& values) { OrtValue* out; @@ -886,11 +890,13 @@ inline bool Value::IsTensor() const { return out != 0; } +#if !defined(DISABLE_SPARSE_TENSORS) inline bool Value::IsSparseTensor() const { int out; ThrowOnError(GetApi().IsSparseTensor(p_, &out)); return out != 0; } +#endif inline size_t Value::GetCount() const { size_t out; @@ -946,12 +952,14 @@ const T* Value::GetTensorData() const { return out; } +#if !defined(DISABLE_SPARSE_TENSORS) template inline const T* Value::GetSparseTensorValues() const { const void* out; ThrowOnError(GetApi().GetSparseTensorValues(p_, &out)); return reinterpret_cast(out); } +#endif // !defined(DISABLE_SPARSE_TENSORS) template inline T& Value::At(const std::vector& location) { diff --git a/onnxruntime/contrib_ops/cpu/cpu_contrib_kernels.cc b/onnxruntime/contrib_ops/cpu/cpu_contrib_kernels.cc index 966dd2b795..67fc7387e4 100644 --- a/onnxruntime/contrib_ops/cpu/cpu_contrib_kernels.cc +++ b/onnxruntime/contrib_ops/cpu/cpu_contrib_kernels.cc @@ -23,7 +23,9 @@ class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, WordC class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, GatherND); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, TransposeMatMul); // backward compatibility class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, FusedMatMul); +#if !defined(DISABLE_SPARSE_TENSORS) class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, SparseToDenseMatMul); +#endif class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, MurmurHash3); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, MaxpoolWithMask); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, Pad); @@ -189,7 +191,9 @@ Status RegisterCpuContribKernels(KernelRegistry& kernel_registry) { BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, + #if !defined(DISABLE_SPARSE_TENSORS) BuildKernelCreateInfo, + #endif BuildKernelCreateInfo, BuildKernelCreateInfo, // backward compatibility BuildKernelCreateInfo, diff --git a/onnxruntime/contrib_ops/cpu/math/sparse_dense_matmul.cc b/onnxruntime/contrib_ops/cpu/math/sparse_dense_matmul.cc index ae0b7c217f..8f99364ba5 100644 --- a/onnxruntime/contrib_ops/cpu/math/sparse_dense_matmul.cc +++ b/onnxruntime/contrib_ops/cpu/math/sparse_dense_matmul.cc @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +#if !defined(DISABLE_SPARSE_TENSORS) + #include "core/framework/sparse_tensor.h" #include "core/providers/cpu/math/gemm_matmul_common.h" #include "core/providers/cpu/math/matmul_helper.h" @@ -208,3 +210,5 @@ Status SparseToDenseMatMul::Compute(OpKernelContext* ctx) const { } // namespace contrib } // namespace onnxruntime + +#endif //!defined(DISABLE_SPARSE_TENSORS) \ No newline at end of file diff --git a/onnxruntime/core/framework/data_transfer.cc b/onnxruntime/core/framework/data_transfer.cc index c16afefe9d..39b18f994b 100644 --- a/onnxruntime/core/framework/data_transfer.cc +++ b/onnxruntime/core/framework/data_transfer.cc @@ -23,12 +23,14 @@ common::Status IDataTransfer::CopyTensors(const std::vector& src_dst_pairs) const { for (const auto& pair : src_dst_pairs) { ORT_RETURN_IF_ERROR(pair.src.get().Copy(*this, pair.dst, pair.exec_queue_id)); } return Status::OK(); } +#endif bool CPUDataTransfer::CanCopy(const OrtDevice& src_device, const OrtDevice& dst_device) const { return src_device.Type() == OrtDevice::CPU && dst_device.Type() == OrtDevice::CPU; diff --git a/onnxruntime/core/framework/data_transfer.h b/onnxruntime/core/framework/data_transfer.h index 82eea0cf92..7fc3c1968a 100644 --- a/onnxruntime/core/framework/data_transfer.h +++ b/onnxruntime/core/framework/data_transfer.h @@ -11,8 +11,10 @@ struct OrtDevice; namespace onnxruntime { #ifndef SHARED_PROVIDER class Tensor; +#if !defined(DISABLE_SPARSE_TENSORS) class SparseTensor; #endif +#endif namespace common { class Status; } @@ -36,6 +38,7 @@ class IDataTransfer { // batched copy. default implementation copies each entry sequentially, and returns on first failure. virtual common::Status CopyTensors(const std::vector& src_dst_pairs) const; +#if !defined(DISABLE_SPARSE_TENSORS) struct SparseSrcDstPair { std::reference_wrapper src; std::reference_wrapper dst; @@ -43,6 +46,7 @@ class IDataTransfer { }; virtual common::Status CopySparseTensors(const std::vector& src_dst_pairs) const; +#endif }; class CPUDataTransfer : public IDataTransfer { diff --git a/onnxruntime/core/framework/data_transfer_manager.cc b/onnxruntime/core/framework/data_transfer_manager.cc index dd10660a69..fecfe34530 100644 --- a/onnxruntime/core/framework/data_transfer_manager.cc +++ b/onnxruntime/core/framework/data_transfer_manager.cc @@ -31,9 +31,11 @@ Status DataTransferManager::CopyTensor(const Tensor& src, Tensor& dst) const { return CopyTensor(src, dst, 0); } +#if !defined(DISABLE_SPARSE_TENSORS) common::Status DataTransferManager::CopySparseTensor(const SparseTensor& src, SparseTensor& dst) const { return CopySparseTensor(src, dst, 0); } +#endif Status DataTransferManager::CopyTensor(const Tensor& src, Tensor& dst, int exec_queue_id) const { if (src.Shape().Size() != dst.Shape().Size()) { @@ -56,6 +58,7 @@ Status DataTransferManager::CopyTensor(const Tensor& src, Tensor& dst, int exec_ dst.Location().device.ToString()); } +#if !defined(DISABLE_SPARSE_TENSORS) Status DataTransferManager::CopySparseTensor(const SparseTensor& src, SparseTensor& dst, int exec_queue_id) const { if (src.DenseShape().Size() != dst.DenseShape().Size()) { return Status(ONNXRUNTIME, FAIL, "Tensor size mismatch"); @@ -76,6 +79,7 @@ Status DataTransferManager::CopySparseTensor(const SparseTensor& src, SparseTens " to ", dst.Location().device.ToString()); } +#endif common::Status DataTransferManager::CopyTensors(const std::vector& src_dst_pairs) const { if (src_dst_pairs.empty()) @@ -128,6 +132,7 @@ common::Status DataTransferManager::CopyTensors(const std::vector& src_dst_pairs) const { if (src_dst_pairs.empty()) return Status::OK(); @@ -178,5 +183,6 @@ common::Status DataTransferManager::CopySparseTensors(const std::vector& src_dst_pairs) const; + #if !defined(DISABLE_SPARSE_TENSORS) common::Status CopySparseTensor(const SparseTensor& src, SparseTensor& dst) const; common::Status CopySparseTensor(const SparseTensor& src, SparseTensor& dst, int exec_queue_id) const; common::Status CopySparseTensors(const std::vector& src_dst_pairs) const; + #endif private: ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(DataTransferManager); diff --git a/onnxruntime/core/framework/data_types.cc b/onnxruntime/core/framework/data_types.cc index b397f6cffb..736346819d 100644 --- a/onnxruntime/core/framework/data_types.cc +++ b/onnxruntime/core/framework/data_types.cc @@ -46,11 +46,13 @@ MLDataType DataTypeImpl::GetType() { namespace onnxruntime { +#if !defined(DISABLE_SPARSE_TENSORS) // Return the MLDataType used for a generic SparseTensor template <> MLDataType DataTypeImpl::GetType() { return SparseTensorTypeBase::Type(); } +#endif template <> MLDataType DataTypeImpl::GetType() { @@ -69,9 +71,12 @@ struct TensorElementTypeSetter { static void SetTensorElementType(ONNX_NAMESPACE::TypeProto& proto) { proto.mutable_tensor_type()->set_elem_type(utils::ToTensorProtoElementType()); } + +#if !defined(DISABLE_SPARSE_TENSORS) static void SetSparseTensorElementType(ONNX_NAMESPACE::TypeProto& proto) { proto.mutable_sparse_tensor_type()->set_elem_type(utils::ToTensorProtoElementType()); } +#endif #if !defined(DISABLE_ML_OPS) static void SetMapKeyType(ONNX_NAMESPACE::TypeProto& proto) { @@ -136,8 +141,10 @@ void AssignOpaqueDomainName(const char* domain, const char* name, bool IsCompatible(const ONNX_NAMESPACE::TypeProto_Tensor& tensor_proto, const ONNX_NAMESPACE::TypeProto_Tensor& type_proto); +#if !defined(DISABLE_SPARSE_TENSORS) bool IsCompatible(const ONNX_NAMESPACE::TypeProto_SparseTensor& tensor_proto, const ONNX_NAMESPACE::TypeProto_SparseTensor& type_proto); +#endif #if !defined(DISABLE_ML_OPS) bool IsCompatible(const ONNX_NAMESPACE::TypeProto_Map& map_proto, @@ -180,9 +187,11 @@ bool IsCompatible(const ONNX_NAMESPACE::TypeProto_Map& map_proto, case TypeProto::ValueCase::kOpaqueType: result = IsCompatible(lhs.value_type().opaque_type(), rhs.value_type().opaque_type()); break; +#if !defined(DISABLE_SPARSE_TENSORS) case TypeProto::ValueCase::kSparseTensorType: result = IsCompatible(lhs.value_type().sparse_tensor_type(), rhs.value_type().sparse_tensor_type()); break; +#endif default: ORT_ENFORCE(false); break; @@ -215,9 +224,11 @@ bool IsCompatible(const ONNX_NAMESPACE::TypeProto_Sequence& sequence_proto, case TypeProto::ValueCase::kOpaqueType: result = IsCompatible(lhs.elem_type().opaque_type(), rhs.elem_type().opaque_type()); break; +#if !defined(DISABLE_SPARSE_TENSORS) case TypeProto::ValueCase::kSparseTensorType: result = IsCompatible(lhs.elem_type().sparse_tensor_type(), rhs.elem_type().sparse_tensor_type()); break; +#endif default: ORT_ENFORCE(false); break; @@ -227,6 +238,7 @@ bool IsCompatible(const ONNX_NAMESPACE::TypeProto_Sequence& sequence_proto, } return result; } + bool IsCompatible(const ONNX_NAMESPACE::TypeProto_Opaque& opaque_proto, const ONNX_NAMESPACE::TypeProto_Opaque& type_proto) { const auto& lhs = opaque_proto; @@ -245,11 +257,12 @@ bool IsCompatible(const ONNX_NAMESPACE::TypeProto_Opaque& opaque_proto, return !((lhs_name != rhs_name) || (lhs_name && rhs_name && lhs.name() != rhs.name())); } - +#if !defined(DISABLE_SPARSE_TENSORS) bool IsCompatible(const ONNX_NAMESPACE::TypeProto_SparseTensor& tensor_proto, const ONNX_NAMESPACE::TypeProto_SparseTensor& type_proto) { return type_proto.elem_type() == tensor_proto.elem_type(); } +#endif void RegisterAllProtos(const std::function& /*reg_fn*/); @@ -368,6 +381,8 @@ MLDataType TensorTypeBase::Type() { return &tensor_base; } +#if !defined(DISABLE_SPARSE_TENSORS) + /// SparseTensor struct SparseTensorTypeBase::Impl : public data_types_internal::TypeProtoImpl { @@ -413,6 +428,7 @@ MLDataType SparseTensorTypeBase::Type() { static SparseTensorTypeBase sparse_tensor_base; return &sparse_tensor_base; } +#endif // !defined(DISABLE_SPARSE_TENSORS) ///// SequenceTensorTypeBase @@ -544,6 +560,7 @@ ORT_REGISTER_TENSOR_TYPE(uint64_t); ORT_REGISTER_TENSOR_TYPE(MLFloat16); ORT_REGISTER_TENSOR_TYPE(BFloat16); +#if !defined(DISABLE_SPARSE_TENSORS) ORT_REGISTER_SPARSE_TENSOR_TYPE(int32_t); ORT_REGISTER_SPARSE_TENSOR_TYPE(float); ORT_REGISTER_SPARSE_TENSOR_TYPE(bool); @@ -558,6 +575,7 @@ ORT_REGISTER_SPARSE_TENSOR_TYPE(uint32_t); ORT_REGISTER_SPARSE_TENSOR_TYPE(uint64_t); ORT_REGISTER_SPARSE_TENSOR_TYPE(MLFloat16); ORT_REGISTER_SPARSE_TENSOR_TYPE(BFloat16); +#endif #if !defined(DISABLE_ML_OPS) ORT_REGISTER_MAP(MapStringToString); @@ -603,11 +621,13 @@ ORT_REGISTER_SEQ(VectorMapInt64ToFloat); reg_fn(mltype); \ } +#if !defined(DISABLE_SPARSE_TENSORS) #define REGISTER_SPARSE_TENSOR_PROTO(TYPE, reg_fn) \ { \ MLDataType mltype = DataTypeImpl::GetSparseTensorType(); \ reg_fn(mltype); \ } +#endif #define REGISTER_ONNX_PROTO(TYPE, reg_fn) \ { \ @@ -633,6 +653,7 @@ void RegisterAllProtos(const std::function& reg_fn) { REGISTER_TENSOR_PROTO(MLFloat16, reg_fn); REGISTER_TENSOR_PROTO(BFloat16, reg_fn); +#if !defined(DISABLE_SPARSE_TENSORS) REGISTER_SPARSE_TENSOR_PROTO(int32_t, reg_fn); REGISTER_SPARSE_TENSOR_PROTO(float, reg_fn); REGISTER_SPARSE_TENSOR_PROTO(bool, reg_fn); @@ -647,6 +668,7 @@ void RegisterAllProtos(const std::function& reg_fn) { REGISTER_SPARSE_TENSOR_PROTO(uint64_t, reg_fn); REGISTER_SPARSE_TENSOR_PROTO(MLFloat16, reg_fn); REGISTER_SPARSE_TENSOR_PROTO(BFloat16, reg_fn); +#endif #if !defined(DISABLE_ML_OPS) REGISTER_ONNX_PROTO(MapStringToString, reg_fn); @@ -819,6 +841,7 @@ const NonTensorTypeBase* DataTypeImpl::SequenceTensorTypeFromONNXEnum(int type) } } +#if !defined(DISABLE_SPARSE_TENSORS) const SparseTensorTypeBase* DataTypeImpl::SparseTensorTypeFromONNXEnum(int type) { switch (type) { case TensorProto_DataType_FLOAT: @@ -830,7 +853,7 @@ const SparseTensorTypeBase* DataTypeImpl::SparseTensorTypeFromONNXEnum(int type) case TensorProto_DataType_DOUBLE: return reinterpret_cast(DataTypeImpl::GetSparseTensorType()); case TensorProto_DataType_STRING: - return reinterpret_cast(DataTypeImpl::GetSparseTensorType()); + return reinterpret_cast(DataTypeImpl::GetSparseTensorType()); case TensorProto_DataType_UINT8: return reinterpret_cast(DataTypeImpl::GetSparseTensorType()); case TensorProto_DataType_UINT16: @@ -853,6 +876,7 @@ const SparseTensorTypeBase* DataTypeImpl::SparseTensorTypeFromONNXEnum(int type) ORT_NOT_IMPLEMENTED("sparse tensor type ", type, " is not supported"); } } +#endif MLDataType DataTypeImpl::TypeFromProto(const ONNX_NAMESPACE::TypeProto& proto) { const auto& registry = data_types_internal::DataTypeRegistry::instance(); diff --git a/onnxruntime/core/framework/execution_frame.cc b/onnxruntime/core/framework/execution_frame.cc index 7a0798782a..e291cae1e1 100644 --- a/onnxruntime/core/framework/execution_frame.cc +++ b/onnxruntime/core/framework/execution_frame.cc @@ -153,10 +153,12 @@ Status IExecutionFrame::GetOrCreateNodeOutputMLValue(const int output_index, int "OrtValue shape verification failed. Current shape:", tensor.Shape(), " Requested shape:", shape ? shape->ToString() : "null"); } else if (p_ort_value->IsSparseTensor()) { +#if !defined(DISABLE_SPARSE_TENSORS) const SparseTensor& sp_tensor = p_ort_value->Get(); ORT_ENFORCE(shape && sp_tensor.DenseShape() == *shape, "OrtValue shape verification failed. Current shape:", sp_tensor.DenseShape(), " Requested shape:", shape ? shape->ToString() : "null"); +#endif } } else { // shape is nullptr for traditional ML output values @@ -252,10 +254,11 @@ void IExecutionFrame::Init(const std::vector& feed_mlvalue_idxs, const std: if (IsOutput(ort_value_index)) { std::string name; ORT_THROW_IF_ERROR(ort_value_idx_map_.GetName(ort_value_index, name)); - const bool is_sparse_initializer = is_initializer_sparse_func(name); const Tensor& src = entry.second.Get(); // all initializers in ONNX are tensors OrtValue& dest = all_values_[ort_value_index]; +#if !defined(DISABLE_SPARSE_TENSORS) + const bool is_sparse_initializer = is_initializer_sparse_func(name); if (is_sparse_initializer) { if (!dest.IsAllocated()) { auto p_tensor = std::make_unique(); @@ -270,6 +273,7 @@ void IExecutionFrame::Init(const std::vector& feed_mlvalue_idxs, const std: cpu_allocator, allocator, has_linear_coo_index, *dest.GetMutable())); } else { +#endif // !defined(DISABLE_SPARSE_TENSORS) if (!dest.IsAllocated()) { // NOTE: This doesn't need to support ExecutionFrame custom allocators as they only come into play // for a subgraph with an output of unknown shape that needs to be accumulated by the control flow node. @@ -278,7 +282,9 @@ void IExecutionFrame::Init(const std::vector& feed_mlvalue_idxs, const std: Tensor::InitOrtValue(src.DataType(), src.Shape(), std::move(allocator), dest); } ORT_THROW_IF_ERROR(CopyTensor(src, *dest.GetMutable())); +#if !defined(DISABLE_SPARSE_TENSORS) } +#endif } else { all_values_[ort_value_index] = entry.second; } @@ -327,6 +333,7 @@ ExecutionFrame::ExecutionFrame(const std::vector& feed_mlvalue_idxs, const planner_(nullptr) { Init( feed_mlvalue_idxs, feeds, session_state.GetInitializedTensors(), +#if !defined(DISABLE_SPARSE_TENSORS) [&session_state](const std::string& name) -> bool { int idx = -1; if (session_state.GetOrtValueNameIdxMap().GetIdx(name, idx).IsOK()) { @@ -334,7 +341,13 @@ ExecutionFrame::ExecutionFrame(const std::vector& feed_mlvalue_idxs, const } return false; }, +#else + [&](const std::string& /*name*/) -> bool { + return false; + }, +#endif fetches); + #if !defined(ORT_MINIMAL_BUILD) && defined(ORT_MEMORY_PROFILE) MemoryInfo::IncreaseIteration(); #endif @@ -355,7 +368,7 @@ ExecutionFrame::ExecutionFrame(const std::vector& feed_mlvalue_idxs, const // and we have execution plan generated, try to setup // memory pattern optimization. if (session_state.GetEnableMemoryPattern() && session_state.GetExecutionPlan()) { - std::vector> input_shapes; + std::vector > input_shapes; bool all_tensors = true; // Reserve mem to avoid re-allocation. input_shapes.reserve(feeds.size()); @@ -614,6 +627,7 @@ static Status AllocateTensorSequence(OrtValue& ort_value) { return Status::OK(); } +#if !defined(DISABLE_SPARSE_TENSORS) static Status AllocateSparseTensor(OrtValue& mlvalue, const DataTypeImpl& ml_type, AllocatorPtr allocator, const TensorShape& shape, bool create_fence, const SessionState& session_state) { @@ -629,6 +643,7 @@ static Status AllocateSparseTensor(OrtValue& mlvalue, const DataTypeImpl& ml_typ return Status::OK(); } +#endif // This method is not thread safe! Status ExecutionFrame::AllocateAsPerAllocationPlan(OrtValue& ort_value, int ort_value_index, const TensorShape* shape) { @@ -705,8 +720,13 @@ Status ExecutionFrame::AllocateAsPerAllocationPlan(OrtValue& ort_value, int ort_ return Status::OK(); } else if (ml_type->IsSparseTensorType()) { +#if !defined(DISABLE_SPARSE_TENSORS) return AllocateSparseTensor(ort_value, *ml_type, GetAllocator(alloc_info), *shape, per_alloc_plan.create_fence_if_async, session_state_); +#else + // Model load should have failed so this should be unreachable + ORT_THROW("SparseTensor is not supported in this build."); +#endif } else if (ml_type->IsTensorSequenceType()) { return AllocateTensorSequence(ort_value); } else { diff --git a/onnxruntime/core/framework/memcpy.cc b/onnxruntime/core/framework/memcpy.cc index 36dbe4bb42..4443ccc2d1 100644 --- a/onnxruntime/core/framework/memcpy.cc +++ b/onnxruntime/core/framework/memcpy.cc @@ -25,7 +25,9 @@ Status Memcpy::Compute(OpKernelContext* ctx) const { " Input shape:", X->Shape(), " Output shape:", Y->Shape(), " X data:", X->DataRaw(), " Y data:", Y->DataRaw()); } - } else if (input_type_0->IsSparseTensorType()) { + } +#if !defined(DISABLE_SPARSE_TENSORS) + else if (input_type_0->IsSparseTensorType()) { const auto* X = ctx->Input(0); SparseTensor* Y = ctx->OutputSparse(0, X->DenseShape()); retval = X->Copy(Info().GetDataTransferManager(), Info().GetKernelDef().ExecQueueId(), *Y); @@ -35,7 +37,9 @@ Status Memcpy::Compute(OpKernelContext* ctx) const { " to ", Node().OutputDefs()[0]->Name(), " Input shape:", X->DenseShape(), " Output shape:", Y->DenseShape()); } - } else { + } +#endif + else { ORT_NOT_IMPLEMENTED("Input type not supported: ", DataTypeImpl::ToString(input_type_0)); } diff --git a/onnxruntime/core/framework/onnxruntime_typeinfo.cc b/onnxruntime/core/framework/onnxruntime_typeinfo.cc index bcfc36d131..1079f5cac9 100644 --- a/onnxruntime/core/framework/onnxruntime_typeinfo.cc +++ b/onnxruntime/core/framework/onnxruntime_typeinfo.cc @@ -20,7 +20,9 @@ using onnxruntime::BFloat16; using onnxruntime::DataTypeImpl; using onnxruntime::MLFloat16; +#if !defined(DISABLE_SPARSE_TENSORS) using onnxruntime::SparseTensor; +#endif using onnxruntime::Tensor; using onnxruntime::TensorShape; @@ -119,6 +121,7 @@ OrtStatus* OrtTypeInfo::FromOrtValue(const OrtValue& value, OrtTypeInfo** out) { } if (type->IsSparseTensorType()) { +#if !defined(DISABLE_SPARSE_TENSORS) OrtTensorTypeAndShapeInfo* info = nullptr; const SparseTensor& tensor = value.Get(); const auto* tensor_data_type = tensor.DataType(); @@ -128,6 +131,9 @@ OrtStatus* OrtTypeInfo::FromOrtValue(const OrtValue& value, OrtTypeInfo** out) { } *out = new OrtTypeInfo(ONNX_TYPE_SPARSETENSOR, info); return nullptr; +#else + return OrtApis::CreateStatus(ORT_FAIL, "SparseTensor is not supported in this build."); +#endif } if (type->IsTensorSequenceType()) { @@ -157,17 +163,21 @@ OrtStatus* OrtTypeInfo::FromOrtValue(const OrtValue& value, OrtTypeInfo** out) { *out = new OrtTypeInfo(ONNX_TYPE_OPAQUE); return nullptr; } +#if !defined(DISABLE_ML_OPS) case on::TypeProto::kMapType: { return OrtTypeInfo::FromTypeProto(type_proto, out); } +#endif case on::TypeProto::kSequenceType: { return OrtTypeInfo::FromTypeProto(type_proto, out); } // Real Tensor support case on::TypeProto::kTensorType: +#if !defined(DISABLE_SPARSE_TENSORS) case on::TypeProto::kSparseTensorType: { return OrtApis::CreateStatus(ORT_FAIL, "Tensor types should have been handled already"); } +#endif default: // NOT_IMPLEMENTED break; @@ -220,7 +230,9 @@ OrtStatus* OrtTypeInfo::FromTypeProto(const ONNX_NAMESPACE::TypeProto* input, Or case on::TypeProto::kSparseTensorType: { ONNXType ten_type = ONNX_TYPE_UNKNOWN; const on::TypeProto_Tensor* tensor_type = nullptr; +#if !defined(DISABLE_SPARSE_TENSORS) const on::TypeProto_SparseTensor* sparse_type = nullptr; +#endif const on::TensorShapeProto* sp = nullptr; if (value_case == on::TypeProto::kTensorType) { tensor_type = &input->tensor_type(); @@ -229,11 +241,13 @@ OrtStatus* OrtTypeInfo::FromTypeProto(const ONNX_NAMESPACE::TypeProto* input, Or sp = &tensor_type->shape(); } } else if (value_case == on::TypeProto::kSparseTensorType) { +#if !defined(DISABLE_SPARSE_TENSORS) sparse_type = &input->sparse_tensor_type(); ten_type = ONNX_TYPE_SPARSETENSOR; if (onnxruntime::utils::HasShape(*sparse_type)) { sp = &sparse_type->shape(); } +#endif } OrtStatus* st = nullptr; @@ -312,6 +326,7 @@ OrtStatus* OrtTypeInfo::Clone(OrtTypeInfo** out) { switch (type) { case ONNX_TYPE_TENSOR: case ONNX_TYPE_SPARSETENSOR: { +#if !defined(DISABLE_SPARSE_TENSORS) OrtTensorTypeAndShapeInfo* clone; if (auto status = data->Clone(&clone)) { return status; @@ -319,6 +334,9 @@ OrtStatus* OrtTypeInfo::Clone(OrtTypeInfo** out) { *out = new OrtTypeInfo(type, clone); (*out)->denotation = denotation; return nullptr; +#else + return OrtApis::CreateStatus(ORT_FAIL, "SparseTensor is not supported in this build."); +#endif } case ONNX_TYPE_SEQUENCE: { OrtSequenceTypeInfo* clone; diff --git a/onnxruntime/core/framework/op_kernel.cc b/onnxruntime/core/framework/op_kernel.cc index 94b9984690..61bf9c4471 100644 --- a/onnxruntime/core/framework/op_kernel.cc +++ b/onnxruntime/core/framework/op_kernel.cc @@ -49,10 +49,12 @@ Tensor* OpKernelContext::Output(int index, const std::initializer_list& return Output(index, TensorShape(shape)); } +#if !defined(DISABLE_SPARSE_TENSORS) SparseTensor* OpKernelContext::OutputSparse(int index, const TensorShape& shape) { auto p_ml_value = OutputMLValue(index, shape); return p_ml_value ? p_ml_value->GetMutable() : nullptr; } +#endif bool OpKernelContext::TryGetInferredInputShape(int index, TensorShape& shape) const { return execution_frame_->TryGetInferredShape(GetInputArgIndex(index), shape); diff --git a/onnxruntime/core/framework/session_state.cc b/onnxruntime/core/framework/session_state.cc index fc894437b5..611564f370 100644 --- a/onnxruntime/core/framework/session_state.cc +++ b/onnxruntime/core/framework/session_state.cc @@ -201,9 +201,11 @@ Status SessionState::AddInitializedTensor(int ort_value_index, const OrtValue& o constant_initialized_tensors_.insert({ort_value_index, ort_value}); } +#if !defined(DISABLE_SPARSE_TENSORS) if (sparse) { sparse_initialized_tensors_.insert(ort_value_index); } +#endif return Status::OK(); } @@ -214,9 +216,11 @@ const std::unordered_map& SessionState::GetConstantInitializedTen return constant_initialized_tensors_; } +#if !defined(DISABLE_SPARSE_TENSORS) bool SessionState::IsSparseInitializer(int ort_value_index) const { return sparse_initialized_tensors_.count(ort_value_index) > 0; } +#endif #ifdef ENABLE_TRAINING Status SessionState::GetInitializedTensors( diff --git a/onnxruntime/core/framework/session_state.h b/onnxruntime/core/framework/session_state.h index d238b656aa..d1aa5a9d3a 100644 --- a/onnxruntime/core/framework/session_state.h +++ b/onnxruntime/core/framework/session_state.h @@ -164,7 +164,9 @@ class SessionState { */ const std::unordered_map& GetConstantInitializedTensors() const; +#if !defined(DISABLE_SPARSE_TENSORS) bool IsSparseInitializer(int ort_value_index) const; +#endif #ifdef ENABLE_TRAINING /** @@ -436,11 +438,13 @@ class SessionState { // subset of initialized_tensors_ that are constant and cannot be overridden at runtime std::unordered_map constant_initialized_tensors_; +#if !defined(DISABLE_SPARSE_TENSORS) // This is an auxiliary lookup to check if the OrtValue was actually a sparse tensor // this is needed because we currently convert all sparse initializer into dense Tensors // if and when we actually place SparseTensor instances (we should) into OrtValues, we // will not need this structure. std::unordered_set sparse_initialized_tensors_; +#endif // This data structure is for uninitializing string tensors and // munmap memory region and close file descriptor diff --git a/onnxruntime/core/framework/session_state_utils.cc b/onnxruntime/core/framework/session_state_utils.cc index ae22c4a3c2..6ec20d0981 100644 --- a/onnxruntime/core/framework/session_state_utils.cc +++ b/onnxruntime/core/framework/session_state_utils.cc @@ -253,8 +253,12 @@ common::Status SaveInitializedTensors( // any outer scope value is shadowed by a local value and can't override it. // due to that check_outer_scope is false const bool constant = graph.IsConstantInitializer(name, /* check_outer_scope */ false); +#if !defined(DISABLE_SPARSE_TENSORS) const bool sparse = graph.GetGraph().IsSparseInitializer(name); ORT_RETURN_IF_ERROR(save_tensor_func(ort_value_index, ort_value, deleter, constant, sparse)); +#else + ORT_RETURN_IF_ERROR(save_tensor_func(ort_value_index, ort_value, deleter, constant, false)); +#endif VLOGS(logger, 1) << "Added weight with name : " << name << " with index: " << ort_value_index; } diff --git a/onnxruntime/core/framework/sparse_tensor.cc b/onnxruntime/core/framework/sparse_tensor.cc index 0f405b2d77..223670392a 100644 --- a/onnxruntime/core/framework/sparse_tensor.cc +++ b/onnxruntime/core/framework/sparse_tensor.cc @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +#if !defined(DISABLE_SPARSE_TENSORS) + #include "core/framework/data_types.h" #include "core/framework/sparse_tensor.h" #include "core/framework/data_transfer_manager.h" @@ -588,3 +590,5 @@ Status SparseTensor::Copy(const IDataTransfer& data_transfer, SparseTensor& dst_ } } // namespace onnxruntime + +#endif // !defined(DISABLE_SPARSE_TENSORS) \ No newline at end of file diff --git a/onnxruntime/core/framework/sparse_utils.cc b/onnxruntime/core/framework/sparse_utils.cc index dd352a1718..2726d468d6 100644 --- a/onnxruntime/core/framework/sparse_utils.cc +++ b/onnxruntime/core/framework/sparse_utils.cc @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +#if !defined(DISABLE_SPARSE_TENSORS) + #include "core/framework/sparse_utils.h" #include "core/common/status.h" #include "core/framework/tensor.h" @@ -308,8 +310,8 @@ Status SparseCooToDenseTensor(const DataTransferManager& data_manager, const Spa const int64_t* indices = nullptr; const auto num_values = src.Values().Shape().Size(); const auto num_indices = src.AsCoo().Indices().Shape().Size(); - ORT_RETURN_IF_NOT((num_values == num_indices || 2 * num_values == num_indices), - "Expecting indices to be equal the number of values or be twice as many"); + ORT_RETURN_IF_NOT((num_values == num_indices || 2 * num_values == num_indices), + "Expecting indices to be equal the number of values or be twice as many"); SparseTensor src_cpu; if (src.Location().device.Type() != OrtDevice::CPU) { @@ -400,13 +402,12 @@ void ScanAndRecordCoo(gsl::span src_span, } } ++index; - } - } + } +} Status DenseTensorToSparseCoo(const DataTransferManager& data_manager, const Tensor& src, const AllocatorPtr& cpu_allocator, const AllocatorPtr& dst_allocator, bool linear_index, SparseTensor& dst) { - const IDataTransfer* data_transfer = data_manager.GetDataTransfer(cpu_allocator->Info().device, dst_allocator->Info().device); ORT_RETURN_IF_NOT(data_transfer != nullptr, "Unable to find a data transfer for copying from device type: ", @@ -509,4 +510,6 @@ Status DenseTensorToSparseCoo(const DataTransferManager& data_manager, const Ten } } // namespace sparse_utils -} // namespace onnxruntime \ No newline at end of file +} // namespace onnxruntime + +#endif // !defined(DISABLE_SPARSE_TENSORS) \ No newline at end of file diff --git a/onnxruntime/core/framework/sparse_utils.h b/onnxruntime/core/framework/sparse_utils.h index 89cbe72ab5..bc0111755b 100644 --- a/onnxruntime/core/framework/sparse_utils.h +++ b/onnxruntime/core/framework/sparse_utils.h @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +#if !defined(DISABLE_SPARSE_TENSORS) + #pragma once #include "core/framework/allocator.h" @@ -77,7 +79,7 @@ Status SparseCsrToDenseTensor(const DataTransferManager& data_manager, const Spa /// Status instance Status SparseCooToDenseTensor(const DataTransferManager& data_manager, const SparseTensor& src, const AllocatorPtr& cpu_allocator, const AllocatorPtr& dst_allocator, Tensor& dst); -#endif //ORT_MINIMAL_BUILD +#endif //!ORT_MINIMAL_BUILD /// /// Convert Dense Tensor to COO format. @@ -97,3 +99,5 @@ Status DenseTensorToSparseCoo(const DataTransferManager& data_manager, const Ten } // namespace sparse_utils } // namespace onnxruntime + +#endif //!defined(DISABLE_SPARSE_TENSORS) \ No newline at end of file diff --git a/onnxruntime/core/framework/tensor_type_and_shape.cc b/onnxruntime/core/framework/tensor_type_and_shape.cc index f818ef7a26..a0098b6a17 100644 --- a/onnxruntime/core/framework/tensor_type_and_shape.cc +++ b/onnxruntime/core/framework/tensor_type_and_shape.cc @@ -20,7 +20,9 @@ using onnxruntime::BFloat16; using onnxruntime::DataTypeImpl; using onnxruntime::MLFloat16; +#if !defined(DISABLE_SPARSE_TENSORS) using onnxruntime::SparseTensor; +#endif using onnxruntime::Tensor; ORT_API_STATUS_IMPL(OrtApis::CreateTensorTypeAndShapeInfo, _Outptr_ OrtTensorTypeAndShapeInfo** out) { @@ -214,9 +216,11 @@ ORT_API_STATUS_IMPL(OrtApis::GetTensorTypeAndShape, _In_ const OrtValue* v, _Out shape = &tensor.Shape(); data_type = tensor.DataType(); } else { +#if !defined(DISABLE_SPARSE_TENSORS) const SparseTensor& tensor = v->Get(); shape = &tensor.DenseShape(); data_type = tensor.DataType(); +#endif } return GetTensorShapeAndType(*shape, *data_type, out); } else { @@ -228,12 +232,17 @@ ORT_API_STATUS_IMPL(OrtApis::GetTensorTypeAndShape, _In_ const OrtValue* v, _Out ORT_API_STATUS_IMPL(OrtApis::GetSparseTensorValuesTypeAndShape, _In_ const OrtValue* v, _Outptr_ OrtTensorTypeAndShapeInfo** out) { API_IMPL_BEGIN +#if !defined(DISABLE_SPARSE_TENSORS) const auto& sparse_tensor = SparseTensor::GetSparseTensorFromOrtValue(*v); const auto& values = sparse_tensor.Values(); return GetTensorShapeAndType(values.Shape(), *values.DataType(), out); +#else + return OrtApis::CreateStatus(ORT_FAIL, "SparseTensor is not supported in this build."); +#endif API_IMPL_END } +#if !defined(DISABLE_SPARSE_TENSORS) namespace { const Tensor& GetIndicesTensor(const OrtValue& v, OrtSparseIndicesFormat indices_format) { const auto& sparse_tensor = SparseTensor::GetSparseTensorFromOrtValue(v); @@ -257,22 +266,31 @@ const Tensor& GetIndicesTensor(const OrtValue& v, OrtSparseIndicesFormat indices return *indices_tensor; } } // namespace +#endif // !defined(DISABLE_SPARSE_TENSORS) ORT_API_STATUS_IMPL(OrtApis::GetSparseTensorIndicesTypeShape, _In_ const OrtValue* v, OrtSparseIndicesFormat indices_format, _Outptr_ OrtTensorTypeAndShapeInfo** out) { API_IMPL_BEGIN +#if !defined(DISABLE_SPARSE_TENSORS) const Tensor& indices_tensor = GetIndicesTensor(*v, indices_format); return GetTensorShapeAndType(indices_tensor.Shape(), *indices_tensor.DataType(), out); +#else + return OrtApis::CreateStatus(ORT_FAIL, "SparseTensor is not supported in this build."); +#endif API_IMPL_END } ORT_API_STATUS_IMPL(OrtApis::GetSparseTensorIndices, _In_ const OrtValue* v, enum OrtSparseIndicesFormat indices_format, _Out_ size_t* num_indices, _Outptr_ const void** indices) { API_IMPL_BEGIN +#if !defined(DISABLE_SPARSE_TENSORS) const Tensor& indices_tensor = GetIndicesTensor(*v, indices_format); *num_indices = gsl::narrow(indices_tensor.Shape().Size()); *indices = indices_tensor.DataRaw(); return nullptr; +#else + return OrtApis::CreateStatus(ORT_FAIL, "SparseTensor is not supported in this build."); +#endif API_IMPL_END } diff --git a/onnxruntime/core/framework/tensorprotoutils.cc b/onnxruntime/core/framework/tensorprotoutils.cc index 5534f5f347..6071bfd022 100644 --- a/onnxruntime/core/framework/tensorprotoutils.cc +++ b/onnxruntime/core/framework/tensorprotoutils.cc @@ -823,11 +823,13 @@ common::Status ConstantNodeProtoToTensorProto(const ONNX_NAMESPACE::NodeProto& n *tensor.mutable_string_data() = constant_attribute.strings(); break; } +#if !defined(DISABLE_SPARSE_TENSORS) case AttributeProto_AttributeType_SPARSE_TENSOR: { auto& s = constant_attribute.sparse_tensor(); ORT_RETURN_IF_ERROR(SparseTensorProtoToDenseTensorProto(s, model_path, tensor)); break; } +#endif default: ORT_THROW("Unsupported attribute value type of ", constant_attribute.type(), " in 'Constant' node '", node.name(), "'"); @@ -839,6 +841,7 @@ common::Status ConstantNodeProtoToTensorProto(const ONNX_NAMESPACE::NodeProto& n return Status::OK(); } +#if !defined(DISABLE_SPARSE_TENSORS) template static Status CopySparseData(size_t n_sparse_elements, const ONNX_NAMESPACE::TensorProto& indices, @@ -902,14 +905,16 @@ static Status CopySparseData(size_t n_sparse_elements, return status; } +#endif // !defined(DISABLE_SPARSE_TENSORS) namespace conversion_internal { +#if !defined(DISABLE_SPARSE_TENSORS) struct UnsupportedSparseDataType { void operator()(int32_t dt_type, Status& status) const { status = ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Unsupported sparse tensor data type of ", dt_type); } }; - +#endif template struct GetElementSize { Status operator()(size_t& element_size) const { @@ -922,6 +927,7 @@ using SupportedConversionTypeList = onnxruntime::TypeList; } // namespace conversion_internal +#if !defined(DISABLE_SPARSE_TENSORS) common::Status SparseTensorProtoToDenseTensorProto(const ONNX_NAMESPACE::SparseTensorProto& sparse, const Path& model_path, ONNX_NAMESPACE::TensorProto& dense) { @@ -1150,6 +1156,7 @@ common::Status DenseTensorToSparseTensorProto(const ONNX_NAMESPACE::TensorProto& } #endif // !ORT_MINIMAL_BUILD +#endif // !defined(DISABLE_SPARSE_TENSORS) template common::Status GetSizeInBytesFromTensorProto(const ONNX_NAMESPACE::TensorProto& tensor_proto, size_t* out); diff --git a/onnxruntime/core/framework/tensorprotoutils.h b/onnxruntime/core/framework/tensorprotoutils.h index abfe16081b..ade59cc301 100644 --- a/onnxruntime/core/framework/tensorprotoutils.h +++ b/onnxruntime/core/framework/tensorprotoutils.h @@ -85,6 +85,7 @@ common::Status ConstantNodeProtoToTensorProto(const ONNX_NAMESPACE::NodeProto& n const Path& model_path, ONNX_NAMESPACE::TensorProto& tensor); +#if !defined(DISABLE_SPARSE_TENSORS) // Convert a SparseTensorProto to a dense TensorProto // If the SparseTensorProto contains external data then it loads the data and converts to dense tensor proto // The resulting TensorProto will contain the data as raw data. @@ -102,6 +103,7 @@ common::Status DenseTensorToSparseTensorProto(const ONNX_NAMESPACE::TensorProto& const Path& model_path, ONNX_NAMESPACE::SparseTensorProto& sparse); #endif // !ORT_MINIMAL_BUILD +#endif // !defined(DISABLE_SPARSE_TENSORS) #endif inline bool HasDimValue(const ONNX_NAMESPACE::TensorShapeProto_Dimension& dim) { @@ -125,6 +127,7 @@ inline bool HasShape(const ONNX_NAMESPACE::TypeProto_Tensor& ten_proto) { return ten_proto.has_shape(); } +#if !defined(DISABLE_SPARSE_TENSORS) inline bool HasSparseTensorType(const ONNX_NAMESPACE::TypeProto& type_proto) { return type_proto.value_case() == ONNX_NAMESPACE::TypeProto::kSparseTensorType; } @@ -137,28 +140,41 @@ inline bool HasShape(const ONNX_NAMESPACE::TypeProto_SparseTensor& ten_proto) { inline bool HasElemType(const ONNX_NAMESPACE::TypeProto_SparseTensor& ten_proto) { return ten_proto.elem_type() != ONNX_NAMESPACE::TensorProto::UNDEFINED; } - -inline bool HasShape(const ONNX_NAMESPACE::TypeProto& type_proto) { - if (HasTensorType(type_proto) && HasShape(type_proto.tensor_type())) { - return true; - } - return HasSparseTensorType(type_proto) && HasShape(type_proto.sparse_tensor_type()); -} +#endif // !defined(DISABLE_SPARSE_TENSORS) inline bool HasElementType(const ONNX_NAMESPACE::TypeProto& type_proto) { if (HasTensorType(type_proto) && HasElemType(type_proto.tensor_type())) { return true; } - return HasSparseTensorType(type_proto) && HasElemType(type_proto.sparse_tensor_type()); +#if !defined(DISABLE_SPARSE_TENSORS) + if (HasSparseTensorType(type_proto) && HasElemType(type_proto.sparse_tensor_type())) { + return true; + } +#endif // !defined(DISABLE_SPARSE_TENSORS) + return false; +} + +inline bool HasShape(const ONNX_NAMESPACE::TypeProto& type_proto) { + if (HasTensorType(type_proto) && HasShape(type_proto.tensor_type())) { + return true; + } +#if !defined(DISABLE_SPARSE_TENSORS) + if (HasSparseTensorType(type_proto) && HasShape(type_proto.sparse_tensor_type())) { + return true; + } +#endif + return false; } inline const ONNX_NAMESPACE::TensorShapeProto& GetShape(const ONNX_NAMESPACE::TypeProto& type_proto) { if (HasTensorType(type_proto) && HasShape(type_proto.tensor_type())) { return type_proto.tensor_type().shape(); } +#if !defined(DISABLE_SPARSE_TENSORS) if (HasSparseTensorType(type_proto) && HasShape(type_proto.sparse_tensor_type())) { return type_proto.sparse_tensor_type().shape(); } +#endif ORT_THROW("TypeProto must have shape for this to run"); } diff --git a/onnxruntime/core/framework/utils.cc b/onnxruntime/core/framework/utils.cc index 8af2b89af9..2eb64d6603 100644 --- a/onnxruntime/core/framework/utils.cc +++ b/onnxruntime/core/framework/utils.cc @@ -139,8 +139,10 @@ static common::Status AllocateHelper(const AllocatorPtr& allocator, source_tensor.Shape(), allocator, target_mlvalue); } else if (source_mlvalue.IsSparseTensor()) { +#if !defined(DISABLE_SPARSE_TENSORS) const SparseTensor& source_tensor = source_mlvalue.Get(); SparseTensor::InitOrtValue(source_tensor.DataType(), source_tensor.DenseShape(), allocator, target_mlvalue); +#endif } else if (source_mlvalue.IsTensorSequence()) { const TensorSeq& source_tensor_seq = source_mlvalue.Get(); auto target_tensor_seq = std::make_unique(source_tensor_seq.DataType()); @@ -182,8 +184,13 @@ static Status BatchOrCopyMLValue(const SessionState& session_state, const MLValueCopyInfo& copy_info, const OrtValue& source_mlvalue, OrtValue& target_mlvalue, +#if !defined(DISABLE_SPARSE_TENSORS) std::vector* copy_tensor_pairs = nullptr, - std::vector* copy_sparse_pairs = nullptr) { + std::vector* copy_sparse_pairs = nullptr) +#else + std::vector* copy_tensor_pairs = nullptr) +#endif +{ // same device so direct copy if (copy_info.source_device == copy_info.target_device) { target_mlvalue = source_mlvalue; @@ -206,6 +213,7 @@ static Status BatchOrCopyMLValue(const SessionState& session_state, ORT_RETURN_IF_ERROR(session_state.GetDataTransferMgr().CopyTensor(source_tensor, *p_output_tensor)); } } else if (source_mlvalue.IsSparseTensor()) { +#if !defined(DISABLE_SPARSE_TENSORS) const auto& source_tensor = source_mlvalue.Get(); SparseTensor* p_output_tensor = target_mlvalue.GetMutable(); if (copy_sparse_pairs != nullptr) { @@ -213,6 +221,7 @@ static Status BatchOrCopyMLValue(const SessionState& session_state, } else { ORT_RETURN_IF_ERROR(session_state.GetDataTransferMgr().CopySparseTensor(source_tensor, *p_output_tensor)); } +#endif } else if (source_mlvalue.IsTensorSequence()) { const TensorSeq& source_tensor_seq = source_mlvalue.Get(); TensorSeq& target_tensor_seq = const_cast(target_mlvalue.Get()); @@ -242,7 +251,7 @@ static Status BatchOrCopyMLValue(const SessionState& session_state, } return Status::OK(); -} +} // namespace utils static bool HaveCpuExecutionProvidersOnly(const ExecutionProviders& execution_providers) { for (const auto& execution_provider : execution_providers) { @@ -448,7 +457,9 @@ static void FinalizeFeedFetchCopyInfo(FeedsFetchesManager& feeds_fetches_manager if (feed.IsTensor()) { feed_locations[i] = feed.Get().Location().device; } else if (feed.IsSparseTensor()) { +#if !defined(DISABLE_SPARSE_TENSORS) feed_locations[i] = feed.Get().Location().device; +#endif } } @@ -461,7 +472,9 @@ static void FinalizeFeedFetchCopyInfo(FeedsFetchesManager& feeds_fetches_manager if (fetch.IsTensor()) { fetch_alloc_info[i] = &fetch.Get().Location(); } else if (fetch.IsSparseTensor()) { +#if !defined(DISABLE_SPARSE_TENSORS) fetch_alloc_info[i] = &fetch.Get().Location(); +#endif } } } @@ -478,20 +491,29 @@ static common::Status CopyInputsAcrossDevices(const SessionState& session_state, new_feeds.resize(num_feeds); std::vector batched_data_transfers; +#if !defined(DISABLE_SPARSE_TENSORS) std::vector batched_sparse_data_transfers; +#endif for (size_t idx = 0; idx < num_feeds; ++idx) { +#if !defined(DISABLE_SPARSE_TENSORS) ORT_RETURN_IF_ERROR(BatchOrCopyMLValue(session_state, copy_info[idx], orig_feeds[idx], new_feeds[idx], &batched_data_transfers, &batched_sparse_data_transfers)); +#else + ORT_RETURN_IF_ERROR(BatchOrCopyMLValue(session_state, copy_info[idx], orig_feeds[idx], new_feeds[idx], + &batched_data_transfers)); +#endif } if (!batched_data_transfers.empty()) { ORT_RETURN_IF_ERROR(session_state.GetDataTransferMgr().CopyTensors(batched_data_transfers)); } +#if !defined(DISABLE_SPARSE_TENSORS) if (!batched_sparse_data_transfers.empty()) { ORT_RETURN_IF_ERROR(session_state.GetDataTransferMgr().CopySparseTensors(batched_sparse_data_transfers)); } +#endif return Status::OK(); } @@ -506,9 +528,13 @@ common::Status CopyOneInputAcrossDevices(const SessionState& session_state, cons MLValueCopyInfo copy_info; ORT_RETURN_IF_ERROR(CalculateStaticCopyInfoForFeed(session_state, input_name, copy_info)); +#if !defined(DISABLE_SPARSE_TENSORS) copy_info.source_device = (orig_mlvalue.IsTensor()) ? orig_mlvalue.Get().Location().device : orig_mlvalue.Get().Location().device; +#else + copy_info.source_device = orig_mlvalue.Get().Location().device; +#endif return BatchOrCopyMLValue(session_state, copy_info, orig_mlvalue, new_mlvalue); } @@ -521,20 +547,29 @@ static common::Status CopyOutputsAcrossDevices(const SessionState& session_state user_fetches.resize(num_outputs); std::vector batched_data_transfers; +#if !defined(DISABLE_SPARSE_TENSORS) std::vector batched_sparse_data_transfers; +#endif for (size_t idx = 0; idx < num_outputs; ++idx) { +#if !defined(DISABLE_SPARSE_TENSORS) ORT_RETURN_IF_ERROR(BatchOrCopyMLValue(session_state, copy_info[idx], fetches[idx], user_fetches[idx], &batched_data_transfers, &batched_sparse_data_transfers)); +#else + ORT_RETURN_IF_ERROR(BatchOrCopyMLValue(session_state, copy_info[idx], fetches[idx], user_fetches[idx], + &batched_data_transfers)); +#endif } if (!batched_data_transfers.empty()) { ORT_RETURN_IF_ERROR(session_state.GetDataTransferMgr().CopyTensors(batched_data_transfers)); } +#if !defined(DISABLE_SPARSE_TENSORS) if (!batched_sparse_data_transfers.empty()) { ORT_RETURN_IF_ERROR(session_state.GetDataTransferMgr().CopySparseTensors(batched_sparse_data_transfers)); } +#endif return Status::OK(); } @@ -633,7 +668,6 @@ common::Status ExecutePartialGraph(const SessionState& session_state, FeedsFetch const std::vector& feeds, std::vector& fetches, const logging::Logger& logger, PartialGraphExecutionState& state, const OrtValueCachePtr& cache) { - // finalize the copy info using the provided feeds and fetches. will update device_copy_checks in the background FinalizeFeedFetchCopyInfo(feeds_fetches_manager, feeds, fetches); PartialExecutor executor{state, cache}; diff --git a/onnxruntime/core/graph/graph.cc b/onnxruntime/core/graph/graph.cc index 4465a38213..e205f00784 100644 --- a/onnxruntime/core/graph/graph.cc +++ b/onnxruntime/core/graph/graph.cc @@ -69,20 +69,30 @@ static bool UsingLatestOnnxOpset(const DomainToVersionMap& opset_versions) { static Status MergeShapeInfo(const std::string& output_name, const TypeProto& source, TypeProto& target, bool strict, const logging::Logger& logger) { +#if !defined(DISABLE_SPARSE_TENSORS) if (!(utils::HasTensorType(source) && utils::HasTensorType(target)) && !(utils::HasSparseTensorType(source) && utils::HasSparseTensorType(target))) { return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Source and target must both be either tensors or sparse tensors"); } +#else + if (!(utils::HasTensorType(source) && utils::HasTensorType(target))) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, + "Source and target must both be tensors"); + } +#endif auto status = Status::OK(); ORT_TRY { if (utils::HasTensorType(source)) { ONNX_NAMESPACE::mergeInShapeInfo(source.tensor_type(), *target.mutable_tensor_type()); - } else { + } +#if !defined(DISABLE_SPARSE_TENSORS) + else { ONNX_NAMESPACE::mergeInShapeInfo(source.sparse_tensor_type(), *target.mutable_sparse_tensor_type()); } +#endif } ORT_CATCH(const ONNX_NAMESPACE::InferenceError& ex) { // if this model was not created with the latest onnx version, allow the shape inferencing failure (strict == false). @@ -97,9 +107,12 @@ static Status MergeShapeInfo(const std::string& output_name, << ". Falling back to lenient merge."; if (utils::HasTensorType(source)) { ONNX_NAMESPACE::UnionShapeInfo(utils::GetShape(source), *target.mutable_tensor_type()); - } else { + } +#if !defined(DISABLE_SPARSE_TENSORS) + else { ONNX_NAMESPACE::UnionShapeInfo(utils::GetShape(source), *target.mutable_sparse_tensor_type()); } +#endif } else { ORT_UNUSED_PARAMETER(logger); ORT_UNUSED_PARAMETER(strict); @@ -205,12 +218,14 @@ const TensorShapeProto* NodeArg::Shape() const { } return nullptr; } +#if !defined(DISABLE_SPARSE_TENSORS) case TypeProto::kSparseTensorType: { if (utils::HasShape(type->sparse_tensor_type())) { return &(type->sparse_tensor_type().shape()); } return nullptr; } +#endif case TypeProto::kSequenceType: case TypeProto::kMapType: case TypeProto::kOpaqueType: @@ -226,11 +241,13 @@ bool NodeArg::HasTensorOrScalarShape() const { const auto type_case = type->value_case(); switch (type_case) { case TypeProto::kTensorType: +#if !defined(DISABLE_SPARSE_TENSORS) case TypeProto::kSparseTensorType: // Standard tensor has a valid shape field while // scalar's shape is empty. Thus, we don't need to // check shape here. return true; +#endif case TypeProto::kSequenceType: case TypeProto::kMapType: case TypeProto::kOpaqueType: @@ -247,9 +264,11 @@ void NodeArg::SetShape(const TensorShapeProto& shape) { case TypeProto::kTensorType: *(node_arg_info_.mutable_type()->mutable_tensor_type()->mutable_shape()) = shape; break; +#if !defined(DISABLE_SPARSE_TENSORS) case TypeProto::kSparseTensorType: *(node_arg_info_.mutable_type()->mutable_sparse_tensor_type()->mutable_shape()) = shape; break; +#endif case TypeProto::kSequenceType: case TypeProto::kMapType: case TypeProto::kOpaqueType: @@ -265,9 +284,11 @@ void NodeArg::ClearShape() { case TypeProto::kTensorType: node_arg_info_.mutable_type()->mutable_tensor_type()->clear_shape(); break; +#if !defined(DISABLE_SPARSE_TENSORS) case TypeProto::kSparseTensorType: node_arg_info_.mutable_type()->mutable_sparse_tensor_type()->clear_shape(); break; +#endif case TypeProto::kSequenceType: case TypeProto::kMapType: case TypeProto::kOpaqueType: @@ -328,6 +349,8 @@ common::Status NodeArg::UpdateTypeAndShape(const ONNX_NAMESPACE::TypeProto& inpu break; } + +#if !defined(DISABLE_SPARSE_TENSORS) case TypeProto::kSparseTensorType: { const auto& input_tensor_type = input_type.sparse_tensor_type(); const auto input_tensor_elem_type = input_tensor_type.elem_type(); @@ -357,12 +380,16 @@ common::Status NodeArg::UpdateTypeAndShape(const ONNX_NAMESPACE::TypeProto& inpu *current_type.mutable_sparse_tensor_type() = input_tensor_type; } } - } break; + + break; + } +#endif case TypeProto::kSequenceType: case TypeProto::kMapType: case TypeProto::kOptionalType: case TypeProto::kOpaqueType: case TypeProto::VALUE_NOT_SET: + default: break; } @@ -812,15 +839,17 @@ ADD_BASIC_ATTR_IMPL(float, AttributeProto_AttributeType::AttributeProto_Attribut ADD_BASIC_ATTR_IMPL(int64_t, AttributeProto_AttributeType::AttributeProto_AttributeType_INT, i) ADD_BASIC_ATTR_IMPL(std::string, AttributeProto_AttributeType::AttributeProto_AttributeType_STRING, s) ADD_ATTR_IMPL(TensorProto, AttributeProto_AttributeType::AttributeProto_AttributeType_TENSOR, t) -ADD_ATTR_IMPL(SparseTensorProto, AttributeProto_AttributeType::AttributeProto_AttributeType_SPARSE_TENSOR, sparse_tensor) ADD_ATTR_IMPL(TypeProto, AttributeProto_AttributeType::AttributeProto_AttributeType_TYPE_PROTO, tp) ADD_LIST_ATTR_IMPL(float, AttributeProto_AttributeType::AttributeProto_AttributeType_FLOATS, floats) ADD_LIST_ATTR_IMPL(int64_t, AttributeProto_AttributeType::AttributeProto_AttributeType_INTS, ints) ADD_LIST_ATTR_IMPL(std::string, AttributeProto_AttributeType::AttributeProto_AttributeType_STRINGS, strings) ADD_LIST_ATTR_IMPL(TensorProto, AttributeProto_AttributeType::AttributeProto_AttributeType_TENSORS, tensors) ADD_LIST_ATTR_IMPL(GraphProto, AttributeProto_AttributeType::AttributeProto_AttributeType_GRAPHS, graphs) -ADD_LIST_ATTR_IMPL(SparseTensorProto, AttributeProto_AttributeType::AttributeProto_AttributeType_SPARSE_TENSORS, sparse_tensors) ADD_LIST_ATTR_IMPL(TypeProto, AttributeProto_AttributeType::AttributeProto_AttributeType_TYPE_PROTOS, type_protos) +#if !defined(DISABLE_SPARSE_TENSORS) +ADD_ATTR_IMPL(SparseTensorProto, AttributeProto_AttributeType::AttributeProto_AttributeType_SPARSE_TENSOR, sparse_tensor) +ADD_LIST_ATTR_IMPL(SparseTensorProto, AttributeProto_AttributeType::AttributeProto_AttributeType_SPARSE_TENSORS, sparse_tensors) +#endif #if !defined(ORT_MINIMAL_BUILD) bool Node::ClearAttribute(const std::string& attr_name) { @@ -998,10 +1027,12 @@ Graph::Graph(const Model& owning_model, const gsl::not_null tensor{graph_proto_->add_initializer()}; auto status = utils::ConstantNodeProtoToTensorProto(node, model_path, *tensor); ORT_ENFORCE(status.IsOK(), status.ToString()); +#if !defined(DISABLE_SPARSE_TENSORS) if (node.attribute(0).type() == AttributeProto_AttributeType_SPARSE_TENSOR) { auto p = sparse_tensor_names_.emplace(tensor->name()); ORT_ENFORCE(p.second, "Duplicate constant node sparse initializer name: '", tensor->name(), "' Model is invalid."); } +#endif } // Remove constant nodes as they're replaced with initializers above. @@ -1013,6 +1044,7 @@ Graph::Graph(const Model& owning_model, }), graph_mutable_nodes->end()); +#if !defined(DISABLE_SPARSE_TENSORS) // For now we convert sparse_intializer to dense tensors // since there are currently no supported ops that consume sparse // initializers directly. We remove them from graph_proto. We will reconstitute them @@ -1034,6 +1066,7 @@ Graph::Graph(const Model& owning_model, delete graph_proto_->mutable_sparse_initializer()->ReleaseCleared(); } } +#endif // Collect all node arg name, type, shape information in the graph. // type/shape information will be assigned to each node arg when going @@ -1780,10 +1813,12 @@ bool FullyDefinedType(const TypeProto& type_proto) { auto& tensor_type = type_proto.tensor_type(); return utils::HasElemType(tensor_type); } +#if !defined(DISABLE_SPARSE_TENSORS) case TypeProto::kSparseTensorType: { auto& tensor_type = type_proto.sparse_tensor_type(); return utils::HasElemType(tensor_type); } +#endif case TypeProto::kSequenceType: { auto& seq_type = type_proto.sequence_type(); return utils::HasElemType(seq_type) && FullyDefinedType(seq_type.elem_type()); @@ -2224,9 +2259,12 @@ Status Graph::InferAndVerifyTypeMatch(Node& node, const OpSchema& op, const Reso TypeProto merge_target; if (utils::HasTensorType(onnx_inferred_type)) { *merge_target.mutable_tensor_type()->mutable_shape() = *output_def->Shape(); - } else if (utils::HasSparseTensorType(onnx_inferred_type)) { + } +#if !defined(DISABLE_SPARSE_TENSORS) + else if (utils::HasSparseTensorType(onnx_inferred_type)) { *merge_target.mutable_sparse_tensor_type()->mutable_shape() = *output_def->Shape(); } +#endif auto status = MergeShapeInfo(output_def->Name(), onnx_inferred_type, merge_target, using_latest_onnx_opset_, logger_); if (!status.IsOK()) { return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Node:", node_name, " ", status.ErrorMessage()); @@ -2674,9 +2712,11 @@ bool Graph::IsInitializedTensor(const std::string& name) const { return name_to_initial_tensor_.count(name) > 0; } +#if !defined(DISABLE_SPARSE_TENSORS) bool Graph::IsSparseInitializer(const std::string& name) const { return sparse_tensor_names_.count(name) > 0; } +#endif void Graph::RemoveInitializedTensor(const std::string& tensor_name) { bool found = false; @@ -2684,10 +2724,14 @@ void Graph::RemoveInitializedTensor(const std::string& tensor_name) { found = iter != name_to_initial_tensor_.end(); if (found) { name_to_initial_tensor_.erase(iter); +#if !defined(DISABLE_SPARSE_TENSORS) sparse_tensor_names_.erase(tensor_name); +#endif SetGraphResolveNeeded(); } else { +#if !defined(DISABLE_SPARSE_TENSORS) ORT_ENFORCE(sparse_tensor_names_.count(tensor_name) == 0, "sparse_tensor_names_ not in sync with name_to_initial_tensor_"); +#endif } auto& mutable_initializers = *(graph_proto_->mutable_initializer()); @@ -2753,7 +2797,9 @@ bool Graph::GetInitializedTensor(const std::string& tensor_name, const TensorPro void Graph::CleanAllInitializedTensors() noexcept { name_to_initial_tensor_.clear(); +#if !defined(DISABLE_SPARSE_TENSORS) sparse_tensor_names_.clear(); +#endif // Clearing RepeatedPtrFields does not free objects' memory. The memory is retained // and can be reused. Need to explicitly release the cleared objects and free the @@ -2902,13 +2948,19 @@ common::Status Graph::SaveToOrtFormat(flatbuffers::FlatBufferBuilder& builder, auto inputs = SaveInputsOutputsToOrtFormat(builder, graph_inputs_including_initializers_); auto outputs = SaveInputsOutputsToOrtFormat(builder, graph_outputs_); +#if !defined(DISABLE_SPARSE_TENSORS) std::vector> sparse_initializers_data; sparse_initializers_data.reserve(sparse_tensor_names_.size()); +#endif const auto sparse_end = sparse_tensor_names_.end(); std::vector> initializers_data; +#if !defined(DISABLE_SPARSE_TENSORS) assert(sparse_tensor_names_.size() <= name_to_initial_tensor_.size()); initializers_data.reserve(name_to_initial_tensor_.size() - sparse_tensor_names_.size()); +#else + initializers_data.reserve(name_to_initial_tensor_.size()); +#endif const auto& model_path = ModelPath(); for (const auto& pair : name_to_initial_tensor_) { @@ -2917,7 +2969,9 @@ common::Status Graph::SaveToOrtFormat(flatbuffers::FlatBufferBuilder& builder, ORT_RETURN_IF_ERROR( experimental::utils::SaveInitializerOrtFormat(builder, *pair.second, model_path, fbs_tensor)); initializers_data.push_back(fbs_tensor); - } else { + } +#if !defined(DISABLE_SPARSE_TENSORS) + else { SparseTensorProto sparse_initializer; ORT_RETURN_IF_ERROR(utils::DenseTensorToSparseTensorProto(*pair.second, model_path, sparse_initializer)); flatbuffers::Offset fbs_sparse_tensor; @@ -2925,9 +2979,12 @@ common::Status Graph::SaveToOrtFormat(flatbuffers::FlatBufferBuilder& builder, experimental::utils::SaveSparseInitializerOrtFormat(builder, sparse_initializer, model_path, fbs_sparse_tensor)); sparse_initializers_data.push_back(fbs_sparse_tensor); } +#endif } - auto initializers = builder.CreateVector(initializers_data); +#if !defined(DISABLE_SPARSE_TENSORS) auto sparse_initializers = builder.CreateVector(sparse_initializers_data); +#endif + auto initializers = builder.CreateVector(initializers_data); std::vector> node_args_data; node_args_data.reserve(node_args_.size()); @@ -2961,7 +3018,9 @@ common::Status Graph::SaveToOrtFormat(flatbuffers::FlatBufferBuilder& builder, gb.add_node_edges(node_edges); gb.add_inputs(inputs); gb.add_outputs(outputs); +#if !defined(DISABLE_SPARSE_TENSORS) gb.add_sparse_initializers(sparse_initializers); +#endif fbs_graph = gb.Finish(); return Status::OK(); } @@ -3090,17 +3149,24 @@ const ONNX_NAMESPACE::GraphProto& Graph::ToGraphProto() { } ONNX_NAMESPACE::GraphProto Graph::ToGraphProto() const { +#if !defined(DISABLE_SPARSE_TENSORS) if (!GraphProtoSyncNeeded() && sparse_tensor_names_.empty()) { return *graph_proto_; } +#else + if (!GraphProtoSyncNeeded()) { + return *graph_proto_; + } +#endif GraphProto result; ToGraphProtoInternal(result); // Path of the owning model // This is used for constructing full path for external data // if it exists - const auto& model_path = ModelPath(); +#if !defined(DISABLE_SPARSE_TENSORS) + const auto& model_path = ModelPath(); // We want to make sure that sparse initializers do not appear // as dense duplicates within the initializers list. if (!sparse_tensor_names_.empty()) { @@ -3118,6 +3184,9 @@ ONNX_NAMESPACE::GraphProto Graph::ToGraphProto() const { } else { *result.mutable_initializer() = graph_proto_->initializer(); } +#else + *result.mutable_initializer() = graph_proto_->initializer(); +#endif return result; } @@ -3126,21 +3195,26 @@ ONNX_NAMESPACE::GraphProto Graph::ToGraphProtoWithExternalInitializers(const std size_t initializer_size_threshold) const { GraphProto result; ToGraphProtoInternal(result); - const auto& model_path = ModelPath(); std::ofstream external_stream(external_file_name, std::ofstream::out | std::ofstream::binary); ORT_ENFORCE(external_stream.is_open()); int64_t external_offset = 0; // Add the initializers to the result graph. +#if !defined(DISABLE_SPARSE_TENSORS) + const auto& model_path = ModelPath(); const auto sparse_end = sparse_tensor_names_.end(); +#endif + for (const auto& initializer : graph_proto_->initializer()) { +#if !defined(DISABLE_SPARSE_TENSORS) if (sparse_end != sparse_tensor_names_.find(initializer.name())) { // Sparse tensors are added to the ONNX file. auto& sparse_initializer = *result.add_sparse_initializer(); auto status = utils::DenseTensorToSparseTensorProto(initializer, model_path, sparse_initializer); ORT_ENFORCE(status.IsOK(), "Failed to convert dense initializer to sparse"); } else { +#endif // Dense tensors larger than the threshold are added to the external file. TensorProto* output_proto = result.add_initializer(); @@ -3175,7 +3249,9 @@ ONNX_NAMESPACE::GraphProto Graph::ToGraphProtoWithExternalInitializers(const std output_proto->set_doc_string(initializer.doc_string()); external_offset += tensor_bytes_size; +#if !defined(DISABLE_SPARSE_TENSORS) } +#endif } return result; @@ -3882,9 +3958,13 @@ common::Status Graph::LoadFromOrtFormat(const onnxruntime::experimental::fbs::Gr // Initializers auto fbs_initializers = fbs_graph.initializers(); +#if !defined(DISABLE_SPARSE_TENSORS) auto fbs_sparse_initializers = fbs_graph.sparse_initializers(); flatbuffers::uoffset_t map_size = (fbs_initializers != nullptr ? fbs_initializers->size() : 0U) + (fbs_sparse_initializers != nullptr ? fbs_sparse_initializers->size() : 0U); +#else + flatbuffers::uoffset_t map_size = (fbs_initializers != nullptr ? fbs_initializers->size() : 0U); +#endif if (map_size > 0) { name_to_initial_tensor_.reserve(map_size); @@ -3905,6 +3985,7 @@ common::Status Graph::LoadFromOrtFormat(const onnxruntime::experimental::fbs::Gr } } +#if !defined(DISABLE_SPARSE_TENSORS) if (fbs_sparse_initializers) { sparse_tensor_names_.reserve(fbs_sparse_initializers->size()); const auto& model_path = ModelPath(); @@ -3925,6 +4006,7 @@ common::Status Graph::LoadFromOrtFormat(const onnxruntime::experimental::fbs::Gr sparse_tensor_names_.emplace(initializer.name()); } } +#endif // NodeArgs auto fbs_node_args = fbs_graph.node_args(); diff --git a/onnxruntime/core/optimizer/constant_folding.cc b/onnxruntime/core/optimizer/constant_folding.cc index af585db08d..ce011b4750 100644 --- a/onnxruntime/core/optimizer/constant_folding.cc +++ b/onnxruntime/core/optimizer/constant_folding.cc @@ -94,9 +94,11 @@ Status ConstantFolding::ApplyImpl(Graph& graph, bool& modified, int graph_level, GraphViewer graph_viewer(graph); auto& order = graph_viewer.GetNodesInTopologicalOrder(); +#if !defined(DISABLE_SPARSE_TENSORS) std::function is_sparse_initializer_check = [&graph](const std::string& name) -> bool { return graph.IsSparseInitializer(name); }; +#endif for (NodeIndex i : order) { auto* node = graph.GetNode(i); @@ -146,9 +148,15 @@ Status ConstantFolding::ApplyImpl(Graph& graph, bool& modified, int graph_level, continue; } +#if !defined(DISABLE_SPARSE_TENSORS) // Create execution frame for executing constant nodes. OptimizerExecutionFrame::Info info({node}, constant_inputs, graph.ModelPath(), execution_provider_, is_sparse_initializer_check); +#else + // Create execution frame for executing constant nodes. + OptimizerExecutionFrame::Info info({node}, constant_inputs, graph.ModelPath(), execution_provider_, + [](std::string const&) { return false; }); +#endif std::vector fetch_mlvalue_idxs; for (const auto* node_out : node->OutputDefs()) { diff --git a/onnxruntime/core/optimizer/optimizer_execution_frame.cc b/onnxruntime/core/optimizer/optimizer_execution_frame.cc index 8bc84056b6..00e76fbef0 100644 --- a/onnxruntime/core/optimizer/optimizer_execution_frame.cc +++ b/onnxruntime/core/optimizer/optimizer_execution_frame.cc @@ -145,9 +145,13 @@ Status OptimizerExecutionFrame::CreateNodeOutputMLValueImpl(OrtValue& ort_value, return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Tried to allocate without valid type information, ort_value index=" + std::to_string(ort_value_idx)); if (ml_type->IsSparseTensorType()) { +#if !defined(DISABLE_SPARSE_TENSORS) auto element_type = ml_type->AsSparseTensorType()->GetElementType(); SparseTensor::InitOrtValue(element_type, *shape, info_.GetAllocator(), ort_value); return Status::OK(); +#else + return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Sparse tensor is not supported in this build"); +#endif } if (ml_type->IsTensorSequenceType()) { diff --git a/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc b/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc index 61a030ea1f..a731a77015 100644 --- a/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc +++ b/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc @@ -104,8 +104,10 @@ AllocatorPtr AllocatorManager::GetAllocator(int id, OrtMemType mem_type) const { template <> MLDataType DataTypeImpl::GetType() { return Provider_GetHost()->DataTypeImpl__GetType_Tensor(); } +#if !defined(DISABLE_SPARSE_TENSORS) template <> MLDataType DataTypeImpl::GetType() { return Provider_GetHost()->DataTypeImpl__GetType_SparseTensor(); } +#endif template <> MLDataType DataTypeImpl::GetType() { return Provider_GetHost()->DataTypeImpl__GetType_TensorSeq(); } MLDataType DataTypeImpl::GetTypeFromOnnxType(int onnx_type) { return Provider_GetHost()->DataTypeImpl__GetTypeFromOnnxType(onnx_type); } @@ -164,6 +166,7 @@ MLDataType DataTypeImpl::GetTensorType() { return Provider_GetHost()-> template <> MLDataType DataTypeImpl::GetTensorType() { return Provider_GetHost()->DataTypeImpl__GetTensorType_MLFloat16(); } +#if !defined(DISABLE_SPARSE_TENSORS) template <> MLDataType DataTypeImpl::GetSparseTensorType() { return Provider_GetHost()->DataTypeImpl__GetSparseTensorType_bool(); } template <> @@ -192,6 +195,7 @@ template <> MLDataType DataTypeImpl::GetSparseTensorType() { return Provider_GetHost()->DataTypeImpl__GetSparseTensorType_BFloat16(); } template <> MLDataType DataTypeImpl::GetSparseTensorType() { return Provider_GetHost()->DataTypeImpl__GetSparseTensorType_MLFloat16(); } +#endif Status IDataTransfer::CopyTensor(const Tensor& src, Tensor& dst) const { return g_host->IDataTransfer__CopyTensor(this, src, dst); @@ -200,10 +204,11 @@ Status IDataTransfer::CopyTensor(const Tensor& src, Tensor& dst) const { Status IDataTransfer::CopyTensors(const std::vector& src_dst_pairs) const { return g_host->IDataTransfer__CopyTensors(this, src_dst_pairs); } - +#if !defined(DISABLE_SPARSE_TENSORS) Status IDataTransfer::CopySparseTensors(const std::vector& src_dst_pairs) const { return g_host->IDataTransfer__CopySparseTensors(this, src_dst_pairs); } +#endif const Node& OpKernel::Node() const { return g_host->OpKernel__Node(this); } @@ -373,6 +378,7 @@ float halfToFloat(uint16_t h) { return g_host->math__halfToFloat(h); } } // namespace math namespace sparse_utils { +#if !defined(DISABLE_SPARSE_TENSORS) #if !defined(ORT_MINIMAL_BUILD) Status DenseTensorToSparseCsr(const DataTransferManager& data_manager, const Tensor& src, const AllocatorPtr& cpu_allocator, const AllocatorPtr& dst_allocator, SparseTensor& dst) { @@ -388,11 +394,14 @@ Status SparseCooToDenseTensor(const DataTransferManager& data_manager, const Spa const AllocatorPtr& dst_allocator, Tensor& dst) { return g_host->sparse_utils__SparseCooToDenseTensor(data_manager, src, cpu_allocator, dst_allocator, dst); } -#endif // ORT_MINIMAL_BUILD +#endif // !ORT_MINIMAL_BUILD + Status DenseTensorToSparseCoo(const DataTransferManager& data_manager, const Tensor& src, const AllocatorPtr& cpu_allocator, const AllocatorPtr& dst_allocator, bool linear_indexs, SparseTensor& dst) { return g_host->sparse_utils__DenseTensorToSparseCoo(data_manager, src, cpu_allocator, dst_allocator, linear_indexs, dst); } +#endif // !defined(DISABLE_SPARSE_TENSORS) + } // namespace sparse_utils float MLFloat16::ToFloat() const { diff --git a/onnxruntime/core/providers/shared_library/provider_interfaces.h b/onnxruntime/core/providers/shared_library/provider_interfaces.h index 77f40f5abd..dfeb3b06f8 100644 --- a/onnxruntime/core/providers/shared_library/provider_interfaces.h +++ b/onnxruntime/core/providers/shared_library/provider_interfaces.h @@ -172,6 +172,7 @@ struct ProviderHost { virtual float math__halfToFloat(uint16_t h) = 0; // sparse_utils +#if !defined(DISABLE_SPARSE_TENSORS) #if !defined(ORT_MINIMAL_BUILD) virtual Status sparse_utils__DenseTensorToSparseCsr(const DataTransferManager& data_manager, const Tensor& src, const AllocatorPtr& cpu_allocator, const AllocatorPtr& dst_allocator, SparseTensor& dst) = 0; @@ -180,9 +181,10 @@ struct ProviderHost { virtual Status sparse_utils__SparseCooToDenseTensor(const DataTransferManager& data_manager, const SparseTensor& src, const AllocatorPtr& cpu_allocator, const AllocatorPtr& dst_allocator, Tensor& dst) = 0; -#endif // ORT_MINIMAL_BUILD +#endif // !ORT_MINIMAL_BUILD virtual Status sparse_utils__DenseTensorToSparseCoo(const DataTransferManager& data_manager, const Tensor& src, const AllocatorPtr& cpu_allocator, const AllocatorPtr& dst_allocator, bool linear_indexs, SparseTensor& dst) = 0; +#endif // !defined(DISABLE_SPARSE_TENSORS) // IAllocator virtual bool IAllocator__CalcMemSizeForArrayWithAlignment(size_t nmemb, size_t size, size_t alignment, size_t* out) = 0; @@ -239,18 +241,22 @@ struct ProviderHost { virtual ONNX_NAMESPACE::TensorShapeProto* TypeProto_Tensor__mutable_shape(ONNX_NAMESPACE::TypeProto_Tensor* p) = 0; virtual int32_t TypeProto_Tensor__elem_type(const ONNX_NAMESPACE::TypeProto_Tensor* p) = 0; +#if !defined(DISABLE_SPARSE_TENSORS) // TypeProto_SparseTensor virtual bool TypeProto_SparseTensor__has_shape(const ONNX_NAMESPACE::TypeProto_SparseTensor* p) = 0; virtual const ONNX_NAMESPACE::TensorShapeProto& TypeProto_SparseTensor__shape(const ONNX_NAMESPACE::TypeProto_SparseTensor* p) = 0; virtual ONNX_NAMESPACE::TensorShapeProto* TypeProto_SparseTensor__mutable_shape(ONNX_NAMESPACE::TypeProto_SparseTensor* p) = 0; virtual int32_t TypeProto_SparseTensor__elem_type(const ONNX_NAMESPACE::TypeProto_SparseTensor* p) = 0; +#endif // TypeProto virtual const ONNX_NAMESPACE::TypeProto_Tensor& TypeProto__tensor_type(const ONNX_NAMESPACE::TypeProto* p) = 0; virtual ONNX_NAMESPACE::TypeProto_Tensor* TypeProto__mutable_tensor_type(ONNX_NAMESPACE::TypeProto* p) = 0; +#if !defined(DISABLE_SPARSE_TENSORS) virtual const ONNX_NAMESPACE::TypeProto_SparseTensor& TypeProto__sparse_tensor_type(const ONNX_NAMESPACE::TypeProto* p) = 0; virtual ONNX_NAMESPACE::TypeProto_SparseTensor* TypeProto__mutable_sparse_tensor_type(ONNX_NAMESPACE::TypeProto* p) = 0; +#endif virtual int TypeProto__value_case(const ONNX_NAMESPACE::TypeProto* p) = 0; // AttributeProto @@ -362,15 +368,19 @@ struct ProviderHost { // DataTransferManager virtual Status DataTransferManager__CopyTensor(const DataTransferManager* p, const Tensor& src, Tensor& dst, int exec_queue_id) = 0; virtual Status DataTransferManager__CopyTensor(const DataTransferManager* p, const Tensor& src, Tensor& dst) = 0; +#if !defined(DISABLE_SPARSE_TENSORS) virtual Status DataTransferManager__CopySparseTensor(const DataTransferManager* p, const SparseTensor& src, SparseTensor& dst) = 0; virtual Status DataTransferManager__CopySparseTensor(const DataTransferManager* p, const SparseTensor& src, SparseTensor& dst, int exec_queue_id) = 0; virtual Status DataTransferManager__CopySparseTensors(const DataTransferManager* p, const std::vector& src_dst_pairs) = 0; +#endif virtual const IDataTransfer* DataTransferManager__GetDataTransfer(const DataTransferManager* p, const OrtDevice& src_device, const OrtDevice& dst_device) = 0; // IDataTransfer virtual Status IDataTransfer__CopyTensor(const IDataTransfer* p, const Tensor& src, Tensor& dst) = 0; virtual Status IDataTransfer__CopyTensors(const IDataTransfer* p, const std::vector& src_dst_pairs) = 0; +#if !defined(DISABLE_SPARSE_TENSORS) virtual Status IDataTransfer__CopySparseTensors(const IDataTransfer* p, const std::vector& src_dst_pairs) = 0; +#endif // IndexedSubGraph_MetaDef virtual std::unique_ptr IndexedSubGraph_MetaDef__construct() = 0; @@ -436,7 +446,9 @@ struct ProviderHost { // DataTypeImpl virtual MLDataType DataTypeImpl__GetType_Tensor() = 0; +#if !defined(DISABLE_SPARSE_TENSORS) virtual MLDataType DataTypeImpl__GetType_SparseTensor() = 0; +#endif virtual MLDataType DataTypeImpl__GetType_TensorSeq() = 0; virtual MLDataType DataTypeImpl__GetTypeFromOnnxType(int) = 0; virtual MLDataType DataTypeImpl__GetType_bool() = 0; @@ -467,6 +479,7 @@ struct ProviderHost { virtual MLDataType DataTypeImpl__GetTensorType_BFloat16() = 0; virtual MLDataType DataTypeImpl__GetTensorType_MLFloat16() = 0; +#if !defined(DISABLE_SPARSE_TENSORS) virtual MLDataType DataTypeImpl__GetSparseTensorType_bool() = 0; virtual MLDataType DataTypeImpl__GetSparseTensorType_int8() = 0; virtual MLDataType DataTypeImpl__GetSparseTensorType_uint8() = 0; @@ -481,11 +494,14 @@ struct ProviderHost { virtual MLDataType DataTypeImpl__GetSparseTensorType_string() = 0; virtual MLDataType DataTypeImpl__GetSparseTensorType_BFloat16() = 0; virtual MLDataType DataTypeImpl__GetSparseTensorType_MLFloat16() = 0; +#endif virtual const char* DataTypeImpl__ToString(MLDataType type) = 0; virtual bool DataTypeImpl__IsTensorType(const DataTypeImpl* p) = 0; virtual bool DataTypeImpl__IsTensorSequenceType(const DataTypeImpl* p) = 0; +#if !defined(DISABLE_SPARSE_TENSORS) virtual bool DataTypeImpl__IsSparseTensorType(const DataTypeImpl* p) = 0; +#endif virtual DeleteFunc DataTypeImpl__GetDeleteFunc(const DataTypeImpl* p) = 0; virtual const std::vector& DataTypeImpl__AllFixedSizeTensorTypes() = 0; virtual const std::vector& DataTypeImpl__AllTensorTypes() = 0; @@ -612,13 +628,17 @@ struct ProviderHost { // OpKernelContext virtual const Tensor* OpKernelContext__Input_Tensor(const OpKernelContext* p, int index) = 0; +#if !defined(DISABLE_SPARSE_TENSORS) virtual const SparseTensor* OpKernelContext__Input_SparseTensor(const OpKernelContext* p, int index) = 0; +#endif virtual const TensorSeq* OpKernelContext__Input_TensorSeq(const OpKernelContext* p, int index) = 0; virtual const Tensor& OpKernelContext__RequiredInput_Tensor(const OpKernelContext* p, int index) = 0; virtual Tensor* OpKernelContext__Output_Tensor(OpKernelContext* p, int index) = 0; virtual TensorSeq* OpKernelContext__Output_TensorSeq(OpKernelContext* p, int index) = 0; virtual Tensor* OpKernelContext__Output(OpKernelContext* p, int index, const TensorShape& shape) = 0; +#if !defined(DISABLE_SPARSE_TENSORS) virtual SparseTensor* OpKernelContext__OutputSparse(OpKernelContext* p, int index, const TensorShape& shape) = 0; +#endif virtual Tensor& OpKernelContext__RequiredOutput(OpKernelContext* p, int index, const TensorShape& shape) = 0; virtual MLDataType OpKernelContext__InputType(const OpKernelContext* p, int index) = 0; virtual int OpKernelContext__InputCount(const OpKernelContext* p) = 0; @@ -718,9 +738,11 @@ struct ProviderHost { virtual int32_t Tensor__GetElementType(const Tensor* p) = 0; virtual MLDataType Tensor__DataType(const Tensor* p) = 0; +#if !defined(DISABLE_SPARSE_TENSORS) // SparseTensor virtual const TensorShape& SparseTensor__DenseShape(const SparseTensor*) = 0; virtual Status SparseTensor__Copy(const SparseTensor*, const DataTransferManager&, int, SparseTensor&) = 0; +#endif // TensorSeq virtual MLDataType TensorSeq__DataType(const TensorSeq* p) noexcept = 0; @@ -756,7 +778,7 @@ struct ProviderHost { #endif #endif - virtual ProviderHostCPU& GetProviderHostCPU()=0; + virtual ProviderHostCPU& GetProviderHostCPU() = 0; }; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h b/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h index 75716bb523..b9c816c09c 100644 --- a/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h +++ b/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h @@ -40,7 +40,7 @@ struct Capture final { void operator=(const Capture&) = delete; }; } // namespace logging -} +} // namespace onnxruntime namespace ONNX_NAMESPACE { @@ -80,13 +80,17 @@ struct AttributeProto final { static constexpr AttributeType STRING = AttributeProto_AttributeType_STRING; static constexpr AttributeType TENSOR = AttributeProto_AttributeType_TENSOR; static constexpr AttributeType GRAPH = AttributeProto_AttributeType_GRAPH; +#if !defined(DISABLE_SPARSE_TENSORS) static constexpr AttributeType SPARSE_TENSOR = AttributeProto_AttributeType_SPARSE_TENSOR; +#endif static constexpr AttributeType FLOATS = AttributeProto_AttributeType_FLOATS; static constexpr AttributeType INTS = AttributeProto_AttributeType_INTS; static constexpr AttributeType STRINGS = AttributeProto_AttributeType_STRINGS; static constexpr AttributeType TENSORS = AttributeProto_AttributeType_TENSORS; static constexpr AttributeType GRAPHS = AttributeProto_AttributeType_GRAPHS; +#if !defined(DISABLE_SPARSE_TENSORS) static constexpr AttributeType SPARSE_TENSORS = AttributeProto_AttributeType_SPARSE_TENSORS; +#endif AttributeProto() = delete; AttributeProto(const AttributeProto&) = delete; @@ -210,6 +214,7 @@ struct TypeProto_Tensor final { PROVIDER_DISALLOW_ALL(TypeProto_Tensor) }; +#if !defined(DISABLE_SPARSE_TENSORS) struct TypeProto_SparseTensor final { bool has_shape() const { return g_host->TypeProto_SparseTensor__has_shape(this); } const TensorShapeProto& shape() const { return g_host->TypeProto_SparseTensor__shape(this); } @@ -218,13 +223,16 @@ struct TypeProto_SparseTensor final { PROVIDER_DISALLOW_ALL(TypeProto_SparseTensor) }; +#endif struct TypeProto final { const TypeProto_Tensor& tensor_type() const { return g_host->TypeProto__tensor_type(this); } TypeProto_Tensor* mutable_tensor_type() { return g_host->TypeProto__mutable_tensor_type(this); } +#if !defined(DISABLE_SPARSE_TENSORS) const TypeProto_SparseTensor& sparse_tensor_type() const { return g_host->TypeProto__sparse_tensor_type(this); } TypeProto_SparseTensor* mutable_sparse_tensor_type() { return g_host->TypeProto__mutable_sparse_tensor_type(this); } +#endif enum ValueCase { kTensorType = 1, @@ -291,9 +299,11 @@ struct ComputeCapability final { struct DataTransferManager final { Status CopyTensor(const Tensor& src, Tensor& dst, int exec_queue_id) const { return g_host->DataTransferManager__CopyTensor(this, src, dst, exec_queue_id); } Status CopyTensor(const Tensor& src, Tensor& dst) const { return g_host->DataTransferManager__CopyTensor(this, src, dst); } +#if !defined(DISABLE_SPARSE_TENSORS) Status CopySparseTensor(const SparseTensor& src, SparseTensor& dst) const { return g_host->DataTransferManager__CopySparseTensor(this, src, dst); } Status CopySparseTensor(const SparseTensor& src, SparseTensor& dst, int exec_queue_id) const { return g_host->DataTransferManager__CopySparseTensor(this, src, dst, exec_queue_id); } Status CopySparseTensors(const std::vector& src_dst_pairs) const { return g_host->DataTransferManager__CopySparseTensors(this, src_dst_pairs); } +#endif const IDataTransfer* GetDataTransfer(const OrtDevice& src_device, const OrtDevice& dst_device) const { return g_host->DataTransferManager__GetDataTransfer(this, src_device, dst_device); } PROVIDER_DISALLOW_ALL(DataTransferManager) @@ -466,14 +476,18 @@ class DataTypeImpl final { static MLDataType GetType(); template static MLDataType GetTensorType(); +#if !defined(DISABLE_SPARSE_TENSORS) template static MLDataType GetSparseTensorType(); +#endif static MLDataType GetTypeFromOnnxType(int); bool IsTensorType() const { return g_host->DataTypeImpl__IsTensorType(this); } bool IsTensorSequenceType() const { return g_host->DataTypeImpl__IsTensorSequenceType(this); } +#if !defined(DISABLE_SPARSE_TENSORS) bool IsSparseTensorType() const { return g_host->DataTypeImpl__IsSparseTensorType(this); } +#endif DeleteFunc GetDeleteFunc() const { return g_host->DataTypeImpl__GetDeleteFunc(this); } static const std::vector& AllFixedSizeTensorTypes() { return g_host->DataTypeImpl__AllFixedSizeTensorTypes(); } @@ -680,7 +694,9 @@ struct OpKernelContext final { T* Output(int index); Tensor* Output(int index, const TensorShape& shape) { return g_host->OpKernelContext__Output(this, index, shape); } - SparseTensor* OutputSparse(int index, const TensorShape& shape) { return g_host->OpKernelContext__OutputSparse(this, index, shape); } +#if !defined(DISABLE_SPARSE_TENSORS) + SparseTensor* OutputSparse(int index, const TensorShape& shape) { return g_host->OpKernelContext__OutputSparse(this, index, shape); } +#endif int OutputCount() const { return g_host->OpKernelContext__OutputCount(this); } Status GetTempSpaceAllocator(AllocatorPtr* output) const { return g_host->OpKernelContext__GetTempSpaceAllocator(this, output); } @@ -698,10 +714,12 @@ inline const Tensor* OpKernelContext::Input(int index) const { return g_host->OpKernelContext__Input_Tensor(this, index); } +#if !defined(DISABLE_SPARSE_TENSORS) template <> inline const SparseTensor* OpKernelContext::Input(int index) const { return g_host->OpKernelContext__Input_SparseTensor(this, index); } +#endif template <> inline const TensorSeq* OpKernelContext::Input(int index) const { @@ -919,10 +937,12 @@ template <> inline const MLFloat16* Tensor::Data() const { return g_host->Tensor__Data_MLFloat16(this); } // SparseTensor +#if !defined(DISABLE_SPARSE_TENSORS) struct SparseTensor final { const TensorShape& DenseShape() const noexcept { return g_host->SparseTensor__DenseShape(this); } Status Copy(const DataTransferManager& dtm, int exec_q_id, SparseTensor& dst) const { return g_host->SparseTensor__Copy(this, dtm, exec_q_id, dst); } }; +#endif //TensorSeq struct TensorSeq final { @@ -936,4 +956,4 @@ struct TensorSeq final { template <> inline gsl::span Tensor::DataAsSpan() const { return g_host->Tensor__DataAsSpan_int64(this); } -} \ No newline at end of file +} // namespace onnxruntime \ No newline at end of file diff --git a/onnxruntime/core/session/inference_session.cc b/onnxruntime/core/session/inference_session.cc index fed696497b..323ba65ceb 100644 --- a/onnxruntime/core/session/inference_session.cc +++ b/onnxruntime/core/session/inference_session.cc @@ -1587,6 +1587,7 @@ common::Status InferenceSession::ValidateInputs(const std::vector& ORT_RETURN_IF_ERROR_SESSIONID_(CheckShapes(feed_name, input_shape, expected_shape)); } } else if (input_ml_value.IsSparseTensor()) { +#if !defined(DISABLE_SPARSE_TENSORS) if (!expected_type->IsSparseTensorType()) { return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input with name: ", feed_name, " is not expected to be of type sparse tensor."); @@ -1601,6 +1602,11 @@ common::Status InferenceSession::ValidateInputs(const std::vector& const auto& input_shape = sparse_tensor.DenseShape(); ORT_RETURN_IF_ERROR_SESSIONID_(CheckShapes(feed_name, input_shape, expected_shape)); } +#else + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input with name ", feed_name, + " is a sparse tensor, which is not supported in this build."); +#endif + } else if (input_ml_value.IsTensorSequence()) { if (!expected_type->IsTensorSequenceType()) { return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input with name: ", feed_name, diff --git a/onnxruntime/core/session/onnxruntime_c_api.cc b/onnxruntime/core/session/onnxruntime_c_api.cc index 8118a49d88..ea1777b1ea 100644 --- a/onnxruntime/core/session/onnxruntime_c_api.cc +++ b/onnxruntime/core/session/onnxruntime_c_api.cc @@ -231,6 +231,7 @@ ORT_API_STATUS_IMPL(OrtApis::CreateTensorAsOrtValue, _Inout_ OrtAllocator* alloc ORT_API_STATUS_IMPL(OrtApis::CreateSparseTensorAsOrtValue, _Inout_ OrtAllocator* allocator, _In_ const int64_t* dense_shape, size_t dense_shape_len, ONNXTensorElementDataType type, _Outptr_ OrtValue** out) { API_IMPL_BEGIN +#if !defined(DISABLE_SPARSE_TENSORS) auto sparse_tensor_type = DataTypeImpl::SparseTensorTypeFromONNXEnum(type); auto element_type = sparse_tensor_type->GetElementType(); assert(element_type->AsPrimitiveDataType() != nullptr); @@ -245,10 +246,14 @@ ORT_API_STATUS_IMPL(OrtApis::CreateSparseTensorAsOrtValue, _Inout_ OrtAllocator* SparseTensor::InitOrtValue(element_type, shape, std::move(alloc_ptr), *value); *out = value.release(); return nullptr; +#else + return OrtApis::CreateStatus(ORT_FAIL, "SparseTensor is not supported in this build."); +#endif API_IMPL_END } namespace { +#if !defined(DISABLE_SPARSE_TENSORS) std::unique_ptr GetDataTransfer(const OrtDevice& src_device, const OrtDevice& dst_device) { if (src_device.Type() == OrtDevice::CPU && dst_device.Type() == OrtDevice::CPU) { return std::make_unique(); @@ -284,12 +289,14 @@ union PtrConvert { const char** strings; }; +#endif // !defined(DISABLE_SPARSE_TENSORS) } // namespace ORT_API_STATUS_IMPL(OrtApis::FillSparseTensorCoo, _Inout_ OrtValue* ort_value, _In_ const OrtMemoryInfo* data_mem_info, _In_ const int64_t* values_shape, size_t values_shape_len, _In_ const void* values, _In_ const int64_t* indices_data, size_t indices_num) { API_IMPL_BEGIN +#if !defined(DISABLE_SPARSE_TENSORS) TensorShape values_t_shape(values_shape, values_shape_len); auto& sparse_tensor = ValidateFillInputArgs(ort_value, values_t_shape, data_mem_info); @@ -305,6 +312,9 @@ ORT_API_STATUS_IMPL(OrtApis::FillSparseTensorCoo, _Inout_ OrtValue* ort_value, _ values, indices_span)); } return nullptr; +#else + return OrtApis::CreateStatus(ORT_FAIL, "SparseTensor is not supported in this build."); +#endif API_IMPL_END } @@ -313,6 +323,7 @@ ORT_API_STATUS_IMPL(OrtApis::FillSparseTensorCsr, _Inout_ OrtValue* ort_value, _ _In_ const int64_t* inner_indices_data, size_t inner_indices_num, _In_ const int64_t* outer_indices_data, size_t outer_indices_num) { API_IMPL_BEGIN +#if !defined(DISABLE_SPARSE_TENSORS) TensorShape values_t_shape(values_shape, values_shape_len); auto& sparse_tensor = ValidateFillInputArgs(ort_value, values_t_shape, data_mem_info); auto values_size = gsl::narrow(values_t_shape.Size()); @@ -328,6 +339,9 @@ ORT_API_STATUS_IMPL(OrtApis::FillSparseTensorCsr, _Inout_ OrtValue* ort_value, _ values, inner_indices_span, outer_indices_span)); } return nullptr; +#else + return OrtApis::CreateStatus(ORT_FAIL, "SparseTensor is not supported in this build."); +#endif API_IMPL_END } @@ -336,6 +350,7 @@ ORT_API_STATUS_IMPL(OrtApis::FillSparseTensorBlockSparse, _Inout_ OrtValue* ort_ _In_ const int64_t* indices_shape_data, size_t indices_shape_len, _In_ const int32_t* indices_data) { API_IMPL_BEGIN +#if !defined(DISABLE_SPARSE_TENSORS) TensorShape values_t_shape(values_shape, values_shape_len); auto& sparse_tensor = ValidateFillInputArgs(ort_value, values_t_shape, data_mem_info); @@ -354,6 +369,9 @@ ORT_API_STATUS_IMPL(OrtApis::FillSparseTensorBlockSparse, _Inout_ OrtValue* ort_ values, indices_t_shape, indices_data)); } return nullptr; +#else + return OrtApis::CreateStatus(ORT_FAIL, "SparseTensor is not supported in this build."); +#endif API_IMPL_END } @@ -362,6 +380,7 @@ ORT_API_STATUS_IMPL(OrtApis::CreateSparseTensorWithValuesAsOrtValue, _In_ const _In_ const int64_t* values_shape, size_t values_shape_len, ONNXTensorElementDataType type, _Outptr_ OrtValue** out) { API_IMPL_BEGIN +#if !defined(DISABLE_SPARSE_TENSORS) auto sparse_tensor_type = DataTypeImpl::SparseTensorTypeFromONNXEnum(type); auto element_type = sparse_tensor_type->GetElementType(); assert(element_type->AsPrimitiveDataType() != nullptr); @@ -380,11 +399,15 @@ ORT_API_STATUS_IMPL(OrtApis::CreateSparseTensorWithValuesAsOrtValue, _In_ const SparseTensor::InitOrtValue(element_type, tensor_dense_shape, tensor_values_shape, p_data, *info, *value); *out = value.release(); return nullptr; +#else + return OrtApis::CreateStatus(ORT_FAIL, "SparseTensor is not supported in this build."); +#endif API_IMPL_END } ORT_API_STATUS_IMPL(OrtApis::UseCooIndices, _Inout_ OrtValue* ort_value, _Inout_ int64_t* indices_data, size_t indices_num) { API_IMPL_BEGIN +#if !defined(DISABLE_SPARSE_TENSORS) auto v = reinterpret_cast<::OrtValue*>(ort_value); auto& sparse_tensor = SparseTensor::GetSparseTensorFromOrtValue(*v); auto indices_span = (indices_num == 0 || indices_data == nullptr) @@ -393,6 +416,9 @@ ORT_API_STATUS_IMPL(OrtApis::UseCooIndices, _Inout_ OrtValue* ort_value, _Inout_ ORT_THROW_IF_ERROR(sparse_tensor.UseCooIndices(indices_span)); return nullptr; +#else + return OrtApis::CreateStatus(ORT_FAIL, "SparseTensor is not supported in this build."); +#endif API_IMPL_END } @@ -400,6 +426,7 @@ ORT_API_STATUS_IMPL(OrtApis::UseCsrIndices, _Inout_ OrtValue* ort_value, _Inout_ int64_t* inner_data, size_t inner_num, _Inout_ int64_t* outer_data, size_t outer_num) { API_IMPL_BEGIN +#if !defined(DISABLE_SPARSE_TENSORS) auto& sparse_tensor = SparseTensor::GetSparseTensorFromOrtValue(*ort_value); auto inner_span = (inner_num == 0 || inner_data == nullptr) ? gsl::span() @@ -409,21 +436,29 @@ ORT_API_STATUS_IMPL(OrtApis::UseCsrIndices, _Inout_ OrtValue* ort_value, : gsl::make_span(outer_data, outer_num); ORT_THROW_IF_ERROR(sparse_tensor.UseCsrIndices(inner_span, outer_span)); return nullptr; +#else + return OrtApis::CreateStatus(ORT_FAIL, "SparseTensor is not supported in this build."); +#endif API_IMPL_END } ORT_API_STATUS_IMPL(OrtApis::UseBlockSparseIndices, _Inout_ OrtValue* ort_value, const int64_t* indices_shape, size_t indices_shape_len, _Inout_ int32_t* indices_data) { API_IMPL_BEGIN +#if !defined(DISABLE_SPARSE_TENSORS) auto& sparse_tensor = SparseTensor::GetSparseTensorFromOrtValue(*ort_value); TensorShape ind_shape(indices_shape, indices_shape_len); ORT_THROW_IF_ERROR(sparse_tensor.UseBlockSparseIndices(ind_shape, indices_data)); return nullptr; +#else + return OrtApis::CreateStatus(ORT_FAIL, "SparseTensor is not supported in this build."); +#endif API_IMPL_END } ORT_API_STATUS_IMPL(OrtApis::GetSparseTensorFormat, _In_ const OrtValue* ort_value, _Out_ enum OrtSparseFormat* out) { API_IMPL_BEGIN +#if !defined(DISABLE_SPARSE_TENSORS) auto v = reinterpret_cast(ort_value); if (!v->IsAllocated()) { return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, "the ort_value must contain a constructed tensor"); @@ -431,11 +466,15 @@ ORT_API_STATUS_IMPL(OrtApis::GetSparseTensorFormat, _In_ const OrtValue* ort_val const auto& sparse_tensor = v->Get(); *out = static_cast(sparse_tensor.Format()); return nullptr; +#else + return OrtApis::CreateStatus(ORT_FAIL, "SparseTensor is not supported in this build."); +#endif API_IMPL_END } ORT_API_STATUS_IMPL(OrtApis::GetSparseTensorValues, _In_ const OrtValue* ort_value, _Outptr_ const void** out) { API_IMPL_BEGIN +#if !defined(DISABLE_SPARSE_TENSORS) const auto& sparse_tensor = SparseTensor::GetSparseTensorFromOrtValue(*ort_value); if (sparse_tensor.IsDataTypeString()) { return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, "Use GetStringTensor*() API to retrieve strings"); @@ -443,6 +482,9 @@ ORT_API_STATUS_IMPL(OrtApis::GetSparseTensorValues, _In_ const OrtValue* ort_val const auto& values = sparse_tensor.Values(); *out = values.DataRaw(); return nullptr; +#else + return OrtApis::CreateStatus(ORT_FAIL, "SparseTensor is not supported in this build."); +#endif API_IMPL_END } @@ -883,9 +925,13 @@ ORT_API_STATUS_IMPL(OrtApis::IsTensor, _In_ const OrtValue* value, _Out_ int* ou } ORT_API_STATUS_IMPL(OrtApis::IsSparseTensor, _In_ const OrtValue* value, _Out_ int* out) { +#if !defined(DISABLE_SPARSE_TENSORS) auto v = reinterpret_cast(value); *out = v->IsSparseTensor() ? 1 : 0; return nullptr; +#else + return OrtApis::CreateStatus(ORT_FAIL, "SparseTensor is not supported in this build."); +#endif } ORT_API_STATUS_IMPL(OrtApis::GetTensorMutableData, _Inout_ OrtValue* value, _Outptr_ void** output) { @@ -943,7 +989,9 @@ OrtStatusPtr GetTensorStringSpan(const ::OrtValue& v, gsl::span= 0) { str_span = tensor.DataAsSpan(); } - } else if (v.IsSparseTensor()) { + } +#if !defined(DISABLE_SPARSE_TENSORS) + else if (v.IsSparseTensor()) { const auto& sparse_tensor = v.Get(); if (sparse_tensor.Format() == onnxruntime::SparseFormat::kUndefined) { return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, "Sparse Tensor does not contain sparse data"); @@ -952,7 +1000,9 @@ OrtStatusPtr GetTensorStringSpan(const ::OrtValue& v, gsl::span= 0) { str_span = sparse_tensor.Values().DataAsSpan(); } - } else { + } +#endif + else { return OrtApis::CreateStatus(ORT_NOT_IMPLEMENTED, "This API supports Tensors or SparseTensors"); } diff --git a/onnxruntime/core/session/provider_bridge_ort.cc b/onnxruntime/core/session/provider_bridge_ort.cc index befaf7ed63..ee196644e5 100644 --- a/onnxruntime/core/session/provider_bridge_ort.cc +++ b/onnxruntime/core/session/provider_bridge_ort.cc @@ -198,6 +198,7 @@ struct ProviderHostImpl : ProviderHost { float math__halfToFloat(uint16_t h) override { return math::halfToFloat(h); } // sparse_utils +#if !defined(DISABLE_SPARSE_TENSORS) #if !defined(ORT_MINIMAL_BUILD) Status sparse_utils__DenseTensorToSparseCsr(const DataTransferManager& data_manager, const Tensor& src, const AllocatorPtr& cpu_allocator, const AllocatorPtr& dst_allocator, SparseTensor& dst) override { @@ -213,12 +214,15 @@ struct ProviderHostImpl : ProviderHost { const AllocatorPtr& dst_allocator, Tensor& dst) override { return sparse_utils::SparseCooToDenseTensor(data_manager, src, cpu_allocator, dst_allocator, dst); } + #endif // ORT_MINIMAL_BUILD Status sparse_utils__DenseTensorToSparseCoo(const DataTransferManager& data_manager, const Tensor& src, const AllocatorPtr& cpu_allocator, const AllocatorPtr& dst_allocator, bool linear_indexs, SparseTensor& dst) override { return sparse_utils::DenseTensorToSparseCoo(data_manager, src, cpu_allocator, dst_allocator, linear_indexs, dst); } +#endif // !defined(DISABLE_SPARSE_TENSORS) + // IAllocator (direct) bool IAllocator__CalcMemSizeForArrayWithAlignment(size_t nmemb, size_t size, size_t alignment, size_t* out) override { return IAllocator::CalcMemSizeForArrayWithAlignment(nmemb, size, alignment, out); } @@ -290,6 +294,7 @@ struct ProviderHostImpl : ProviderHost { int32_t TypeProto_Tensor__elem_type(const ONNX_NAMESPACE::TypeProto_Tensor* p) override { return p->elem_type(); } //TypeProto_SparseTensor (wrapped) +#if !defined(DISABLE_SPARSE_TENSORS) bool TypeProto_SparseTensor__has_shape(const ONNX_NAMESPACE::TypeProto_SparseTensor* p) override { return p->has_shape(); } const ONNX_NAMESPACE::TensorShapeProto& TypeProto_SparseTensor__shape(const ONNX_NAMESPACE::TypeProto_SparseTensor* p) override { return p->shape(); @@ -300,17 +305,20 @@ struct ProviderHostImpl : ProviderHost { int32_t TypeProto_SparseTensor__elem_type(const ONNX_NAMESPACE::TypeProto_SparseTensor* p) override { return p->elem_type(); } +#endif // TypeProto (wrapped) const ONNX_NAMESPACE::TypeProto_Tensor& TypeProto__tensor_type(const ONNX_NAMESPACE::TypeProto* p) override { return p->tensor_type(); } ONNX_NAMESPACE::TypeProto_Tensor* TypeProto__mutable_tensor_type(ONNX_NAMESPACE::TypeProto* p) override { return p->mutable_tensor_type(); } int TypeProto__value_case(const ONNX_NAMESPACE::TypeProto* p) override { return p->value_case(); } +#if !defined(DISABLE_SPARSE_TENSORS) const ONNX_NAMESPACE::TypeProto_SparseTensor& TypeProto__sparse_tensor_type(const ONNX_NAMESPACE::TypeProto* p) override { return p->sparse_tensor_type(); } ONNX_NAMESPACE::TypeProto_SparseTensor* TypeProto__mutable_sparse_tensor_type(ONNX_NAMESPACE::TypeProto* p) override { return p->mutable_sparse_tensor_type(); } +#endif // AttributeProto (wrapped) std::unique_ptr AttributeProto__construct() override { return std::make_unique(); } @@ -427,17 +435,21 @@ struct ProviderHostImpl : ProviderHost { // DataTransferManager (wrapped) Status DataTransferManager__CopyTensor(const DataTransferManager* p, const Tensor& src, Tensor& dst, int exec_queue_id) override { return p->CopyTensor(src, dst, exec_queue_id); } Status DataTransferManager__CopyTensor(const DataTransferManager* p, const Tensor& src, Tensor& dst) override { return p->CopyTensor(src, dst); } +#if !defined(DISABLE_SPARSE_TENSORS) Status DataTransferManager__CopySparseTensor(const DataTransferManager* p, const SparseTensor& src, SparseTensor& dst) override { return p->CopySparseTensor(src, dst); } Status DataTransferManager__CopySparseTensor(const DataTransferManager* p, const SparseTensor& src, SparseTensor& dst, int exec_queue_id) override { return p->CopySparseTensor(src, dst, exec_queue_id); } Status DataTransferManager__CopySparseTensors(const DataTransferManager* p, const std::vector& src_dst_pairs) override { return p->CopySparseTensors(src_dst_pairs); }; +#endif const IDataTransfer* DataTransferManager__GetDataTransfer(const DataTransferManager* p, const OrtDevice& src_device, const OrtDevice& dst_device) override { return p->GetDataTransfer(src_device, dst_device); } // IDataTransfer (direct) Status IDataTransfer__CopyTensor(const IDataTransfer* p, const Tensor& src, Tensor& dst) override { return p->IDataTransfer::CopyTensor(src, dst); } Status IDataTransfer__CopyTensors(const IDataTransfer* p, const std::vector& src_dst_pairs) override { return p->IDataTransfer::CopyTensors(src_dst_pairs); } +#if !defined(DISABLE_SPARSE_TENSORS) Status IDataTransfer__CopySparseTensors(const IDataTransfer* p, const std::vector& src_dst_pairs) override { return p->CopySparseTensors(src_dst_pairs); } +#endif // IndexedSubGraph_MetaDef (wrapped) std::unique_ptr IndexedSubGraph_MetaDef__construct() override { return std::make_unique(); } @@ -506,7 +518,9 @@ struct ProviderHostImpl : ProviderHost { // DataTypeImpl (wrapped) MLDataType DataTypeImpl__GetType_Tensor() override { return DataTypeImpl::GetType(); } +#if !defined(DISABLE_SPARSE_TENSORS) MLDataType DataTypeImpl__GetType_SparseTensor() override { return DataTypeImpl::GetType(); } +#endif MLDataType DataTypeImpl__GetType_TensorSeq() override { return DataTypeImpl::GetType(); } MLDataType DataTypeImpl__GetTypeFromOnnxType(int onnx_type) override { return DataTypeImpl::TensorTypeFromONNXEnum(onnx_type)->GetElementType(); } MLDataType DataTypeImpl__GetType_bool() override { return DataTypeImpl::GetType(); } @@ -537,6 +551,7 @@ struct ProviderHostImpl : ProviderHost { MLDataType DataTypeImpl__GetTensorType_BFloat16() override { return DataTypeImpl::GetTensorType(); } MLDataType DataTypeImpl__GetTensorType_MLFloat16() override { return DataTypeImpl::GetTensorType(); } +#if !defined(DISABLE_SPARSE_TENSORS) MLDataType DataTypeImpl__GetSparseTensorType_bool() override { return DataTypeImpl::GetSparseTensorType(); } MLDataType DataTypeImpl__GetSparseTensorType_int8() override { return DataTypeImpl::GetSparseTensorType(); } MLDataType DataTypeImpl__GetSparseTensorType_uint8() override { return DataTypeImpl::GetSparseTensorType(); } @@ -551,11 +566,14 @@ struct ProviderHostImpl : ProviderHost { MLDataType DataTypeImpl__GetSparseTensorType_string() override { return DataTypeImpl::GetSparseTensorType(); } MLDataType DataTypeImpl__GetSparseTensorType_BFloat16() override { return DataTypeImpl::GetSparseTensorType(); } MLDataType DataTypeImpl__GetSparseTensorType_MLFloat16() override { return DataTypeImpl::GetSparseTensorType(); } +#endif const char* DataTypeImpl__ToString(MLDataType type) override { return DataTypeImpl::ToString(type); } bool DataTypeImpl__IsTensorType(const DataTypeImpl* p) override { return p->IsTensorType(); } bool DataTypeImpl__IsTensorSequenceType(const DataTypeImpl* p) override { return p->IsTensorSequenceType(); } +#if !defined(DISABLE_SPARSE_TENSORS) bool DataTypeImpl__IsSparseTensorType(const DataTypeImpl* p) override { return p->IsSparseTensorType(); } +#endif DeleteFunc DataTypeImpl__GetDeleteFunc(const DataTypeImpl* p) override { return p->GetDeleteFunc(); } const std::vector& DataTypeImpl__AllFixedSizeTensorTypes() override { return DataTypeImpl::AllFixedSizeTensorTypes(); } const std::vector& DataTypeImpl__AllTensorTypes() override { return DataTypeImpl::AllTensorTypes(); } @@ -694,14 +712,18 @@ struct ProviderHostImpl : ProviderHost { // OpKernelContext (wrapped) const Tensor* OpKernelContext__Input_Tensor(const OpKernelContext* p, int index) override { return p->Input(index); } +#if !defined(DISABLE_SPARSE_TENSORS) const SparseTensor* OpKernelContext__Input_SparseTensor(const OpKernelContext* p, int index) override { return p->Input(index); } +#endif const TensorSeq* OpKernelContext__Input_TensorSeq(const OpKernelContext* p, int index) override { return p->Input(index); } const Tensor& OpKernelContext__RequiredInput_Tensor(const OpKernelContext* p, int index) override { return p->RequiredInput(index); } MLDataType OpKernelContext__InputType(const OpKernelContext* p, int index) override { return p->InputType(index); } Tensor* OpKernelContext__Output_Tensor(OpKernelContext* p, int index) override { return p->Output(index); } TensorSeq* OpKernelContext__Output_TensorSeq(OpKernelContext* p, int index) override { return p->Output(index); } Tensor* OpKernelContext__Output(OpKernelContext* p, int index, const TensorShape& shape) override { return p->Output(index, shape); } +#if !defined(DISABLE_SPARSE_TENSORS) SparseTensor* OpKernelContext__OutputSparse(OpKernelContext* p, int index, const TensorShape& shape) override { return p->OutputSparse(index, shape); } +#endif Tensor& OpKernelContext__RequiredOutput(OpKernelContext* p, int index, const TensorShape& shape) override { return p->RequiredOutput(index, shape); } int OpKernelContext__InputCount(const OpKernelContext* p) override { return p->InputCount(); } int OpKernelContext__OutputCount(const OpKernelContext* p) override { return p->OutputCount(); } @@ -806,8 +828,10 @@ struct ProviderHostImpl : ProviderHost { MLDataType Tensor__DataType(const Tensor* p) override { return p->DataType(); } // SparseTensor(wrapped) +#if !defined(DISABLE_SPARSE_TENSORS) const TensorShape& SparseTensor__DenseShape(const SparseTensor* p) override { return p->DenseShape(); } Status SparseTensor__Copy(const SparseTensor* p, const DataTransferManager& dtm, int exec_q_id, SparseTensor& dst) override { return p->Copy(dtm, exec_q_id, dst); } +#endif // TensorSeq(wrapped) MLDataType TensorSeq__DataType(const TensorSeq* p) noexcept override { return p->DataType(); } diff --git a/onnxruntime/test/contrib_ops/math/matmul_sparse_test.cc b/onnxruntime/test/contrib_ops/math/matmul_sparse_test.cc index c8ab4fa08d..6e3ead7013 100644 --- a/onnxruntime/test/contrib_ops/math/matmul_sparse_test.cc +++ b/onnxruntime/test/contrib_ops/math/matmul_sparse_test.cc @@ -141,7 +141,7 @@ void resize(Index size, double reserveSizeFactor = 0) { m_size = size; } */ - +#if !defined(DISABLE_SPARSE_TENSORS) #if !defined(__i386__) && !defined(_M_IX86) && !defined(__wasm__) && !defined(__ANDROID__) TEST(SparseToDenseMatMul, TestCsr) { constexpr int64_t rows = 9; @@ -383,6 +383,7 @@ TEST(SparseToDenseMatMul, TestCoo) { tester.Run(OpTester::ExpectResult::kExpectSuccess); } } +#endif // !defined(DISABLE_SPARSE_TENSORS) } // namespace test } // namespace onnxruntime \ No newline at end of file diff --git a/onnxruntime/test/framework/execution_frame_test.cc b/onnxruntime/test/framework/execution_frame_test.cc index 6064a1edfc..80ef95673d 100644 --- a/onnxruntime/test/framework/execution_frame_test.cc +++ b/onnxruntime/test/framework/execution_frame_test.cc @@ -106,8 +106,8 @@ TEST_F(ExecutionFrameTest, TensorAllocationTest) { } TEST_F(ExecutionFrameTest, OutputShapeValidationTest) { - onnxruntime::Model model("test", false, ModelMetaData(), PathString(), IOnnxRuntimeOpSchemaRegistryList(), - {{kOnnxDomain, 12}}, {}, DefaultLoggingManager().DefaultLogger()); + onnxruntime::Model model("test", false, ModelMetaData(), PathString(), IOnnxRuntimeOpSchemaRegistryList(), + {{kOnnxDomain, 12}}, {}, DefaultLoggingManager().DefaultLogger()); onnxruntime::Graph& graph = model.MainGraph(); TypeProto tensor_float; tensor_float.mutable_tensor_type()->set_elem_type(TensorProto_DataType_FLOAT); @@ -145,7 +145,7 @@ TEST_F(ExecutionFrameTest, OutputShapeValidationTest) { ASSERT_EQ(start_index, 0); TensorShape actual_shape_same_as_input(std::vector{2, 3}); TensorShape actual_shape_diff_from_input(std::vector{2, 9}); - + OrtValue* p_ml_value = frame.GetMutableNodeInputOrOutputMLValue(0); ASSERT_TRUE(p_ml_value != nullptr); @@ -467,6 +467,7 @@ TEST(ExecutionFrameTestInit, InitializerAsOutput) { } } +#if !defined(DISABLE_SPARSE_TENSORS) TEST(ExecutionFrameTestInit, SparseInitializerAsOutput) { const std::vector dense_shape{3, 3}; @@ -508,6 +509,7 @@ TEST(ExecutionFrameTestInit, SparseInitializerAsOutput) { EXPECT_THAT(coo_view.Indices().DataAsSpan(), ::testing::ContainerEq(gsl::make_span(expected_linear_indices))); } } +#endif // !defined(DISABLE_SPARSE_TENSORS) } // namespace test } // namespace onnxruntime diff --git a/onnxruntime/test/framework/sparse_kernels_test.cc b/onnxruntime/test/framework/sparse_kernels_test.cc index 01a5adc387..83aa35101f 100644 --- a/onnxruntime/test/framework/sparse_kernels_test.cc +++ b/onnxruntime/test/framework/sparse_kernels_test.cc @@ -129,6 +129,7 @@ This operator constructs a sparse tensor from three tensors that provide a COO TensorShape shape(shape_tensor.Data(), shape_shape.Size()); +#if !defined(DISABLE_SPARSE_TENSORS) SparseTensor* output = ctx->OutputSparse(0, shape); ORT_ENFORCE(output != nullptr); const auto& dtm = Info().GetDataTransferManager(); @@ -138,6 +139,7 @@ This operator constructs a sparse tensor from three tensors that provide a COO static_cast(val_shape.Size()), values.DataRaw(), indices.DataAsSpan())); +#endif return Status::OK(); } }; @@ -147,8 +149,12 @@ This operator constructs a sparse tensor from three tensors that provide a COO def.SetName(SparseFromCOO::OpName()) .TypeConstraint("values", DataTypeImpl::GetTensorType()) .TypeConstraint("indices", DataTypeImpl::GetTensorType()) +#if !defined(DISABLE_SPARSE_TENSORS) .TypeConstraint("shape", DataTypeImpl::GetTensorType()) .TypeConstraint("sparse_rep", DataTypeImpl::GetSparseTensorType()); +#else + .TypeConstraint("shape", DataTypeImpl::GetTensorType()); +#endif return def; } }; @@ -190,6 +196,7 @@ This operator applies the Abs op element-wise to the input sparse-tensor. Status Compute(OpKernelContext* ctx) const override { ORT_ENFORCE(ctx->InputCount() == 1, "Expecting 1 input"); +#if !defined(DISABLE_SPARSE_TENSORS) const SparseTensor* input = ctx->Input(0); const auto* input_values = input->Values().Data(); const auto nnz = input->NumValues(); @@ -211,6 +218,7 @@ This operator applies the Abs op element-wise to the input sparse-tensor. // TODO: Extend allocation-planner to enable such sharing. const auto& input_indices = input_coo_view.Indices(); memcpy(output_mutator.Indices().MutableData(), input_indices.Data(), input_indices.SizeInBytes()); +#endif return Status::OK(); } }; @@ -218,8 +226,10 @@ This operator applies the Abs op element-wise to the input sparse-tensor. // A KernelDefBuilder for SparseAbs: static KernelDefBuilder KernelDef() { KernelDefBuilder def; +#if !defined(DISABLE_SPARSE_TENSORS) def.SetName(OpName()) .TypeConstraint("T", DataTypeImpl::GetSparseTensorType()); +#endif return def; } }; @@ -261,6 +271,8 @@ struct SparseToValues { Status Compute(OpKernelContext* ctx) const override { ORT_ENFORCE(ctx->InputCount() == 1, "Expecting a single SparseTensorSample input"); + +#if !defined(DISABLE_SPARSE_TENSORS) const SparseTensor* sparse_input = ctx->Input(0); const auto* values = sparse_input->Values().Data(); auto nnz = sparse_input->Values().Shape().Size(); @@ -272,7 +284,7 @@ struct SparseToValues { ORT_ENFORCE(output_data != nullptr); memcpy(output_data, values, sparse_input->Values().SizeInBytes()); - +#endif return Status::OK(); } }; @@ -280,9 +292,11 @@ struct SparseToValues { // A KernelDefBuilder for SparseToValues static KernelDefBuilder KernelDef() { KernelDefBuilder def; +#if !defined(DISABLE_SPARSE_TENSORS) def.SetName(OpName()) .TypeConstraint("sparse_rep", DataTypeImpl::GetSparseTensorType()) .TypeConstraint("values", DataTypeImpl::GetTensorType()); +#endif return def; } }; @@ -347,12 +361,14 @@ class SparseTensorTests : public testing::Test { EXPECT_TRUE(session_object.Initialize().IsOK()); } +#if !defined(DISABLE_SPARSE_TENSORS) NodeArg* Sparse(const std::string& name) { types.push_back(*DataTypeImpl::GetSparseTensorType()->GetTypeProto()); Graph& graph = model->MainGraph(); auto& arg = graph.GetOrCreateNodeArg(name, &types.back()); return &arg; } +#endif NodeArg* Dense(const std::string& name) { types.push_back(*DataTypeImpl::GetTensorType()->GetTypeProto()); @@ -431,6 +447,7 @@ class SparseTensorTests : public testing::Test { } }; +#if !defined(DISABLE_SPARSE_TENSORS) // Test ops SparseFromCOO, SparseAbs, and SparseToValues. // Tests 1-dimensional int64 sparse tensor. TEST_F(SparseTensorTests, Test1) { @@ -606,6 +623,7 @@ TEST(SparseCrcsFormatTests, Test1) { csr_wrap.Outer().Data(), outer_indices.size() * sizeof(int64_t))); } +#endif // !defined(DISABLE_SPARSE_TENSORS) // Code below depends on the values being size 4 template @@ -928,6 +946,7 @@ TEST(SparseTensorConversionTests, TestConstantNodeConversion) { } /// Dense to Sparse conversion tests +#if !defined(DISABLE_SPARSE_TENSORS) #if !defined(ORT_MINIMAL_BUILD) template @@ -1570,7 +1589,6 @@ TEST(SparseTensorConversionTests, CooConversion) { ASSERT_TRUE(std::equal(expected_linear_indices.cbegin(), expected_linear_indices.cend(), indices.cbegin(), indices.cend())); } - { // test where both src and destination are on CPU. 2-D index SparseTensor dst; @@ -1718,7 +1736,6 @@ TEST(SparseTensorConversionTests, BlockSparse) { const std::string expected_strings[] = { "1", "2", "3", "4", "5", "6", "7", "8"}; - const TensorShape indices_shape{2, 2}; // two blocks by two coordinates // (0, 0), (0,1) std::vector blocksparse_indices = { @@ -1783,8 +1800,9 @@ TEST(SparseTensorConversionTests, BlockSparse) { auto indices_span = indices.DataAsSpan(); ASSERT_TRUE(std::equal(blocksparse_indices.cbegin(), blocksparse_indices.cend(), indices_span.cbegin(), indices_span.cend())); - } } +#endif // !defined(DISABLE_SPARSE_TENSORS) + } // namespace test } // namespace onnxruntime diff --git a/onnxruntime/test/ir/graph_test.cc b/onnxruntime/test/ir/graph_test.cc index 558beed2e3..2c2fc774cf 100644 --- a/onnxruntime/test/ir/graph_test.cc +++ b/onnxruntime/test/ir/graph_test.cc @@ -144,8 +144,7 @@ static bool RegisterCustomSchemas() { node.set_domain(kMSNchwcDomain); } } - return nodes; - }(), + return nodes; }(), []() { std::vector operator_sets(2); auto& onnx_opset = operator_sets[0]; @@ -217,6 +216,7 @@ const std::vector values = {13.f, const std::vector indices = {9, 30, 50}; // Not to exceed 59 } // namespace sparse_details +#if !defined(DISABLE_SPARSE_TENSORS) // To match a simple Add graph above static void ConstructSparseTensor(const std::string& name, SparseTensorProto& sparse_proto) { @@ -274,6 +274,7 @@ static void ValidateSparseTensorProto(const SparseTensorProto& proto) { auto expected_shape = gsl::make_span(sparse_details::shape); EXPECT_THAT(actual_shape, testing::ContainerEq(expected_shape)); } +#endif // !defined(DISABLE_SPARSE_TENSORS) TEST_F(GraphTest, SimpleAddWithoutDomain) { ModelProto m; @@ -1304,6 +1305,7 @@ TEST_F(GraphTest, UnusedInitializerIsIgnored) { ASSERT_TRUE(graph.GetAllInitializedTensors().empty()); } +#if !defined(DISABLE_SPARSE_TENSORS) TEST_F(GraphTest, UnusedSparseInitializerIsIgnored) { std::string s1; { @@ -1334,6 +1336,7 @@ TEST_F(GraphTest, UnusedSparseInitializerIsIgnored) { auto& graph_proto = graph2.ToGraphProto(); ASSERT_TRUE(graph_proto.sparse_initializer().empty()); } +#endif // !defined(DISABLE_SPARSE_TENSORS) TEST_F(GraphTest, GraphConstruction_CheckIsNotAcyclic) { // A cyclic graph @@ -1798,6 +1801,7 @@ TEST_F(GraphTest, AddRemoveInitializerHandling) { << num_initializers << " remain."; } +#if !defined(DISABLE_SPARSE_TENSORS) TEST_F(GraphTest, SparseInitializerHandling) { const char* const input_initializer_name = "x"; Model model("SparseInitializerHandling", false, *logger_); @@ -1848,6 +1852,7 @@ TEST_F(GraphTest, SparseInitializerHandling) { ValidateSparseTensorProto(model_proto_get.graph().sparse_initializer().at(0)); } } +#endif //!defined(DISABLE_SPARSE_TENSORS) TEST_F(GraphTest, SetInputsAndSetOutputs_NewInputAndOutput) { std::shared_ptr model; diff --git a/onnxruntime/test/optimizer/optimizer_test.cc b/onnxruntime/test/optimizer/optimizer_test.cc index b3b4f4c2b5..3a6edc1e47 100644 --- a/onnxruntime/test/optimizer/optimizer_test.cc +++ b/onnxruntime/test/optimizer/optimizer_test.cc @@ -66,12 +66,19 @@ TEST(OptimizerTest, Basic) { std::unique_ptr cpu_execution_provider = std::make_unique(CPUExecutionProviderInfo()); +#if !defined(DISABLE_SPARSE_TENSORS) OptimizerExecutionFrame::Info info(nodes, initialized_tensor_set, graph.ModelPath(), *cpu_execution_provider.get(), [&graph](const std::string& name) -> bool { return graph.IsSparseInitializer(name); }); +#else + OptimizerExecutionFrame::Info info(nodes, initialized_tensor_set, + graph.ModelPath(), + *cpu_execution_provider.get(), + [](std::string const& ) { return false; }); +#endif //!defined(DISABLE_SPARSE_TENSORS) std::vector fetch_mlvalue_idxs{info.GetMLValueIndex("out")}; OptimizerExecutionFrame frame(info, fetch_mlvalue_idxs); diff --git a/onnxruntime/test/providers/internal_testing/internal_testing_tests.cc b/onnxruntime/test/providers/internal_testing/internal_testing_tests.cc index 2b5be8c7a4..cdd2eb6445 100644 --- a/onnxruntime/test/providers/internal_testing/internal_testing_tests.cc +++ b/onnxruntime/test/providers/internal_testing/internal_testing_tests.cc @@ -81,6 +81,7 @@ static void ExecuteMnist(InferenceSessionWrapper& session, bool custom_ep_enable } } +#if !defined(DISABLE_SPARSE_TENSORS) #if !defined(ORT_MINIMAL_BUILD) TEST(InternalTestingEP, TestSaveAndLoadOrtModel) { const ORTCHAR_T* ort_model_path = ORT_TSTR("testdata/mnist.internal_testing_ep.test_output.ort"); @@ -145,6 +146,7 @@ TEST(InternalTestingEP, PreventSaveOfModelWithCompiledOps) { ASSERT_THAT(status.ErrorMessage(), ::testing::HasSubstr("Unable to serialize model as it contains compiled nodes")); } #endif // !defined(ORT_MINIMAL_BUILD) +#endif // !defined(DISABLE_SPARSE_TENSORS) // test to validate a minimal build TEST(InternalTestingEP, TestLoadOrtModel) { diff --git a/onnxruntime/test/providers/provider_test_utils.cc b/onnxruntime/test/providers/provider_test_utils.cc index 51f5e269cd..a9933c4e1e 100644 --- a/onnxruntime/test/providers/provider_test_utils.cc +++ b/onnxruntime/test/providers/provider_test_utils.cc @@ -548,6 +548,7 @@ void OpTester::AddShapeToTensorData(NodeArg& node_arg, const std::vector MakeSparseTensor(MLDataType data_type, const std::vector& dims) { TensorShape shape{dims}; auto allocator = test::AllocatorManager::Instance().GetAllocator(CPU); @@ -673,7 +674,7 @@ void OpTester::AddSparseCsrTensorStrings(std::vector& data, NodeArg node_arg = MakeSparseNodeArg(dtype, name, dims, dim_params); AddSparseTensorData(data, std::move(node_arg), std::move(p_tensor), CheckParams()); } - +#endif // !defined(DISABLE_SPARSE_TENSORS) void OpTester::AddInitializers(onnxruntime::Graph& graph) { for (auto index : initializer_index_) { diff --git a/onnxruntime/test/providers/provider_test_utils.h b/onnxruntime/test/providers/provider_test_utils.h index a6f0e3f1e8..8de7332f29 100644 --- a/onnxruntime/test/providers/provider_test_utils.h +++ b/onnxruntime/test/providers/provider_test_utils.h @@ -149,6 +149,7 @@ struct TTensorType { template const TTypeProto TTensorType::s_type_proto; +#if !defined(DISABLE_SPARSE_TENSORS) struct TSparseTensorProto { explicit TSparseTensorProto(int32_t dtype, const std::vector* shape = nullptr) { proto.mutable_sparse_tensor_type()->set_elem_type(dtype); @@ -165,6 +166,7 @@ struct TSparseTensorProto { } ONNX_NAMESPACE::TypeProto proto; }; +#endif // TypeProto for map template @@ -295,6 +297,7 @@ class OpTester { AddData(input_data_, name, dims, p_values, size, is_initializer, false, dim_params); } +#if !defined(DISABLE_SPARSE_TENSORS) // Useful to add boolean data template void AddSparseCooInput(const char* name, const std::vector& dims, @@ -394,6 +397,7 @@ class OpTester { gsl::make_span(outer_indices), dim_params); } +#endif // Add other registered types, possibly experimental template @@ -475,7 +479,8 @@ class OpTester { AddData(output_data_, name, dims, p_values, size, false, sort_output, nullptr /* dim_params */, rel_error, abs_error); } - + +#if !defined(DISABLE_SPARSE_TENSORS) template void AddSparseCooOutput(const char* name, const std::vector& dims, const std::initializer_list& expected_values, @@ -571,6 +576,7 @@ class OpTester { gsl::make_span(expected_inner_indices), gsl::make_span(expected_outer_indices)); } +#endif /* * Use this API to add an output *edge* to the node/op being tested that shouldn't have any @@ -585,7 +591,7 @@ class OpTester { output_data_.push_back(Data(NodeArg(name, &TTensorType::s_type_proto.proto), OrtValue(), optional(), optional())); } - + // Add other registered types, possibly experimental template void AddOutput(const char* name, const T& val) { @@ -858,6 +864,7 @@ class OpTester { void CopyDataToTensor(gsl::span data, Tensor& dst); +#if !defined(DISABLE_SPARSE_TENSORS) NodeArg MakeSparseNodeArg(int32_t dtype, const char* name, const std::vector& dims, const std::vector* dim_params); @@ -899,6 +906,7 @@ class OpTester { void AddSparseTensorData(std::vector& data, NodeArg node_arg, std::unique_ptr p_tensor, const CheckParams& check_params); +#endif const char* domain_; int opset_version_; diff --git a/onnxruntime/test/shared_lib/test_inference.cc b/onnxruntime/test/shared_lib/test_inference.cc index 4c9316e6dd..579c6c3dbf 100644 --- a/onnxruntime/test/shared_lib/test_inference.cc +++ b/onnxruntime/test/shared_lib/test_inference.cc @@ -176,10 +176,12 @@ static constexpr PATH_TYPE VARIED_INPUT_CUSTOM_OP_MODEL_URI_2 = TSTR("testdata/f static constexpr PATH_TYPE OPTIONAL_INPUT_OUTPUT_CUSTOM_OP_MODEL_URI = TSTR("testdata/foo_bar_1.onnx"); static constexpr PATH_TYPE OPTIONAL_INPUT_OUTPUT_CUSTOM_OP_MODEL_URI_2 = TSTR("testdata/foo_bar_2.onnx"); static constexpr PATH_TYPE CUSTOM_OP_MODEL_WITH_ATTRIBUTES_URI = TSTR("testdata/foo_bar_3.onnx"); +#if !defined(DISABLE_SPARSE_TENSORS) static constexpr PATH_TYPE SPARSE_OUTPUT_MODEL_URI = TSTR("testdata/sparse_initializer_as_output.onnx"); #ifndef DISABLE_CONTRIB_OPS static constexpr PATH_TYPE SPARSE_INPUT_MATMUL_MODEL_URI = TSTR("testdata/sparse_to_dense_matmul.onnx"); #endif +#endif // !defined(DISABLE_SPARSE_TENSORS) #ifdef ENABLE_EXTENSION_CUSTOM_OPS static constexpr PATH_TYPE ORT_CUSTOM_OPS_MODEL_URI = TSTR("testdata/custom_op_string_lower.onnx"); @@ -244,6 +246,7 @@ INSTANTIATE_TEST_SUITE_P(CApiTestWithProviders, CApiTestWithProvider, ::testing::Values(0, 1, 2, 3, 4)); +#if !defined(DISABLE_SPARSE_TENSORS) TEST(CApiTest, SparseOutputModel) { std::vector dense_shape{3, 3}; std::vector values{1.764052391052246, 0.40015721321105957, 0.978738009929657}; @@ -358,6 +361,7 @@ TEST(CApiTest, SparseInputModel) { ASSERT_TRUE(std::equal(Y_result.cbegin(), Y_result.cend(), result_span.cbegin(), result_span.cend())); } #endif // DISABLE_CONTRIB_OPS +#endif // !defined(DISABLE_SPARSE_TENSORS) TEST(CApiTest, custom_op_handler) { std::cout << "Running custom op inference" << std::endl; diff --git a/onnxruntime/test/shared_lib/test_nontensor_types.cc b/onnxruntime/test/shared_lib/test_nontensor_types.cc index 232b1d8a62..cf3e9a320c 100644 --- a/onnxruntime/test/shared_lib/test_nontensor_types.cc +++ b/onnxruntime/test/shared_lib/test_nontensor_types.cc @@ -309,6 +309,7 @@ TEST(CApiTest, TypeInfoSequence) { ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64); } +#if !defined(DISABLE_SPARSE_TENSORS) TEST(CApiTest, SparseTensorUsingAPI) { Ort::MemoryInfo info("Cpu", OrtDeviceAllocator, 0, OrtMemTypeDefault); @@ -921,4 +922,5 @@ TEST(CApiTest, SparseTensorFillSparseFormatStringsAPI) { ASSERT_TRUE(std::equal(blocksparse_indices.cbegin(), blocksparse_indices.cend(), ind_span.cbegin(), ind_span.cend())); } } -} \ No newline at end of file +} +#endif // !defined(DISABLE_SPARSE_TENSORS) \ No newline at end of file diff --git a/onnxruntime/test/util/compare_ortvalue.cc b/onnxruntime/test/util/compare_ortvalue.cc index 67e20821f0..cd55df10b4 100644 --- a/onnxruntime/test/util/compare_ortvalue.cc +++ b/onnxruntime/test/util/compare_ortvalue.cc @@ -251,6 +251,7 @@ std::pair CompareSeqOfMapToFloat(const T& real_outp return std::make_pair(COMPARE_RESULT::SUCCESS, ""); } +#if !defined(DISABLE_SPARSE_TENSORS) std::pair CompareSparseTensors(const SparseTensor& actual, const SparseTensor& expected, double per_sample_tolerance, double relative_per_sample_tolerance, bool post_processing) { @@ -290,6 +291,7 @@ std::pair CompareSparseTensors(const SparseTensor& return std::make_pair(COMPARE_RESULT::SUCCESS, ""); } +#endif // !defined(DISABLE_SPARSE_TENSORS) // The expected_shape could contain unknown dimensions, but the real_shape cannot bool AreShapesEqual(const std::vector& real_shape, const ::ONNX_NAMESPACE::TensorShapeProto& expected_shape) { @@ -353,12 +355,14 @@ std::pair CompareOrtValue(const OrtValue& o, const return CompareTwoTensors(outvalue, expected_tensor, per_sample_tolerance, relative_per_sample_tolerance, post_processing); } else if (o.IsSparseTensor()) { +#if !defined(DISABLE_SPARSE_TENSORS) TEST_RETURN_IF_NOT(expected_mlvalue.IsSparseTensor(), COMPARE_RESULT::TYPE_MISMATCH, "SparseTensor is not expected as output"); TEST_RETURN_IF_ERROR(CompareSparseTensors(o.Get(), expected_mlvalue.Get(), per_sample_tolerance, relative_per_sample_tolerance, post_processing), "while comaring sparse tensors"); +#endif return std::make_pair(COMPARE_RESULT::SUCCESS, ""); } else if (o.IsTensorSequence()) { auto& expected_tensor_seq = expected_mlvalue.Get();