diff --git a/include/onnxruntime/core/graph/graph.h b/include/onnxruntime/core/graph/graph.h index 0063c5687b..52fd6f477b 100644 --- a/include/onnxruntime/core/graph/graph.h +++ b/include/onnxruntime/core/graph/graph.h @@ -519,6 +519,14 @@ class Graph { return graph_inputs_including_initializers_; } + /** Gets the Graph inputs that are initializers + These are overridable initializers. This is a difference between + graph_inputs_including_initializers_ and graph_inputs_excluding_initializers_ + @remarks Contains no nullptr values. */ + const std::vector& GetOverridableInitializers () const { + return graph_overridable_initializers_; + } + /** Gets the Graph outputs. @remarks Contains no nullptr values.*/ const std::vector& GetOutputs() const noexcept { return graph_outputs_; } @@ -852,6 +860,9 @@ class Graph { // Initialize all the graph inputs, initializers and outputs common::Status InitInputsInitializersOutputs(); + // Initialize overridable initializers container + void ComputeOverridableInitializers(); + // recursively accumulate and set the outer scope node args in the resolve context for all subgraphs // so they can be used to resolve outer scope dependencies when running BuildConnections for the subgraphs. common::Status SetOuterScopeNodeArgs(const std::unordered_set& outer_scope_node_args); @@ -965,6 +976,10 @@ class Graph { // Graph inputs excluding initializers. std::vector graph_inputs_excluding_initializers_; + // Overridable Initializers. The difference between graph_inputs_including_initializers_ + // and graph_inputs_excluding_initializers_ + std::vector graph_overridable_initializers_; + // Graph outputs. std::vector graph_outputs_; bool graph_outputs_manually_set_ = false; diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index e5ff24087e..0c8e574d73 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -269,6 +269,7 @@ ORT_API_STATUS(OrtSetInterOpNumThreads, _Inout_ OrtSessionOptions* options, int ORT_API_STATUS(OrtSessionGetInputCount, _In_ const OrtSession* sess, _Out_ size_t* out); ORT_API_STATUS(OrtSessionGetOutputCount, _In_ const OrtSession* sess, _Out_ size_t* out); +ORT_API_STATUS(OrtSessionGetOverridableInitializerCount, _In_ const OrtSession* sess, _Out_ size_t* out); /** * \param out should be freed by OrtReleaseTypeInfo after use @@ -280,6 +281,12 @@ ORT_API_STATUS(OrtSessionGetInputTypeInfo, _In_ const OrtSession* sess, size_t i */ ORT_API_STATUS(OrtSessionGetOutputTypeInfo, _In_ const OrtSession* sess, size_t index, _Outptr_ OrtTypeInfo** type_info); +/** + * \param out should be freed by OrtReleaseTypeInfo after use + */ +ORT_API_STATUS(OrtSessionGetOverridableInitializerTypeInfo, _In_ const OrtSession* sess, size_t index, _Outptr_ OrtTypeInfo** type_info); + + /** * \param value is set to a null terminated string allocated using 'allocator'. The caller is responsible in freeing it. */ @@ -287,6 +294,8 @@ ORT_API_STATUS(OrtSessionGetInputName, _In_ const OrtSession* sess, size_t index _Inout_ OrtAllocator* allocator, _Outptr_ char** value); ORT_API_STATUS(OrtSessionGetOutputName, _In_ const OrtSession* sess, size_t index, _Inout_ OrtAllocator* allocator, _Outptr_ char** value); +ORT_API_STATUS(OrtSessionGetOverridableInitializerName, _In_ const OrtSession* sess, size_t index, + _Inout_ OrtAllocator* allocator, _Outptr_ char** value); /** * \return A pointer to the newly created object. The pointer should be freed by OrtReleaseRunOptions after use diff --git a/include/onnxruntime/core/session/onnxruntime_cxx_api.h b/include/onnxruntime/core/session/onnxruntime_cxx_api.h index 92ae73efba..fe433dd9cd 100644 --- a/include/onnxruntime/core/session/onnxruntime_cxx_api.h +++ b/include/onnxruntime/core/session/onnxruntime_cxx_api.h @@ -72,6 +72,7 @@ struct Base { protected: Base(const Base&) = delete; + Base& operator=(const Base&) = delete; Base(Base&& v) noexcept : p_{v.p_} { v.p_ = nullptr; } void operator=(Base&& v) noexcept { OrtRelease(p_); @@ -175,12 +176,15 @@ struct Session : Base { size_t GetInputCount() const; size_t GetOutputCount() const; + size_t GetOverridableInitializerCount() const; char* GetInputName(size_t index, OrtAllocator* allocator) const; char* GetOutputName(size_t index, OrtAllocator* allocator) const; + char* GetOverridableInitializerName(size_t index, OrtAllocator* allocator) const; TypeInfo GetInputTypeInfo(size_t index) const; TypeInfo GetOutputTypeInfo(size_t index) const; + TypeInfo GetOverridableInitializerTypeInfo(size_t index) const; }; struct TensorTypeAndShapeInfo : Base { @@ -223,6 +227,8 @@ struct Value : Base { explicit Value(nullptr_t) {} explicit Value(OrtValue* p) : Base{p} {} + Value(Value&&) = default; + Value& operator=(Value&&) = default; bool IsTensor() const; size_t GetCount() const; // If a non tensor, returns 2 for map and N for sequence, where N is the number of elements diff --git a/include/onnxruntime/core/session/onnxruntime_cxx_inline.h b/include/onnxruntime/core/session/onnxruntime_cxx_inline.h index e1f23c0e5d..6832862c7d 100644 --- a/include/onnxruntime/core/session/onnxruntime_cxx_inline.h +++ b/include/onnxruntime/core/session/onnxruntime_cxx_inline.h @@ -38,6 +38,8 @@ template <> struct TypeToTensorType { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32; }; template <> struct TypeToTensorType { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64; }; +template <> +struct TypeToTensorType { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL; }; inline AllocatorWithDefaultOptions::AllocatorWithDefaultOptions() { ORT_THROW_ON_ERROR(OrtGetAllocatorWithDefaultOptions(&p_)); @@ -242,6 +244,12 @@ inline size_t Session::GetOutputCount() const { return out; } +inline size_t Session::GetOverridableInitializerCount () const { + size_t out; + ORT_THROW_ON_ERROR(OrtSessionGetOverridableInitializerCount(p_, &out)); + return out; +} + inline char* Session::GetInputName(size_t index, OrtAllocator* allocator) const { char* out; ORT_THROW_ON_ERROR(OrtSessionGetInputName(p_, index, allocator, &out)); @@ -254,6 +262,12 @@ inline char* Session::GetOutputName(size_t index, OrtAllocator* allocator) const return out; } +inline char* Session::GetOverridableInitializerName(size_t index, OrtAllocator* allocator) const { + char* out; + ORT_THROW_ON_ERROR(OrtSessionGetOverridableInitializerName(p_, index, allocator, &out)); + return out; +} + inline TypeInfo Session::GetInputTypeInfo(size_t index) const { OrtTypeInfo* out; ORT_THROW_ON_ERROR(OrtSessionGetInputTypeInfo(p_, index, &out)); @@ -266,6 +280,12 @@ inline TypeInfo Session::GetOutputTypeInfo(size_t index) const { return TypeInfo{out}; } +inline TypeInfo Session::GetOverridableInitializerTypeInfo(size_t index) const { + OrtTypeInfo* out; + ORT_THROW_ON_ERROR(OrtSessionGetOverridableInitializerTypeInfo(p_, index, &out)); + return TypeInfo{out}; +} + inline ONNXTensorElementDataType TensorTypeAndShapeInfo::GetElementType() const { ONNXTensorElementDataType out; ORT_THROW_ON_ERROR(OrtGetTensorElementType(p_, &out)); diff --git a/onnxruntime/core/graph/graph.cc b/onnxruntime/core/graph/graph.cc index 938aa54f53..da7e21d7fc 100644 --- a/onnxruntime/core/graph/graph.cc +++ b/onnxruntime/core/graph/graph.cc @@ -126,7 +126,6 @@ const TensorShapeProto* NodeArg::Shape() const { } void NodeArg::SetShape(const TensorShapeProto& shape) { - const auto type_case = node_arg_info_.type().value_case(); switch (type_case) { case TypeProto::kTensorType: @@ -1356,7 +1355,7 @@ Status Graph::InferAndVerifySubgraphTypes(const Node& node, Graph& subgraph, " inputs and requires ", num_required_subgraph_inputs, " inputs. Either provide all subgraph inputs, or just the required inputs."); } - + subgraph_inputs = &required_subgraph_inputs; num_subgraph_inputs = num_required_subgraph_inputs; } @@ -2459,9 +2458,34 @@ Status Graph::SetGraphInputsOutputs() { } } + ComputeOverridableInitializers(); + return Status::OK(); } +void Graph::ComputeOverridableInitializers() { + graph_overridable_initializers_.clear(); + if (CanOverrideInitializer()) { + // graph_inputs_excluding_initializers_ and graph_inputs_including_initializers_ + // are inserted in the same order. So we walk and compute the difference. + auto f_incl = graph_inputs_including_initializers_.cbegin(); + const auto l_incl = graph_inputs_including_initializers_.cend(); + auto f_excl = graph_inputs_excluding_initializers_.cbegin(); + const auto l_excl = graph_inputs_excluding_initializers_.cend(); + + while (f_incl != l_incl) { + // Equal means not an initializer + if (f_excl != l_excl && *f_incl == *f_excl) { + ++f_incl; + ++f_excl; + continue; + } + graph_overridable_initializers_.push_back(*f_incl); + ++f_incl; + } + } +} + // calling private ctor GSL_SUPPRESS(r .11) gsl::not_null Graph::AllocateNode() { diff --git a/onnxruntime/core/providers/cpu/symbols.txt b/onnxruntime/core/providers/cpu/symbols.txt index bce73b938c..42baf19c34 100644 --- a/onnxruntime/core/providers/cpu/symbols.txt +++ b/onnxruntime/core/providers/cpu/symbols.txt @@ -77,6 +77,9 @@ OrtSessionGetInputTypeInfo OrtSessionGetOutputCount OrtSessionGetOutputName OrtSessionGetOutputTypeInfo +OrtSessionGetOverridableInitializerCount +OrtSessionGetOverridableInitializerName +OrtSessionGetOverridableInitializerTypeInfo OrtSessionOptionsAppendExecutionProvider_CPU OrtSetDimensions OrtSetSessionGraphOptimizationLevel diff --git a/onnxruntime/core/session/inference_session.cc b/onnxruntime/core/session/inference_session.cc index 9802464c28..5a02b4393a 100644 --- a/onnxruntime/core/session/inference_session.cc +++ b/onnxruntime/core/session/inference_session.cc @@ -814,6 +814,20 @@ std::pair InferenceSession::GetModelInputs( return std::make_pair(common::Status::OK(), &model_->MainGraph().GetInputs()); } +std::pair InferenceSession::GetOverridableInitializers() const { + { + std::lock_guard l(session_mutex_); + if (!is_model_loaded_) { + LOGS(*session_logger_, ERROR) << "Model was not loaded"; + return std::make_pair(common::Status(common::ONNXRUNTIME, common::FAIL, "Model was not loaded."), + nullptr); + } + } + + // returns a list of initializers that can be overriden. + return std::make_pair(common::Status::OK(), &model_->MainGraph().GetOverridableInitializers()); +} + std::pair InferenceSession::GetModelOutputs() const { { std::lock_guard l(session_mutex_); diff --git a/onnxruntime/core/session/inference_session.h b/onnxruntime/core/session/inference_session.h index f7229cce8f..5dc5b26fc6 100644 --- a/onnxruntime/core/session/inference_session.h +++ b/onnxruntime/core/session/inference_session.h @@ -271,6 +271,15 @@ class InferenceSession { */ std::pair GetModelInputs() const; + /** + * Get all definitions of the model for overridable initializers. + * This does not include weights. Use this to get the name/type/shapes of the overridable initializers. + * @return pair.first = OK; FAIL otherwise. pair.second is non-NULL when pair.first = OK. + * @note lifetime of the returned pointer is valid as long as the Session object is live. + * @note for IR < 4 returned list will always be empty. + */ + std::pair GetOverridableInitializers() const; + /** * Get all output definitions of the model. Use this to get the name/type/shapes of the outputs. * @return pair.first = OK; FAIL otherwise. pair.second is non-NULL when pair.first = OK. diff --git a/onnxruntime/core/session/onnxruntime_c_api.cc b/onnxruntime/core/session/onnxruntime_c_api.cc index b13ea9b4dc..14e3d96c04 100644 --- a/onnxruntime/core/session/onnxruntime_c_api.cc +++ b/onnxruntime/core/session/onnxruntime_c_api.cc @@ -525,10 +525,16 @@ ORT_API_STATUS_IMPL(OrtGetStringTensorContent, _In_ const OrtValue* value, delete reinterpret_cast(value); \ } -ORT_API_STATUS_IMPL(OrtSessionGetInputCount, _In_ const OrtSession* sess, _Out_ size_t* out) { +using DefListResult = std::pair; +using GetDefListFn = DefListResult (*)(const ::onnxruntime::InferenceSession*); +const auto get_inputs_fn = [](const ::onnxruntime::InferenceSession* session) -> DefListResult { return session->GetModelInputs(); }; +const auto get_outputs_fn = [](const ::onnxruntime::InferenceSession* session) -> DefListResult { return session->GetModelOutputs(); }; +const auto get_overridable_initializers_fn = [](const ::onnxruntime::InferenceSession* session) -> DefListResult { return session->GetOverridableInitializers(); }; + +static OrtStatus* GetNodeDefListCountHelper(const OrtSession* sess, GetDefListFn get_fn, size_t* out) { API_IMPL_BEGIN auto session = reinterpret_cast(sess); - std::pair p = session->GetModelInputs(); + std::pair p = get_fn(session); if (!p.first.IsOK()) return ToOrtStatus(p.first); *out = p.second->size(); @@ -536,40 +542,41 @@ ORT_API_STATUS_IMPL(OrtSessionGetInputCount, _In_ const OrtSession* sess, _Out_ API_IMPL_END } +ORT_API_STATUS_IMPL(OrtSessionGetInputCount, _In_ const OrtSession* sess, _Out_ size_t* out) { + return GetNodeDefListCountHelper(sess, get_inputs_fn, out); +} + ORT_API_STATUS_IMPL(OrtSessionGetOutputCount, _In_ const OrtSession* sess, _Out_ size_t* out) { + return GetNodeDefListCountHelper(sess, get_outputs_fn, out); +} + +ORT_API_STATUS_IMPL(OrtSessionGetOverridableInitializerCount, _In_ const OrtSession* sess, _Out_ size_t* out) { + return GetNodeDefListCountHelper(sess, get_overridable_initializers_fn, out); +} + +static OrtStatus* GetNodeDefTypeInfoHelper(const OrtSession* sess, GetDefListFn get_fn, size_t index, _Outptr_ struct OrtTypeInfo** out) { API_IMPL_BEGIN auto session = reinterpret_cast(sess); - std::pair p = session->GetModelOutputs(); + std::pair p = get_fn(session); if (!p.first.IsOK()) return ToOrtStatus(p.first); - *out = p.second->size(); - return nullptr; + if (p.second->size() <= index) + return OrtCreateStatus(ORT_FAIL, "out of index"); + const ONNX_NAMESPACE::TypeProto* type_proto = (*p.second)[index]->TypeAsProto(); + return OrtTypeInfo::FromDataTypeImpl(type_proto, out); API_IMPL_END } ORT_API_STATUS_IMPL(OrtSessionGetInputTypeInfo, _In_ const OrtSession* sess, size_t index, _Outptr_ struct OrtTypeInfo** out) { - API_IMPL_BEGIN - auto session = reinterpret_cast(sess); - std::pair p = session->GetModelInputs(); - if (!p.first.IsOK()) - return ToOrtStatus(p.first); - if (p.second->size() <= index) - return OrtCreateStatus(ORT_FAIL, "out of index"); - const ONNX_NAMESPACE::TypeProto* type_proto = (*p.second)[index]->TypeAsProto(); - return OrtTypeInfo::FromDataTypeImpl(type_proto, out); - API_IMPL_END + return GetNodeDefTypeInfoHelper(sess, get_inputs_fn, index, out); } + ORT_API_STATUS_IMPL(OrtSessionGetOutputTypeInfo, _In_ const OrtSession* sess, size_t index, _Outptr_ struct OrtTypeInfo** out) { - API_IMPL_BEGIN - auto session = reinterpret_cast(sess); - std::pair p = session->GetModelOutputs(); - if (!p.first.IsOK()) - return ToOrtStatus(p.first); - if (p.second->size() <= index) - return OrtCreateStatus(ORT_FAIL, "out of index"); - const ONNX_NAMESPACE::TypeProto* type_proto = (*p.second)[index]->TypeAsProto(); - return OrtTypeInfo::FromDataTypeImpl(type_proto, out); - API_IMPL_END + return GetNodeDefTypeInfoHelper(sess, get_outputs_fn, index, out); +} + +ORT_API_STATUS_IMPL(OrtSessionGetOverridableInitializerTypeInfo, _In_ const OrtSession* sess, size_t index, _Outptr_ struct OrtTypeInfo** out) { + return GetNodeDefTypeInfoHelper(sess, get_overridable_initializers_fn, index, out); } static char* StrDup(const std::string& str, OrtAllocator* allocator) { @@ -579,11 +586,11 @@ static char* StrDup(const std::string& str, OrtAllocator* allocator) { return output_string; } -static OrtStatus* GetInputOutputNameImpl(_In_ const OrtSession* sess, size_t index, - _Inout_ OrtAllocator* allocator, bool is_input, - _Outptr_ char** output) { +static OrtStatus* GetNodeDefNameImpl(_In_ const OrtSession* sess, size_t index, + _Inout_ OrtAllocator* allocator, GetDefListFn get_fn, + _Outptr_ char** output) { auto session = reinterpret_cast(sess); - std::pair p = is_input ? session->GetModelInputs() : session->GetModelOutputs(); + std::pair p = get_fn(session); if (!p.first.IsOK()) return ToOrtStatus(p.first); if (p.second == nullptr) @@ -625,14 +632,21 @@ ORT_API_STATUS_IMPL(OrtAllocatorGetInfo, _In_ const OrtAllocator* ptr, _Outptr_ ORT_API_STATUS_IMPL(OrtSessionGetInputName, _In_ const OrtSession* sess, size_t index, _Inout_ OrtAllocator* allocator, _Outptr_ char** output) { API_IMPL_BEGIN - return GetInputOutputNameImpl(sess, index, allocator, true, output); + return GetNodeDefNameImpl(sess, index, allocator, get_inputs_fn, output); API_IMPL_END } ORT_API_STATUS_IMPL(OrtSessionGetOutputName, _In_ const OrtSession* sess, size_t index, _Inout_ OrtAllocator* allocator, _Outptr_ char** output) { API_IMPL_BEGIN - return GetInputOutputNameImpl(sess, index, allocator, false, output); + return GetNodeDefNameImpl(sess, index, allocator, get_outputs_fn, output); + API_IMPL_END +} + +ORT_API_STATUS_IMPL(OrtSessionGetOverridableInitializerName, _In_ const OrtSession* sess, size_t index, + _Inout_ OrtAllocator* allocator, _Outptr_ char** output) { + API_IMPL_BEGIN + return GetNodeDefNameImpl(sess, index, allocator, get_overridable_initializers_fn, output); API_IMPL_END } @@ -1073,13 +1087,13 @@ ORT_API_STATUS_IMPL(OrtCreateValue, const OrtValue* const* in, size_t num_values } ORT_API_STATUS_IMPL(OrtCreateOpaqueValue, const char* domain_name, const char* type_name, const void* data_container, - size_t data_container_size, OrtValue** out) { + size_t data_container_size, OrtValue** out) { API_IMPL_BEGIN std::string dtype("opaque("); dtype.append(domain_name).append(",").append(type_name).append(")"); MLDataType ml_type = DataTypeImpl::GetDataType(dtype); ORT_ENFORCE(ml_type != nullptr, - "Specified domain and type names combination does not refer to a registered opaque type"); + "Specified domain and type names combination does not refer to a registered opaque type"); const auto* non_tensor_base = ml_type->AsNonTensorTypeBase(); ORT_ENFORCE(non_tensor_base != nullptr, "Opaque type is not a non_tensor type!!!"); std::unique_ptr ort_val(new OrtValue); @@ -1089,8 +1103,8 @@ ORT_API_STATUS_IMPL(OrtCreateOpaqueValue, const char* domain_name, const char* t return nullptr; } -ORT_API_STATUS_IMPL(OrtGetOpaqueValue, const char* domain_name, const char* type_name, const OrtValue* in, - void* data_container, size_t data_container_size) { +ORT_API_STATUS_IMPL(OrtGetOpaqueValue, const char* domain_name, const char* type_name, const OrtValue* in, + void* data_container, size_t data_container_size) { API_IMPL_BEGIN std::string dtype("opaque("); dtype.append(domain_name).append(",").append(type_name).append(")"); @@ -1104,7 +1118,6 @@ ORT_API_STATUS_IMPL(OrtGetOpaqueValue, const char* domain_name, const char* type return nullptr; } - // End support for non-tensor types DEFINE_RELEASE_ORT_OBJECT_FUNCTION(Env, OrtEnv) diff --git a/onnxruntime/python/onnxruntime_pybind_state.cc b/onnxruntime/python/onnxruntime_pybind_state.cc index 1bc04f660c..21251d0b3c 100644 --- a/onnxruntime/python/onnxruntime_pybind_state.cc +++ b/onnxruntime/python/onnxruntime_pybind_state.cc @@ -768,6 +768,14 @@ including arg name, arg type (contains both type and shape).)pbdoc") return *(res.second); } }) + .def_property_readonly("overridable_initializers", [](const InferenceSession* sess) -> const std::vector& { + auto res = sess->GetOverridableInitializers(); + if (!res.first.IsOK()) { + throw std::runtime_error(res.first.ToString().c_str()); + } else { + return *res.second; + } + }) .def_property_readonly("model_meta", [](const InferenceSession* sess) -> const onnxruntime::ModelMetadata& { auto res = sess->GetModelMetadata(); if (!res.first.IsOK()) { diff --git a/onnxruntime/python/session.py b/onnxruntime/python/session.py index b875dc54d2..e46ed675d4 100644 --- a/onnxruntime/python/session.py +++ b/onnxruntime/python/session.py @@ -42,6 +42,7 @@ class InferenceSession: self._inputs_meta = self._sess.inputs_meta self._outputs_meta = self._sess.outputs_meta + self._overridable_initializers = self._sess.overridable_initializers self._model_meta = self._sess.model_meta self._providers = self._sess.get_providers() @@ -51,6 +52,7 @@ class InferenceSession: # so they must be set to None to decrement _sess reference count. self._inputs_meta = None self._outputs_meta = None + self._overridable_initializers = None self._model_meta = None self._providers = None self._sess = None @@ -63,6 +65,10 @@ class InferenceSession: "Return the outputs metadata as a list of :class:`onnxruntime.NodeArg`." return self._outputs_meta + def get_overridable_initializers(self): + "Return the inputs (including initializers) metadata as a list of :class:`onnxruntime.NodeArg`." + return self._overridable_initializers + def get_modelmeta(self): "Return the metadata. See :class:`onnxruntime.ModelMetadata`." return self._model_meta diff --git a/onnxruntime/python/tools/onnxruntime_test.py b/onnxruntime/python/tools/onnxruntime_test.py index 56223b1981..a430ca9bd4 100644 --- a/onnxruntime/python/tools/onnxruntime_test.py +++ b/onnxruntime/python/tools/onnxruntime_test.py @@ -74,6 +74,25 @@ def main(): input_meta.type, input_meta.name)) sys.exit(-1) + # Starting with IR4 some initializers provide default values + # and can be overridden (available in IR4). For IR < 4 models + # the list would be empty + for initializer in sess.get_overridable_initializers(): + shape = [dim if dim else 1 for dim in initializer.shape] + if initializer.type in float_dict: + feeds[initializer.name] = np.random.rand( + *shape).astype(float_dict[initializer.type]) + elif initializer.type in integer_dict: + feeds[initializer.name] = np.random.uniform( + high=1000, size=tuple(shape)).astype(integer_dict[initializer.type]) + elif initializer.type == 'tensor(bool)': + feeds[initializer.name] = np.random.randint( + 2, size=tuple(shape)).astype('bool') + else: + print("unsupported initializer type {} for initializer {}".format( + initializer.type, initializer.name)) + sys.exit(-1) + start = timer() for i in range(iters): sess.run([], feeds) # fetch all outputs diff --git a/onnxruntime/test/shared_lib/test_inference.cc b/onnxruntime/test/shared_lib/test_inference.cc index 3e36e600b8..32fb292184 100644 --- a/onnxruntime/test/shared_lib/test_inference.cc +++ b/onnxruntime/test/shared_lib/test_inference.cc @@ -116,6 +116,7 @@ void TestInference(Ort::Env& env, T model_uri, static constexpr PATH_TYPE MODEL_URI = TSTR("testdata/mul_1.onnx"); static constexpr PATH_TYPE CUSTOM_OP_MODEL_URI = TSTR("testdata/foo_1.onnx"); +static constexpr PATH_TYPE OVERRIDABLE_INITIALIZER_MODEL_URI = TSTR("testdata/overridable_initializer.onnx"); #ifdef ENABLE_LANGUAGE_INTEROP_OPS static constexpr PATH_TYPE PYOP_FLOAT_MODEL_URI = TSTR("testdata/pyop_1.onnx"); #endif @@ -290,6 +291,61 @@ TEST_F(CApiTest, create_tensor_with_data) { ASSERT_EQ(1, tensor_info.GetDimensionsCount()); } +TEST_F(CApiTest, override_initializer) { + Ort::MemoryInfo info("Cpu", OrtDeviceAllocator, 0, OrtMemTypeDefault); + auto allocator = std::make_unique(); + // CreateTensor which is not owning this ptr + bool Label_input[] = {true}; + std::vector dims = {1, 1}; + Ort::Value label_input_tensor = Ort::Value::CreateTensor(info, Label_input, 1U, dims.data(), dims.size()); + + std::string f2_data{"f2_string"}; + // Place a string into Tensor OrtValue and assign to the + Ort::Value f2_input_tensor = Ort::Value::CreateTensor(allocator.get(), dims.data(), dims.size(), ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING); + // No C++ Api to either create a string Tensor or to fill one with string, so we use C + const char* const input_char_string[] = {f2_data.c_str()}; + ORT_THROW_ON_ERROR(OrtFillStringTensor(static_cast(f2_input_tensor), input_char_string, 1U)); + + Ort::SessionOptions session_options; + Ort::Session session(env_, OVERRIDABLE_INITIALIZER_MODEL_URI, session_options); + + // Get Overrideable initializers + size_t init_count = session.GetOverridableInitializerCount(); + ASSERT_EQ(init_count, 1U); + + char* f1_init_name = session.GetOverridableInitializerName(0, allocator.get()); + ASSERT_TRUE(strcmp("F1", f1_init_name) == 0); + allocator->Free(f1_init_name); + + Ort::TypeInfo init_type_info = session.GetOverridableInitializerTypeInfo(0); + ASSERT_EQ(ONNX_TYPE_TENSOR, init_type_info.GetONNXType()); + + // Let's override the initializer + float f11_input_data[] = {2.0f}; + Ort::Value f11_input_tensor = Ort::Value::CreateTensor(info, f11_input_data, 1U, dims.data(), dims.size()); + + std::vector ort_inputs; + ort_inputs.push_back(std::move(label_input_tensor)); + ort_inputs.push_back(std::move(f2_input_tensor)); + ort_inputs.push_back(std::move(f11_input_tensor)); + + std::vector input_names = {"Label", "F2", "F1"}; + + const char* const output_names[] = {"Label0", "F20", "F11"}; + std::vector ort_outputs = session.Run(Ort::RunOptions{nullptr}, input_names.data(), + ort_inputs.data(), ort_inputs.size(), + output_names, countof(output_names)); + + ASSERT_EQ(ort_outputs.size(), 3U); + // Expecting the last output would be the overridden value of the initializer + auto type_info = ort_outputs[2].GetTensorTypeAndShapeInfo(); + ASSERT_EQ(type_info.GetShape(), dims); + ASSERT_EQ(type_info.GetElementType(), ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT); + ASSERT_EQ(type_info.GetElementCount(), 1U); + float* output_data = ort_outputs[2].GetTensorMutableData(); + ASSERT_EQ(*output_data, f11_input_data[0]); +} + int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); int ret = RUN_ALL_TESTS(); diff --git a/onnxruntime/test/testdata/overridable_initializer.onnx b/onnxruntime/test/testdata/overridable_initializer.onnx new file mode 100644 index 0000000000..5c80850469 Binary files /dev/null and b/onnxruntime/test/testdata/overridable_initializer.onnx differ