From e915f01b413cf35b43d6450ba37418aaf5a73859 Mon Sep 17 00:00:00 2001 From: Scott McKay Date: Sat, 18 Jan 2025 11:22:58 +1000 Subject: [PATCH] Merge --- .../core/session/onnxruntime_c_api.h | 20 +++--- .../core/session/onnxruntime_cxx_api.h | 4 +- .../onnxruntime_session_options_config_keys.h | 10 +++ .../test/shared_lib/test_model_builder_api.cc | 66 +++++++++++++++++-- 4 files changed, 84 insertions(+), 16 deletions(-) diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index e0beb2f22b..ab020c6658 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -5282,14 +5282,14 @@ struct OrtModelBuilderApi { /** \brief Create an OrtSession to augment an existing model. * - * Create an OrtSession with an existing model that will be augmented with additional nodes. + * Create an OrtSession with an existing model that will be augmented with additional nodes and initializers. * Nodes can be added before or after the existing nodes in the model. ONNX Runtime will connect the nodes when the * model is finalized. * - * To add nodes to the existing model, first create an OrtModel using CreateModel. - * Add additional nodes and initializers to the OrtModel using AddNodeToGraph and AddInitializerToGraph. - * Graph inputs/outputs should be updated with SetGraphInputs and SetGraphOutputs to reflect changes made by the new - * nodes. The list of inputs/outputs should be for the overall model and not just the new nodes. + * To add nodes and initializers to the existing model, first create an OrtModel using CreateModel. + * Add nodes and initializers to the OrtModel using AddNodeToGraph and AddInitializerToGraph. + * Graph inputs/outputs should be updated with SetGraphInputs and SetGraphOutputs as needed to reflect changes made + * by the new nodes. The list of graph inputs/outputs should be for the overall model and not just the new nodes. * * Add the new information from the OrtModel to the original model using ApplyModelToSession, and prepare the * session for inferencing by calling FinalizeModelBuilderSession. @@ -5304,14 +5304,14 @@ struct OrtModelBuilderApi { /** \brief Create an OrtSession to augment an existing model. * - * Create an OrtSession with an existing model that will be augmented with additional nodes. + * Create an OrtSession with an existing model that will be augmented with additional nodes and initializers. * Nodes can be added before or after the existing nodes in the model. ONNX Runtime will connect the nodes when the * model is finalized. * - * To add nodes to the existing model, first create an OrtModel using CreateModel. - * Add additional nodes and initializers to the OrtModel using AddNodeToGraph and AddInitializerToGraph. - * Graph inputs/outputs should be updated with SetGraphInputs and SetGraphOutputs to reflect changes made by the new - * nodes. The list of inputs/outputs should be for the overall model and not just the new nodes. + * To add nodes and initializers to the existing model, first create an OrtModel using CreateModel. + * Add nodes and initializers to the OrtModel using AddNodeToGraph and AddInitializerToGraph. + * Graph inputs/outputs should be updated with SetGraphInputs and SetGraphOutputs as needed to reflect changes made + * by the new nodes. The list of graph inputs/outputs should be for the overall model and not just the new nodes. * * Add the new information from the OrtModel to the original model using ApplyModelToSession, and prepare the * session for inferencing by calling FinalizeModelBuilderSession. diff --git a/include/onnxruntime/core/session/onnxruntime_cxx_api.h b/include/onnxruntime/core/session/onnxruntime_cxx_api.h index 061c24ba4c..28dc8dfdf4 100644 --- a/include/onnxruntime/core/session/onnxruntime_cxx_api.h +++ b/include/onnxruntime/core/session/onnxruntime_cxx_api.h @@ -1122,8 +1122,8 @@ struct ConstSessionImpl : Base { int GetOpset(const std::string& domain) const; ///< Wraps OrtApi::SessionGetOpsetForDomain - // TODO: Should we move ValueInfo from ModelBuilderAPI to the ORT API? It will also be relevant to the Plugin EP API - // although the internal implementation may differ there. + // NOTE: We will probably move ValueInfo from ModelBuilderAPI to the ORT API as it will also be relevant to the Plugin EP API. + // Will move before checkin if that's the case. std::vector GetInputs() const; std::vector GetOutputs() const; }; diff --git a/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h b/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h index 64a4dd19c1..0c3ebebc52 100644 --- a/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h +++ b/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h @@ -300,3 +300,13 @@ static const char* const kOrtSessionOptionsQDQMatMulNBitsAccuracyLevel = "sessio // “Default”: OS determines the scheduling priority and processor performance to service this workload. [Default] // “Efficient”: OS treats this workload is efficiency oriented with low scheduling priority and efficient processor performance. static const char* const kOrtEpDynamicOptionsWorkloadType = "ep.dynamic.workload_type"; + +// Create an Inference Session that will use the Model Builder API to create/update the model. +// This flag will create the session but not fully initialize it. A model, if provided, will be loaded. +// A session logger will be created, and execution providers will be registered. +// Any device specific allocators and IDataTransfer objects will be registered. +// This allows CreateAllocator to return device specific allocators registered by EPs. +// FUTURE: This will also allow CopyTensors to utilize the IDataTransfer objects +// "0": Disabled. [DEFAULT] +// "1": Enable Model Builder Session +static const char* const kOrtSessionOptionsEnableModelBuilder = "session.model_builder_session"; diff --git a/onnxruntime/test/shared_lib/test_model_builder_api.cc b/onnxruntime/test/shared_lib/test_model_builder_api.cc index 852636b2c5..8e2d93dfe8 100644 --- a/onnxruntime/test/shared_lib/test_model_builder_api.cc +++ b/onnxruntime/test/shared_lib/test_model_builder_api.cc @@ -369,8 +369,6 @@ TEST(ModelBuilderAPITest, Basic_CxxApi) { ModelBuilderAPI::Model model(opsets); model.AddGraph(graph); - auto session = CreateSession(*ort_env, model); - std::vector> inputs(1); auto& input = inputs[0]; input.name = "X"; @@ -381,6 +379,7 @@ TEST(ModelBuilderAPITest, Basic_CxxApi) { std::vector expected_dims = {3, 8}; + auto session = CreateSession(*ort_env, model); TestInference(session, inputs, "Z", expected_dims, {340.0f, 360.0f, 380.0f, 400.0f, 420.0f, 440.0f, 460.0f, 480.0f, 596.0f, 648.0f, 700.0f, 752.0f, 804.0f, 856.0f, 908.0f, 960.0f, @@ -416,8 +415,8 @@ TEST(ModelBuilderAPITest, BasicModelEdit_CxxApi) { ASSERT_EQ(graph_inputs[0].TypeInfo().GetTensorTypeAndShapeInfo().GetElementType(), ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT); - // typically this isn't needed, but we want to replace this input and read info from it later on in the test - // validation so we move it out of the vector so it's saved locally. + // typically this isn't needed. we replace this input but need to read info from it later on in the test + // validation so we save the info locally to keep it accessible. auto orig_input_name = graph_inputs[0].Name(); auto input_shape = graph_inputs[0].TypeInfo().GetTensorTypeAndShapeInfo().GetShape(); const std::string new_input_name = "Int64Input"; @@ -622,11 +621,70 @@ TEST(ModelBuilderAPITest, InvalidModelEdit) { } TEST(ModelBuilderAPITest, CreateTypeInfo) { + const auto& api = Ort::GetApi(); + TensorTypeAndShapeInfo base_tensor_info(ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, + {2, 4}); + + OrtTypeInfo* base_tensor_type_info = nullptr; + Ort::ThrowOnError(api.CreateTensorTypeInfo(base_tensor_info, &base_tensor_type_info)); + + ONNXType onnx_type = ONNX_TYPE_UNKNOWN; + OrtTypeInfo* sparse_tensor_type_info = nullptr; + const OrtTensorTypeAndShapeInfo* tensor_info = nullptr; + ONNXTensorElementDataType onnx_element_type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED; + // sparse tensor + Ort::ThrowOnError(api.CreateSparseTensorTypeInfo(base_tensor_info, &sparse_tensor_type_info)); + Ort::ThrowOnError(api.GetOnnxTypeFromTypeInfo(sparse_tensor_type_info, &onnx_type)); + ASSERT_EQ(onnx_type, ONNXType::ONNX_TYPE_SPARSETENSOR); + Ort::ThrowOnError(api.CastTypeInfoToTensorInfo(sparse_tensor_type_info, &tensor_info)); + Ort::ThrowOnError(api.GetTensorElementType(tensor_info, &onnx_element_type)); + ASSERT_EQ(onnx_element_type, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT); // sequence + OrtTypeInfo* sequence_type_info = nullptr; + const OrtSequenceTypeInfo* sequence_info = nullptr; + OrtTypeInfo* sequence_element_type_info = nullptr; + + Ort::ThrowOnError(api.CreateSequenceTypeInfo(base_tensor_type_info, &sequence_type_info)); + Ort::ThrowOnError(api.GetOnnxTypeFromTypeInfo(sequence_type_info, &onnx_type)); + ASSERT_EQ(onnx_type, ONNXType::ONNX_TYPE_SEQUENCE); + Ort::ThrowOnError(api.CastTypeInfoToSequenceTypeInfo(sequence_type_info, &sequence_info)); + Ort::ThrowOnError(api.GetSequenceElementType(sequence_info, &sequence_element_type_info)); + Ort::ThrowOnError(api.GetOnnxTypeFromTypeInfo(sequence_element_type_info, &onnx_type)); + ASSERT_EQ(onnx_type, ONNXType::ONNX_TYPE_TENSOR); + Ort::ThrowOnError(api.CastTypeInfoToTensorInfo(sequence_element_type_info, &tensor_info)); + Ort::ThrowOnError(api.GetTensorElementType(tensor_info, &onnx_element_type)); + ASSERT_EQ(onnx_element_type, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT); // map + OrtTypeInfo* map_type_info = nullptr; + const OrtMapTypeInfo* map_info = nullptr; + ONNXTensorElementDataType map_key_type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED; + OrtTypeInfo* map_value_type_info = nullptr; + Ort::ThrowOnError(api.CreateMapTypeInfo(ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64, base_tensor_type_info, + &map_type_info)); + Ort::ThrowOnError(api.GetOnnxTypeFromTypeInfo(map_type_info, &onnx_type)); + ASSERT_EQ(onnx_type, ONNXType::ONNX_TYPE_MAP); + Ort::ThrowOnError(api.CastTypeInfoToMapTypeInfo(map_type_info, &map_info)); + Ort::ThrowOnError(api.GetMapKeyType(map_info, &map_key_type)); + ASSERT_EQ(map_key_type, ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64); + Ort::ThrowOnError(api.GetMapValueType(map_info, &map_value_type_info)); + Ort::ThrowOnError(api.GetOnnxTypeFromTypeInfo(map_value_type_info, &onnx_type)); + ASSERT_EQ(onnx_type, ONNXType::ONNX_TYPE_TENSOR); + Ort::ThrowOnError(api.CastTypeInfoToTensorInfo(map_value_type_info, &tensor_info)); + Ort::ThrowOnError(api.GetTensorElementType(tensor_info, &onnx_element_type)); + ASSERT_EQ(onnx_element_type, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT); // optional + OrtTypeInfo* optional_type_info = nullptr; + const OrtOptionalTypeInfo* optional_info = nullptr; + OrtTypeInfo* optional_contained_type_info = nullptr; + Ort::ThrowOnError(api.CreateOptionalTypeInfo(base_tensor_type_info, &optional_type_info)); + Ort::ThrowOnError(api.GetOnnxTypeFromTypeInfo(optional_type_info, &onnx_type)); + ASSERT_EQ(onnx_type, ONNXType::ONNX_TYPE_OPTIONAL); + Ort::ThrowOnError(api.CastTypeInfoToOptionalTypeInfo(optional_type_info, &optional_info)); + Ort::ThrowOnError(api.GetOptionalContainedTypeInfo(optional_info, &optional_contained_type_info)); + Ort::ThrowOnError(api.GetOnnxTypeFromTypeInfo(optional_contained_type_info, &onnx_type)); + ASSERT_EQ(onnx_type, ONNXType::ONNX_TYPE_TENSOR); }