diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index 06b83fdf73..fcd42323ef 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -5158,16 +5158,29 @@ struct OrtModelBuilderApi { * * Two options: * - * Pre-existing memory: - * Use CreateTensorWithDataAsOrtValue or CreateTensorWithDataAndDeleterAsOrtValue to create an OrtValue - * with a tensor that contains a pointer to the existing data. - * If using CreateTensorWithDataAsOrtValue you must keep the pointer valid for lifetime of the inference session. - * Set `data_is_external` to true. - * * Allocated memory: * Use CreateTensorAsOrtValue (allocates memory) and populate the tensor with the data. * Set `data_is_external` to false. * + * Pre-existing memory: + * Use CreateTensorWithDataAsOrtValue or CreateTensorWithDataAndDeleterAsOrtValue to create an OrtValue + * with a tensor that contains a pointer to the existing data. + * Set `data_is_external` to true. + * + * The pointer must remain valid for the duration of the inference session. + * If using CreateTensorWithDataAsOrtValue you are responsible for freeing the memory after the inference session + * is released. + * If using CreateTensorWithDataAndDeleterAsOrtValue, ORT will free the memory using the provided deleter as + * soon as the OrtValue is no longer in use. + * + * NOTE: A tensor containing pre-existing memory MUST have 128 bytes of data or more. + * For smaller tensors use CreateTensorAsOrtValue. + * + * ONNX shape inferencing does not support external data. An initializer involved in shape inferencing is + * typically small (a single value or limited by the rank of a tensor) and uses less than 128 bytes of + * memory, so this limit acts as a simple catch-all rule to avoid issues. + * e.g. Reshape's `shape`, Clip's `min` and `max`, various ops `axes`. + * * \param[in] graph The OrtGraph instance to update. * \param[in] name The value name for the initializer. * \param[in] tensor The OrtValue instance containing the tensor data. diff --git a/include/onnxruntime/core/session/onnxruntime_cxx_inline.h b/include/onnxruntime/core/session/onnxruntime_cxx_inline.h index 1de5db2669..7365d39938 100644 --- a/include/onnxruntime/core/session/onnxruntime_cxx_inline.h +++ b/include/onnxruntime/core/session/onnxruntime_cxx_inline.h @@ -2418,10 +2418,8 @@ template <> inline void GraphImpl::SetInputs(std::vector& inputs) { std::vector inputs_ptrs; inputs_ptrs.reserve(inputs.size()); - - // Graph takes ownership. std::transform(inputs.begin(), inputs.end(), std::back_inserter(inputs_ptrs), - [](ValueInfo& vi) -> OrtValueInfo* { return vi.release(); }); + [](ValueInfo& vi) -> OrtValueInfo* { return vi; }); ThrowOnError(GetModelBuilderApi().SetGraphInputs(p_, inputs_ptrs.data(), inputs_ptrs.size())); diff --git a/onnxruntime/core/graph/graph.cc b/onnxruntime/core/graph/graph.cc index 41a3c28e01..660b0cf288 100644 --- a/onnxruntime/core/graph/graph.cc +++ b/onnxruntime/core/graph/graph.cc @@ -7,21 +7,24 @@ #include #include #include -#include #include +#include + +#include #include "core/common/common.h" -#include #include "core/common/inlined_containers.h" #include "core/common/logging/logging.h" #include "core/common/narrow.h" #include "core/flatbuffers/flatbuffers_utils.h" #include "core/framework/tensor_type_and_shape.h" #include "core/flatbuffers/schema/ort.fbs.h" -#include "core/framework/tensor_shape.h" #include "core/framework/tensor_external_data_info.h" +#include "core/framework/tensor_shape.h" +#include "core/framework/tensor_type_and_shape.h" #include "core/framework/tensorprotoutils.h" #include "core/framework/utils.h" +#include "core/graph/function_utils.h" #include "core/graph/graph_flatbuffers_utils.h" #include "core/graph/graph_viewer.h" #include "core/graph/indexed_sub_graph.h" @@ -32,7 +35,6 @@ #include "core/graph/node_attr_utils.h" #include "core/graph/op.h" #include "core/graph/runtime_optimization_record_container.h" -#include "core/graph/function_utils.h" #if !defined(ORT_MINIMAL_BUILD) #include "core/graph/function.h" @@ -4096,27 +4098,51 @@ ONNX_NAMESPACE::GraphProto Graph::ToGraphProto() const { // This is used for constructing full path for external data // if it exists + auto add_initializer = [](TensorList& output_initializers, const TensorProto& initializer) -> void { + TensorProto& output = *output_initializers.Add(); + output = initializer; + + // copy any in-memory external data into raw data + if (utils::HasExternalData(initializer)) { + const std::filesystem::path ignored; + std::basic_string location; + onnxruntime::FileOffsetType file_offset; + SafeInt tensor_byte_size; + + ORT_THROW_IF_ERROR(utils::GetExternalDataInfo(initializer, ignored, location, file_offset, tensor_byte_size)); + + if (location == onnxruntime::utils::kTensorProtoMemoryAddressTag) { + // file_offset is address + void* data = reinterpret_cast(file_offset); + + // set in raw data + output.clear_data_location(); + output.set_raw_data(data, tensor_byte_size); + } + } + }; + + auto* mutable_initializers = result.mutable_initializer(); + #if !defined(DISABLE_SPARSE_TENSORS) const auto& model_path = ModelPath(); // We want to make sure that sparse initializers do not appear // as dense duplicates within the initializers list. - if (!sparse_tensor_names_.empty()) { - const auto sparse_end = sparse_tensor_names_.end(); - auto* mutable_initializer = result.mutable_initializer(); - for (const auto& initializer : graph_proto_->initializer()) { - if (sparse_end == sparse_tensor_names_.find(initializer.name())) { - *mutable_initializer->Add() = initializer; - } else { - auto& sparse_initializer = *result.add_sparse_initializer(); - auto status = utils::DenseTensorToSparseTensorProto(initializer, model_path, sparse_initializer); - ORT_ENFORCE(status.IsOK(), "Failed to convert dense initializer to sparse"); - } + const bool has_sparse_initializers = !sparse_tensor_names_.empty(); + const auto sparse_end = sparse_tensor_names_.end(); + for (const auto& initializer : graph_proto_->initializer()) { + if (!has_sparse_initializers || sparse_end == sparse_tensor_names_.find(initializer.name())) { + add_initializer(*mutable_initializers, initializer); + } else { + auto& sparse_initializer = *result.add_sparse_initializer(); + auto status = utils::DenseTensorToSparseTensorProto(initializer, model_path, sparse_initializer); + ORT_ENFORCE(status.IsOK(), "Failed to convert dense initializer to sparse"); } - } else { - *result.mutable_initializer() = graph_proto_->initializer(); } #else - *result.mutable_initializer() = graph_proto_->initializer(); + for (const auto& initializer : graph_proto_->initializer()) { + add_initializer(*mutable_initializers, initializer); + } #endif return result; diff --git a/onnxruntime/core/session/inference_session.h b/onnxruntime/core/session/inference_session.h index f89eacb633..89a2693d19 100644 --- a/onnxruntime/core/session/inference_session.h +++ b/onnxruntime/core/session/inference_session.h @@ -627,6 +627,12 @@ class InferenceSession { /// convenience pointer to logger. should always be the same as session_state_.Logger(); const logging::Logger* session_logger_; + // The list of execution providers. + // This MUST be prior to model_ in case there are values in the model that were allocated using an allocator + // provided by the EP. If that is the case the allocator's `free` implementation may depend on other parts of the + // EP instance. + ExecutionProviders execution_providers_; + // The model served by this inference session instance. // Currently this has to be a shared ptr because the Model::Load method // returns a shared_ptr only. Ideally factory functions should always return @@ -637,9 +643,6 @@ class InferenceSession { // The file path of where the model was loaded. e.g. /tmp/test_squeezenet/model.onnx PathString model_location_; - // The list of execution providers. - ExecutionProviders execution_providers_; - private: ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(InferenceSession); void SetLoggingManager(const SessionOptions& session_options, diff --git a/onnxruntime/core/session/model_builder_c_api.cc b/onnxruntime/core/session/model_builder_c_api.cc index 25e2409805..8eac1ebce3 100644 --- a/onnxruntime/core/session/model_builder_c_api.cc +++ b/onnxruntime/core/session/model_builder_c_api.cc @@ -93,6 +93,9 @@ ORT_API_STATUS_IMPL(OrtModelBuilderAPI::CreateNode, const char* operator_name, c n->attributes.reserve(attribs_len); for (size_t i = 0; i < attribs_len; ++i) { n->attributes.push_back(*reinterpret_cast(attributes[i])); + // take ownership. as we took a copy that means releasing the original value + OrtApis::ReleaseOpAttr(attributes[i]); + attributes[i] = nullptr; } } @@ -156,12 +159,31 @@ ORT_API_STATUS_IMPL(OrtModelBuilderAPI::SetGraphOutputs, _In_ OrtGraph* graph, ORT_API_STATUS_IMPL(OrtModelBuilderAPI::AddInitializerToGraph, _In_ OrtGraph* graph, _In_ const char* name, _Inout_ OrtValue* tensor, bool data_is_external) { API_IMPL_BEGIN + if (!tensor->IsTensor()) { + return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, "Only Tensor is currently supported."); + } + + if (!tensor->IsAllocated()) { + return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, "Tensor must be allocated."); + } + + const auto& t = tensor->Get(); + if (t.Location().device.Type() != OrtDevice::CPU) { + return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, "Only CPU based tensors are currently supported."); + } + if (data_is_external) { -#if !defined(DISABLE_EXTERNAL_INITIALIZERS) + // enforce that an external initializer is not used if the data size is < 128 bytes. + // the reason for this is to avoid potential shape inferencing errors if this initializer is providing an + // input involved in that. the ONNX shape inferencing does not support external data for those values. + // e.g. Reshape's `shape` input, Reduce's `axes', Slice's `starts`, `ends`, `steps`, Clip's `min`, `max`, etc. + if (t.SizeInBytes() < 128) { + return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, + "External initializer should only be used for data >= 128 bytes. " + "Please use CreateTensorAsOrtValue instead."); + } + graph->external_initializers[name] = std::unique_ptr(tensor); // take ownership -#else - return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, "External initializers are not supported in this build"); -#endif } else { graph->initializers[name] = std::unique_ptr(tensor); // take ownership } diff --git a/onnxruntime/test/shared_lib/test_model_builder_api.cc b/onnxruntime/test/shared_lib/test_model_builder_api.cc index e4870809de..b6314f48a0 100644 --- a/onnxruntime/test/shared_lib/test_model_builder_api.cc +++ b/onnxruntime/test/shared_lib/test_model_builder_api.cc @@ -141,14 +141,14 @@ TEST(ModelBuilderAPITest, Basic_CApi) { Ort::ThrowOnError(model_builder_api.CreateGraph(&graph)); // - // Create OrtModel with a Gemm. X input is 3x2, Y input is 2x3, Z output is 3x3. + // Create OrtModel with a Gemm. X input is 3x4, Y input is 4x8, Z output is 3x8. // X is model input. Y is initializer. // Set the alpha attribute of the Gemm node to 2.0 to test attribute handling. // // model input OrtTensorTypeAndShapeInfo* tensor_type_info = nullptr; - std::vector input_dims = {3, 2}; + std::vector input_dims = {3, 4}; // can use api.SetSymbolicDimensions to set symbolic dimensions. // the input array should have the same rank as the call to SetDimensions. // e.g. call SetDimensions with {-1, 3, 2} and SetSymbolicDimensions with {"N", nullptr, nullptr} to create @@ -170,7 +170,7 @@ TEST(ModelBuilderAPITest, Basic_CApi) { // model outputs OrtTypeInfo* output_type_info = nullptr; - std::vector output_dims = {3, 3}; + std::vector output_dims = {3, 8}; Ort::ThrowOnError(api.CreateTensorTypeAndShapeInfo(&tensor_type_info)); Ort::ThrowOnError(api.SetTensorElementType(tensor_type_info, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT)); @@ -203,24 +203,22 @@ TEST(ModelBuilderAPITest, Basic_CApi) { std::vector node_output_names = {gemm_output_name.c_str()}; std::vector node_attributes{alpha_attr}; OrtNode* node = CreateNode(model_builder_api, "Gemm", "Gemm1", node_input_names, node_output_names, node_attributes); - - api.ReleaseOpAttr(alpha_attr); // CreateNode copies all OrtOpAttr instances + alpha_attr = nullptr; // Node now owns Ort::ThrowOnError(model_builder_api.AddNodeToGraph(graph, node)); node = nullptr; // graph now owns node // Y input - std::vector y_dims = {2, 3}; - deleter.weights.emplace_back( - std::make_unique>(std::initializer_list{1.0f, 2.0f, 3.0f, - 4.0f, 5.0f, 6.0f})); + // As it's 128 bytes it could either be allocated using CreateTensorAsOrtValue or use existing memory. + // Under 128 bytes must use CreateTensorAsOrtValue. + std::vector y_dims = {4, 8}; + + deleter.weights.emplace_back(std::make_unique>(32)); auto& y_values = *deleter.weights.back(); + std::iota(y_values.begin(), y_values.end(), 1.0f); - // create an initializer for the Y input. add to `weights` so the memory remains valid + // create an initializer for the Y input. add to `weights` so the memory remains valid. OrtValue* y_tensor = nullptr; - auto info = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeDefault); - - // if you use this API the initializer data MUST remain valid for the lifetime of the InferenceSession Ort::ThrowOnError( api.CreateTensorWithDataAndDeleterAsOrtValue(&deleter, y_values.data(), y_values.size() * sizeof(y_values[0]), @@ -232,18 +230,24 @@ TEST(ModelBuilderAPITest, Basic_CApi) { y_tensor = nullptr; // graph now owns if (use_constant_node) { - // Test that a Constant node is converted to an intializer + // Test that a Constant node is converted to an initializer - // create Constant node that is used as the Max in a Clip to limit the output - OrtOpAttr* value_attr = nullptr; - float max = 60.0f; - Ort::ThrowOnError(api.CreateOpAttr("value", &max, sizeof(max), ORT_OP_ATTR_FLOAT, &value_attr)); - - node = CreateNode(model_builder_api, "Constant", "clip_max", {}, {"max"}, {value_attr}); + // create Constant nodes for min/max to limit output range + OrtOpAttr* min_attr = nullptr; + float min = 400.0f; + Ort::ThrowOnError(api.CreateOpAttr("value", &min, sizeof(min), ORT_OP_ATTR_FLOAT, &min_attr)); + node = CreateNode(model_builder_api, "Constant", "clip_min", {}, {"min"}, {min_attr}); Ort::ThrowOnError(model_builder_api.AddNodeToGraph(graph, node)); node = nullptr; // graph now owns node - node = CreateNode(model_builder_api, "Clip", "Clip1", {gemm_output_name.c_str(), "", "max"}, {"Z"}); + OrtOpAttr* max_attr = nullptr; + float max = 900.0f; + Ort::ThrowOnError(api.CreateOpAttr("value", &max, sizeof(max), ORT_OP_ATTR_FLOAT, &max_attr)); + node = CreateNode(model_builder_api, "Constant", "clip_max", {}, {"max"}, {max_attr}); + Ort::ThrowOnError(model_builder_api.AddNodeToGraph(graph, node)); + node = nullptr; // graph now owns node + + node = CreateNode(model_builder_api, "Clip", "Clip1", {gemm_output_name.c_str(), "min", "max"}, {"Z"}); Ort::ThrowOnError(model_builder_api.AddNodeToGraph(graph, node)); node = nullptr; // graph now owns node } @@ -265,22 +269,25 @@ TEST(ModelBuilderAPITest, Basic_CApi) { std::vector> inputs(1); auto& input = inputs[0]; input.name = "X"; - input.dims = {3, 2}; - input.values = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f}; + input.dims = {3, 4}; + input.values = {1.0f, 2.0f, 3.0f, 4.0f, + 8.0f, 7.0f, 6.0f, 5.0f, + 9.0f, 3.0f, 5.0f, 7.0f}; - std::vector expected_dims = {3, 3}; + std::vector expected_dims = {3, 8}; ModelBuilderAPI::Model cxx_model(model); auto session = CreateSession(*ort_env, cxx_model); std::vector expected_output; if (use_constant_node) { - expected_output = {18.0f, 24.0f, 30.0f, - 38.0f, 52.0f, 60.0f, // clipped - 58.0f, 60.0f, 60.0f}; // clipped + // clipped with min 400 and max 900 + expected_output = {400.0f, 400.0f, 400.0f, 400.0f, 420.0f, 440.0f, 460.0f, 480.0f, + 596.0f, 648.0f, 700.0f, 752.0f, 804.0f, 856.0f, 900.0f, 900.0f, + 592.0f, 640.0f, 688.0f, 736.0f, 784.0f, 832.0f, 880.0f, 900.0f}; } else { - expected_output = {18.0f, 24.0f, 30.0f, - 38.0f, 52.0f, 66.0f, - 58.0f, 80.0f, 102.0f}; + expected_output = {340.0f, 360.0f, 380.0f, 400.0f, 420.0f, 440.0f, 460.0f, 480.0f, + 596.0f, 648.0f, 700.0f, 752.0f, 804.0f, 856.0f, 908.0f, 960.0f, + 592.0f, 640.0f, 688.0f, 736.0f, 784.0f, 832.0f, 880.0f, 928.0f}; } TestInference(session, inputs, "Z", expected_dims, expected_output); @@ -301,7 +308,7 @@ TEST(ModelBuilderAPITest, Basic_CxxApi) { Ort::ModelBuilderAPI::Graph graph; // - // Create OrtModel with a Gemm. X input is 3x2, Y input is 2x3, Z output is 3x3. + // Create OrtModel with a Gemm. X input is 3x4, Y input is 4x8, Z output is 3x8. // X is model input. Y is initializer. // Set the alpha attribute of the Gemm node to 2.0 to test attribute handling. // @@ -309,8 +316,8 @@ TEST(ModelBuilderAPITest, Basic_CxxApi) { std::vector graph_inputs; std::vector graph_outputs; - // model input. it's {3, 2} but use a symbolic dim to test that works. - std::vector input_dims({-1, 2}); + // model input. it's {3, 4} but use a symbolic dim to test that works. + std::vector input_dims({-1, 4}); std::vector input_symbolic_dims({"multiple_of_3", ""}); TensorTypeAndShapeInfo input_tensor_info(ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, input_dims, @@ -319,7 +326,7 @@ TEST(ModelBuilderAPITest, Basic_CxxApi) { graph_inputs.emplace_back("X", input_type_info.GetConst()); // model outputs - std::vector output_dims = {-1, 3}; + std::vector output_dims = {-1, 8}; std::vector output_symbolic_dims({"multiple_of_3", ""}); TensorTypeAndShapeInfo output_tensor_info(ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, output_dims, @@ -344,10 +351,14 @@ TEST(ModelBuilderAPITest, Basic_CxxApi) { // create an initializer for the Y input. // add to `weights` so it remains valid for the lifetime of the session and we can avoid copying the data. - std::vector y_dims = {2, 3}; - weights.emplace_back(std::make_unique>(std::initializer_list{1.0f, 2.0f, 3.0f, - 4.0f, 5.0f, 6.0f})); + // As it's 128 bytes it could either be allocated using CreateTensorAsOrtValue or use existing memory. + // Under 128 bytes must use CreateTensorAsOrtValue. + std::vector y_dims = {4, 8}; + + weights.emplace_back(std::make_unique>(32)); auto& y_values = *weights.back(); + std::iota(y_values.begin(), y_values.end(), 1.0f); + auto info = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeDefault); // if you use this API the initializer data MUST remain valid for the lifetime of the InferenceSession @@ -361,16 +372,18 @@ TEST(ModelBuilderAPITest, Basic_CxxApi) { std::vector> inputs(1); auto& input = inputs[0]; input.name = "X"; - input.dims = {3, 2}; - input.values = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f}; + input.dims = {3, 4}; + input.values = {1.0f, 2.0f, 3.0f, 4.0f, + 8.0f, 7.0f, 6.0f, 5.0f, + 9.0f, 3.0f, 5.0f, 7.0f}; - std::vector expected_dims = {3, 3}; + std::vector expected_dims = {3, 8}; auto session = CreateSession(*ort_env, model); TestInference(session, inputs, "Z", expected_dims, - {18.0f, 24.0f, 30.0f, - 38.0f, 52.0f, 66.0f, - 58.0f, 80.0f, 102.0f}); + {340.0f, 360.0f, 380.0f, 400.0f, 420.0f, 440.0f, 460.0f, 480.0f, + 596.0f, 648.0f, 700.0f, 752.0f, 804.0f, 856.0f, 908.0f, 960.0f, + 592.0f, 640.0f, 688.0f, 736.0f, 784.0f, 832.0f, 880.0f, 928.0f}); } TEST(ModelBuilderAPITest, BasicModelEdit_CxxApi) {