saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-29 20:14:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

- take ownership of OrtOpAttr in CreateNode

Browse source 
- enforce 128 byte minimum for tensors with external data to avoid shape inferencing issues
  - update unit tests to use 128 byte initializer so external data can be tested
- support saving initializer with in-memory external data to ONNX model by copying into TensorProto's raw_data property.
This commit is contained in:
Scott McKay 2025-01-08 10:23:49 +10:00
parent dc2caa6772
commit 4e2d061977
6 changed files with 152 additions and 77 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
include/onnxruntime/core/session/onnxruntime_c_api.h
View file

@ -5158,16 +5158,29 @@ struct OrtModelBuilderApi {
*
* Two options:
*
* Pre-existing memory:
* Use CreateTensorWithDataAsOrtValue or CreateTensorWithDataAndDeleterAsOrtValue to create an OrtValue
* with a tensor that contains a pointer to the existing data.
* If using CreateTensorWithDataAsOrtValue you must keep the pointer valid for lifetime of the inference session.
* Set `data_is_external` to true.
*
* Allocated memory:
* Use CreateTensorAsOrtValue (allocates memory) and populate the tensor with the data.
* Set `data_is_external` to false.
*
* Pre-existing memory:
* Use CreateTensorWithDataAsOrtValue or CreateTensorWithDataAndDeleterAsOrtValue to create an OrtValue
* with a tensor that contains a pointer to the existing data.
* Set `data_is_external` to true.
*
* The pointer must remain valid for the duration of the inference session.
* If using CreateTensorWithDataAsOrtValue you are responsible for freeing the memory after the inference session
* is released.
* If using CreateTensorWithDataAndDeleterAsOrtValue, ORT will free the memory using the provided deleter as
* soon as the OrtValue is no longer in use.
*
* NOTE: A tensor containing pre-existing memory MUST have 128 bytes of data or more.
* For smaller tensors use CreateTensorAsOrtValue.
*
* ONNX shape inferencing does not support external data. An initializer involved in shape inferencing is
* typically small (a single value or limited by the rank of a tensor) and uses less than 128 bytes of
* memory, so this limit acts as a simple catch-all rule to avoid issues.
* e.g. Reshape's `shape`, Clip's `min` and `max`, various ops `axes`.
*
* \param[in] graph The OrtGraph instance to update.
* \param[in] name The value name for the initializer.
* \param[in] tensor The OrtValue instance containing the tensor data.

4
include/onnxruntime/core/session/onnxruntime_cxx_inline.h
View file

@ -2418,10 +2418,8 @@ template <>
inline void GraphImpl<OrtGraph>::SetInputs(std::vector<ValueInfo>& inputs) {
std::vector<OrtValueInfo*> inputs_ptrs;
inputs_ptrs.reserve(inputs.size());
// Graph takes ownership.
std::transform(inputs.begin(), inputs.end(), std::back_inserter(inputs_ptrs),
[](ValueInfo& vi) -> OrtValueInfo* { return vi.release(); });
[](ValueInfo& vi) -> OrtValueInfo* { return vi; });
ThrowOnError(GetModelBuilderApi().SetGraphInputs(p_, inputs_ptrs.data(), inputs_ptrs.size()));

62
onnxruntime/core/graph/graph.cc
View file

@ -7,21 +7,24 @@
#include <fstream>
#include <iostream>
#include <numeric>
#include <stack>
#include <queue>
#include <stack>
#include <gsl/gsl>
#include "core/common/common.h"
#include <gsl/gsl>
#include "core/common/inlined_containers.h"
#include "core/common/logging/logging.h"
#include "core/common/narrow.h"
#include "core/flatbuffers/flatbuffers_utils.h"
#include "core/framework/tensor_type_and_shape.h"
#include "core/flatbuffers/schema/ort.fbs.h"
#include "core/framework/tensor_shape.h"
#include "core/framework/tensor_external_data_info.h"
#include "core/framework/tensor_shape.h"
#include "core/framework/tensor_type_and_shape.h"
#include "core/framework/tensorprotoutils.h"
#include "core/framework/utils.h"
#include "core/graph/function_utils.h"
#include "core/graph/graph_flatbuffers_utils.h"
#include "core/graph/graph_viewer.h"
#include "core/graph/indexed_sub_graph.h"
@ -32,7 +35,6 @@
#include "core/graph/node_attr_utils.h"
#include "core/graph/op.h"
#include "core/graph/runtime_optimization_record_container.h"
#include "core/graph/function_utils.h"
#if !defined(ORT_MINIMAL_BUILD)
#include "core/graph/function.h"
@ -4096,27 +4098,51 @@ ONNX_NAMESPACE::GraphProto Graph::ToGraphProto() const {
// This is used for constructing full path for external data
// if it exists
auto add_initializer = [](TensorList& output_initializers, const TensorProto& initializer) -> void {
TensorProto& output = *output_initializers.Add();
output = initializer;
// copy any in-memory external data into raw data
if (utils::HasExternalData(initializer)) {
const std::filesystem::path ignored;
std::basic_string<ORTCHAR_T> location;
onnxruntime::FileOffsetType file_offset;
SafeInt<size_t> tensor_byte_size;
ORT_THROW_IF_ERROR(utils::GetExternalDataInfo(initializer, ignored, location, file_offset, tensor_byte_size));
if (location == onnxruntime::utils::kTensorProtoMemoryAddressTag) {
// file_offset is address
void* data = reinterpret_cast<void*>(file_offset);
// set in raw data
output.clear_data_location();
output.set_raw_data(data, tensor_byte_size);
}
}
};
auto* mutable_initializers = result.mutable_initializer();
#if !defined(DISABLE_SPARSE_TENSORS)
const auto& model_path = ModelPath();
// We want to make sure that sparse initializers do not appear
// as dense duplicates within the initializers list.
if (!sparse_tensor_names_.empty()) {
const auto sparse_end = sparse_tensor_names_.end();
auto* mutable_initializer = result.mutable_initializer();
for (const auto& initializer : graph_proto_->initializer()) {
if (sparse_end == sparse_tensor_names_.find(initializer.name())) {
*mutable_initializer->Add() = initializer;
} else {
auto& sparse_initializer = *result.add_sparse_initializer();
auto status = utils::DenseTensorToSparseTensorProto(initializer, model_path, sparse_initializer);
ORT_ENFORCE(status.IsOK(), "Failed to convert dense initializer to sparse");
}
const bool has_sparse_initializers = !sparse_tensor_names_.empty();
const auto sparse_end = sparse_tensor_names_.end();
for (const auto& initializer : graph_proto_->initializer()) {
if (!has_sparse_initializers || sparse_end == sparse_tensor_names_.find(initializer.name())) {
add_initializer(*mutable_initializers, initializer);
} else {
auto& sparse_initializer = *result.add_sparse_initializer();
auto status = utils::DenseTensorToSparseTensorProto(initializer, model_path, sparse_initializer);
ORT_ENFORCE(status.IsOK(), "Failed to convert dense initializer to sparse");
}
} else {
*result.mutable_initializer() = graph_proto_->initializer();
}
#else
*result.mutable_initializer() = graph_proto_->initializer();
for (const auto& initializer : graph_proto_->initializer()) {
add_initializer(*mutable_initializers, initializer);
}
#endif
return result;

9
onnxruntime/core/session/inference_session.h
View file

@ -627,6 +627,12 @@ class InferenceSession {
/// convenience pointer to logger. should always be the same as session_state_.Logger();
const logging::Logger* session_logger_;
// The list of execution providers.
// This MUST be prior to model_ in case there are values in the model that were allocated using an allocator
// provided by the EP. If that is the case the allocator's `free` implementation may depend on other parts of the
// EP instance.
ExecutionProviders execution_providers_;
// The model served by this inference session instance.
// Currently this has to be a shared ptr because the Model::Load method
// returns a shared_ptr only. Ideally factory functions should always return
@ -637,9 +643,6 @@ class InferenceSession {
// The file path of where the model was loaded. e.g. /tmp/test_squeezenet/model.onnx
PathString model_location_;
// The list of execution providers.
ExecutionProviders execution_providers_;
private:
ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(InferenceSession);
void SetLoggingManager(const SessionOptions& session_options,

30
onnxruntime/core/session/model_builder_c_api.cc
View file

@ -93,6 +93,9 @@ ORT_API_STATUS_IMPL(OrtModelBuilderAPI::CreateNode, const char* operator_name, c
n->attributes.reserve(attribs_len);
for (size_t i = 0; i < attribs_len; ++i) {
n->attributes.push_back(*reinterpret_cast<const ONNX_NAMESPACE::AttributeProto*>(attributes[i]));
// take ownership. as we took a copy that means releasing the original value
OrtApis::ReleaseOpAttr(attributes[i]);
attributes[i] = nullptr;
}
}
@ -156,12 +159,31 @@ ORT_API_STATUS_IMPL(OrtModelBuilderAPI::SetGraphOutputs, _In_ OrtGraph* graph,
ORT_API_STATUS_IMPL(OrtModelBuilderAPI::AddInitializerToGraph, _In_ OrtGraph* graph, _In_ const char* name,
_Inout_ OrtValue* tensor, bool data_is_external) {
API_IMPL_BEGIN
if (!tensor->IsTensor()) {
return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, "Only Tensor is currently supported.");
}
if (!tensor->IsAllocated()) {
return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, "Tensor must be allocated.");
}
const auto& t = tensor->Get<onnxruntime::Tensor>();
if (t.Location().device.Type() != OrtDevice::CPU) {
return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, "Only CPU based tensors are currently supported.");
}
if (data_is_external) {
#if !defined(DISABLE_EXTERNAL_INITIALIZERS)
// enforce that an external initializer is not used if the data size is < 128 bytes.
// the reason for this is to avoid potential shape inferencing errors if this initializer is providing an
// input involved in that. the ONNX shape inferencing does not support external data for those values.
// e.g. Reshape's `shape` input, Reduce's `axes', Slice's `starts`, `ends`, `steps`, Clip's `min`, `max`, etc.
if (t.SizeInBytes() < 128) {
return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT,
"External initializer should only be used for data >= 128 bytes. "
"Please use CreateTensorAsOrtValue instead.");
}
graph->external_initializers[name] = std::unique_ptr<OrtValue>(tensor); // take ownership
#else
return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, "External initializers are not supported in this build");
#endif
} else {
graph->initializers[name] = std::unique_ptr<OrtValue>(tensor); // take ownership
}

99
onnxruntime/test/shared_lib/test_model_builder_api.cc
View file

@ -141,14 +141,14 @@ TEST(ModelBuilderAPITest, Basic_CApi) {
Ort::ThrowOnError(model_builder_api.CreateGraph(&graph));
//
// Create OrtModel with a Gemm. X input is 3x2, Y input is 2x3, Z output is 3x3.
// Create OrtModel with a Gemm. X input is 3x4, Y input is 4x8, Z output is 3x8.
// X is model input. Y is initializer.
// Set the alpha attribute of the Gemm node to 2.0 to test attribute handling.
//
// model input
OrtTensorTypeAndShapeInfo* tensor_type_info = nullptr;
std::vector<int64_t> input_dims = {3, 2};
std::vector<int64_t> input_dims = {3, 4};
// can use api.SetSymbolicDimensions to set symbolic dimensions.
// the input array should have the same rank as the call to SetDimensions.
// e.g. call SetDimensions with {-1, 3, 2} and SetSymbolicDimensions with {"N", nullptr, nullptr} to create
@ -170,7 +170,7 @@ TEST(ModelBuilderAPITest, Basic_CApi) {
// model outputs
OrtTypeInfo* output_type_info = nullptr;
std::vector<int64_t> output_dims = {3, 3};
std::vector<int64_t> output_dims = {3, 8};
Ort::ThrowOnError(api.CreateTensorTypeAndShapeInfo(&tensor_type_info));
Ort::ThrowOnError(api.SetTensorElementType(tensor_type_info, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT));
@ -203,24 +203,22 @@ TEST(ModelBuilderAPITest, Basic_CApi) {
std::vector<const char*> node_output_names = {gemm_output_name.c_str()};
std::vector<OrtOpAttr*> node_attributes{alpha_attr};
OrtNode* node = CreateNode(model_builder_api, "Gemm", "Gemm1", node_input_names, node_output_names, node_attributes);
api.ReleaseOpAttr(alpha_attr); // CreateNode copies all OrtOpAttr instances
alpha_attr = nullptr; // Node now owns
Ort::ThrowOnError(model_builder_api.AddNodeToGraph(graph, node));
node = nullptr; // graph now owns node
// Y input
std::vector<int64_t> y_dims = {2, 3};
deleter.weights.emplace_back(
std::make_unique<std::vector<float>>(std::initializer_list<float>{1.0f, 2.0f, 3.0f,
4.0f, 5.0f, 6.0f}));
// As it's 128 bytes it could either be allocated using CreateTensorAsOrtValue or use existing memory.
// Under 128 bytes must use CreateTensorAsOrtValue.
std::vector<int64_t> y_dims = {4, 8};
deleter.weights.emplace_back(std::make_unique<std::vector<float>>(32));
auto& y_values = *deleter.weights.back();
std::iota(y_values.begin(), y_values.end(), 1.0f);
// create an initializer for the Y input. add to `weights` so the memory remains valid
// create an initializer for the Y input. add to `weights` so the memory remains valid.
OrtValue* y_tensor = nullptr;
auto info = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeDefault);
// if you use this API the initializer data MUST remain valid for the lifetime of the InferenceSession
Ort::ThrowOnError(
api.CreateTensorWithDataAndDeleterAsOrtValue(&deleter,
y_values.data(), y_values.size() * sizeof(y_values[0]),
@ -232,18 +230,24 @@ TEST(ModelBuilderAPITest, Basic_CApi) {
y_tensor = nullptr; // graph now owns
if (use_constant_node) {
// Test that a Constant node is converted to an intializer
// Test that a Constant node is converted to an initializer
// create Constant node that is used as the Max in a Clip to limit the output
OrtOpAttr* value_attr = nullptr;
float max = 60.0f;
Ort::ThrowOnError(api.CreateOpAttr("value", &max, sizeof(max), ORT_OP_ATTR_FLOAT, &value_attr));
node = CreateNode(model_builder_api, "Constant", "clip_max", {}, {"max"}, {value_attr});
// create Constant nodes for min/max to limit output range
OrtOpAttr* min_attr = nullptr;
float min = 400.0f;
Ort::ThrowOnError(api.CreateOpAttr("value", &min, sizeof(min), ORT_OP_ATTR_FLOAT, &min_attr));
node = CreateNode(model_builder_api, "Constant", "clip_min", {}, {"min"}, {min_attr});
Ort::ThrowOnError(model_builder_api.AddNodeToGraph(graph, node));
node = nullptr; // graph now owns node
node = CreateNode(model_builder_api, "Clip", "Clip1", {gemm_output_name.c_str(), "", "max"}, {"Z"});
OrtOpAttr* max_attr = nullptr;
float max = 900.0f;
Ort::ThrowOnError(api.CreateOpAttr("value", &max, sizeof(max), ORT_OP_ATTR_FLOAT, &max_attr));
node = CreateNode(model_builder_api, "Constant", "clip_max", {}, {"max"}, {max_attr});
Ort::ThrowOnError(model_builder_api.AddNodeToGraph(graph, node));
node = nullptr; // graph now owns node
node = CreateNode(model_builder_api, "Clip", "Clip1", {gemm_output_name.c_str(), "min", "max"}, {"Z"});
Ort::ThrowOnError(model_builder_api.AddNodeToGraph(graph, node));
node = nullptr; // graph now owns node
}
@ -265,22 +269,25 @@ TEST(ModelBuilderAPITest, Basic_CApi) {
std::vector<Input<float>> inputs(1);
auto& input = inputs[0];
input.name = "X";
input.dims = {3, 2};
input.values = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f};
input.dims = {3, 4};
input.values = {1.0f, 2.0f, 3.0f, 4.0f,
8.0f, 7.0f, 6.0f, 5.0f,
9.0f, 3.0f, 5.0f, 7.0f};
std::vector<int64_t> expected_dims = {3, 3};
std::vector<int64_t> expected_dims = {3, 8};
ModelBuilderAPI::Model cxx_model(model);
auto session = CreateSession(*ort_env, cxx_model);
std::vector<float> expected_output;
if (use_constant_node) {
expected_output = {18.0f, 24.0f, 30.0f,
38.0f, 52.0f, 60.0f, // clipped
58.0f, 60.0f, 60.0f}; // clipped
// clipped with min 400 and max 900
expected_output = {400.0f, 400.0f, 400.0f, 400.0f, 420.0f, 440.0f, 460.0f, 480.0f,
596.0f, 648.0f, 700.0f, 752.0f, 804.0f, 856.0f, 900.0f, 900.0f,
592.0f, 640.0f, 688.0f, 736.0f, 784.0f, 832.0f, 880.0f, 900.0f};
} else {
expected_output = {18.0f, 24.0f, 30.0f,
38.0f, 52.0f, 66.0f,
58.0f, 80.0f, 102.0f};
expected_output = {340.0f, 360.0f, 380.0f, 400.0f, 420.0f, 440.0f, 460.0f, 480.0f,
596.0f, 648.0f, 700.0f, 752.0f, 804.0f, 856.0f, 908.0f, 960.0f,
592.0f, 640.0f, 688.0f, 736.0f, 784.0f, 832.0f, 880.0f, 928.0f};
}
TestInference<float>(session, inputs, "Z", expected_dims, expected_output);
@ -301,7 +308,7 @@ TEST(ModelBuilderAPITest, Basic_CxxApi) {
Ort::ModelBuilderAPI::Graph graph;
//
// Create OrtModel with a Gemm. X input is 3x2, Y input is 2x3, Z output is 3x3.
// Create OrtModel with a Gemm. X input is 3x4, Y input is 4x8, Z output is 3x8.
// X is model input. Y is initializer.
// Set the alpha attribute of the Gemm node to 2.0 to test attribute handling.
//
@ -309,8 +316,8 @@ TEST(ModelBuilderAPITest, Basic_CxxApi) {
std::vector<ModelBuilderAPI::ValueInfo> graph_inputs;
std::vector<ModelBuilderAPI::ValueInfo> graph_outputs;
// model input. it's {3, 2} but use a symbolic dim to test that works.
std::vector<int64_t> input_dims({-1, 2});
// model input. it's {3, 4} but use a symbolic dim to test that works.
std::vector<int64_t> input_dims({-1, 4});
std::vector<std::string> input_symbolic_dims({"multiple_of_3", ""});
TensorTypeAndShapeInfo input_tensor_info(ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT,
input_dims,
@ -319,7 +326,7 @@ TEST(ModelBuilderAPITest, Basic_CxxApi) {
graph_inputs.emplace_back("X", input_type_info.GetConst());
// model outputs
std::vector<int64_t> output_dims = {-1, 3};
std::vector<int64_t> output_dims = {-1, 8};
std::vector<std::string> output_symbolic_dims({"multiple_of_3", ""});
TensorTypeAndShapeInfo output_tensor_info(ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT,
output_dims,
@ -344,10 +351,14 @@ TEST(ModelBuilderAPITest, Basic_CxxApi) {
// create an initializer for the Y input.
// add to `weights` so it remains valid for the lifetime of the session and we can avoid copying the data.
std::vector<int64_t> y_dims = {2, 3};
weights.emplace_back(std::make_unique<std::vector<float>>(std::initializer_list<float>{1.0f, 2.0f, 3.0f,
4.0f, 5.0f, 6.0f}));
// As it's 128 bytes it could either be allocated using CreateTensorAsOrtValue or use existing memory.
// Under 128 bytes must use CreateTensorAsOrtValue.
std::vector<int64_t> y_dims = {4, 8};
weights.emplace_back(std::make_unique<std::vector<float>>(32));
auto& y_values = *weights.back();
std::iota(y_values.begin(), y_values.end(), 1.0f);
auto info = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeDefault);
// if you use this API the initializer data MUST remain valid for the lifetime of the InferenceSession
@ -361,16 +372,18 @@ TEST(ModelBuilderAPITest, Basic_CxxApi) {
std::vector<Input<float>> inputs(1);
auto& input = inputs[0];
input.name = "X";
input.dims = {3, 2};
input.values = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f};
input.dims = {3, 4};
input.values = {1.0f, 2.0f, 3.0f, 4.0f,
8.0f, 7.0f, 6.0f, 5.0f,
9.0f, 3.0f, 5.0f, 7.0f};
std::vector<int64_t> expected_dims = {3, 3};
std::vector<int64_t> expected_dims = {3, 8};
auto session = CreateSession(*ort_env, model);
TestInference<float>(session, inputs, "Z", expected_dims,
{18.0f, 24.0f, 30.0f,
38.0f, 52.0f, 66.0f,
58.0f, 80.0f, 102.0f});
{340.0f, 360.0f, 380.0f, 400.0f, 420.0f, 440.0f, 460.0f, 480.0f,
596.0f, 648.0f, 700.0f, 752.0f, 804.0f, 856.0f, 908.0f, 960.0f,
592.0f, 640.0f, 688.0f, 736.0f, 784.0f, 832.0f, 880.0f, 928.0f});
}
TEST(ModelBuilderAPITest, BasicModelEdit_CxxApi) {