From b8575dda7be471af14607de0230767d38f79606e Mon Sep 17 00:00:00 2001 From: Hariharan Seshadri Date: Thu, 12 Mar 2020 18:38:10 -0700 Subject: [PATCH] Avoid some heap allocations in the InferenceSession and Model classes (#3103) * Avoid some heap allocations in the InferenceSession and Model classes --- onnxruntime/core/graph/model.cc | 87 ++++++++--------- onnxruntime/core/graph/model.h | 10 +- .../core/optimizer/graph_transformer_mgr.cc | 17 +++- .../core/optimizer/graph_transformer_mgr.h | 17 +++- onnxruntime/core/session/inference_session.cc | 90 +++++++++-------- onnxruntime/core/session/inference_session.h | 24 ++--- onnxruntime/test/ir/graph_test.cc | 96 +++++++++---------- onnxruntime/test/ir/onnx_model_test.cc | 4 +- .../optimizer/free_dimension_override_test.cc | 12 +-- .../test/optimizer/graph_transform_test.cc | 18 ++-- .../rule_based_graph_transformer_test.cc | 20 +++- 11 files changed, 210 insertions(+), 185 deletions(-) diff --git a/onnxruntime/core/graph/model.cc b/onnxruntime/core/graph/model.cc index e5d49c141e..cc59386621 100644 --- a/onnxruntime/core/graph/model.cc +++ b/onnxruntime/core/graph/model.cc @@ -35,12 +35,11 @@ Model::Model(const std::string& graph_name, const std::vector& model_functions, const logging::Logger& logger) : model_path_(Path::Parse(model_path)) { - model_proto_ = onnxruntime::make_unique(); - model_proto_->set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION); - model_proto_->mutable_graph()->set_name(graph_name); + model_proto_.set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION); + model_proto_.mutable_graph()->set_name(graph_name); model_metadata_ = model_metadata; for (auto& metadata : model_metadata_) { - const gsl::not_null prop{model_proto_->add_metadata_props()}; + const gsl::not_null prop{model_proto_.add_metadata_props()}; prop->set_key(metadata.first); prop->set_value(metadata.second); } @@ -58,52 +57,48 @@ Model::Model(const std::string& graph_name, } for (const auto& domain : *p_domain_to_version) { - const gsl::not_null opset_id_proto{model_proto_->add_opset_import()}; + const gsl::not_null opset_id_proto{model_proto_.add_opset_import()}; opset_id_proto->set_domain(domain.first); opset_id_proto->set_version(domain.second); } std::unordered_map model_functions_map; for (auto& func : model_functions) { - auto func_ptr = model_proto_->add_functions(); + auto func_ptr = model_proto_.add_functions(); func_ptr->CopyFrom(func); model_functions_map[func_ptr->name()] = func_ptr; } // need to call private ctor so can't use make_shared GSL_SUPPRESS(r .11) - graph_.reset(new Graph(*this, model_proto_->mutable_graph(), *p_domain_to_version, IrVersion(), schema_registry, + graph_.reset(new Graph(*this, model_proto_.mutable_graph(), *p_domain_to_version, IrVersion(), schema_registry, logger, model_functions_map)); } Model::Model(const ModelProto& model_proto, const PathString& model_path, const IOnnxRuntimeOpSchemaRegistryList* local_registries, const logging::Logger& logger) - : Model(onnxruntime::make_unique(model_proto), model_path, local_registries, logger) { + : Model(ModelProto(model_proto), model_path, local_registries, logger) { } -Model::Model(std::unique_ptr model_proto, const PathString& model_path, const IOnnxRuntimeOpSchemaRegistryList* local_registries, +Model::Model(ModelProto&& model_proto, const PathString& model_path, const IOnnxRuntimeOpSchemaRegistryList* local_registries, const logging::Logger& logger) : model_path_(Path::Parse(model_path)) { - if (!model_proto) { - throw std::invalid_argument("ModelProto was null."); - } - - if (!utils::HasGraph(*model_proto)) { + if (!utils::HasGraph(model_proto)) { throw std::invalid_argument("ModelProto does not have a graph."); } - if (model_proto->opset_import_size() == 0) { + if (model_proto.opset_import_size() == 0) { throw std::invalid_argument( "Missing opset in the model. All ModelProtos MUST have at least one entry that" " specifies which version of the ONNX OperatorSet is being imported."); } - if (!model_proto->has_ir_version() || model_proto->ir_version() > ONNX_NAMESPACE::Version::IR_VERSION) { + if (!model_proto.has_ir_version() || model_proto.ir_version() > ONNX_NAMESPACE::Version::IR_VERSION) { throw std::invalid_argument("Unknown model file format version."); } model_proto_ = std::move(model_proto); - for (auto& prop : model_proto_->metadata_props()) { + for (auto& prop : model_proto_.metadata_props()) { model_metadata_[prop.key()] = prop.value(); } @@ -115,7 +110,7 @@ Model::Model(std::unique_ptr model_proto, const PathString& model_pa } std::unordered_map domain_to_version; - for (auto& opSet : model_proto_->opset_import()) { + for (auto& opSet : model_proto_.opset_import()) { const auto& domain = opSet.domain(); const auto version = opSet.version(); // empty domain and 'ai.onnx' are equivalent @@ -144,71 +139,71 @@ Model::Model(std::unique_ptr model_proto, const PathString& model_pa for (const auto& domain : domain_map) { if (domain_to_version.find(domain.first) == domain_to_version.end()) { domain_to_version[domain.first] = domain.second; - const gsl::not_null opset_id_proto{model_proto_->add_opset_import()}; + const gsl::not_null opset_id_proto{model_proto_.add_opset_import()}; opset_id_proto->set_domain(domain.first); opset_id_proto->set_version(domain.second); } } std::unordered_map model_functions_map; - for (auto& func : model_proto_->functions()) { + for (auto& func : model_proto_.functions()) { model_functions_map[func.name()] = &func; } // create instance. need to call private ctor so can't use make_unique GSL_SUPPRESS(r .11) - graph_.reset(new Graph(*this, model_proto_->mutable_graph(), domain_to_version, IrVersion(), schema_registry, logger, + graph_.reset(new Graph(*this, model_proto_.mutable_graph(), domain_to_version, IrVersion(), schema_registry, logger, model_functions_map)); } Version Model::IrVersion() const { - if (utils::HasIrVersion(*model_proto_)) { - return model_proto_->ir_version(); + if (utils::HasIrVersion(model_proto_)) { + return model_proto_.ir_version(); } return kNoVersion; } const std::string& Model::ProducerName() const { - return model_proto_->producer_name(); + return model_proto_.producer_name(); } void Model::SetProducerName(const std::string& producer_name) { - model_proto_->set_producer_name(producer_name); + model_proto_.set_producer_name(producer_name); } const std::string& Model::ProducerVersion() const { - return model_proto_->producer_version(); + return model_proto_.producer_version(); } void Model::SetProducerVersion(const std::string& producer_version) { - model_proto_->set_producer_version(producer_version); + model_proto_.set_producer_version(producer_version); } const std::string& Model::Domain() const { - return model_proto_->domain(); + return model_proto_.domain(); } void Model::SetDomain(const std::string& domain) { - model_proto_->set_domain(domain); + model_proto_.set_domain(domain); } Version Model::ModelVersion() const { - if (utils::HasModelVersion(*model_proto_)) { - return model_proto_->model_version(); + if (utils::HasModelVersion(model_proto_)) { + return model_proto_.model_version(); } return kNoVersion; } void Model::SetModelversion(onnxruntime::Version version) { - model_proto_->set_model_version(version); + model_proto_.set_model_version(version); } const std::string& Model::DocString() const { - return model_proto_->doc_string(); + return model_proto_.doc_string(); } void Model::SetDocString(const std::string& doc_string) { - model_proto_->set_doc_string(doc_string); + model_proto_.set_doc_string(doc_string); } const ModelMetaData& Model::MetaData() const noexcept { @@ -224,14 +219,14 @@ const Graph& Model::MainGraph() const noexcept { } void Model::AddFunction(const ONNX_NAMESPACE::FunctionProto& func_proto) { - auto func_ptr = model_proto_->add_functions(); + auto func_ptr = model_proto_.add_functions(); func_ptr->CopyFrom(func_proto); graph_->AddFunction(func_ptr); } ModelProto Model::ToProto() { - *(model_proto_->mutable_graph()) = graph_->ToGraphProto(); - return *model_proto_; + *(model_proto_.mutable_graph()) = graph_->ToGraphProto(); + return model_proto_; } Status Model::Load(std::istream& model_istream, ModelProto* p_model_proto) { @@ -279,27 +274,27 @@ Status Model::Load(const ModelProto& model_proto, return Status::OK(); } -Status Model::Load(std::unique_ptr p_model_proto, +Status Model::Load(ModelProto&& model_proto, std::shared_ptr& model, const IOnnxRuntimeOpSchemaRegistryList* local_registries, const logging::Logger& logger) { - return Model::Load(std::move(p_model_proto), PathString{}, model, local_registries, logger); + return Model::Load(std::move(model_proto), PathString{}, model, local_registries, logger); } -Status Model::Load(std::unique_ptr p_model_proto, +Status Model::Load(ModelProto&& model_proto, const PathString& model_path, std::shared_ptr& model, const IOnnxRuntimeOpSchemaRegistryList* local_registries, const logging::Logger& logger) { // we expect a graph to be present - if (!utils::HasGraph(*p_model_proto)) { + if (!utils::HasGraph(model_proto)) { return Status(ONNXRUNTIME, INVALID_ARGUMENT, "No graph was found in the protobuf."); } // need to call private ctor so can't use make_shared GSL_SUPPRESS(r .11) try { - model.reset(new Model(std::move(p_model_proto), model_path, local_registries, logger)); + model.reset(new Model(std::move(model_proto), model_path, local_registries, logger)); } catch (const std::exception& ex) { return Status(ONNXRUNTIME, INVALID_ARGUMENT, "Failed to load model with error: " + std::string(ex.what())); } @@ -421,9 +416,9 @@ Status Model::LoadFromBytes(int count, void* p_bytes, /*out*/ std::shared_ptr& p_model, const IOnnxRuntimeOpSchemaRegistryList* local_registries, const logging::Logger& logger) { - auto model_proto = onnxruntime::make_unique(); + ModelProto model_proto; - auto status = LoadFromBytes(count, p_bytes, *model_proto); + auto status = LoadFromBytes(count, p_bytes, model_proto); if (!status.IsOK()) { return status; } @@ -473,9 +468,9 @@ Status Model::Load(int fd, std::shared_ptr& p_model, const IOnnxRuntimeOp Status Model::Load(int fd, const PathString& model_path, std::shared_ptr& p_model, const IOnnxRuntimeOpSchemaRegistryList* local_registries, const logging::Logger& logger) { - auto model_proto = onnxruntime::make_unique(); + ModelProto model_proto; - ORT_RETURN_IF_ERROR(Load(fd, *model_proto)); + ORT_RETURN_IF_ERROR(Load(fd, model_proto)); p_model = std::make_shared(std::move(model_proto), model_path, local_registries, logger); diff --git a/onnxruntime/core/graph/model.h b/onnxruntime/core/graph/model.h index 08b4d6ee27..10c7bfcce7 100644 --- a/onnxruntime/core/graph/model.h +++ b/onnxruntime/core/graph/model.h @@ -56,14 +56,14 @@ class Model { // NOTE: after calling this constructor, <*this> model will // own the . - explicit Model(std::unique_ptr model_proto, + explicit Model(ONNX_NAMESPACE::ModelProto&& model_proto, const IOnnxRuntimeOpSchemaRegistryList* local_registries, const logging::Logger& logger) : Model(std::move(model_proto), PathString(), local_registries, logger) {} // NOTE: after calling this constructor, <*this> model will // own the . - explicit Model(std::unique_ptr model_proto, + explicit Model(ONNX_NAMESPACE::ModelProto&& model_proto, const PathString& model_path, const IOnnxRuntimeOpSchemaRegistryList* local_registries, const logging::Logger& logger); @@ -173,12 +173,12 @@ class Model { const IOnnxRuntimeOpSchemaRegistryList* local_registries, const logging::Logger& logger); - static common::Status Load(std::unique_ptr p_model_proto, + static common::Status Load(ONNX_NAMESPACE::ModelProto&& model_proto, /*out*/ std::shared_ptr& p_model, const IOnnxRuntimeOpSchemaRegistryList* local_registries, const logging::Logger& logger); - static common::Status Load(std::unique_ptr p_model_proto, + static common::Status Load(ONNX_NAMESPACE::ModelProto&& model_proto, const PathString& model_path, /*out*/ std::shared_ptr& p_model, const IOnnxRuntimeOpSchemaRegistryList* local_registries, @@ -186,7 +186,7 @@ class Model { private: // Model data. - std::unique_ptr model_proto_; + ONNX_NAMESPACE::ModelProto model_proto_; // This is a duplication of . // It gives better accessibility. diff --git a/onnxruntime/core/optimizer/graph_transformer_mgr.cc b/onnxruntime/core/optimizer/graph_transformer_mgr.cc index 114b7d86ad..f161fcf0cb 100644 --- a/onnxruntime/core/optimizer/graph_transformer_mgr.cc +++ b/onnxruntime/core/optimizer/graph_transformer_mgr.cc @@ -3,11 +3,22 @@ #include "core/optimizer/graph_transformer_mgr.h" #include "core/optimizer/rule_based_graph_transformer.h" + using namespace onnxruntime; using namespace ::onnxruntime::common; namespace onnxruntime { +common::Status GraphTransformerManager::SetSteps(unsigned steps) { + steps_ = steps; + return Status::OK(); +} + +common::Status GraphTransformerManager::GetSteps(unsigned& steps) const { + steps = steps_; + return Status::OK(); +} + common::Status GraphTransformerManager::ApplyTransformers(Graph& graph, TransformerLevel level, const logging::Logger& logger) const { const auto& transformers = level_to_transformer_map_.find(level); if (transformers == level_to_transformer_map_.end()) { @@ -29,14 +40,14 @@ common::Status GraphTransformerManager::ApplyTransformers(Graph& graph, Transfor return Status::OK(); } -common::Status GraphTransformerManager::Register(std::unique_ptr transformer, TransformerLevel level){ +common::Status GraphTransformerManager::Register(std::unique_ptr transformer, TransformerLevel level) { const auto& name = transformer->Name(); if (transformers_info_.find(name) != transformers_info_.end()) { - return Status(ONNXRUNTIME, FAIL, "This transformer is already registered " + name); + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "This transformer is already registered " + name); } transformers_info_[name] = transformer.get(); level_to_transformer_map_[level].push_back(std::move(transformer)); return Status::OK(); } -} // namespace onnxruntime \ No newline at end of file +} // namespace onnxruntime diff --git a/onnxruntime/core/optimizer/graph_transformer_mgr.h b/onnxruntime/core/optimizer/graph_transformer_mgr.h index 8b60e57cd4..c61f76bb2b 100644 --- a/onnxruntime/core/optimizer/graph_transformer_mgr.h +++ b/onnxruntime/core/optimizer/graph_transformer_mgr.h @@ -17,16 +17,20 @@ class GraphTransformerManager { explicit GraphTransformerManager(unsigned steps) : steps_(steps) { } + // Update (set) the maximum number of graph transformation steps + common::Status SetSteps(unsigned steps); + + // Get the maximum number of graph transformation steps + common::Status GetSteps(unsigned& steps) const; + // Register a transformer with a level. - common::Status Register(std::unique_ptr transformer, TransformerLevel level); + common::Status Register(std::unique_ptr transformer, TransformerLevel level); // Apply all transformers registered for the given level on the given graph common::Status ApplyTransformers(Graph& graph, TransformerLevel level, const logging::Logger& logger) const; private: - ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(GraphTransformerManager); - - const unsigned steps_; + ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(GraphTransformerManager); // Older GCC versions don't support std::hash with enum types // Therefore, std::hash appears to be undefined when T is an enum Type. This is fixed in version 6.1 @@ -38,7 +42,10 @@ class GraphTransformerManager { } }; + // maximum number of graph transformation steps + unsigned steps_; + std::unordered_map>, EnumHashKey> level_to_transformer_map_; std::unordered_map transformers_info_; }; -} // namespace onnxruntime \ No newline at end of file +} // namespace onnxruntime diff --git a/onnxruntime/core/session/inference_session.cc b/onnxruntime/core/session/inference_session.cc index b1d2305594..0de00bad3a 100644 --- a/onnxruntime/core/session/inference_session.cc +++ b/onnxruntime/core/session/inference_session.cc @@ -97,7 +97,8 @@ inline std::basic_string GetCurrentTimeString() { std::atomic InferenceSession::global_session_id_{1}; static Status FinalizeSessionOptions(const SessionOptions& user_provided_session_options, - const ONNX_NAMESPACE::ModelProto* model_proto, + const ONNX_NAMESPACE::ModelProto& model_proto, + bool is_model_proto_parsed, /*out*/ SessionOptions& finalized_session_options) { const logging::Logger& default_logger = logging::LoggingManager::DefaultLogger(); @@ -134,12 +135,12 @@ static Status FinalizeSessionOptions(const SessionOptions& user_provided_session // In theory we should not hit this condition unless this internal class' APIs are being called incorrectly. // This is a good sanity check to enforce that the model has been parsed prior to looking into it for ort config. - ORT_ENFORCE(model_proto, "Model needs to be provided to check for ORT config within it"); + ORT_ENFORCE(is_model_proto_parsed, "ModelProto needs to be parsed to check for ORT config within it"); // Use default logger as the session_logger_ hasn't been initialized yet. InferenceSessionUtils inference_session_utils(default_logger); - auto status = inference_session_utils.ParseOrtConfigJsonInModelProto(*model_proto); + auto status = inference_session_utils.ParseOrtConfigJsonInModelProto(model_proto); if (!status.IsOK()) { return status; } @@ -161,12 +162,13 @@ static Status FinalizeSessionOptions(const SessionOptions& user_provided_session void InferenceSession::ConstructorCommon(const SessionOptions& session_options, logging::LoggingManager* logging_manager) { - auto status = FinalizeSessionOptions(session_options, model_proto_.get(), session_options_); + auto status = FinalizeSessionOptions(session_options, model_proto_, model_loaded_, session_options_); ORT_ENFORCE(status.IsOK(), "Could not finalize session options while constructing the inference session. Error Message: ", status.ErrorMessage()); - graph_transformation_mgr_ = onnxruntime::make_unique( - session_options_.max_num_graph_transformation_steps); + // Update the number of steps for the graph transformer manager using the "finalized" session options + graph_transformation_mgr_.SetSteps(session_options_.max_num_graph_transformation_steps); + logging_manager_ = logging_manager; thread_pool_ = concurrency::CreateThreadPool("intra_op_thread_pool", @@ -199,7 +201,8 @@ void InferenceSession::ConstructorCommon(const SessionOptions& session_options, InferenceSession::InferenceSession(const SessionOptions& session_options, logging::LoggingManager* logging_manager) - : insert_cast_transformer_("CastFloat16Transformer") { + : graph_transformation_mgr_(session_options.max_num_graph_transformation_steps), + insert_cast_transformer_("CastFloat16Transformer") { // Initialize assets of this session instance ConstructorCommon(session_options, logging_manager); } @@ -207,13 +210,13 @@ InferenceSession::InferenceSession(const SessionOptions& session_options, InferenceSession::InferenceSession(const SessionOptions& session_options, const std::string& model_uri, logging::LoggingManager* logging_manager) - : insert_cast_transformer_("CastFloat16Transformer") { + : graph_transformation_mgr_(session_options.max_num_graph_transformation_steps), + insert_cast_transformer_("CastFloat16Transformer") { model_location_ = ToWideString(model_uri); - model_proto_ = onnxruntime::make_unique(); - auto status = Model::Load(model_location_, *model_proto_); + auto status = Model::Load(model_location_, model_proto_); ORT_ENFORCE(status.IsOK(), "Given model could not be parsed while creating inference session. Error message: ", status.ErrorMessage()); - + model_loaded_ = true; // Finalize session options and initialize assets of this session instance ConstructorCommon(session_options, logging_manager); } @@ -222,13 +225,13 @@ InferenceSession::InferenceSession(const SessionOptions& session_options, InferenceSession::InferenceSession(const SessionOptions& session_options, const std::wstring& model_uri, logging::LoggingManager* logging_manager) - : insert_cast_transformer_("CastFloat16Transformer") { + : graph_transformation_mgr_(session_options.max_num_graph_transformation_steps), + insert_cast_transformer_("CastFloat16Transformer") { model_location_ = ToWideString(model_uri); - model_proto_ = onnxruntime::make_unique(); - auto status = Model::Load(model_location_, *model_proto_); + auto status = Model::Load(model_location_, model_proto_); ORT_ENFORCE(status.IsOK(), "Given model could not be parsed while creating inference session. Error message: ", status.ErrorMessage()); - + model_loaded_ = true; // Finalize session options and initialize assets of this session instance ConstructorCommon(session_options, logging_manager); } @@ -237,12 +240,12 @@ InferenceSession::InferenceSession(const SessionOptions& session_options, InferenceSession::InferenceSession(const SessionOptions& session_options, std::istream& model_istream, logging::LoggingManager* logging_manager) - : insert_cast_transformer_("CastFloat16Transformer") { + : graph_transformation_mgr_(session_options.max_num_graph_transformation_steps), + insert_cast_transformer_("CastFloat16Transformer") { google::protobuf::io::IstreamInputStream zero_copy_input(&model_istream); - model_proto_ = onnxruntime::make_unique(); - const bool result = model_proto_->ParseFromZeroCopyStream(&zero_copy_input) && model_istream.eof(); + const bool result = model_proto_.ParseFromZeroCopyStream(&zero_copy_input) && model_istream.eof(); ORT_ENFORCE(result, "Could not parse model successfully while constructing the inference session"); - + model_loaded_ = true; // Finalize session options and initialize assets of this session instance ConstructorCommon(session_options, logging_manager); } @@ -251,11 +254,11 @@ InferenceSession::InferenceSession(const SessionOptions& session_options, const void* model_data, int model_data_len, logging::LoggingManager* logging_manager) - : insert_cast_transformer_("CastFloat16Transformer") { - model_proto_ = onnxruntime::make_unique(); - const bool result = model_proto_->ParseFromArray(model_data, model_data_len); + : graph_transformation_mgr_(session_options.max_num_graph_transformation_steps), + insert_cast_transformer_("CastFloat16Transformer") { + const bool result = model_proto_.ParseFromArray(model_data, model_data_len); ORT_ENFORCE(result, "Could not parse model successfully while constructing the inference session"); - + model_loaded_ = true; // Finalize session options and initialize assets of this session instance ConstructorCommon(session_options, logging_manager); } @@ -315,7 +318,7 @@ common::Status InferenceSession::RegisterGraphTransformer( if (p_graph_transformer == nullptr) { return Status(common::ONNXRUNTIME, common::FAIL, "Received nullptr for graph transformer"); } - return graph_transformation_mgr_->Register(std::move(p_graph_transformer), level); + return graph_transformation_mgr_.Register(std::move(p_graph_transformer), level); } common::Status InferenceSession::AddCustomTransformerList(const std::vector& transformers_to_enable) { @@ -372,11 +375,6 @@ common::Status InferenceSession::Load(std::function& model_uri) { } common::Status InferenceSession::Load(const std::string& model_uri) { - if (model_proto_ != nullptr) { + if (model_loaded_) { return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "ModelProto corresponding to the model to be loaded has already been parsed. " "Invoke Load()."); @@ -428,7 +426,7 @@ common::Status InferenceSession::Load(const std::string& model_uri) { #ifdef _WIN32 common::Status InferenceSession::Load(const std::wstring& model_uri) { - if (model_proto_ != nullptr) { + if (model_loaded_) { return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "ModelProto corresponding to the model to be loaded has already been parsed. " "Invoke Load()."); @@ -439,7 +437,7 @@ common::Status InferenceSession::Load(const std::wstring& model_uri) { #endif common::Status InferenceSession::Load(const ModelProto& model_proto) { - if (model_proto_ != nullptr) { + if (model_loaded_) { return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "ModelProto corresponding to the model to be loaded has already been parsed. " "Invoke Load()."); @@ -461,7 +459,7 @@ common::Status InferenceSession::Load(const ModelProto& model_proto) { } common::Status InferenceSession::Load(std::unique_ptr p_model_proto) { - if (model_proto_ != nullptr) { + if (model_loaded_) { return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "ModelProto corresponding to the model to be loaded has already been parsed. " "Invoke Load()."); @@ -474,7 +472,7 @@ common::Status InferenceSession::Load(std::unique_ptr p_model_proto) AddCustomOpDomains({domain.get()}); } #endif - return onnxruntime::Model::Load(std::move(p_model_proto), PathString(), model, + return onnxruntime::Model::Load(std::move(*p_model_proto), PathString(), model, HasLocalSchema() ? &custom_schema_registries_ : nullptr, *session_logger_); }; @@ -482,23 +480,23 @@ common::Status InferenceSession::Load(std::unique_ptr p_model_proto) } common::Status InferenceSession::Load(std::istream& model_istream) { - if (model_proto_ != nullptr) { + if (model_loaded_) { return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "ModelProto corresponding to the model to be loaded has already been parsed. " "Invoke Load()."); } auto loader = [this, &model_istream](std::shared_ptr& model) { - auto model_proto = onnxruntime::make_unique(); + ModelProto model_proto; google::protobuf::io::IstreamInputStream zero_copy_input(&model_istream); - const bool result = model_proto->ParseFromZeroCopyStream(&zero_copy_input) && model_istream.eof(); + const bool result = model_proto.ParseFromZeroCopyStream(&zero_copy_input) && model_istream.eof(); if (!result) { return Status(common::ONNXRUNTIME, common::INVALID_PROTOBUF, "Failed to load model because protobuf parsing failed."); } #ifdef ENABLE_LANGUAGE_INTEROP_OPS - LoadInterOp(*model_proto, interop_domains_, [&](const char* msg) { LOGS(*session_logger_, WARNING) << msg; }); + LoadInterOp(model_proto, interop_domains_, [&](const char* msg) { LOGS(*session_logger_, WARNING) << msg; }); for (const auto& domain : interop_domains_) { AddCustomOpDomains({domain.get()}); } @@ -511,22 +509,22 @@ common::Status InferenceSession::Load(std::istream& model_istream) { } common::Status InferenceSession::Load(const void* model_data, int model_data_len) { - if (model_proto_ != nullptr) { + if (model_loaded_) { return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "ModelProto corresponding to the model to be loaded has already been parsed. " "Invoke Load()."); } auto loader = [this, model_data, model_data_len](std::shared_ptr& model) { - auto model_proto = onnxruntime::make_unique(); + ModelProto model_proto; - const bool result = model_proto->ParseFromArray(model_data, model_data_len); + const bool result = model_proto.ParseFromArray(model_data, model_data_len); if (!result) { return Status(common::ONNXRUNTIME, common::INVALID_PROTOBUF, "Failed to load model because protobuf parsing failed."); } #ifdef ENABLE_LANGUAGE_INTEROP_OPS - LoadInterOp(*model_proto, interop_domains_, [&](const char* msg) { LOGS(*session_logger_, WARNING) << msg; }); + LoadInterOp(model_proto, interop_domains_, [&](const char* msg) { LOGS(*session_logger_, WARNING) << msg; }); for (const auto& domain : interop_domains_) { AddCustomOpDomains({domain.get()}); } @@ -540,7 +538,7 @@ common::Status InferenceSession::Load(const void* model_data, int model_data_len } common::Status InferenceSession::Load() { - if (model_proto_ == nullptr) { + if (!model_loaded_) { return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "ModelProto corresponding to the model to be loaded has not been parsed yet. " "This API should be called in conjunction with a ctor that takes a model abstraction."); @@ -548,7 +546,7 @@ common::Status InferenceSession::Load() { auto loader = [this](std::shared_ptr& model) { #ifdef ENABLE_LANGUAGE_INTEROP_OPS - LoadInterOp(*this->model_proto_, interop_domains_, [&](const char* msg) { LOGS(*session_logger_, WARNING) << msg; }); + LoadInterOp(this->model_proto_, interop_domains_, [&](const char* msg) { LOGS(*session_logger_, WARNING) << msg; }); for (const auto& domain : interop_domains_) { AddCustomOpDomains({domain.get()}); } @@ -823,7 +821,7 @@ common::Status InferenceSession::Initialize() { } // add predefined transformers - AddPredefinedTransformers(*graph_transformation_mgr_, session_options_.graph_optimization_level, + AddPredefinedTransformers(graph_transformation_mgr_, session_options_.graph_optimization_level, transformers_to_enable_); onnxruntime::Graph& graph = model_->MainGraph(); @@ -845,7 +843,7 @@ common::Status InferenceSession::Initialize() { ORT_RETURN_IF_ERROR_SESSIONID_(CreateSubgraphSessionState(graph, *session_state_)); // apply any transformations to the main graph and any subgraphs - ORT_RETURN_IF_ERROR_SESSIONID_(TransformGraph(graph, *graph_transformation_mgr_, + ORT_RETURN_IF_ERROR_SESSIONID_(TransformGraph(graph, graph_transformation_mgr_, execution_providers_, kernel_registry_manager_, insert_cast_transformer_, *session_state_)); diff --git a/onnxruntime/core/session/inference_session.h b/onnxruntime/core/session/inference_session.h index b4977a9de5..a2f5fa3dc1 100644 --- a/onnxruntime/core/session/inference_session.h +++ b/onnxruntime/core/session/inference_session.h @@ -441,7 +441,7 @@ class InferenceSession { SessionOptions session_options_; - std::unique_ptr graph_transformation_mgr_; + onnxruntime::GraphTransformerManager graph_transformation_mgr_; // List of transformers to run. When this list is not empty only the transformers in this list // will be run regardless of the level set. @@ -510,20 +510,20 @@ class InferenceSession { InterOpDomains interop_domains_; #endif // used to support platform telemetry - static std::atomic global_session_id_; // a monotonically increasing session id - uint32_t session_id_; // the current session's id + static std::atomic global_session_id_; // a monotonically increasing session id + uint32_t session_id_; // the current session's id struct Telemetry { Telemetry() : time_sent_last_(), time_sent_last_evalutation_start_() {} - uint32_t total_runs_since_last_ = 0; // the total number of Run() calls since the last report - long long total_run_duration_since_last_ = 0; // the total duration (us) of Run() calls since the last report - std::string event_name_; // where the model is loaded from: ["model_loading_uri", "model_loading_proto", "model_loading_istream"] + uint32_t total_runs_since_last_ = 0; // the total number of Run() calls since the last report + long long total_run_duration_since_last_ = 0; // the total duration (us) of Run() calls since the last report + std::string event_name_; // where the model is loaded from: ["model_loading_uri", "model_loading_proto", "model_loading_istream"] - TimePoint time_sent_last_; // the TimePoint of the last report + TimePoint time_sent_last_; // the TimePoint of the last report TimePoint time_sent_last_evalutation_start_; - // Event Rate per provider < 20 peak events per second - constexpr static long long kDurationBetweenSending = 1000 * 1000 * 60 * 10; // duration in (us). send a report every 10 mins - constexpr static long long kDurationBetweenSendingEvaluationStart = 1000 * 50; // duration in (us). send a EvaluationStop Event every 50 ms; + // Event Rate per provider < 20 peak events per second + constexpr static long long kDurationBetweenSending = 1000 * 1000 * 60 * 10; // duration in (us). send a report every 10 mins + constexpr static long long kDurationBetweenSendingEvaluationStart = 1000 * 50; // duration in (us). send a EvaluationStop Event every 50 ms; bool isEvaluationStart = false; } telemetry_; @@ -534,6 +534,8 @@ class InferenceSession { #endif // used to hold the ModelProto parsed in an applicable ctor to be used while calling parameter-less Load() - std::unique_ptr model_proto_; + ONNX_NAMESPACE::ModelProto model_proto_; + + bool model_loaded_ = false; }; } // namespace onnxruntime diff --git a/onnxruntime/test/ir/graph_test.cc b/onnxruntime/test/ir/graph_test.cc index fbbfd54dc9..1ea8bc8084 100644 --- a/onnxruntime/test/ir/graph_test.cc +++ b/onnxruntime/test/ir/graph_test.cc @@ -162,76 +162,76 @@ static void ConstructASimpleAddGraph(GraphProto& g, const char* domain) { } TEST_F(GraphTest, SimpleAddWithoutDomain) { - std::unique_ptr m = onnxruntime::make_unique(); - m->set_ir_version(3); - ImportOpset(*m, "", 10); - ConstructASimpleAddGraph(*m->mutable_graph(), nullptr); + ModelProto m; + m.set_ir_version(3); + ImportOpset(m, "", 10); + ConstructASimpleAddGraph(*m.mutable_graph(), nullptr); std::shared_ptr model; ASSERT_STATUS_OK(Model::Load(std::move(m), model, nullptr, *logger_)); } TEST_F(GraphTest, SimpleAddDefaultDomain) { - std::unique_ptr m = onnxruntime::make_unique(); - m->set_ir_version(3); - ImportOpset(*m, "", 10); - ConstructASimpleAddGraph(*m->mutable_graph(), ""); + ModelProto m; + m.set_ir_version(3); + ImportOpset(m, "", 10); + ConstructASimpleAddGraph(*m.mutable_graph(), ""); std::shared_ptr model; ASSERT_STATUS_OK(Model::Load(std::move(m), model, nullptr, *logger_)); } TEST_F(GraphTest, SimpleAddFutureOpSet) { - std::unique_ptr m = onnxruntime::make_unique(); - m->set_ir_version(3); - ImportOpset(*m, "", 9999); - ConstructASimpleAddGraph(*m->mutable_graph(), "ai.onnx"); + ModelProto m; + m.set_ir_version(3); + ImportOpset(m, "", 9999); + ConstructASimpleAddGraph(*m.mutable_graph(), "ai.onnx"); std::shared_ptr model; Status st; ASSERT_FALSE((st = Model::Load(std::move(m), model, nullptr, *logger_)).IsOK()); } TEST_F(GraphTest, SimpleAddONNXDomain) { - std::unique_ptr m = onnxruntime::make_unique(); - m->set_ir_version(3); - ImportOpset(*m, "", 10); - ConstructASimpleAddGraph(*m->mutable_graph(), "ai.onnx"); + ModelProto m; + m.set_ir_version(3); + ImportOpset(m, "", 10); + ConstructASimpleAddGraph(*m.mutable_graph(), "ai.onnx"); std::shared_ptr model; ASSERT_STATUS_OK(Model::Load(std::move(m), model, nullptr, *logger_)); } TEST_F(GraphTest, SimpleAddONNXDomain2) { - std::unique_ptr m = onnxruntime::make_unique(); - m->set_ir_version(3); - ImportOpset(*m, "ai.onnx", 10); - ConstructASimpleAddGraph(*m->mutable_graph(), "ai.onnx"); + ModelProto m; + m.set_ir_version(3); + ImportOpset(m, "ai.onnx", 10); + ConstructASimpleAddGraph(*m.mutable_graph(), "ai.onnx"); std::shared_ptr model; ASSERT_STATUS_OK(Model::Load(std::move(m), model, nullptr, *logger_)); } TEST_F(GraphTest, SimpleAddWrongDomain) { - std::unique_ptr m = onnxruntime::make_unique(); - m->set_ir_version(3); - ImportOpset(*m, "", 10); - ConstructASimpleAddGraph(*m->mutable_graph(), "AAAA"); + ModelProto m; + m.set_ir_version(3); + ImportOpset(m, "", 10); + ConstructASimpleAddGraph(*m.mutable_graph(), "AAAA"); std::shared_ptr model; Status st; ASSERT_FALSE((st = Model::Load(std::move(m), model, nullptr, *logger_)).IsOK()); } TEST_F(GraphTest, SimpleAddWrongDomain2) { - std::unique_ptr m = onnxruntime::make_unique(); - m->set_ir_version(3); - ImportOpset(*m, "AAAA", 10); - ConstructASimpleAddGraph(*m->mutable_graph(), "AAAA"); + ModelProto m; + m.set_ir_version(3); + ImportOpset(m, "AAAA", 10); + ConstructASimpleAddGraph(*m.mutable_graph(), "AAAA"); std::shared_ptr model; Status st; ASSERT_FALSE((st = Model::Load(std::move(m), model, nullptr, *logger_)).IsOK()); } TEST_F(GraphTest, SimpleUnique) { - std::unique_ptr m = onnxruntime::make_unique(); - m->set_ir_version(3); - ImportOpset(*m, "", 11); - GraphProto& g = *m->mutable_graph(); + ModelProto m; + m.set_ir_version(3); + ImportOpset(m, "", 11); + GraphProto& g = *m.mutable_graph(); NodeProto* node = g.add_node(); *node->add_input() = "x"; *node->add_output() = "sum"; @@ -248,11 +248,11 @@ TEST_F(GraphTest, SimpleUnique) { } TEST_F(GraphTest, WrongOpset) { - std::unique_ptr m = onnxruntime::make_unique(); - m->set_ir_version(3); + ModelProto m; + m.set_ir_version(3); //No Op registered for Unique with domain_version of 1 - ImportOpset(*m, "", 1); - GraphProto& g = *m->mutable_graph(); + ImportOpset(m, "", 1); + GraphProto& g = *m.mutable_graph(); NodeProto* node = g.add_node(); *node->add_input() = "x"; *node->add_output() = "sum"; @@ -270,11 +270,11 @@ TEST_F(GraphTest, WrongOpset) { } TEST_F(GraphTest, ExtraInput) { - std::unique_ptr m = onnxruntime::make_unique(); - m->set_ir_version(3); + ModelProto m; + m.set_ir_version(3); //Node () has input size 2 not in range [min=1, max=1]. - ImportOpset(*m, "", 11); - GraphProto& g = *m->mutable_graph(); + ImportOpset(m, "", 11); + GraphProto& g = *m.mutable_graph(); NodeProto* node = g.add_node(); *node->add_input() = "x"; *node->add_input() = "y"; @@ -297,10 +297,10 @@ TEST_F(GraphTest, LocalCustomRegistry) { std::vector schema = { OpSchema().SetName("FakeUnique").Input(0, "X", "A N-D input tensor that is to be processed.", "T").Output(0, "Y", "desc", "T").TypeConstraint("T", OpSchema::all_tensor_types(), "Constrain input and output types to any tensor type.").SetDomain("FakeTestDomain")}; ASSERT_TRUE(registry->RegisterOpSet(schema, "FakeTestDomain", 0, 1).IsOK()); - std::unique_ptr m = onnxruntime::make_unique(); - m->set_ir_version(3); - ImportOpset(*m, "FakeTestDomain", 1); - GraphProto& g = *m->mutable_graph(); + ModelProto m; + m.set_ir_version(3); + ImportOpset(m, "FakeTestDomain", 1); + GraphProto& g = *m.mutable_graph(); NodeProto* node = g.add_node(); *node->add_input() = "x"; *node->add_output() = "sum"; @@ -323,11 +323,11 @@ TEST_F(GraphTest, LocalCustomRegistryWrongOpsetImportVersion) { std::vector schema = { OpSchema().SetName("FakeUnique").Input(0, "X", "A N-D input tensor that is to be processed.", "T").Output(0, "Y", "desc", "T").TypeConstraint("T", OpSchema::all_tensor_types(), "Constrain input and output types to any tensor type.").SetDomain("FakeTestDomain")}; ASSERT_TRUE(registry->RegisterOpSet(schema, "FakeTestDomain", 0, 1).IsOK()); - std::unique_ptr m = onnxruntime::make_unique(); - m->set_ir_version(3); + ModelProto m; + m.set_ir_version(3); //Should be 1, but we put 11 herer so the model loading will fail - ImportOpset(*m, "FakeTestDomain", 11); - GraphProto& g = *m->mutable_graph(); + ImportOpset(m, "FakeTestDomain", 11); + GraphProto& g = *m.mutable_graph(); NodeProto* node = g.add_node(); *node->add_input() = "x"; *node->add_output() = "sum"; diff --git a/onnxruntime/test/ir/onnx_model_test.cc b/onnxruntime/test/ir/onnx_model_test.cc index 20339b7716..55722380e2 100644 --- a/onnxruntime/test/ir/onnx_model_test.cc +++ b/onnxruntime/test/ir/onnx_model_test.cc @@ -142,8 +142,8 @@ TEST_P(ONNXModelsTest1, LoadFromProtobuf) { std::unique_ptr raw_input(new FileInputStream(fd)); std::unique_ptr coded_input(new CodedInputStream(raw_input.get())); coded_input->SetTotalBytesLimit(INT_MAX); - std::unique_ptr model_proto = onnxruntime::make_unique(); - bool result = model_proto->ParseFromCodedStream(coded_input.get()); + ModelProto model_proto; + bool result = model_proto.ParseFromCodedStream(coded_input.get()); coded_input.reset(); raw_input.reset(); ASSERT_TRUE(result); diff --git a/onnxruntime/test/optimizer/free_dimension_override_test.cc b/onnxruntime/test/optimizer/free_dimension_override_test.cc index 0961650a45..71dbc9f8b9 100644 --- a/onnxruntime/test/optimizer/free_dimension_override_test.cc +++ b/onnxruntime/test/optimizer/free_dimension_override_test.cc @@ -30,10 +30,10 @@ TEST(FreeDimensionOverrideTransformerTest, Test) { // and DATA_CHANNEL. Supplying these overrides to the transformer should replace those free // dimensions with values of 1 and 42, respectively. std::vector overrides = - { - FreeDimensionOverride{ onnx::DATA_BATCH, 1 }, - FreeDimensionOverride{ onnx::DATA_CHANNEL, 42 }, - }; + { + FreeDimensionOverride{onnx::DATA_BATCH, 1}, + FreeDimensionOverride{onnx::DATA_CHANNEL, 42}, + }; auto graph_transformer = onnxruntime::make_unique(overrides); @@ -46,10 +46,10 @@ TEST(FreeDimensionOverrideTransformerTest, Test) { // Verify that the shape of the input graph has the correct values const auto& graph_inputs = graph.GetInputs(); - ASSERT_TRUE(graph_inputs.size() == 1); // This model only has a single input ('x') + ASSERT_TRUE(graph_inputs.size() == 1); // This model only has a single input ('x') const auto* input_shape = graph_inputs[0]->Shape(); - ASSERT_TRUE(input_shape->dim_size() == 3); // Model takes a 3D tensor as input; two of those dimensions are (were) free dimensions + ASSERT_TRUE(input_shape->dim_size() == 3); // Model takes a 3D tensor as input; two of those dimensions are (were) free dimensions ASSERT_TRUE(input_shape->dim(0).denotation() == onnx::DATA_BATCH); ASSERT_TRUE(input_shape->dim(0).has_dim_value()); diff --git a/onnxruntime/test/optimizer/graph_transform_test.cc b/onnxruntime/test/optimizer/graph_transform_test.cc index 39c9eed51d..92e4632411 100644 --- a/onnxruntime/test/optimizer/graph_transform_test.cc +++ b/onnxruntime/test/optimizer/graph_transform_test.cc @@ -1502,22 +1502,22 @@ static void TestSkipLayerNormFusion(const std::basic_string& file_pat std::map op_to_count = CountOpsInGraph(graph); ASSERT_TRUE(op_to_count["Div"] == 0); - ASSERT_TRUE(op_to_count["Add"] == add_count ); + ASSERT_TRUE(op_to_count["Add"] == add_count); ASSERT_TRUE(op_to_count["Sub"] == 0); ASSERT_TRUE(op_to_count["ReduceMean"] == 0); ASSERT_TRUE(op_to_count["Pow"] == 0); ASSERT_TRUE(op_to_count["Sqrt"] == 0); - ASSERT_TRUE(op_to_count["LayerNormalization"] == ln_count ); - ASSERT_TRUE(op_to_count["SkipLayerNormalization"] == skip_ln_count ); + ASSERT_TRUE(op_to_count["LayerNormalization"] == ln_count); + ASSERT_TRUE(op_to_count["SkipLayerNormalization"] == skip_ln_count); } TEST(GraphTransformationTests, SkipLayerNormFusionTest) { TestSkipLayerNormFusion(MODEL_FOLDER "fusion/skip_layer_norm_format1.onnx", 0, 0, 1); - TestSkipLayerNormFusion(MODEL_FOLDER "fusion/skip_layer_norm_format2.onnx", 0, 0, 1 ); - TestSkipLayerNormFusion(MODEL_FOLDER "fusion/skip_layer_norm_format3.onnx", 0, 0, 1 ); - TestSkipLayerNormFusion( MODEL_FOLDER "fusion/skip_layer_norm_format1_partial.onnx", 1, 0, 1 ); - TestSkipLayerNormFusion( MODEL_FOLDER "fusion/skip_layer_norm_format2_partial.onnx", 1, 0, 1 ); - TestSkipLayerNormFusion( MODEL_FOLDER "fusion/skip_layer_norm_format3_no_fusion.onnx", 1, 1, 0 ); + TestSkipLayerNormFusion(MODEL_FOLDER "fusion/skip_layer_norm_format2.onnx", 0, 0, 1); + TestSkipLayerNormFusion(MODEL_FOLDER "fusion/skip_layer_norm_format3.onnx", 0, 0, 1); + TestSkipLayerNormFusion(MODEL_FOLDER "fusion/skip_layer_norm_format1_partial.onnx", 1, 0, 1); + TestSkipLayerNormFusion(MODEL_FOLDER "fusion/skip_layer_norm_format2_partial.onnx", 1, 0, 1); + TestSkipLayerNormFusion(MODEL_FOLDER "fusion/skip_layer_norm_format3_no_fusion.onnx", 1, 1, 0); } TEST(GraphTransformationTests, EmbedLayerNormFusionFormat1) { @@ -1666,4 +1666,4 @@ TEST(GraphTransformationTests, EmbedLayerNormFusionFormat5) { #endif } // namespace test -} // namespace onnxruntime +} // namespace onnxruntime \ No newline at end of file diff --git a/onnxruntime/test/optimizer/rule_based_graph_transformer_test.cc b/onnxruntime/test/optimizer/rule_based_graph_transformer_test.cc index 5b5d7cb33f..3de36dc5c4 100644 --- a/onnxruntime/test/optimizer/rule_based_graph_transformer_test.cc +++ b/onnxruntime/test/optimizer/rule_based_graph_transformer_test.cc @@ -25,14 +25,14 @@ TEST(RuleBasedGraphTransformerTest, TestCompatibleProviders) { DefaultLoggingManager().DefaultLogger()) .IsOK()); Graph& graph = model->MainGraph(); - + // Create rule based transformer with a dummy rewrite rule and register it with Cuda as compatible provider std::unordered_set compatible_provider{onnxruntime::kCudaExecutionProvider}; auto dummy_rule = onnxruntime::make_unique("DummyRule"); const auto* dummy_rule_ptr = dummy_rule.get(); auto graph_transformer = onnxruntime::make_unique("CUDATopDownTransformer", compatible_provider); - graph_transformer->Register(std::move(dummy_rule)); + graph_transformer->Register(std::move(dummy_rule)); // Create rule based transformer with a dummy rewrite rule and register it with CPU as compatible provider auto dummy_rule1 = onnxruntime::make_unique("DummyRule1"); @@ -40,7 +40,7 @@ TEST(RuleBasedGraphTransformerTest, TestCompatibleProviders) { auto graph_transformer1 = onnxruntime::make_unique("CPUTopDownTransformer"); - graph_transformer1->Register(std::move(dummy_rule1)); + graph_transformer1->Register(std::move(dummy_rule1)); onnxruntime::GraphTransformerManager graph_transformation_mgr{5}; graph_transformation_mgr.Register(std::move(graph_transformer), TransformerLevel::Level2); @@ -56,5 +56,17 @@ TEST(RuleBasedGraphTransformerTest, TestCompatibleProviders) { ASSERT_TRUE(dummy_rule1_ptr->IsRewriteRuleInvoked()); } +TEST(RuleBasedGraphTransformerTest, TestSettingStepsInGraphTransformerManager) { + // steps provided at object construction time + onnxruntime::GraphTransformerManager graph_transformation_mgr{5}; + unsigned steps_queried; + graph_transformation_mgr.GetSteps(steps_queried); + ASSERT_EQ(steps_queried, static_cast(5)); + + // steps upadted + graph_transformation_mgr.SetSteps(10); + graph_transformation_mgr.GetSteps(steps_queried); + ASSERT_EQ(steps_queried, static_cast (10)); +} } // namespace test -} // namespace onnxruntime +} // namespace onnxruntime \ No newline at end of file