From 197fd9ea3defc52a3534dcaf40bd527b91a14307 Mon Sep 17 00:00:00 2001 From: Ryan Lai Date: Wed, 27 Nov 2019 15:31:30 -0800 Subject: [PATCH] Remove usage of IOBinding in WinML and use C_API Run method (#2504) * remove usage of iobinding * Change data structure to use vector of Ort::Values * Polish bind input / output * Use C APIrun method * Update providers on evaluate getresults * Remove run and IObinding interface from WinMLAdapter * Remove use of IObinding * bind unbound outputs code moved to learningmodelbinding * clean up unneeded istensor adapter function * Fix comment * Check if session is closed before binding and clearing * PR feedback --- winml/lib/Api.Core/WinMLAdapter.cpp | 85 +++------------ winml/lib/Api.Core/inc/WinMLAdapter.h | 21 ++-- winml/lib/Api/LearningModelBinding.cpp | 142 +++++++++++++++++++++---- winml/lib/Api/LearningModelBinding.h | 15 ++- winml/lib/Api/LearningModelSession.cpp | 101 +++++++----------- winml/lib/Api/LearningModelSession.h | 10 +- 6 files changed, 204 insertions(+), 170 deletions(-) diff --git a/winml/lib/Api.Core/WinMLAdapter.cpp b/winml/lib/Api.Core/WinMLAdapter.cpp index b25b92e7c2..8d1a1af801 100644 --- a/winml/lib/Api.Core/WinMLAdapter.cpp +++ b/winml/lib/Api.Core/WinMLAdapter.cpp @@ -30,6 +30,8 @@ #include "google/protobuf/io/zero_copy_stream_impl.h" #include "FeatureDescriptorFactory.h" +#include "core\framework\utils.h" +#include "core\framework\session_state.h" using namespace winrt::Windows::AI::MachineLearning; @@ -499,7 +501,7 @@ class WinMLAdapter : public Microsoft::WRL::RuntimeClass< #endif USE_DML } -HRESULT STDMETHODCALLTYPE GetOperatorRegistry(ILearningModelOperatorProviderNative* operator_provider_native, IMLOperatorRegistry** registry) override { + HRESULT STDMETHODCALLTYPE GetOperatorRegistry(ILearningModelOperatorProviderNative* operator_provider_native, IMLOperatorRegistry** registry) override { #ifdef USE_DML // Retrieve the "operator abi" registry. winrt::com_ptr operator_registry; @@ -509,7 +511,7 @@ HRESULT STDMETHODCALLTYPE GetOperatorRegistry(ILearningModelOperatorProviderNati #else return E_NOTIMPL; #endif USE_DML -} + } void* STDMETHODCALLTYPE CreateGPUAllocationFromD3DResource(ID3D12Resource* pResource) override { #ifdef USE_DML @@ -529,7 +531,7 @@ HRESULT STDMETHODCALLTYPE GetOperatorRegistry(ILearningModelOperatorProviderNati onnxruntime::IExecutionProvider* provider, OrtValue* src, OrtValue* dst) override { -#ifdef USE_DML +#ifdef USE_DML ORT_THROW_IF_ERROR(Dml::CopyTensor(provider, *(src->GetMutable()), *(dst->GetMutable()))); return S_OK; #else @@ -550,7 +552,7 @@ HRESULT STDMETHODCALLTYPE GetOperatorRegistry(ILearningModelOperatorProviderNati }); return S_OK; #else - return S_OK; // needs to return S_OK otherwise everything breaks because this gets called from the learningmodel constructor + return S_OK; // needs to return S_OK otherwise everything breaks because this gets called from the learningmodel constructor #endif USE_DML } @@ -611,7 +613,6 @@ HRESULT STDMETHODCALLTYPE GetOperatorRegistry(ILearningModelOperatorProviderNati return S_OK; } - }; // namespace Windows::AI::MachineLearning::Adapter extern "C" HRESULT STDMETHODCALLTYPE OrtGetWinMLAdapter(IWinMLAdapter** adapter) { @@ -620,52 +621,6 @@ extern "C" HRESULT STDMETHODCALLTYPE OrtGetWinMLAdapter(IWinMLAdapter** adapter) return adapterptr.CopyTo(__uuidof(IWinMLAdapter), reinterpret_cast(adapter)); } -// class IOBinding -// =============== -class IOBinding : public Microsoft::WRL::RuntimeClass< - Microsoft::WRL::RuntimeClassFlags, - IIOBinding> { - private: - std::shared_ptr binding_; - std::vector outputs_weak_; - - public: - IOBinding(onnxruntime::IOBinding* binding) : binding_(binding) { - } - - onnxruntime::IOBinding* STDMETHODCALLTYPE get() override { - return binding_.get(); - } - - HRESULT STDMETHODCALLTYPE BindInput(const std::string& name, OrtValue* ort_value) override { - ORT_THROW_IF_ERROR(binding_->BindInput(name, *ort_value)); - return S_OK; - } - - HRESULT STDMETHODCALLTYPE BindOutput(const std::string& name, OrtValue* ort_value) override { - // this can be null for unbound outputs - if (ort_value == nullptr) { - OrtValue empty_value = {}; - ORT_THROW_IF_ERROR(binding_->BindOutput(name, empty_value)); - } else { - ORT_THROW_IF_ERROR(binding_->BindOutput(name, *ort_value)); - } - return S_OK; - } - - const std::vector& STDMETHODCALLTYPE GetOutputNames() override { - return binding_->GetOutputNames(); - } - std::vector& STDMETHODCALLTYPE GetOutputs() override { - auto& output_inner = binding_->GetOutputs(); - outputs_weak_.clear(); - for (unsigned i = 0; i < output_inner.size(); i++) { - outputs_weak_.push_back(&(output_inner[i])); - } - return outputs_weak_; - } -}; - // InferenceSession // ================ @@ -679,18 +634,7 @@ void STDMETHODCALLTYPE InferenceSession::RegisterGraphTransformers() { #endif USE_DML } -HRESULT STDMETHODCALLTYPE InferenceSession::NewIOBinding(IIOBinding** io_binding) { - std::unique_ptr binding; - ORT_THROW_IF_ERROR(this->session_->NewIOBinding(&binding)); - auto io_binding_outer = wil::MakeOrThrow(binding.release()); - return io_binding_outer.CopyTo(__uuidof(IIOBinding), reinterpret_cast(io_binding)); -} - -HRESULT STDMETHODCALLTYPE InferenceSession::Run(const onnxruntime::RunOptions* run_options, IIOBinding* io_binding) { - ORT_THROW_IF_ERROR(this->session_->Run(*run_options, *(io_binding->get()))); - return S_OK; -} -HRESULT STDMETHODCALLTYPE InferenceSession::StartProfiling() { + HRESULT STDMETHODCALLTYPE InferenceSession::StartProfiling() { this->session_->StartProfiling(PheonixSingleton()->GetDefaultLogger()); return S_OK; } @@ -746,10 +690,15 @@ void STDMETHODCALLTYPE InferenceSession::ReleaseCompletedReferences(onnxruntime: } HRESULT STDMETHODCALLTYPE InferenceSession::CopyOneInputAcrossDevices( - const char* input_name, - const OrtValue* orig_mlvalue, - OrtValue** new_mlvalue) { - return E_NOTIMPL; + const char* input_name, + const OrtValue* orig_mlvalue, + OrtValue** new_mlvalue) { + auto session_protected_load_accessor = + static_cast(session_.get()); + const onnxruntime::SessionState& sessionState = session_protected_load_accessor->GetSessionState(); + auto temp_mlvalue = std::make_unique(); + ORT_THROW_IF_ERROR(onnxruntime::utils::CopyOneInputAcrossDevices(sessionState, input_name, *orig_mlvalue, *temp_mlvalue.get())); + *new_mlvalue = temp_mlvalue.release(); + return S_OK; } - } // namespace Windows::AI::MachineLearning::Adapter \ No newline at end of file diff --git a/winml/lib/Api.Core/inc/WinMLAdapter.h b/winml/lib/Api.Core/inc/WinMLAdapter.h index eeb50a15b9..e6fb12b78a 100644 --- a/winml/lib/Api.Core/inc/WinMLAdapter.h +++ b/winml/lib/Api.Core/inc/WinMLAdapter.h @@ -20,16 +20,6 @@ MIDL_INTERFACE("eaae30b5-7381-432d-9730-322136b02371") IModelInfo : IUnknown{ virtual HRESULT STDMETHODCALLTYPE GetOutputFeatures(ABI::Windows::Foundation::Collections::IVectorView * *features) = 0; }; -MIDL_INTERFACE("438e7719-554a-4058-84d9-eb6226c34887") IIOBinding : IUnknown{ - // this returns a weak ref - virtual onnxruntime::IOBinding* STDMETHODCALLTYPE get() = 0; - virtual HRESULT STDMETHODCALLTYPE BindInput(const std::string& name, OrtValue * ml_value) = 0; - virtual HRESULT STDMETHODCALLTYPE BindOutput(const std::string& name, OrtValue * ml_value) = 0; - virtual const std::vector& STDMETHODCALLTYPE GetOutputNames() = 0; - // this returns a weak ref - virtual std::vector& STDMETHODCALLTYPE GetOutputs() = 0; -}; - MIDL_INTERFACE("a848faf6-5a2e-4a7f-b622-cc036f71e28a") IModelProto : IUnknown{ // this returns a weak ref virtual onnx::ModelProto* STDMETHODCALLTYPE get() = 0; @@ -39,11 +29,11 @@ MIDL_INTERFACE("a848faf6-5a2e-4a7f-b622-cc036f71e28a") IModelProto : IUnknown{ MIDL_INTERFACE("6ec766ef-6365-42bf-b64f-ae85c015adb8") IInferenceSession : IUnknown { virtual onnxruntime::InferenceSession* STDMETHODCALLTYPE get() = 0; + // the below returns a weak ref , DO NOT RELEASE IT + virtual HRESULT STDMETHODCALLTYPE GetOrtSession(OrtSession ** out) = 0; virtual void STDMETHODCALLTYPE RegisterGraphTransformers() = 0; virtual HRESULT STDMETHODCALLTYPE RegisterCustomRegistry(IMLOperatorRegistry * registry) = 0; virtual HRESULT STDMETHODCALLTYPE LoadModel(IModelProto* model_proto) = 0; - virtual HRESULT STDMETHODCALLTYPE NewIOBinding(IIOBinding** io_binding) = 0; - virtual HRESULT STDMETHODCALLTYPE Run(const onnxruntime::RunOptions* run_options, IIOBinding* io_binding) = 0; virtual HRESULT STDMETHODCALLTYPE StartProfiling() = 0; virtual HRESULT STDMETHODCALLTYPE EndProfiling() = 0; virtual void STDMETHODCALLTYPE FlushContext(onnxruntime::IExecutionProvider * dml_provider) = 0; @@ -130,11 +120,14 @@ public: InferenceSession(onnxruntime::InferenceSession * session); onnxruntime::InferenceSession* STDMETHODCALLTYPE get() override { return session_.get(); } + HRESULT STDMETHODCALLTYPE GetOrtSession(OrtSession ** out) override { + // (OrtSession *) are really (InferenceSession *) as well + *out = reinterpret_cast(session_.get()); + return S_OK; + } void STDMETHODCALLTYPE RegisterGraphTransformers() override; HRESULT STDMETHODCALLTYPE RegisterCustomRegistry(IMLOperatorRegistry* registry) override; HRESULT STDMETHODCALLTYPE LoadModel(IModelProto* model_proto) override; - HRESULT STDMETHODCALLTYPE NewIOBinding(IIOBinding** io_binding) override; - HRESULT STDMETHODCALLTYPE Run(const onnxruntime::RunOptions* run_options, IIOBinding* io_binding) override; HRESULT STDMETHODCALLTYPE StartProfiling() override; HRESULT STDMETHODCALLTYPE EndProfiling() override; void STDMETHODCALLTYPE FlushContext(onnxruntime::IExecutionProvider* dml_provider) override; diff --git a/winml/lib/Api/LearningModelBinding.cpp b/winml/lib/Api/LearningModelBinding.cpp index f126ffab7c..f45d3a0aca 100644 --- a/winml/lib/Api/LearningModelBinding.cpp +++ b/winml/lib/Api/LearningModelBinding.cpp @@ -8,13 +8,15 @@ #include "LearningModelBinding.h" #include "LearningModelSession.h" #include "TelemetryEvent.h" +#include +#include "LearningModel.h" using namespace WinML; namespace winrt::Windows::AI::MachineLearning::implementation { LearningModelBinding::LearningModelBinding( - Windows::AI::MachineLearning::LearningModelSession const& session) try : m_session(session) { - m_lotusBinding.attach(session.as()->CreateSessionBinding()); + Windows::AI::MachineLearning::LearningModelSession const& session) try : m_session(session) { + session.as()->CheckClosed(); WINML_THROW_IF_FAILED(OrtGetWinMLAdapter(adapter_.put())); } WINML_CATCH_ALL @@ -40,7 +42,7 @@ static Windows::AI::MachineLearning::ILearningModelFeatureDescriptor FindValidBi using NullableBindingPort = std::optional>; static NullableBindingPort FindValidBinding( - LearningModel model, + winml::LearningModel model, const std::wstring& name) { if (auto descriptor = FindValidBinding(model.InputFeatures(), name)) { return std::make_pair(descriptor, BindingType::kInput); @@ -156,13 +158,13 @@ void LearningModelBinding::Bind( auto featureName = WinML::Strings::UTF8FromHString(name); std::tie(bindingName, binding_value, bindingType) = CreateBinding(featureName, value, properties); - + Ort::Value ortValue = binding_value ? Ort::Value(binding_value) : Ort::Value(nullptr); switch (bindingType) { case BindingType::kInput: - WINML_THROW_IF_FAILED(m_lotusBinding->BindInput(bindingName, binding_value)); + WINML_THROW_IF_FAILED(BindInput(bindingName, ortValue)); break; case BindingType::kOutput: - WINML_THROW_IF_FAILED(m_lotusBinding->BindOutput(bindingName, binding_value)); + WINML_THROW_IF_FAILED(BindOutput(bindingName, ortValue)); break; default: FAIL_FAST(); @@ -171,7 +173,11 @@ void LearningModelBinding::Bind( WINML_CATCH_ALL void LearningModelBinding::Clear() try { - m_lotusBinding.attach(m_session.as()->CreateSessionBinding()); + m_session.as()->CheckClosed(); + inputs_.clear(); + input_names_.clear(); + outputs_.clear(); + output_names_.clear(); m_providers.clear(); } WINML_CATCH_ALL @@ -218,11 +224,6 @@ void LearningModelBinding::Split( throw hresult_not_implemented(); } -_winmla::IIOBinding* LearningModelBinding::BindingCollection() { - _winmla::IIOBinding* p; - m_lotusBinding.copy_to(&p); - return p; -} ONNXTensorElementDataType STDMETHODCALLTYPE GetONNXTensorElementDataType(winml::TensorKind kind) { if (kind == TensorKind::Float) { @@ -387,7 +388,7 @@ ILearningModelFeatureValue LearningModelBinding::CreateUnboundOuputFeatureValue( Windows::Foundation::IInspectable LearningModelBinding::CreateUnboundOutput( const std::string& name, - Ort::Value& ort_value) { + Ort::Value& ort_value) { // Find valid binding port auto bindingPort = FindValidBinding( m_session.Model(), @@ -440,8 +441,8 @@ Windows::Foundation::IInspectable LearningModelBinding::CreateUnboundOutput( std::unordered_map LearningModelBinding::UpdateProviders() { std::unordered_map outputs; - auto& outputNames = m_lotusBinding->GetOutputNames(); - auto& outputMLValues = m_lotusBinding->GetOutputs(); + auto& outputNames = GetOutputNames(); + auto& outputMLValues = GetOutputs(); WINML_THROW_HR_IF_FALSE_MSG( E_UNEXPECTED, outputNames.size() == outputMLValues.size(), @@ -449,7 +450,7 @@ std::unordered_map LearningModel for (unsigned i = 0; i < outputNames.size(); i++) { auto utf8Name = outputNames[i]; - auto mlValue = outputMLValues[i]; + OrtValue* mlValue = outputMLValues[i]; if (m_providers.find(utf8Name) != std::end(m_providers)) { auto& providerInfo = m_providers[utf8Name]; @@ -499,14 +500,13 @@ STDMETHODIMP LearningModelBinding::Bind( auto featureName = WinML::Strings::UTF8FromUnicode(name, cchName); std::tie(bindingName, binding_value_ptr, bindingType) = CreateBinding(featureName, to, nullptr); - Ort::Value bindingValue(binding_value_ptr); - + Ort::Value ortValue = binding_value_ptr ? Ort::Value(binding_value_ptr) : Ort::Value(nullptr); switch (bindingType) { case BindingType::kInput: - WINML_THROW_IF_FAILED(m_lotusBinding->BindInput(bindingName, bindingValue)); + WINML_THROW_IF_FAILED(BindInput(bindingName, ortValue)); break; case BindingType::kOutput: - WINML_THROW_IF_FAILED(m_lotusBinding->BindOutput(bindingName, bindingValue)); + WINML_THROW_IF_FAILED(BindOutput(bindingName, ortValue)); break; default: FAIL_FAST(); @@ -515,4 +515,106 @@ STDMETHODIMP LearningModelBinding::Bind( } WINML_CATCH_ALL_COM } + +static std::pair Contains(const std::vector& names, const std::string& name) { + auto it = std::find(std::begin(names), std::end(names), name); + if (it == std::end(names)) { + return {false, 0}; + } + return {true, it - std::begin(names)}; +} + +// This method releases control of memory of ml_value from caller of BindInput +HRESULT LearningModelBinding::BindInput(const std::string& name, Ort::Value& ml_value) { + auto rc = Contains(input_names_, name); + + auto add_or_replace = [this, &name](const bool exists, size_t index, Ort::Value& value) { + if (exists) { + inputs_[index] = Ort::Value(value.release()); + } else { + input_names_.push_back(name); + inputs_.push_back(Ort::Value(value.release())); + } + }; + if (ml_value.IsTensor()) { + Ort::Value new_mlvalue = Ort::Value(nullptr); + WINML_THROW_IF_FAILED(m_session.as() + ->GetIInferenceSession() + ->CopyOneInputAcrossDevices(name.c_str(), ml_value, new_mlvalue.put())); + add_or_replace(rc.first, rc.second, new_mlvalue); + } else { + add_or_replace(rc.first, rc.second, ml_value); + } + return S_OK; +} + +// This method releases control of memory of ml_value from caller of BindInput +HRESULT LearningModelBinding::BindOutput(const std::string& name, Ort::Value& ml_value) { + auto rc = Contains(output_names_, name); + OrtValue* ml_value_data = ml_value.release(); + if (rc.first) { + outputs_[rc.second] = ml_value_data ? Ort::Value(ml_value_data) : Ort::Value(nullptr); + return S_OK; + } + + output_names_.push_back(name); + outputs_.push_back(ml_value_data ? Ort::Value(ml_value_data) : Ort::Value(nullptr)); + return S_OK; +} + +const std::vector& LearningModelBinding::GetOutputNames() const { + return output_names_; +} + +std::vector& LearningModelBinding::GetOutputs() { return outputs_; } + +const std::vector& LearningModelBinding::GetInputNames() const { + return input_names_; +} + +const std::vector& LearningModelBinding::GetInputs() const { return inputs_; } + +void LearningModelBinding::BindUnboundOutputs() +{ + auto& bound_output_names = GetOutputNames(); + std::unordered_set bound_output_names_set( + bound_output_names.begin(), + bound_output_names.end()); + + // Get model output feature names + auto model_impl = m_session.Model().as(); + auto output_features = model_impl->OutputFeatures(); + std::vector output_descriptors( + begin(output_features), + end(output_features)); + + // Convert all output features to their feature names + std::vector output_feature_names; + std::transform( + std::begin(output_descriptors), + std::end(output_descriptors), + std::back_inserter(output_feature_names), + [&](auto& descriptor) { + auto descriptor_native = descriptor.as(); + const wchar_t* p_name; + uint32_t size; + WINML_THROW_IF_FAILED(descriptor_native->GetName(&p_name, &size)); + return WinML::Strings::UTF8FromUnicode(p_name, size); + }); + + // Find the set difference to determine if there are any unbound output features + std::vector unbound_output_names; + std::copy_if( + std::begin(output_feature_names), std::end(output_feature_names), + std::inserter(unbound_output_names, std::begin(unbound_output_names)), + [&](const auto& outputFeatureName) { + return bound_output_names_set.find(outputFeatureName) == bound_output_names_set.end(); + }); + + // Add all unbound outputs to binding collection + for (const auto& unbound_output : unbound_output_names) { + WINML_THROW_IF_FAILED(BindOutput(unbound_output, Ort::Value(nullptr))); + } +} + } // namespace winrt::Windows::AI::MachineLearning::implementation \ No newline at end of file diff --git a/winml/lib/Api/LearningModelBinding.h b/winml/lib/Api/LearningModelBinding.h index e1b33ad74e..21e9e6b2b8 100644 --- a/winml/lib/Api/LearningModelBinding.h +++ b/winml/lib/Api/LearningModelBinding.h @@ -41,7 +41,6 @@ struct LearningModelBinding : LearningModelBindingT UpdateProviders(); const Windows::AI::MachineLearning::LearningModelSession& GetSession() { return m_session; } @@ -52,6 +51,13 @@ struct LearningModelBinding : LearningModelBindingT& LearningModelBinding::GetOutputNames() const; + std::vector& LearningModelBinding::GetOutputs(); + const std::vector& LearningModelBinding::GetInputNames() const; + const std::vector& LearningModelBinding::GetInputs() const; + HRESULT BindOutput(const std::string& name, Ort::Value& ml_value); + void BindUnboundOutputs(); + private: void CacheProvider(std::string name, ProviderInfo& spProvider); Windows::Foundation::IInspectable CreateUnboundOutput(const std::string& name, Ort::Value& ort_value); @@ -61,15 +67,18 @@ struct LearningModelBinding : LearningModelBindingT m_providers; - com_ptr<_winmla::IIOBinding> m_lotusBinding; com_ptr<_winmla::IWinMLAdapter> adapter_; + std::vector input_names_; + std::vector inputs_; + std::vector output_names_; + std::vector outputs_; }; } // namespace winrt::Windows::AI::MachineLearning::implementation diff --git a/winml/lib/Api/LearningModelSession.cpp b/winml/lib/Api/LearningModelSession.cpp index 9370fb8116..6eaac423ff 100644 --- a/winml/lib/Api/LearningModelSession.cpp +++ b/winml/lib/Api/LearningModelSession.cpp @@ -198,54 +198,20 @@ LearningModelSession::EvaluateFeaturesAsync( return EvaluateAsync(binding, correlation_id); } -static _winmla::IIOBinding* -GetIOBinding( - winrt::com_ptr binding_impl, - winml::LearningModel& model) { - // Get the IOBinding Collection, and bound outputs - com_ptr<_winmla::IIOBinding> io_binding; - io_binding.attach(binding_impl->BindingCollection()); - auto& bound_output_names = io_binding->GetOutputNames(); - std::unordered_set bound_output_names_set( - bound_output_names.begin(), - bound_output_names.end()); - - // Get model output feature names - auto model_impl = model.as(); - auto output_features = model_impl->OutputFeatures(); - std::vector output_descriptors( - begin(output_features), - end(output_features)); - - // Convert all output features to their feature names - std::vector output_feature_names; - std::transform( - std::begin(output_descriptors), - std::end(output_descriptors), - std::back_inserter(output_feature_names), - [&](auto& descriptor) { - auto descriptor_native = descriptor.as(); - const wchar_t* p_name; - uint32_t size; - WINML_THROW_IF_FAILED(descriptor_native->GetName(&p_name, &size)); - return WinML::Strings::UTF8FromUnicode(p_name, size); - }); - - // Find the set difference to determine if there are any unbound output features - std::vector unbound_output_names; - std::copy_if( - std::begin(output_feature_names), std::end(output_feature_names), - std::inserter(unbound_output_names, std::begin(unbound_output_names)), - [&](const auto& outputFeatureName) { - return bound_output_names_set.find(outputFeatureName) == bound_output_names_set.end(); - }); - - // Add all unbound outputs to the iobinding collection - for (const auto& unbound_output : unbound_output_names) { - WINML_THROW_IF_FAILED(io_binding->BindOutput(unbound_output, nullptr)); - } - - return io_binding.detach(); +// copied from onnxruntime_cxx_inline.h +inline OrtStatus* OrtRun( + OrtSession * session, + const Ort::RunOptions& run_options, + const char* const* input_names, + const Ort::Value* input_values, + size_t input_count, + const char* const* output_names, + Ort::Value* output_values, + size_t output_count) { + static_assert(sizeof(Ort::Value) == sizeof(OrtValue*), "Value is really just an array of OrtValue* in memory, so we can reinterpret_cast safely"); + auto ort_input_values = reinterpret_cast(const_cast(input_values)); + auto ort_output_values = reinterpret_cast(output_values); + return Ort::GetApi().Run(session, run_options, input_names, ort_input_values, input_count, output_names, output_count, ort_output_values); } uint64_t @@ -255,13 +221,31 @@ LearningModelSession::Run( auto device = device_.as(); CWinMLAutoLock lock(!device->IsCpuDevice() ? &evaluate_lock_ : nullptr); // TODO : set the run_options - onnxruntime::RunOptions run_options; + Ort::RunOptions run_options; + binding_impl->BindUnboundOutputs(); - com_ptr<_winmla::IIOBinding> io_binding; - io_binding.attach(GetIOBinding(binding_impl, model_)); + std::vector inputNames_c; + for (int i=0; i < binding_impl->GetInputNames().size(); i++) + { + inputNames_c.push_back(binding_impl->GetInputNames()[i].c_str()); + } + std::vector outputNames_c; + for (int i = 0; i < binding_impl->GetOutputNames().size(); i++) { + outputNames_c.push_back(binding_impl->GetOutputNames()[i].c_str()); + } + OrtSession* session = nullptr; + WINML_THROW_IF_FAILED(inference_session_->GetOrtSession(&session)); // Invoke run on the ORT session. - WINML_THROW_IF_FAILED(inference_session_->Run(&run_options, io_binding.get())); + Ort::ThrowOnError(OrtRun( + session, + run_options, + inputNames_c.data(), + binding_impl->GetInputs().data(), + binding_impl->GetInputs().size(), + outputNames_c.data(), + binding_impl->GetOutputs().data(), + binding_impl->GetOutputs().size())); if (!device->IsCpuDevice()) { // Flush the D3D12 work from the DML execution provider and queue a fence before we release the lock. @@ -409,14 +393,6 @@ void LearningModelSession::Close() { inference_session_ = nullptr; } -_winmla::IIOBinding* -LearningModelSession::CreateSessionBinding() { - CheckClosed(); - com_ptr<_winmla::IIOBinding> binding; - WINML_THROW_IF_FAILED(inference_session_->NewIOBinding(binding.put())); - return binding.detach(); -} - void LearningModelSession::ApplyEvaluationProperties() try { if (evaluation_properties_) { auto is_debug_output_enabled = evaluation_properties_.HasKey(c_enable_debug_output); @@ -449,6 +425,11 @@ LearningModelSession::GetExecutionProvider() { return cached_execution_provider_; } +_winmla::IInferenceSession* +LearningModelSession::GetIInferenceSession() { + return inference_session_.get(); +} + void LearningModelSession::CheckClosed() { if (!inference_session_) { WINML_THROW_HR(RO_E_CLOSED); diff --git a/winml/lib/Api/LearningModelSession.h b/winml/lib/Api/LearningModelSession.h index 6934597663..821510e502 100644 --- a/winml/lib/Api/LearningModelSession.h +++ b/winml/lib/Api/LearningModelSession.h @@ -68,8 +68,11 @@ struct LearningModelSession : LearningModelSessionT { onnxruntime::IExecutionProvider* GetExecutionProvider(); - _winmla::IIOBinding* - CreateSessionBinding(); + _winmla::IInferenceSession* + GetIInferenceSession(); + + void + CheckClosed(); private: void @@ -97,9 +100,6 @@ struct LearningModelSession : LearningModelSessionT { void ToggleProfiler(); - void - CheckClosed(); - private: com_ptr<_winmla::IInferenceSession> inference_session_;