mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-30 20:18:08 +00:00
Remove usage of IOBinding in WinML and use C_API Run method (#2504)
* remove usage of iobinding * Change data structure to use vector of Ort::Values * Polish bind input / output * Use C APIrun method * Update providers on evaluate getresults * Remove run and IObinding interface from WinMLAdapter * Remove use of IObinding * bind unbound outputs code moved to learningmodelbinding * clean up unneeded istensor adapter function * Fix comment * Check if session is closed before binding and clearing * PR feedback
This commit is contained in:
Ryan Lai
GitHub
parent
e8e285dd97
commit
197fd9ea3d
6 changed files with 204 additions and 170 deletions
|
|
@ -30,6 +30,8 @@
|
|||
#include "google/protobuf/io/zero_copy_stream_impl.h"
|
||||
|
||||
#include "FeatureDescriptorFactory.h"
|
||||
#include "core\framework\utils.h"
|
||||
#include "core\framework\session_state.h"
|
||||
|
||||
using namespace winrt::Windows::AI::MachineLearning;
|
||||
|
||||
|
|
@ -499,7 +501,7 @@ class WinMLAdapter : public Microsoft::WRL::RuntimeClass<
|
|||
#endif USE_DML
|
||||
}
|
||||
|
||||
HRESULT STDMETHODCALLTYPE GetOperatorRegistry(ILearningModelOperatorProviderNative* operator_provider_native, IMLOperatorRegistry** registry) override {
|
||||
HRESULT STDMETHODCALLTYPE GetOperatorRegistry(ILearningModelOperatorProviderNative* operator_provider_native, IMLOperatorRegistry** registry) override {
|
||||
#ifdef USE_DML
|
||||
// Retrieve the "operator abi" registry.
|
||||
winrt::com_ptr<IMLOperatorRegistry> operator_registry;
|
||||
|
|
@ -509,7 +511,7 @@ HRESULT STDMETHODCALLTYPE GetOperatorRegistry(ILearningModelOperatorProviderNati
|
|||
#else
|
||||
return E_NOTIMPL;
|
||||
#endif USE_DML
|
||||
}
|
||||
}
|
||||
|
||||
void* STDMETHODCALLTYPE CreateGPUAllocationFromD3DResource(ID3D12Resource* pResource) override {
|
||||
#ifdef USE_DML
|
||||
|
|
@ -529,7 +531,7 @@ HRESULT STDMETHODCALLTYPE GetOperatorRegistry(ILearningModelOperatorProviderNati
|
|||
onnxruntime::IExecutionProvider* provider,
|
||||
OrtValue* src,
|
||||
OrtValue* dst) override {
|
||||
#ifdef USE_DML
|
||||
#ifdef USE_DML
|
||||
ORT_THROW_IF_ERROR(Dml::CopyTensor(provider, *(src->GetMutable<onnxruntime::Tensor>()), *(dst->GetMutable<onnxruntime::Tensor>())));
|
||||
return S_OK;
|
||||
#else
|
||||
|
|
@ -550,7 +552,7 @@ HRESULT STDMETHODCALLTYPE GetOperatorRegistry(ILearningModelOperatorProviderNati
|
|||
});
|
||||
return S_OK;
|
||||
#else
|
||||
return S_OK; // needs to return S_OK otherwise everything breaks because this gets called from the learningmodel constructor
|
||||
return S_OK; // needs to return S_OK otherwise everything breaks because this gets called from the learningmodel constructor
|
||||
#endif USE_DML
|
||||
}
|
||||
|
||||
|
|
@ -611,7 +613,6 @@ HRESULT STDMETHODCALLTYPE GetOperatorRegistry(ILearningModelOperatorProviderNati
|
|||
|
||||
return S_OK;
|
||||
}
|
||||
|
||||
}; // namespace Windows::AI::MachineLearning::Adapter
|
||||
|
||||
extern "C" HRESULT STDMETHODCALLTYPE OrtGetWinMLAdapter(IWinMLAdapter** adapter) {
|
||||
|
|
@ -620,52 +621,6 @@ extern "C" HRESULT STDMETHODCALLTYPE OrtGetWinMLAdapter(IWinMLAdapter** adapter)
|
|||
return adapterptr.CopyTo(__uuidof(IWinMLAdapter), reinterpret_cast<void**>(adapter));
|
||||
}
|
||||
|
||||
// class IOBinding
|
||||
// ===============
|
||||
class IOBinding : public Microsoft::WRL::RuntimeClass<
|
||||
Microsoft::WRL::RuntimeClassFlags<Microsoft::WRL::ClassicCom>,
|
||||
IIOBinding> {
|
||||
private:
|
||||
std::shared_ptr<onnxruntime::IOBinding> binding_;
|
||||
std::vector<OrtValue*> outputs_weak_;
|
||||
|
||||
public:
|
||||
IOBinding(onnxruntime::IOBinding* binding) : binding_(binding) {
|
||||
}
|
||||
|
||||
onnxruntime::IOBinding* STDMETHODCALLTYPE get() override {
|
||||
return binding_.get();
|
||||
}
|
||||
|
||||
HRESULT STDMETHODCALLTYPE BindInput(const std::string& name, OrtValue* ort_value) override {
|
||||
ORT_THROW_IF_ERROR(binding_->BindInput(name, *ort_value));
|
||||
return S_OK;
|
||||
}
|
||||
|
||||
HRESULT STDMETHODCALLTYPE BindOutput(const std::string& name, OrtValue* ort_value) override {
|
||||
// this can be null for unbound outputs
|
||||
if (ort_value == nullptr) {
|
||||
OrtValue empty_value = {};
|
||||
ORT_THROW_IF_ERROR(binding_->BindOutput(name, empty_value));
|
||||
} else {
|
||||
ORT_THROW_IF_ERROR(binding_->BindOutput(name, *ort_value));
|
||||
}
|
||||
return S_OK;
|
||||
}
|
||||
|
||||
const std::vector<std::string>& STDMETHODCALLTYPE GetOutputNames() override {
|
||||
return binding_->GetOutputNames();
|
||||
}
|
||||
std::vector<OrtValue*>& STDMETHODCALLTYPE GetOutputs() override {
|
||||
auto& output_inner = binding_->GetOutputs();
|
||||
outputs_weak_.clear();
|
||||
for (unsigned i = 0; i < output_inner.size(); i++) {
|
||||
outputs_weak_.push_back(&(output_inner[i]));
|
||||
}
|
||||
return outputs_weak_;
|
||||
}
|
||||
};
|
||||
|
||||
// InferenceSession
|
||||
// ================
|
||||
|
||||
|
|
@ -679,18 +634,7 @@ void STDMETHODCALLTYPE InferenceSession::RegisterGraphTransformers() {
|
|||
#endif USE_DML
|
||||
}
|
||||
|
||||
HRESULT STDMETHODCALLTYPE InferenceSession::NewIOBinding(IIOBinding** io_binding) {
|
||||
std::unique_ptr<onnxruntime::IOBinding> binding;
|
||||
ORT_THROW_IF_ERROR(this->session_->NewIOBinding(&binding));
|
||||
auto io_binding_outer = wil::MakeOrThrow<IOBinding>(binding.release());
|
||||
return io_binding_outer.CopyTo(__uuidof(IIOBinding), reinterpret_cast<void**>(io_binding));
|
||||
}
|
||||
|
||||
HRESULT STDMETHODCALLTYPE InferenceSession::Run(const onnxruntime::RunOptions* run_options, IIOBinding* io_binding) {
|
||||
ORT_THROW_IF_ERROR(this->session_->Run(*run_options, *(io_binding->get())));
|
||||
return S_OK;
|
||||
}
|
||||
HRESULT STDMETHODCALLTYPE InferenceSession::StartProfiling() {
|
||||
HRESULT STDMETHODCALLTYPE InferenceSession::StartProfiling() {
|
||||
this->session_->StartProfiling(PheonixSingleton<WinML::LotusEnvironment>()->GetDefaultLogger());
|
||||
return S_OK;
|
||||
}
|
||||
|
|
@ -746,10 +690,15 @@ void STDMETHODCALLTYPE InferenceSession::ReleaseCompletedReferences(onnxruntime:
|
|||
}
|
||||
|
||||
HRESULT STDMETHODCALLTYPE InferenceSession::CopyOneInputAcrossDevices(
|
||||
const char* input_name,
|
||||
const OrtValue* orig_mlvalue,
|
||||
OrtValue** new_mlvalue) {
|
||||
return E_NOTIMPL;
|
||||
const char* input_name,
|
||||
const OrtValue* orig_mlvalue,
|
||||
OrtValue** new_mlvalue) {
|
||||
auto session_protected_load_accessor =
|
||||
static_cast<InferenceSessionProtectedLoadAccessor*>(session_.get());
|
||||
const onnxruntime::SessionState& sessionState = session_protected_load_accessor->GetSessionState();
|
||||
auto temp_mlvalue = std::make_unique<OrtValue>();
|
||||
ORT_THROW_IF_ERROR(onnxruntime::utils::CopyOneInputAcrossDevices(sessionState, input_name, *orig_mlvalue, *temp_mlvalue.get()));
|
||||
*new_mlvalue = temp_mlvalue.release();
|
||||
return S_OK;
|
||||
}
|
||||
|
||||
} // namespace Windows::AI::MachineLearning::Adapter
|
||||
|
|
@ -20,16 +20,6 @@ MIDL_INTERFACE("eaae30b5-7381-432d-9730-322136b02371") IModelInfo : IUnknown{
|
|||
virtual HRESULT STDMETHODCALLTYPE GetOutputFeatures(ABI::Windows::Foundation::Collections::IVectorView<winml::ILearningModelFeatureDescriptor> * *features) = 0;
|
||||
};
|
||||
|
||||
MIDL_INTERFACE("438e7719-554a-4058-84d9-eb6226c34887") IIOBinding : IUnknown{
|
||||
// this returns a weak ref
|
||||
virtual onnxruntime::IOBinding* STDMETHODCALLTYPE get() = 0;
|
||||
virtual HRESULT STDMETHODCALLTYPE BindInput(const std::string& name, OrtValue * ml_value) = 0;
|
||||
virtual HRESULT STDMETHODCALLTYPE BindOutput(const std::string& name, OrtValue * ml_value) = 0;
|
||||
virtual const std::vector<std::string>& STDMETHODCALLTYPE GetOutputNames() = 0;
|
||||
// this returns a weak ref
|
||||
virtual std::vector<OrtValue *>& STDMETHODCALLTYPE GetOutputs() = 0;
|
||||
};
|
||||
|
||||
MIDL_INTERFACE("a848faf6-5a2e-4a7f-b622-cc036f71e28a") IModelProto : IUnknown{
|
||||
// this returns a weak ref
|
||||
virtual onnx::ModelProto* STDMETHODCALLTYPE get() = 0;
|
||||
|
|
@ -39,11 +29,11 @@ MIDL_INTERFACE("a848faf6-5a2e-4a7f-b622-cc036f71e28a") IModelProto : IUnknown{
|
|||
|
||||
MIDL_INTERFACE("6ec766ef-6365-42bf-b64f-ae85c015adb8") IInferenceSession : IUnknown {
|
||||
virtual onnxruntime::InferenceSession* STDMETHODCALLTYPE get() = 0;
|
||||
// the below returns a weak ref , DO NOT RELEASE IT
|
||||
virtual HRESULT STDMETHODCALLTYPE GetOrtSession(OrtSession ** out) = 0;
|
||||
virtual void STDMETHODCALLTYPE RegisterGraphTransformers() = 0;
|
||||
virtual HRESULT STDMETHODCALLTYPE RegisterCustomRegistry(IMLOperatorRegistry * registry) = 0;
|
||||
virtual HRESULT STDMETHODCALLTYPE LoadModel(IModelProto* model_proto) = 0;
|
||||
virtual HRESULT STDMETHODCALLTYPE NewIOBinding(IIOBinding** io_binding) = 0;
|
||||
virtual HRESULT STDMETHODCALLTYPE Run(const onnxruntime::RunOptions* run_options, IIOBinding* io_binding) = 0;
|
||||
virtual HRESULT STDMETHODCALLTYPE StartProfiling() = 0;
|
||||
virtual HRESULT STDMETHODCALLTYPE EndProfiling() = 0;
|
||||
virtual void STDMETHODCALLTYPE FlushContext(onnxruntime::IExecutionProvider * dml_provider) = 0;
|
||||
|
|
@ -130,11 +120,14 @@ public:
|
|||
InferenceSession(onnxruntime::InferenceSession * session);
|
||||
|
||||
onnxruntime::InferenceSession* STDMETHODCALLTYPE get() override { return session_.get(); }
|
||||
HRESULT STDMETHODCALLTYPE GetOrtSession(OrtSession ** out) override {
|
||||
// (OrtSession *) are really (InferenceSession *) as well
|
||||
*out = reinterpret_cast<OrtSession*>(session_.get());
|
||||
return S_OK;
|
||||
}
|
||||
void STDMETHODCALLTYPE RegisterGraphTransformers() override;
|
||||
HRESULT STDMETHODCALLTYPE RegisterCustomRegistry(IMLOperatorRegistry* registry) override;
|
||||
HRESULT STDMETHODCALLTYPE LoadModel(IModelProto* model_proto) override;
|
||||
HRESULT STDMETHODCALLTYPE NewIOBinding(IIOBinding** io_binding) override;
|
||||
HRESULT STDMETHODCALLTYPE Run(const onnxruntime::RunOptions* run_options, IIOBinding* io_binding) override;
|
||||
HRESULT STDMETHODCALLTYPE StartProfiling() override;
|
||||
HRESULT STDMETHODCALLTYPE EndProfiling() override;
|
||||
void STDMETHODCALLTYPE FlushContext(onnxruntime::IExecutionProvider* dml_provider) override;
|
||||
|
|
|
|||
|
|
@ -8,13 +8,15 @@
|
|||
#include "LearningModelBinding.h"
|
||||
#include "LearningModelSession.h"
|
||||
#include "TelemetryEvent.h"
|
||||
#include <onnxruntime_c_api.h>
|
||||
#include "LearningModel.h"
|
||||
|
||||
using namespace WinML;
|
||||
|
||||
namespace winrt::Windows::AI::MachineLearning::implementation {
|
||||
LearningModelBinding::LearningModelBinding(
|
||||
Windows::AI::MachineLearning::LearningModelSession const& session) try : m_session(session) {
|
||||
m_lotusBinding.attach(session.as<LearningModelSession>()->CreateSessionBinding());
|
||||
Windows::AI::MachineLearning::LearningModelSession const& session) try : m_session(session) {
|
||||
session.as<winmlp::LearningModelSession>()->CheckClosed();
|
||||
WINML_THROW_IF_FAILED(OrtGetWinMLAdapter(adapter_.put()));
|
||||
}
|
||||
WINML_CATCH_ALL
|
||||
|
|
@ -40,7 +42,7 @@ static Windows::AI::MachineLearning::ILearningModelFeatureDescriptor FindValidBi
|
|||
using NullableBindingPort = std::optional<std::pair<Windows::AI::MachineLearning::ILearningModelFeatureDescriptor, BindingType>>;
|
||||
|
||||
static NullableBindingPort FindValidBinding(
|
||||
LearningModel model,
|
||||
winml::LearningModel model,
|
||||
const std::wstring& name) {
|
||||
if (auto descriptor = FindValidBinding(model.InputFeatures(), name)) {
|
||||
return std::make_pair(descriptor, BindingType::kInput);
|
||||
|
|
@ -156,13 +158,13 @@ void LearningModelBinding::Bind(
|
|||
|
||||
auto featureName = WinML::Strings::UTF8FromHString(name);
|
||||
std::tie(bindingName, binding_value, bindingType) = CreateBinding(featureName, value, properties);
|
||||
|
||||
Ort::Value ortValue = binding_value ? Ort::Value(binding_value) : Ort::Value(nullptr);
|
||||
switch (bindingType) {
|
||||
case BindingType::kInput:
|
||||
WINML_THROW_IF_FAILED(m_lotusBinding->BindInput(bindingName, binding_value));
|
||||
WINML_THROW_IF_FAILED(BindInput(bindingName, ortValue));
|
||||
break;
|
||||
case BindingType::kOutput:
|
||||
WINML_THROW_IF_FAILED(m_lotusBinding->BindOutput(bindingName, binding_value));
|
||||
WINML_THROW_IF_FAILED(BindOutput(bindingName, ortValue));
|
||||
break;
|
||||
default:
|
||||
FAIL_FAST();
|
||||
|
|
@ -171,7 +173,11 @@ void LearningModelBinding::Bind(
|
|||
WINML_CATCH_ALL
|
||||
|
||||
void LearningModelBinding::Clear() try {
|
||||
m_lotusBinding.attach(m_session.as<LearningModelSession>()->CreateSessionBinding());
|
||||
m_session.as<winmlp::LearningModelSession>()->CheckClosed();
|
||||
inputs_.clear();
|
||||
input_names_.clear();
|
||||
outputs_.clear();
|
||||
output_names_.clear();
|
||||
m_providers.clear();
|
||||
}
|
||||
WINML_CATCH_ALL
|
||||
|
|
@ -218,11 +224,6 @@ void LearningModelBinding::Split(
|
|||
throw hresult_not_implemented();
|
||||
}
|
||||
|
||||
_winmla::IIOBinding* LearningModelBinding::BindingCollection() {
|
||||
_winmla::IIOBinding* p;
|
||||
m_lotusBinding.copy_to(&p);
|
||||
return p;
|
||||
}
|
||||
|
||||
ONNXTensorElementDataType STDMETHODCALLTYPE GetONNXTensorElementDataType(winml::TensorKind kind) {
|
||||
if (kind == TensorKind::Float) {
|
||||
|
|
@ -387,7 +388,7 @@ ILearningModelFeatureValue LearningModelBinding::CreateUnboundOuputFeatureValue(
|
|||
|
||||
Windows::Foundation::IInspectable LearningModelBinding::CreateUnboundOutput(
|
||||
const std::string& name,
|
||||
Ort::Value& ort_value) {
|
||||
Ort::Value& ort_value) {
|
||||
// Find valid binding port
|
||||
auto bindingPort = FindValidBinding(
|
||||
m_session.Model(),
|
||||
|
|
@ -440,8 +441,8 @@ Windows::Foundation::IInspectable LearningModelBinding::CreateUnboundOutput(
|
|||
std::unordered_map<std::string, Windows::Foundation::IInspectable> LearningModelBinding::UpdateProviders() {
|
||||
std::unordered_map<std::string, Windows::Foundation::IInspectable> outputs;
|
||||
|
||||
auto& outputNames = m_lotusBinding->GetOutputNames();
|
||||
auto& outputMLValues = m_lotusBinding->GetOutputs();
|
||||
auto& outputNames = GetOutputNames();
|
||||
auto& outputMLValues = GetOutputs();
|
||||
WINML_THROW_HR_IF_FALSE_MSG(
|
||||
E_UNEXPECTED,
|
||||
outputNames.size() == outputMLValues.size(),
|
||||
|
|
@ -449,7 +450,7 @@ std::unordered_map<std::string, Windows::Foundation::IInspectable> LearningModel
|
|||
|
||||
for (unsigned i = 0; i < outputNames.size(); i++) {
|
||||
auto utf8Name = outputNames[i];
|
||||
auto mlValue = outputMLValues[i];
|
||||
OrtValue* mlValue = outputMLValues[i];
|
||||
|
||||
if (m_providers.find(utf8Name) != std::end(m_providers)) {
|
||||
auto& providerInfo = m_providers[utf8Name];
|
||||
|
|
@ -499,14 +500,13 @@ STDMETHODIMP LearningModelBinding::Bind(
|
|||
|
||||
auto featureName = WinML::Strings::UTF8FromUnicode(name, cchName);
|
||||
std::tie(bindingName, binding_value_ptr, bindingType) = CreateBinding(featureName, to, nullptr);
|
||||
Ort::Value bindingValue(binding_value_ptr);
|
||||
|
||||
Ort::Value ortValue = binding_value_ptr ? Ort::Value(binding_value_ptr) : Ort::Value(nullptr);
|
||||
switch (bindingType) {
|
||||
case BindingType::kInput:
|
||||
WINML_THROW_IF_FAILED(m_lotusBinding->BindInput(bindingName, bindingValue));
|
||||
WINML_THROW_IF_FAILED(BindInput(bindingName, ortValue));
|
||||
break;
|
||||
case BindingType::kOutput:
|
||||
WINML_THROW_IF_FAILED(m_lotusBinding->BindOutput(bindingName, bindingValue));
|
||||
WINML_THROW_IF_FAILED(BindOutput(bindingName, ortValue));
|
||||
break;
|
||||
default:
|
||||
FAIL_FAST();
|
||||
|
|
@ -515,4 +515,106 @@ STDMETHODIMP LearningModelBinding::Bind(
|
|||
}
|
||||
WINML_CATCH_ALL_COM
|
||||
}
|
||||
|
||||
static std::pair<bool, size_t> Contains(const std::vector<std::string>& names, const std::string& name) {
|
||||
auto it = std::find(std::begin(names), std::end(names), name);
|
||||
if (it == std::end(names)) {
|
||||
return {false, 0};
|
||||
}
|
||||
return {true, it - std::begin(names)};
|
||||
}
|
||||
|
||||
// This method releases control of memory of ml_value from caller of BindInput
|
||||
HRESULT LearningModelBinding::BindInput(const std::string& name, Ort::Value& ml_value) {
|
||||
auto rc = Contains(input_names_, name);
|
||||
|
||||
auto add_or_replace = [this, &name](const bool exists, size_t index, Ort::Value& value) {
|
||||
if (exists) {
|
||||
inputs_[index] = Ort::Value(value.release());
|
||||
} else {
|
||||
input_names_.push_back(name);
|
||||
inputs_.push_back(Ort::Value(value.release()));
|
||||
}
|
||||
};
|
||||
if (ml_value.IsTensor()) {
|
||||
Ort::Value new_mlvalue = Ort::Value(nullptr);
|
||||
WINML_THROW_IF_FAILED(m_session.as<LearningModelSession>()
|
||||
->GetIInferenceSession()
|
||||
->CopyOneInputAcrossDevices(name.c_str(), ml_value, new_mlvalue.put()));
|
||||
add_or_replace(rc.first, rc.second, new_mlvalue);
|
||||
} else {
|
||||
add_or_replace(rc.first, rc.second, ml_value);
|
||||
}
|
||||
return S_OK;
|
||||
}
|
||||
|
||||
// This method releases control of memory of ml_value from caller of BindInput
|
||||
HRESULT LearningModelBinding::BindOutput(const std::string& name, Ort::Value& ml_value) {
|
||||
auto rc = Contains(output_names_, name);
|
||||
OrtValue* ml_value_data = ml_value.release();
|
||||
if (rc.first) {
|
||||
outputs_[rc.second] = ml_value_data ? Ort::Value(ml_value_data) : Ort::Value(nullptr);
|
||||
return S_OK;
|
||||
}
|
||||
|
||||
output_names_.push_back(name);
|
||||
outputs_.push_back(ml_value_data ? Ort::Value(ml_value_data) : Ort::Value(nullptr));
|
||||
return S_OK;
|
||||
}
|
||||
|
||||
const std::vector<std::string>& LearningModelBinding::GetOutputNames() const {
|
||||
return output_names_;
|
||||
}
|
||||
|
||||
std::vector<Ort::Value>& LearningModelBinding::GetOutputs() { return outputs_; }
|
||||
|
||||
const std::vector<std::string>& LearningModelBinding::GetInputNames() const {
|
||||
return input_names_;
|
||||
}
|
||||
|
||||
const std::vector<Ort::Value>& LearningModelBinding::GetInputs() const { return inputs_; }
|
||||
|
||||
void LearningModelBinding::BindUnboundOutputs()
|
||||
{
|
||||
auto& bound_output_names = GetOutputNames();
|
||||
std::unordered_set<std::string> bound_output_names_set(
|
||||
bound_output_names.begin(),
|
||||
bound_output_names.end());
|
||||
|
||||
// Get model output feature names
|
||||
auto model_impl = m_session.Model().as<winmlp::LearningModel>();
|
||||
auto output_features = model_impl->OutputFeatures();
|
||||
std::vector<ILearningModelFeatureDescriptor> output_descriptors(
|
||||
begin(output_features),
|
||||
end(output_features));
|
||||
|
||||
// Convert all output features to their feature names
|
||||
std::vector<std::string> output_feature_names;
|
||||
std::transform(
|
||||
std::begin(output_descriptors),
|
||||
std::end(output_descriptors),
|
||||
std::back_inserter(output_feature_names),
|
||||
[&](auto& descriptor) {
|
||||
auto descriptor_native = descriptor.as<ILearningModelFeatureDescriptorNative>();
|
||||
const wchar_t* p_name;
|
||||
uint32_t size;
|
||||
WINML_THROW_IF_FAILED(descriptor_native->GetName(&p_name, &size));
|
||||
return WinML::Strings::UTF8FromUnicode(p_name, size);
|
||||
});
|
||||
|
||||
// Find the set difference to determine if there are any unbound output features
|
||||
std::vector<std::string> unbound_output_names;
|
||||
std::copy_if(
|
||||
std::begin(output_feature_names), std::end(output_feature_names),
|
||||
std::inserter(unbound_output_names, std::begin(unbound_output_names)),
|
||||
[&](const auto& outputFeatureName) {
|
||||
return bound_output_names_set.find(outputFeatureName) == bound_output_names_set.end();
|
||||
});
|
||||
|
||||
// Add all unbound outputs to binding collection
|
||||
for (const auto& unbound_output : unbound_output_names) {
|
||||
WINML_THROW_IF_FAILED(BindOutput(unbound_output, Ort::Value(nullptr)));
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace winrt::Windows::AI::MachineLearning::implementation
|
||||
|
|
@ -41,7 +41,6 @@ struct LearningModelBinding : LearningModelBindingT<LearningModelBinding, ILearn
|
|||
const Windows::Foundation::IInspectable& value,
|
||||
Windows::Foundation::Collections::IPropertySet const& properties);
|
||||
|
||||
_winmla::IIOBinding* BindingCollection();
|
||||
std::unordered_map<std::string, Windows::Foundation::IInspectable> UpdateProviders();
|
||||
|
||||
const Windows::AI::MachineLearning::LearningModelSession& GetSession() { return m_session; }
|
||||
|
|
@ -52,6 +51,13 @@ struct LearningModelBinding : LearningModelBindingT<LearningModelBinding, ILearn
|
|||
UINT32 cchName,
|
||||
IUnknown* value);
|
||||
|
||||
const std::vector<std::string>& LearningModelBinding::GetOutputNames() const;
|
||||
std::vector<Ort::Value>& LearningModelBinding::GetOutputs();
|
||||
const std::vector<std::string>& LearningModelBinding::GetInputNames() const;
|
||||
const std::vector<Ort::Value>& LearningModelBinding::GetInputs() const;
|
||||
HRESULT BindOutput(const std::string& name, Ort::Value& ml_value);
|
||||
void BindUnboundOutputs();
|
||||
|
||||
private:
|
||||
void CacheProvider(std::string name, ProviderInfo& spProvider);
|
||||
Windows::Foundation::IInspectable CreateUnboundOutput(const std::string& name, Ort::Value& ort_value);
|
||||
|
|
@ -61,15 +67,18 @@ struct LearningModelBinding : LearningModelBindingT<LearningModelBinding, ILearn
|
|||
bool IsOfTensorType(const Ort::Value& ort_value, TensorKind kind);
|
||||
bool IsOfMapType(const Ort::Value& ort_value, TensorKind key_kind, TensorKind value_kind);
|
||||
bool IsOfVectorMapType(const Ort::Value& ort_value, TensorKind key_kind, TensorKind value_kind);
|
||||
|
||||
HRESULT BindInput(const std::string& name, Ort::Value& ml_value);
|
||||
|
||||
private:
|
||||
const Windows::AI::MachineLearning::LearningModelSession m_session;
|
||||
|
||||
std::unordered_map<std::string, ProviderInfo> m_providers;
|
||||
|
||||
com_ptr<_winmla::IIOBinding> m_lotusBinding;
|
||||
com_ptr<_winmla::IWinMLAdapter> adapter_;
|
||||
std::vector<std::string> input_names_;
|
||||
std::vector<Ort::Value> inputs_;
|
||||
std::vector<std::string> output_names_;
|
||||
std::vector<Ort::Value> outputs_;
|
||||
};
|
||||
} // namespace winrt::Windows::AI::MachineLearning::implementation
|
||||
|
||||
|
|
|
|||
|
|
@ -198,54 +198,20 @@ LearningModelSession::EvaluateFeaturesAsync(
|
|||
return EvaluateAsync(binding, correlation_id);
|
||||
}
|
||||
|
||||
static _winmla::IIOBinding*
|
||||
GetIOBinding(
|
||||
winrt::com_ptr<winmlp::LearningModelBinding> binding_impl,
|
||||
winml::LearningModel& model) {
|
||||
// Get the IOBinding Collection, and bound outputs
|
||||
com_ptr<_winmla::IIOBinding> io_binding;
|
||||
io_binding.attach(binding_impl->BindingCollection());
|
||||
auto& bound_output_names = io_binding->GetOutputNames();
|
||||
std::unordered_set<std::string> bound_output_names_set(
|
||||
bound_output_names.begin(),
|
||||
bound_output_names.end());
|
||||
|
||||
// Get model output feature names
|
||||
auto model_impl = model.as<winmlp::LearningModel>();
|
||||
auto output_features = model_impl->OutputFeatures();
|
||||
std::vector<ILearningModelFeatureDescriptor> output_descriptors(
|
||||
begin(output_features),
|
||||
end(output_features));
|
||||
|
||||
// Convert all output features to their feature names
|
||||
std::vector<std::string> output_feature_names;
|
||||
std::transform(
|
||||
std::begin(output_descriptors),
|
||||
std::end(output_descriptors),
|
||||
std::back_inserter(output_feature_names),
|
||||
[&](auto& descriptor) {
|
||||
auto descriptor_native = descriptor.as<ILearningModelFeatureDescriptorNative>();
|
||||
const wchar_t* p_name;
|
||||
uint32_t size;
|
||||
WINML_THROW_IF_FAILED(descriptor_native->GetName(&p_name, &size));
|
||||
return WinML::Strings::UTF8FromUnicode(p_name, size);
|
||||
});
|
||||
|
||||
// Find the set difference to determine if there are any unbound output features
|
||||
std::vector<std::string> unbound_output_names;
|
||||
std::copy_if(
|
||||
std::begin(output_feature_names), std::end(output_feature_names),
|
||||
std::inserter(unbound_output_names, std::begin(unbound_output_names)),
|
||||
[&](const auto& outputFeatureName) {
|
||||
return bound_output_names_set.find(outputFeatureName) == bound_output_names_set.end();
|
||||
});
|
||||
|
||||
// Add all unbound outputs to the iobinding collection
|
||||
for (const auto& unbound_output : unbound_output_names) {
|
||||
WINML_THROW_IF_FAILED(io_binding->BindOutput(unbound_output, nullptr));
|
||||
}
|
||||
|
||||
return io_binding.detach();
|
||||
// copied from onnxruntime_cxx_inline.h
|
||||
inline OrtStatus* OrtRun(
|
||||
OrtSession * session,
|
||||
const Ort::RunOptions& run_options,
|
||||
const char* const* input_names,
|
||||
const Ort::Value* input_values,
|
||||
size_t input_count,
|
||||
const char* const* output_names,
|
||||
Ort::Value* output_values,
|
||||
size_t output_count) {
|
||||
static_assert(sizeof(Ort::Value) == sizeof(OrtValue*), "Value is really just an array of OrtValue* in memory, so we can reinterpret_cast safely");
|
||||
auto ort_input_values = reinterpret_cast<const OrtValue**>(const_cast<Ort::Value*>(input_values));
|
||||
auto ort_output_values = reinterpret_cast<OrtValue**>(output_values);
|
||||
return Ort::GetApi().Run(session, run_options, input_names, ort_input_values, input_count, output_names, output_count, ort_output_values);
|
||||
}
|
||||
|
||||
uint64_t
|
||||
|
|
@ -255,13 +221,31 @@ LearningModelSession::Run(
|
|||
auto device = device_.as<LearningModelDevice>();
|
||||
CWinMLAutoLock lock(!device->IsCpuDevice() ? &evaluate_lock_ : nullptr);
|
||||
// TODO : set the run_options
|
||||
onnxruntime::RunOptions run_options;
|
||||
Ort::RunOptions run_options;
|
||||
binding_impl->BindUnboundOutputs();
|
||||
|
||||
com_ptr<_winmla::IIOBinding> io_binding;
|
||||
io_binding.attach(GetIOBinding(binding_impl, model_));
|
||||
std::vector<const char*> inputNames_c;
|
||||
for (int i=0; i < binding_impl->GetInputNames().size(); i++)
|
||||
{
|
||||
inputNames_c.push_back(binding_impl->GetInputNames()[i].c_str());
|
||||
}
|
||||
std::vector<const char*> outputNames_c;
|
||||
for (int i = 0; i < binding_impl->GetOutputNames().size(); i++) {
|
||||
outputNames_c.push_back(binding_impl->GetOutputNames()[i].c_str());
|
||||
}
|
||||
OrtSession* session = nullptr;
|
||||
|
||||
WINML_THROW_IF_FAILED(inference_session_->GetOrtSession(&session));
|
||||
// Invoke run on the ORT session.
|
||||
WINML_THROW_IF_FAILED(inference_session_->Run(&run_options, io_binding.get()));
|
||||
Ort::ThrowOnError(OrtRun(
|
||||
session,
|
||||
run_options,
|
||||
inputNames_c.data(),
|
||||
binding_impl->GetInputs().data(),
|
||||
binding_impl->GetInputs().size(),
|
||||
outputNames_c.data(),
|
||||
binding_impl->GetOutputs().data(),
|
||||
binding_impl->GetOutputs().size()));
|
||||
|
||||
if (!device->IsCpuDevice()) {
|
||||
// Flush the D3D12 work from the DML execution provider and queue a fence before we release the lock.
|
||||
|
|
@ -409,14 +393,6 @@ void LearningModelSession::Close() {
|
|||
inference_session_ = nullptr;
|
||||
}
|
||||
|
||||
_winmla::IIOBinding*
|
||||
LearningModelSession::CreateSessionBinding() {
|
||||
CheckClosed();
|
||||
com_ptr<_winmla::IIOBinding> binding;
|
||||
WINML_THROW_IF_FAILED(inference_session_->NewIOBinding(binding.put()));
|
||||
return binding.detach();
|
||||
}
|
||||
|
||||
void LearningModelSession::ApplyEvaluationProperties() try {
|
||||
if (evaluation_properties_) {
|
||||
auto is_debug_output_enabled = evaluation_properties_.HasKey(c_enable_debug_output);
|
||||
|
|
@ -449,6 +425,11 @@ LearningModelSession::GetExecutionProvider() {
|
|||
return cached_execution_provider_;
|
||||
}
|
||||
|
||||
_winmla::IInferenceSession*
|
||||
LearningModelSession::GetIInferenceSession() {
|
||||
return inference_session_.get();
|
||||
}
|
||||
|
||||
void LearningModelSession::CheckClosed() {
|
||||
if (!inference_session_) {
|
||||
WINML_THROW_HR(RO_E_CLOSED);
|
||||
|
|
|
|||
|
|
@ -68,8 +68,11 @@ struct LearningModelSession : LearningModelSessionT<LearningModelSession> {
|
|||
onnxruntime::IExecutionProvider*
|
||||
GetExecutionProvider();
|
||||
|
||||
_winmla::IIOBinding*
|
||||
CreateSessionBinding();
|
||||
_winmla::IInferenceSession*
|
||||
GetIInferenceSession();
|
||||
|
||||
void
|
||||
CheckClosed();
|
||||
|
||||
private:
|
||||
void
|
||||
|
|
@ -97,9 +100,6 @@ struct LearningModelSession : LearningModelSessionT<LearningModelSession> {
|
|||
void
|
||||
ToggleProfiler();
|
||||
|
||||
void
|
||||
CheckClosed();
|
||||
|
||||
private:
|
||||
com_ptr<_winmla::IInferenceSession> inference_session_;
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue