saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-20 19:12:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Delete Ort Allocator in LearningModelBinding (#2653)

Browse source 
* Delete OrtAllocator in LearningModelBinding

* PR comments to make Ort::Allocator a smart pointer

* Small comment change

* PR feedback to clean up code

* PR feedback on move semantics

* Clean up std::move
This commit is contained in:
Ryan Lai 2019-12-17 14:58:46 -08:00 committed by GitHub
parent fe26146311
commit b28a3a92e6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 125 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

58
winml/adapter/WinMLAdapter.h
View file

@ -146,4 +146,60 @@ private:
std::shared_ptr<onnxruntime::InferenceSession> session_;
};
} // namespace Windows::AI::MachineLearning::Adapter
} // namespace Windows::AI::MachineLearning::Adapter
namespace Ort {
// Ort::Allocator is not in the C ABI yet so it will have to be in the WinMLAdapter for now.
// This struct was copied using the Base struct from onnxruntime_cxx_api.h for reference
// Ort::Allocator struct is used as a smart pointer to OrtAllocator.
struct Allocator {
Allocator() {
m_ort_allocator = nullptr;
m_adapter = nullptr;
}
Allocator(winmla::IWinMLAdapter* adapter, OrtAllocator* ort_allocator) :
m_adapter(adapter), m_ort_allocator(ort_allocator) {}
~Allocator() {
if (m_adapter != nullptr && m_ort_allocator != nullptr) {
m_adapter->FreeProviderAllocator(m_ort_allocator);
}
}
operator OrtAllocator*() { return m_ort_allocator; }
operator const OrtAllocator*() const { return m_ort_allocator; }
OrtAllocator* release() {
OrtAllocator* p = m_ort_allocator;
m_ort_allocator = nullptr;
m_adapter = nullptr;
return p;
}
OrtAllocator** put() noexcept {
assert(m_ort_allocator == nullptr);
return &m_ort_allocator;
}
Allocator(const Allocator&) = delete;
Allocator& operator=(const Allocator&) = delete;
Allocator(Allocator&& v) noexcept :
m_adapter{v.m_adapter}, m_ort_allocator{v.m_ort_allocator} {
v.m_adapter = nullptr;
v.m_ort_allocator = nullptr;
}
void operator=(Allocator&& v) noexcept {
if (m_ort_allocator != nullptr && m_adapter != nullptr) {
m_adapter->FreeProviderAllocator(m_ort_allocator);
}
m_adapter = v.m_adapter;
m_ort_allocator = v.m_ort_allocator;
v.m_adapter = nullptr;
v.m_ort_allocator = nullptr;
}
private:
winmla::IWinMLAdapter* m_adapter;
OrtAllocator* m_ort_allocator;
};
} // namespace Ort

14
winml/lib/Api/ImageFeatureValue.cpp
View file

@ -173,12 +173,6 @@ ImageFeatureValue::ImageFeatureValue(IVectorView<Windows::Media::VideoFrame> con
Initialize();
}
ImageFeatureValue::~ImageFeatureValue() {
for (auto allocator : m_tensorAllocators) {
m_adapter->FreeProviderAllocator(allocator);
}
}
static std::optional<BitmapPixelFormat> GetBitmapPixelFormatFromMetadata(const IPropertySet& properties) {
if (properties != nullptr && properties.HasKey(L"BitmapPixelFormat")) {
if (auto pixelFormatInspectable = properties.Lookup(L"BitmapPixelFormat")) {
@ -496,7 +490,7 @@ std::optional<ImageFeatureValue::ImageResourceMetadata> ImageFeatureValue::GetIn
return ImageResourceMetadata{bounds, imageTensorDescriptor};
}
HRESULT ImageFeatureValue::GetOrtValue(WinML::BindingContext& context, OrtValue** ort_value) try {
HRESULT ImageFeatureValue::GetOrtValue(WinML::BindingContext& context, OrtValue** ort_value, OrtAllocator** ort_allocator) try {
FAIL_FAST_IF(!(std::all_of(m_widths.begin(), m_widths.end(), [](int i) { return i != 0; })));
FAIL_FAST_IF(!(std::all_of(m_heights.begin(), m_heights.end(), [](int i) { return i != 0; })));
@ -516,8 +510,8 @@ HRESULT ImageFeatureValue::GetOrtValue(WinML::BindingContext& context, OrtValue*
}
// create the OrtValue
OrtAllocator* dml_allocator;
WINML_THROW_IF_FAILED(m_adapter->GetProviderAllocator(provider, &dml_allocator));
Ort::Allocator dml_allocator(m_adapter.get(), nullptr);
WINML_THROW_IF_FAILED(m_adapter->GetProviderAllocator(provider, dml_allocator.put()));
// create the OrtValue as a tensor letting ort know that we own the data buffer
Ort::Value ort_tensor = Ort::Value::CreateTensor(
@ -525,7 +519,6 @@ HRESULT ImageFeatureValue::GetOrtValue(WinML::BindingContext& context, OrtValue*
&(resourceMetadata.TensorDescriptor.sizes[0]),
sizeof(resourceMetadata.TensorDescriptor.sizes) / sizeof(resourceMetadata.TensorDescriptor.sizes[0]),
(resourceMetadata.TensorDescriptor.dataType == kImageTensorDataTypeFloat32) ? ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT : ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16);
m_tensorAllocators.emplace_back(dml_allocator);
// Get the tensor raw data
void* pAllocatedResource = nullptr;
@ -545,6 +538,7 @@ HRESULT ImageFeatureValue::GetOrtValue(WinML::BindingContext& context, OrtValue*
}
*ort_value = ort_tensor.release();
*ort_allocator = dml_allocator.release();
return S_OK;
}
WINML_CATCH_ALL_COM

5
winml/lib/Api/ImageFeatureValue.h
View file

@ -14,7 +14,6 @@ struct ImageFeatureValue : ImageFeatureValueT<ImageFeatureValue, WinML::ILotusVa
struct ImageResourceMetadata;
ImageFeatureValue() = delete;
~ImageFeatureValue();
ImageFeatureValue(Windows::Media::VideoFrame const& image);
ImageFeatureValue(winrt::Windows::Foundation::Collections::IVector<Windows::Media::VideoFrame> const& images);
ImageFeatureValue(winrt::Windows::Foundation::Collections::IVectorView<Windows::Media::VideoFrame> const& images);
@ -34,7 +33,7 @@ struct ImageFeatureValue : ImageFeatureValueT<ImageFeatureValue, WinML::ILotusVa
// ILotusValueProviderPrivate implementation
STDMETHOD(GetOrtValue)
(WinML::BindingContext& context, OrtValue** ort_value);
(WinML::BindingContext& context, OrtValue** ort_value, OrtAllocator** ort_allocator);
STDMETHOD(IsPlaceholder)
(bool* pIsPlaceHolder);
STDMETHOD(UpdateSourceResourceData)
@ -45,13 +44,11 @@ struct ImageFeatureValue : ImageFeatureValueT<ImageFeatureValue, WinML::ILotusVa
std::vector<uint32_t> Widths() { return m_widths; }
std::vector<uint32_t> Heights() { return m_heights; }
bool IsBatch() { return m_batchSize > 1; }
private:
com_ptr<winmla::IWinMLAdapter> m_adapter;
winrt::Windows::Foundation::Collections::IVector<Windows::Media::VideoFrame> m_videoFrames;
std::vector<uint32_t> m_widths = {};
std::vector<uint32_t> m_heights = {};
std::vector<OrtAllocator *> m_tensorAllocators;
uint32_t m_batchSize = 1;
// Crop the image with desired aspect ratio.
// This function does not crop image to desried width and height, but crops to center for desired ratio

75
winml/lib/Api/LearningModelBinding.cpp
View file

@ -39,6 +39,10 @@ static Windows::AI::MachineLearning::ILearningModelFeatureDescriptor FindValidBi
return nullptr;
}
LearningModelBinding::~LearningModelBinding() {
Clear();
}
using NullableBindingPort = std::optional<std::pair<Windows::AI::MachineLearning::ILearningModelFeatureDescriptor, BindingType>>;
static NullableBindingPort FindValidBinding(
@ -59,7 +63,7 @@ void LearningModelBinding::CacheProvider(
m_providers[name] = providerInfo;
}
std::tuple<std::string, OrtValue*, BindingType> LearningModelBinding::CreateBinding(
std::tuple<std::string, OrtValue*, BindingType, OrtAllocator*> LearningModelBinding::CreateBinding(
const std::string& name,
const Windows::Foundation::IInspectable& inspectable,
Windows::Foundation::Collections::IPropertySet const& properties) {
@ -99,6 +103,7 @@ std::tuple<std::string, OrtValue*, BindingType> LearningModelBinding::CreateBind
// Get the bound tensor
Ort::Value value(nullptr);
Ort::Allocator ort_allocator(adapter_.get(), nullptr);
// Get the native ORT interface for the given bind value
auto spLotusValueProvider = featureValue.as<WinML::ILotusValueProviderPrivate>();
@ -121,7 +126,7 @@ std::tuple<std::string, OrtValue*, BindingType> LearningModelBinding::CreateBind
if (!isPlaceHolder || shouldAlwaysTensorize) {
// If not a placeholder, attempt to get the underlying resource
WINML_THROW_IF_FAILED_MSG(
spLotusValueProvider->GetOrtValue(context, value.put()),
spLotusValueProvider->GetOrtValue(context, value.put(), ort_allocator.put()),
"The model variable %s failed tensorization.",
name.c_str());
} else {
@ -136,7 +141,7 @@ std::tuple<std::string, OrtValue*, BindingType> LearningModelBinding::CreateBind
auto providerInfo = ProviderInfo{inspectable, spLotusValueProvider, context};
CacheProvider(name, providerInfo);
return std::make_tuple(name, value.release(), bindingType);
return std::make_tuple(name, value.release(), bindingType, ort_allocator.release());
}
void LearningModelBinding::Bind(
@ -155,16 +160,23 @@ void LearningModelBinding::Bind(
BindingType bindingType;
std::string bindingName;
OrtValue* binding_value = nullptr;
OrtAllocator* ort_allocator = nullptr;
auto featureName = WinML::Strings::UTF8FromHString(name);
std::tie(bindingName, binding_value, bindingType) = CreateBinding(featureName, value, properties);
std::tie(bindingName, binding_value, bindingType, ort_allocator) = CreateBinding(featureName, value, properties);
Ort::Value ortValue = binding_value ? Ort::Value(binding_value) : Ort::Value(nullptr);
Ort::Allocator ortAllocator(adapter_.get(), ort_allocator);
switch (bindingType) {
case BindingType::kInput:
WINML_THROW_IF_FAILED(BindInput(bindingName, ortValue));
WINML_THROW_IF_FAILED(BindInput(
bindingName,
std::move(ortValue),
std::move(ortAllocator)));
break;
case BindingType::kOutput:
WINML_THROW_IF_FAILED(BindOutput(bindingName, ortValue));
WINML_THROW_IF_FAILED(BindOutput(
bindingName,
std::move(ortValue),
std::move(ortAllocator)));
break;
default:
FAIL_FAST();
@ -179,6 +191,8 @@ void LearningModelBinding::Clear() try {
outputs_.clear();
output_names_.clear();
m_providers.clear();
input_allocators_.clear();
output_allocators_.clear();
}
WINML_CATCH_ALL
@ -219,7 +233,7 @@ bool LearningModelBinding::HasKey(hstring const& key) {
void LearningModelBinding::Split(
Windows::Foundation::Collections::IMapView<hstring, Windows::Foundation::IInspectable>& first,
Windows::Foundation::Collections::IMapView<hstring, Windows::Foundation::IInspectable>& second) {
// the winrt api guide states:
// the winrt api guide states:
// If the IMapView instance cannot be split, then both the first and second parameters are null when the method returns.
first = nullptr;
second = nullptr;
@ -490,21 +504,28 @@ STDMETHODIMP LearningModelBinding::Bind(
BindingType bindingType;
std::string bindingName;
OrtValue* binding_value_ptr = nullptr;
OrtAllocator* ort_allocator = nullptr;
winrt::Windows::Foundation::IInspectable to;
RETURN_IF_FAILED(value->QueryInterface(
winrt::guid_of<winrt::Windows::Foundation::IInspectable>(),
reinterpret_cast<void**>(winrt::put_abi(to))));
auto featureName = WinML::Strings::UTF8FromUnicode(name, cchName);
std::tie(bindingName, binding_value_ptr, bindingType) = CreateBinding(featureName, to, nullptr);
std::tie(bindingName, binding_value_ptr, bindingType, ort_allocator) = CreateBinding(featureName, to, nullptr);
Ort::Value ortValue = binding_value_ptr ? Ort::Value(binding_value_ptr) : Ort::Value(nullptr);
Ort::Allocator ortAllocator(adapter_.get(), ort_allocator);
switch (bindingType) {
case BindingType::kInput:
WINML_THROW_IF_FAILED(BindInput(bindingName, ortValue));
WINML_THROW_IF_FAILED(BindInput(
bindingName,
std::move(ortValue),
std::move(ortAllocator)));
break;
case BindingType::kOutput:
WINML_THROW_IF_FAILED(BindOutput(bindingName, ortValue));
WINML_THROW_IF_FAILED(BindOutput(
bindingName,
std::move(ortValue),
std::move(ortAllocator)));
break;
default:
FAIL_FAST();
@ -523,40 +544,43 @@ static std::pair<bool, size_t> Contains(const std::vector<std::string>& names, c
}
// This method releases control of memory of ml_value from caller of BindInput
HRESULT LearningModelBinding::BindInput(const std::string& name, Ort::Value& ml_value) {
HRESULT LearningModelBinding::BindInput(const std::string& name, Ort::Value&& ml_value, Ort::Allocator&& ort_allocator) {
auto rc = Contains(input_names_, name);
auto add_or_replace = [this, &name](const bool exists, size_t index, Ort::Value& value) {
auto add_or_replace = [this, &name](const bool exists, size_t index, Ort::Value&& value, Ort::Allocator&& ort_allocator) {
if (exists) {
inputs_[index] = Ort::Value(value.release());
inputs_[index] = std::move(value);
input_allocators_[index] = std::move(ort_allocator);
} else {
input_names_.push_back(name);
inputs_.push_back(Ort::Value(value.release()));
inputs_.push_back(std::move(value));
input_allocators_.push_back(std::move(ort_allocator));
}
};
if (ml_value.IsTensor()) {
Ort::Value new_mlvalue = Ort::Value(nullptr);
OrtValue* new_mlvalue;
WINML_THROW_IF_FAILED(m_session.as<LearningModelSession>()
->GetIInferenceSession()
->CopyOneInputAcrossDevices(name.c_str(), ml_value, new_mlvalue.put()));
add_or_replace(rc.first, rc.second, new_mlvalue);
->CopyOneInputAcrossDevices(name.c_str(), ml_value, &new_mlvalue));
add_or_replace(rc.first, rc.second, Ort::Value(new_mlvalue), std::move(ort_allocator));
} else {
add_or_replace(rc.first, rc.second, ml_value);
add_or_replace(rc.first, rc.second, Ort::Value(ml_value.release()), std::move(ort_allocator));
}
return S_OK;
}
// This method releases control of memory of ml_value from caller of BindInput
HRESULT LearningModelBinding::BindOutput(const std::string& name, Ort::Value& ml_value) {
HRESULT LearningModelBinding::BindOutput(const std::string& name, Ort::Value&& ml_value, Ort::Allocator&& ort_allocator) {
auto rc = Contains(output_names_, name);
OrtValue* ml_value_data = ml_value.release();
if (rc.first) {
outputs_[rc.second] = ml_value_data ? Ort::Value(ml_value_data) : Ort::Value(nullptr);
outputs_[rc.second] = std::move(ml_value);
output_allocators_[rc.second] = std::move(ort_allocator);
return S_OK;
}
output_names_.push_back(name);
outputs_.push_back(ml_value_data ? Ort::Value(ml_value_data) : Ort::Value(nullptr));
outputs_.push_back(std::move(ml_value));
output_allocators_.push_back(std::move(ort_allocator));
return S_OK;
}
@ -610,8 +634,7 @@ void LearningModelBinding::BindUnboundOutputs() {
// Add all unbound outputs to binding collection
for (const auto& unbound_output : unbound_output_names) {
Ort::Value out(nullptr);
WINML_THROW_IF_FAILED(BindOutput(unbound_output, out));
WINML_THROW_IF_FAILED(BindOutput(unbound_output, Ort::Value(nullptr), Ort::Allocator()));
}
}

10
winml/lib/Api/LearningModelBinding.h
View file

@ -22,7 +22,7 @@ struct LearningModelBinding : LearningModelBindingT<LearningModelBinding, ILearn
Windows::Foundation::Collections::IKeyValuePair<hstring, Windows::Foundation::IInspectable>;
LearningModelBinding() = delete;
~LearningModelBinding();
LearningModelBinding(Windows::AI::MachineLearning::LearningModelSession const& session);
void Bind(hstring const& name, Windows::Foundation::IInspectable const& value);
@ -36,7 +36,7 @@ struct LearningModelBinding : LearningModelBindingT<LearningModelBinding, ILearn
Windows::Foundation::Collections::IMapView<hstring, Windows::Foundation::IInspectable>& first,
Windows::Foundation::Collections::IMapView<hstring, Windows::Foundation::IInspectable>& second);
std::tuple<std::string, OrtValue*, WinML::BindingType> CreateBinding(
std::tuple<std::string, OrtValue*, WinML::BindingType, OrtAllocator*> CreateBinding(
const std::string& name,
const Windows::Foundation::IInspectable& value,
Windows::Foundation::Collections::IPropertySet const& properties);
@ -55,7 +55,7 @@ struct LearningModelBinding : LearningModelBindingT<LearningModelBinding, ILearn
std::vector<Ort::Value>& LearningModelBinding::GetOutputs();
const std::vector<std::string>& LearningModelBinding::GetInputNames() const;
const std::vector<Ort::Value>& LearningModelBinding::GetInputs() const;
HRESULT BindOutput(const std::string& name, Ort::Value& ml_value);
HRESULT BindOutput(const std::string& name, Ort::Value&& ml_value, Ort::Allocator&& ort_allocator);
void BindUnboundOutputs();
private:
@ -67,7 +67,7 @@ struct LearningModelBinding : LearningModelBindingT<LearningModelBinding, ILearn
bool IsOfTensorType(const Ort::Value& ort_value, TensorKind kind);
bool IsOfMapType(const Ort::Value& ort_value, TensorKind key_kind, TensorKind value_kind);
bool IsOfVectorMapType(const Ort::Value& ort_value, TensorKind key_kind, TensorKind value_kind);
HRESULT BindInput(const std::string& name, Ort::Value& ml_value);
HRESULT BindInput(const std::string& name, Ort::Value&& ml_value, Ort::Allocator&& ort_allocator);
private:
const Windows::AI::MachineLearning::LearningModelSession m_session;
@ -77,8 +77,10 @@ struct LearningModelBinding : LearningModelBindingT<LearningModelBinding, ILearn
com_ptr<winmla::IWinMLAdapter> adapter_;
std::vector<std::string> input_names_;
std::vector<Ort::Value> inputs_;
std::vector<Ort::Allocator> input_allocators_;
std::vector<std::string> output_names_;
std::vector<Ort::Value> outputs_;
std::vector<Ort::Allocator> output_allocators_;
};
} // namespace winrt::Windows::AI::MachineLearning::implementation

3
winml/lib/Api/impl/MapBase.h
View file

@ -158,7 +158,8 @@ struct MapBase : winrt::implements<
}
STDMETHOD(GetOrtValue)
(WinML::BindingContext& context, OrtValue** ort_value) {
(WinML::BindingContext& context, OrtValue** ort_value, OrtAllocator** ort_allocator) {
ORT_UNUSED_PARAMETER(ort_allocator);
ORT_UNUSED_PARAMETER(context);
// TODO: Tensorized data should be cached so multiple bindings work more efficiently

4
winml/lib/Api/impl/SequenceBase.h
View file

@ -195,7 +195,9 @@ struct SequenceBase : public winrt::implements<
STDMETHOD(GetOrtValue)(
WinML::BindingContext& context,
OrtValue** ort_value) {
OrtValue** ort_value,
OrtAllocator** ort_allocator) {
ORT_UNUSED_PARAMETER(ort_allocator);
// TODO: Tensorized data should be cached so multiple bindings work more efficiently
// TODO : we need to handle inputs. for now only handle outputs and don't pre allocate anything

3
winml/lib/Api/impl/TensorBase.h
View file

@ -208,7 +208,8 @@ struct TensorBase : TBase {
// ILotusValueProviderPrivate::GetOrtValue
STDMETHOD(GetOrtValue)
(WinML::BindingContext& context, OrtValue** ort_value) {
(WinML::BindingContext& context, OrtValue** ort_value, OrtAllocator** ort_allocator) {
ORT_UNUSED_PARAMETER(ort_allocator);
RETURN_HR_IF_NULL_MSG(
WINML_ERR_INVALID_BINDING,
m_resources,

2
winml/lib/Api/inc/ILotusValueProviderPrivate.h
View file

@ -24,7 +24,7 @@ struct BindingContext {
};
struct __declspec(uuid("27e2f437-0112-4693-849e-e04323a620fb")) __declspec(novtable) ILotusValueProviderPrivate : IUnknown {
virtual HRESULT __stdcall GetOrtValue(BindingContext& binding_context, OrtValue ** ort_value) = 0;
virtual HRESULT __stdcall GetOrtValue(BindingContext& binding_context, OrtValue** ort_value, OrtAllocator** ort_allocator) = 0;
virtual HRESULT __stdcall IsPlaceholder(bool* is_placeholder) = 0;
virtual HRESULT __stdcall UpdateSourceResourceData(BindingContext& binding_context, OrtValue* ort_value) = 0;
virtual HRESULT __stdcall AbiRepresentation(winrt::Windows::Foundation::IInspectable& abi_representation) = 0;