diff --git a/winml/adapter/WinMLAdapter.cpp b/winml/adapter/WinMLAdapter.cpp index e45065aada..6271cfd1df 100644 --- a/winml/adapter/WinMLAdapter.cpp +++ b/winml/adapter/WinMLAdapter.cpp @@ -639,7 +639,7 @@ class WinMLAdapter : public Microsoft::WRL::RuntimeClass< onnxruntime::IExecutionProvider* provider, OrtAllocator** allocator) override try { auto allocator_ptr = provider->GetAllocator(0, ::OrtMemType::OrtMemTypeDefault); - *allocator = new AllocatorWrapper(allocator_ptr); + *allocator = new (std::nothrow) AllocatorWrapper(allocator_ptr); if (*allocator == nullptr) { return E_OUTOFMEMORY; } @@ -647,6 +647,13 @@ class WinMLAdapter : public Microsoft::WRL::RuntimeClass< return S_OK; } WINMLA_CATCH_ALL_COM + + HRESULT STDMETHODCALLTYPE FreeProviderAllocator( + OrtAllocator* allocator) override try { + delete static_cast(allocator); + return S_OK; + } + WINMLA_CATCH_ALL_COM }; // namespace Windows::AI::MachineLearning::Adapter std::shared_ptr WinMLAdapter::lotus_environment_ = nullptr; diff --git a/winml/adapter/WinMLAdapter.h b/winml/adapter/WinMLAdapter.h index 378b3054c4..6df268244d 100644 --- a/winml/adapter/WinMLAdapter.h +++ b/winml/adapter/WinMLAdapter.h @@ -101,6 +101,7 @@ MIDL_INTERFACE("b19385e7-d9af-441a-ba7f-3993c7b1c9db") IWinMLAdapter : IUnknown // proposed adapter. uses the cross plat ABI currencies virtual HRESULT STDMETHODCALLTYPE GetProviderMemoryInfo(onnxruntime::IExecutionProvider * provider, OrtMemoryInfo** memory_info) = 0; virtual HRESULT STDMETHODCALLTYPE GetProviderAllocator(onnxruntime::IExecutionProvider * provider, OrtAllocator** allocator) = 0; + virtual HRESULT STDMETHODCALLTYPE FreeProviderAllocator(OrtAllocator* allocator) = 0; virtual HRESULT STDMETHODCALLTYPE GetValueMemoryInfo(const OrtValue * value, OrtMemoryInfo** memory_info) = 0; virtual HRESULT STDMETHODCALLTYPE GetMapType(const OrtValue * ort_value, ONNXTensorElementDataType * key_type, ONNXTensorElementDataType * value_type) = 0; virtual HRESULT STDMETHODCALLTYPE GetVectorMapType(const OrtValue * ort_value, ONNXTensorElementDataType * key_type, ONNXTensorElementDataType * value_type) = 0; diff --git a/winml/lib/Api/ImageFeatureValue.cpp b/winml/lib/Api/ImageFeatureValue.cpp index 5f13a58212..5cc0e1b4ce 100644 --- a/winml/lib/Api/ImageFeatureValue.cpp +++ b/winml/lib/Api/ImageFeatureValue.cpp @@ -173,6 +173,12 @@ ImageFeatureValue::ImageFeatureValue(IVectorView con Initialize(); } +ImageFeatureValue::~ImageFeatureValue() { + for (auto allocator : m_tensorAllocators) { + m_adapter->FreeProviderAllocator(allocator); + } +} + static std::optional GetBitmapPixelFormatFromMetadata(const IPropertySet& properties) { if (properties != nullptr && properties.HasKey(L"BitmapPixelFormat")) { if (auto pixelFormatInspectable = properties.Lookup(L"BitmapPixelFormat")) { @@ -505,12 +511,13 @@ HRESULT ImageFeatureValue::GetOrtValue(WinML::BindingContext& context, OrtValue* auto provider = spSession->GetExecutionProvider(); // and the adapter - com_ptr adapter; - WINML_THROW_IF_FAILED(OrtGetWinMLAdapter(adapter.put())); + if (!m_adapter) { + WINML_THROW_IF_FAILED(OrtGetWinMLAdapter(m_adapter.put())); + } // create the OrtValue OrtAllocator* dml_allocator; - WINML_THROW_IF_FAILED(adapter->GetProviderAllocator(provider, &dml_allocator)); + WINML_THROW_IF_FAILED(m_adapter->GetProviderAllocator(provider, &dml_allocator)); // create the OrtValue as a tensor letting ort know that we own the data buffer Ort::Value ort_tensor = Ort::Value::CreateTensor( @@ -518,6 +525,7 @@ HRESULT ImageFeatureValue::GetOrtValue(WinML::BindingContext& context, OrtValue* &(resourceMetadata.TensorDescriptor.sizes[0]), sizeof(resourceMetadata.TensorDescriptor.sizes) / sizeof(resourceMetadata.TensorDescriptor.sizes[0]), (resourceMetadata.TensorDescriptor.dataType == kImageTensorDataTypeFloat32) ? ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT : ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16); + m_tensorAllocators.emplace_back(dml_allocator); // Get the tensor raw data void* pAllocatedResource = nullptr; @@ -552,8 +560,9 @@ HRESULT ImageFeatureValue::UpdateSourceResourceData(BindingContext& context, Ort auto spSession = context.session.as(); auto spDevice = spSession->Device().as(); - com_ptr adapter; - WINML_THROW_IF_FAILED(OrtGetWinMLAdapter(adapter.put())); + if (!m_adapter) { + WINML_THROW_IF_FAILED(OrtGetWinMLAdapter(m_adapter.put())); + } // Get the output tensor raw data void* pAllocatedResource = nullptr; @@ -568,7 +577,7 @@ HRESULT ImageFeatureValue::UpdateSourceResourceData(BindingContext& context, Ort descriptor.height = static_cast(resourceMetadata.TensorDescriptor.sizes[2]); Ort::MemoryInfo memory_info(nullptr); - adapter->GetValueMemoryInfo(ort_value, memory_info.put()); + m_adapter->GetValueMemoryInfo(ort_value, memory_info.put()); if (!strcmp(memory_info.Name(), onnxruntime::CPU) || memory_info.MemType() == ::OrtMemType::OrtMemTypeCPUOutput || @@ -596,7 +605,7 @@ HRESULT ImageFeatureValue::UpdateSourceResourceData(BindingContext& context, Ort auto pooledConverter = PoolObjectWrapper::Create(spDevice->DetensorizerStore()->Fetch(descriptor)); auto pProvider = spSession->GetExecutionProvider(); - auto d3dResource = adapter->GetD3D12ResourceFromAllocation(pProvider, pAllocatedResource); + auto d3dResource = m_adapter->GetD3D12ResourceFromAllocation(pProvider, pAllocatedResource); for (uint32_t batchIdx = 0; batchIdx < m_batchSize; ++batchIdx) { auto videoFrame = m_videoFrames.GetAt(batchIdx); diff --git a/winml/lib/Api/ImageFeatureValue.h b/winml/lib/Api/ImageFeatureValue.h index fea1e7363e..31e19b7e5d 100644 --- a/winml/lib/Api/ImageFeatureValue.h +++ b/winml/lib/Api/ImageFeatureValue.h @@ -14,6 +14,7 @@ struct ImageFeatureValue : ImageFeatureValueT const& images); ImageFeatureValue(winrt::Windows::Foundation::Collections::IVectorView const& images); @@ -46,9 +47,11 @@ struct ImageFeatureValue : ImageFeatureValueT 1; } private: + com_ptr m_adapter; winrt::Windows::Foundation::Collections::IVector m_videoFrames; std::vector m_widths = {}; std::vector m_heights = {}; + std::vector m_tensorAllocators; uint32_t m_batchSize = 1; // Crop the image with desired aspect ratio. // This function does not crop image to desried width and height, but crops to center for desired ratio diff --git a/winml/test/scenario/cppwinrt/scenariotestscppwinrt.cpp b/winml/test/scenario/cppwinrt/scenariotestscppwinrt.cpp index c037685217..0f7f56e0c6 100644 --- a/winml/test/scenario/cppwinrt/scenariotestscppwinrt.cpp +++ b/winml/test/scenario/cppwinrt/scenariotestscppwinrt.cpp @@ -171,7 +171,7 @@ static void BindFeatures(LearningModelBinding binding, IVectorView DoEva return session.EvaluateAsync(binding, L""); } -TEST_F(ScenarioCppWinrtTest, Scenario5_AsyncEval) +TEST_F(ScenarioCppWinrtTest, Scenario5AsyncEval) { auto task = DoEvalAsync(); @@ -273,7 +273,7 @@ TEST_F(ScenarioCppWinrtTest, Scenario5_AsyncEval) //! Scenario6: use BindInputWithProperties - BitmapBounds, BitmapPixelFormat // apparently this scenario is cut for rs5. - not cut, just rewprked. move props // to the image value when that is checked in. -TEST_F(ScenarioCppWinrtGpuTest, Scenario6_BindWithProperties) +TEST_F(ScenarioCppWinrtGpuTest, Scenario6BindWithProperties) { // load a model std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; @@ -319,7 +319,7 @@ TEST_F(ScenarioCppWinrtGpuTest, Scenario6_BindWithProperties) } //! Scenario7: run eval without creating a binding object -TEST_F(ScenarioCppWinrtTest, Scenario7_EvalWithNoBind) +TEST_F(ScenarioCppWinrtTest, Scenario7EvalWithNoBind) { auto map = winrt::single_threaded_map(); @@ -341,7 +341,7 @@ TEST_F(ScenarioCppWinrtTest, Scenario7_EvalWithNoBind) //! Scenario8: choose which device to run the model on - PreferredDeviceType, PreferredDevicePerformance, SetDeviceFromSurface, SetDevice // create a session on the default device -TEST_F(ScenarioCppWinrtTest, Scenario8_SetDeviceSample_Default) +TEST_F(ScenarioCppWinrtTest, Scenario8SetDeviceSampleDefault) { // load a model std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; @@ -352,7 +352,7 @@ TEST_F(ScenarioCppWinrtTest, Scenario8_SetDeviceSample_Default) } // create a session on the CPU device -TEST_F(ScenarioCppWinrtTest, Scenario8_SetDeviceSample_CPU) +TEST_F(ScenarioCppWinrtTest, Scenario8SetDeviceSampleCPU) { // load a model std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; @@ -363,7 +363,7 @@ TEST_F(ScenarioCppWinrtTest, Scenario8_SetDeviceSample_CPU) } // create a session on the default DML device -TEST_F(ScenarioCppWinrtGpuTest, Scenario8_SetDeviceSample_DefaultDirectX) +TEST_F(ScenarioCppWinrtGpuTest, Scenario8SetDeviceSampleDefaultDirectX) { // load a model std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; @@ -374,7 +374,7 @@ TEST_F(ScenarioCppWinrtGpuTest, Scenario8_SetDeviceSample_DefaultDirectX) } // create a session on the DML device that provides best power -TEST_F(ScenarioCppWinrtGpuTest, Scenario8_SetDeviceSample_MinPower) +TEST_F(ScenarioCppWinrtGpuTest, Scenario8SetDeviceSampleMinPower) { // load a model std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; @@ -385,7 +385,7 @@ TEST_F(ScenarioCppWinrtGpuTest, Scenario8_SetDeviceSample_MinPower) } // create a session on the DML device that provides best perf -TEST_F(ScenarioCppWinrtGpuTest, Scenario8_SetDeviceSample_MaxPerf) +TEST_F(ScenarioCppWinrtGpuTest, Scenario8SetDeviceSampleMaxPerf) { // load a model std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; @@ -396,7 +396,7 @@ TEST_F(ScenarioCppWinrtGpuTest, Scenario8_SetDeviceSample_MaxPerf) } // create a session on the same device my camera is on -TEST_F(ScenarioCppWinrtGpuTest, Scenario8_SetDeviceSample_MyCameraDevice) +TEST_F(ScenarioCppWinrtGpuTest, Scenario8SetDeviceSampleMyCameraDevice) { // load a model std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; @@ -427,7 +427,7 @@ TEST_F(ScenarioCppWinrtGpuTest, Scenario8_SetDeviceSample_MyCameraDevice) } // create a device from D3D11 Device -TEST_F(ScenarioCppWinrtGpuSkipEdgeCoreTest, Scenario8_SetDeviceSample_D3D11Device) +TEST_F(ScenarioCppWinrtGpuSkipEdgeCoreTest, Scenario8SetDeviceSampleD3D11Device) { // load a model std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; @@ -457,7 +457,7 @@ TEST_F(ScenarioCppWinrtGpuSkipEdgeCoreTest, Scenario8_SetDeviceSample_D3D11Devic } // create a session on the a specific dx device that I chose some other way , note we have to use native interop here and pass a cmd queue -TEST_F(ScenarioCppWinrtGpuTest, Scenario8_SetDeviceSample_CustomCommandQueue) +TEST_F(ScenarioCppWinrtGpuTest, Scenario8SetDeviceSampleCustomCommandQueue) { // load a model std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; @@ -510,7 +510,7 @@ TEST_F(ScenarioCppWinrtGpuTest, Scenario8_SetDeviceSample_CustomCommandQueue) //pass a Tensor in as an input GPU -TEST_F(ScenarioCppWinrtGpuTest, DISABLED_Scenario9_LoadBindEval_InputTensorGPU) +TEST_F(ScenarioCppWinrtGpuTest, DISABLED_Scenario9LoadBindEvalInputTensorGPU) { // load a model std::wstring filePath = FileHelpers::GetModulePath() + L"fns-candy.onnx"; @@ -597,7 +597,7 @@ TEST_F(ScenarioCppWinrtGpuTest, DISABLED_Scenario9_LoadBindEval_InputTensorGPU) } -TEST_F(ScenarioCppWinrtGpuTest, Scenario13_SingleModelOnCPUandGPU) +TEST_F(ScenarioCppWinrtGpuTest, Scenario13SingleModelOnCPUandGPU) { std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; LearningModel model = LearningModel::LoadFromFilePath(filePath); @@ -624,7 +624,7 @@ TEST_F(ScenarioCppWinrtGpuTest, Scenario13_SingleModelOnCPUandGPU) } // Validates when binding input image with free dimensions, the binding step is executed correctly. -TEST_F(ScenarioCppWinrtGpuTest, Scenario11_FreeDimenions_tensor) +TEST_F(ScenarioCppWinrtGpuTest, Scenario11FreeDimensionsTensor) { std::wstring filePath = FileHelpers::GetModulePath() + L"free_dimensional_image_input.onnx"; // load a model with expected input size: -1 x -1 @@ -643,7 +643,7 @@ TEST_F(ScenarioCppWinrtGpuTest, Scenario11_FreeDimenions_tensor) session.Evaluate(binding, L""); } -TEST_F(ScenarioCppWinrtGpuTest, Scenario11_FreeDimenions_image) +TEST_F(ScenarioCppWinrtGpuTest, Scenario11FreeDimensionsImage) { std::wstring filePath = FileHelpers::GetModulePath() + L"free_dimensional_imageDes.onnx"; // load a model with expected input size: -1 x -1 @@ -695,7 +695,7 @@ void SubmitEval(LearningModel model, SwapChainEntry *sessionBindings, int swapch } //Scenario14:Load single model, run it mutliple times on a single gpu device using a fast swapchain pattern -TEST_F(ScenarioCppWinrtGpuTest, Scenario14_RunModelSwapchain) +TEST_F(ScenarioCppWinrtGpuTest, Scenario14RunModelSwapchain) { const int swapchainentrycount = 3; SwapChainEntry sessionBindings[swapchainentrycount]; @@ -758,7 +758,7 @@ static void LoadBindEval_CustomOperator_CPU(const wchar_t* fileName) } //! Scenario17 : Control the dev diagnostics features of WinML Tracing -TEST_F(ScenarioCppWinrtTest, Scenario17_DevDiagnostics) +TEST_F(ScenarioCppWinrtTest, Scenario17DevDiagnostics) { // load a model std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; @@ -781,21 +781,21 @@ TEST_F(ScenarioCppWinrtTest, Scenario17_DevDiagnostics) } // create a session that loads a model with a branch new operator, register the custom operator, and load/bind/eval -TEST_F(ScenarioCppWinrtTest, Scenario20a_LoadBindEval_CustomOperator_CPU) +TEST_F(ScenarioCppWinrtTest, Scenario20aLoadBindEvalCustomOperatorCPU) { std::wstring filePath = FileHelpers::GetModulePath() + L"noisy_relu.onnx"; LoadBindEval_CustomOperator_CPU(filePath.c_str()); } // create a session that loads a model with an overridden operator, register the replacement custom operator, and load/bind/eval -TEST_F(ScenarioCppWinrtTest, Scenario20b_LoadBindEval_ReplacementCustomOperator_CPU) +TEST_F(ScenarioCppWinrtTest, Scenario20bLoadBindEvalReplacementCustomOperatorCPU) { std::wstring filePath = FileHelpers::GetModulePath() + L"relu.onnx"; LoadBindEval_CustomOperator_CPU(filePath.c_str()); } //! Scenario21: Load two models, set them up to run chained after one another on the same gpu hardware device -TEST_F(ScenarioCppWinrtGpuTest, DISABLED_Scenario21_RunModel2ChainZ) +TEST_F(ScenarioCppWinrtGpuTest, DISABLED_Scenario21RunModel2ChainZ) { // load a model, TODO: get a model that has an image descriptor std::wstring filePath = FileHelpers::GetModulePath() + L"fns-candy.onnx"; @@ -878,7 +878,7 @@ bool VerifyHelper(ImageFeatureValue actual, ImageFeatureValue expected) return ((float)errors / size < cMaxErrorRate); } -TEST_F(ScenarioCppWinrtTest, DISABLED_Scenario22_ImageBindingAsCPUTensor) +TEST_F(ScenarioCppWinrtTest, DISABLED_Scenario22ImageBindingAsCPUTensor) { std::wstring modulePath = FileHelpers::GetModulePath(); std::wstring inputImagePath = modulePath + L"fish_720.png"; @@ -953,7 +953,7 @@ TEST_F(ScenarioCppWinrtTest, DISABLED_Scenario22_ImageBindingAsCPUTensor) encoder.FlushAsync().get(); } -TEST_F(ScenarioCppWinrtGpuTest, DISABLED_Scenario22_ImageBindingAsGPUTensor) +TEST_F(ScenarioCppWinrtGpuTest, DISABLED_Scenario22ImageBindingAsGPUTensor) { std::wstring modulePath = FileHelpers::GetModulePath(); std::wstring inputImagePath = modulePath + L"fish_720.png";