From 64709b1335380259b6b6e40cd02f5c3aeee7b38d Mon Sep 17 00:00:00 2001 From: Edward Chen <18449977+edgchen1@users.noreply.github.com> Date: Tue, 15 Dec 2020 11:32:43 -0800 Subject: [PATCH] Deprecate Python global configuration functions [Part 1] (#5923) Enable options to be set via execution provider (EP)-specific options and log deprecation warning from current global configuration functions. --- include/onnxruntime/core/common/common.h | 31 +- .../onnxruntime/core/common/string_utils.h | 100 +++++++ .../core/framework/execution_provider.h | 27 +- .../onnxruntime/core/framework/ortdevice.h | 2 + .../core/framework/provider_options.h | 18 ++ .../core/framework/provider_options_utils.h | 39 +++ .../core/framework/arena_extend_strategy.h | 44 +++ onnxruntime/core/framework/bfc_arena.h | 6 +- .../core/framework/execution_provider.cc | 2 + onnxruntime/core/platform/env_var_utils.h | 5 +- .../providers/cpu/cpu_provider_factory.cc | 8 +- .../cpu/cpu_provider_factory_creator.h | 14 + .../providers/cuda/cuda_execution_provider.cc | 46 +-- .../providers/cuda/cuda_execution_provider.h | 32 +- .../cuda/cuda_execution_provider_info.cc | 62 ++++ .../cuda/cuda_execution_provider_info.h | 25 ++ .../providers/cuda/cuda_provider_factory.cc | 65 ++-- .../cuda/cuda_provider_factory_creator.h | 16 + .../providers/rocm/rocm_execution_provider.cc | 41 +-- .../providers/rocm/rocm_execution_provider.h | 23 +- .../rocm/rocm_execution_provider_info.cc | 36 +++ .../rocm/rocm_execution_provider_info.h | 22 ++ .../providers/rocm/rocm_provider_factory.cc | 43 ++- .../rocm/rocm_provider_factory_creator.h | 16 + .../python/onnxruntime_pybind_state.cc | 277 ++++++++---------- onnxruntime/test/common/string_utils_test.cc | 86 ++++++ onnxruntime/test/onnx/tensorprotoutils.cc | 32 +- onnxruntime/test/util/default_providers.cc | 25 +- orttraining/orttraining/models/bert/main.cc | 47 ++- orttraining/orttraining/models/gpt2/main.cc | 21 +- orttraining/orttraining/models/mnist/main.cc | 13 +- tools/ci_build/amd_hipify.py | 41 +-- tools/ci_build/build.py | 0 33 files changed, 778 insertions(+), 487 deletions(-) create mode 100644 include/onnxruntime/core/common/string_utils.h create mode 100644 include/onnxruntime/core/framework/provider_options.h create mode 100644 include/onnxruntime/core/framework/provider_options_utils.h create mode 100644 onnxruntime/core/framework/arena_extend_strategy.h create mode 100644 onnxruntime/core/providers/cpu/cpu_provider_factory_creator.h create mode 100644 onnxruntime/core/providers/cuda/cuda_execution_provider_info.cc create mode 100644 onnxruntime/core/providers/cuda/cuda_execution_provider_info.h create mode 100644 onnxruntime/core/providers/cuda/cuda_provider_factory_creator.h create mode 100644 onnxruntime/core/providers/rocm/rocm_execution_provider_info.cc create mode 100644 onnxruntime/core/providers/rocm/rocm_execution_provider_info.h create mode 100644 onnxruntime/core/providers/rocm/rocm_provider_factory_creator.h create mode 100644 onnxruntime/test/common/string_utils_test.cc mode change 100644 => 100755 tools/ci_build/build.py diff --git a/include/onnxruntime/core/common/common.h b/include/onnxruntime/core/common/common.h index 96924eb847..9071906337 100644 --- a/include/onnxruntime/core/common/common.h +++ b/include/onnxruntime/core/common/common.h @@ -35,6 +35,7 @@ #include "core/common/exceptions.h" #include "core/common/make_unique.h" #include "core/common/status.h" +#include "core/common/string_utils.h" #ifdef USE_MIMALLOC_ARENA_ALLOCATOR #include @@ -261,36 +262,6 @@ void LogRuntimeError(uint32_t session_id, const common::Status& status, const ch #define GSL_SUPPRESS(tag) #endif -inline void MakeStringInternal(std::ostringstream& /*ss*/) noexcept { -} - -template -inline void MakeStringInternal(std::ostringstream& ss, const T& t) noexcept { - ss << t; -} - -template -inline void MakeStringInternal(std::ostringstream& ss, const T& t, const Args&... args) noexcept { - ::onnxruntime::MakeStringInternal(ss, t); - ::onnxruntime::MakeStringInternal(ss, args...); -} - -template -std::string MakeString(const Args&... args) { - std::ostringstream ss; - ::onnxruntime::MakeStringInternal(ss, args...); - return std::string(ss.str()); -} - -// Specializations for already-a-string types. -template <> -inline std::string MakeString(const std::string& str) { - return str; -} -inline std::string MakeString(const char* p_str) { - return p_str; -} - inline long long TimeDiffMicroSeconds(TimePoint start_time) { auto end_time = std::chrono::high_resolution_clock::now(); return std::chrono::duration_cast(end_time - start_time).count(); diff --git a/include/onnxruntime/core/common/string_utils.h b/include/onnxruntime/core/common/string_utils.h new file mode 100644 index 0000000000..bb3dd62cc9 --- /dev/null +++ b/include/onnxruntime/core/common/string_utils.h @@ -0,0 +1,100 @@ +/** + * Copyright (c) 2016-present, Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// Portions Copyright (c) Microsoft Corporation + +#pragma once + +#include +#include +#include + +namespace onnxruntime { + +namespace detail { +inline void MakeStringImpl(std::ostringstream& /*ss*/) noexcept { +} + +template +inline void MakeStringImpl(std::ostringstream& ss, const T& t) noexcept { + ss << t; +} + +template +inline void MakeStringImpl(std::ostringstream& ss, const T& t, const Args&... args) noexcept { + MakeStringImpl(ss, t); + MakeStringImpl(ss, args...); +} +} // namespace detail + +/** + * Makes a string by concatenating string representations of the arguments. + */ +template +std::string MakeString(const Args&... args) { + std::ostringstream ss; + ss.imbue(std::locale::classic()); + detail::MakeStringImpl(ss, args...); + return ss.str(); +} + +// MakeString versions for already-a-string types. + +inline std::string MakeString(const std::string& str) { + return str; +} + +inline std::string MakeString(const char* cstr) { + return cstr; +} + +/** + * Tries to parse a value from an entire string. + */ +template +bool TryParse(const std::string& str, T& value) { + if (std::is_integral::value && std::is_unsigned::value) { + // if T is unsigned integral type, reject negative values which will wrap + if (!str.empty() && str[0] == '-') { + return false; + } + } + + // don't allow leading whitespace + if (!str.empty() && std::isspace(str[0], std::locale::classic())) { + return false; + } + + std::istringstream is{str}; + is.imbue(std::locale::classic()); + T parsed_value{}; + + const bool parse_successful = + is >> parsed_value && + is.get() == std::istringstream::traits_type::eof(); // don't allow trailing characters + if (!parse_successful) { + return false; + } + + value = std::move(parsed_value); + return true; +} + +inline bool TryParse(const std::string& str, std::string& value) { + value = str; + return true; +} + +} // namespace onnxruntime diff --git a/include/onnxruntime/core/framework/execution_provider.h b/include/onnxruntime/core/framework/execution_provider.h index 2be3f20767..b3a276c645 100644 --- a/include/onnxruntime/core/framework/execution_provider.h +++ b/include/onnxruntime/core/framework/execution_provider.h @@ -6,11 +6,12 @@ #include #include "gsl/gsl" -#include "core/common/status.h" #include "core/common/logging/logging.h" -#include "core/framework/tensor.h" -#include "core/framework/func_api.h" +#include "core/common/status.h" #include "core/framework/data_transfer.h" +#include "core/framework/func_api.h" +#include "core/framework/provider_options.h" +#include "core/framework/tensor.h" namespace onnxruntime { class GraphViewer; @@ -31,13 +32,6 @@ using CreateFunctionStateFunc = std::function; using DestroyFunctionStateFunc = std::function; -//unordered maps -using UnorderedMapStringToString = std::unordered_map; - -//data types for execution provider options -using ProviderOptionsVector = std::vector; -using ProviderOptionsMap = std::unordered_map; - struct NodeComputeInfo { CreateFunctionStateFunc create_state_func; ComputeFunc compute_func; @@ -107,16 +101,9 @@ class IExecutionProvider { virtual int GetDeviceId() const { return -1; }; /** - Get execution provider's configurations. + Get execution provider's configuration options. */ - const UnorderedMapStringToString& GetProviderOptions() const { return provider_options_; } - - /** - Store execution provider's configurations. - */ - void SetProviderOptions(UnorderedMapStringToString& options) { - provider_options_ = options; - } + virtual ProviderOptions GetProviderOptions() const; /** Returns an opaque handle whose exact type varies based on the provider @@ -248,7 +235,5 @@ class IExecutionProvider { // convenience list of the allocators so GetAllocatorList doesn't have to build a new vector each time // contains the same instances as allocators_ std::vector allocator_list_; - // It will be set when constructor is being called - UnorderedMapStringToString provider_options_; }; } // namespace onnxruntime diff --git a/include/onnxruntime/core/framework/ortdevice.h b/include/onnxruntime/core/framework/ortdevice.h index c0b186ed5c..afc15187e1 100644 --- a/include/onnxruntime/core/framework/ortdevice.h +++ b/include/onnxruntime/core/framework/ortdevice.h @@ -3,6 +3,8 @@ #pragma once +#include + // Struct to represent a physical device. struct OrtDevice { using DeviceType = int8_t; diff --git a/include/onnxruntime/core/framework/provider_options.h b/include/onnxruntime/core/framework/provider_options.h new file mode 100644 index 0000000000..aab13e808e --- /dev/null +++ b/include/onnxruntime/core/framework/provider_options.h @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include +#include + +namespace onnxruntime { + +// data types for execution provider options + +using ProviderOptions = std::unordered_map; +using ProviderOptionsVector = std::vector; +using ProviderOptionsMap = std::unordered_map; + +} // namespace onnxruntime diff --git a/include/onnxruntime/core/framework/provider_options_utils.h b/include/onnxruntime/core/framework/provider_options_utils.h new file mode 100644 index 0000000000..8804dde7ad --- /dev/null +++ b/include/onnxruntime/core/framework/provider_options_utils.h @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/framework/provider_options.h" + +namespace onnxruntime { +/** + * Reads the named provider option. + * Returns true if the option is present, false otherwise. + */ +template +bool ReadProviderOption(const ProviderOptions& options, const std::string& key, T& value) { + auto it = options.find(key); + if (it != options.end()) { + ORT_ENFORCE( + TryParse(it->second, value), + "Failed to parse provider option \"", key, "\" with value \"", it->second, "\"."); + return true; + } + return false; +} + +/** + * Reads the named provider option. + * Returns the value if the option is present or the specified default value otherwise. + */ +template +T ReadProviderOptionOrDefault( + const ProviderOptions& options, const std::string& key, const T& default_value) { + T value{}; + if (ReadProviderOption(options, key, value)) { + return value; + } + return default_value; +} +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/arena_extend_strategy.h b/onnxruntime/core/framework/arena_extend_strategy.h new file mode 100644 index 0000000000..ebdd981eb9 --- /dev/null +++ b/onnxruntime/core/framework/arena_extend_strategy.h @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include + +namespace onnxruntime { + +enum class ArenaExtendStrategy : int32_t { + kNextPowerOfTwo = 0, + kSameAsRequested, +}; + +inline std::istream& operator>>(std::istream& is, ArenaExtendStrategy& value) { + std::string value_str; + if (is >> value_str) { + if (value_str == "kNextPowerOfTwo") { + value = ArenaExtendStrategy::kNextPowerOfTwo; + } else if (value_str == "kSameAsRequested") { + value = ArenaExtendStrategy::kSameAsRequested; + } else { + is.setstate(std::ios_base::failbit); + } + } + return is; +} + +inline std::ostream& operator<<(std::ostream& os, ArenaExtendStrategy value) { + switch (value) { + case ArenaExtendStrategy::kNextPowerOfTwo: + os << "kNextPowerOfTwo"; + break; + case ArenaExtendStrategy::kSameAsRequested: + os << "kSameAsRequested"; + break; + default: + os << "unknown"; + break; + } + return os; +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/bfc_arena.h b/onnxruntime/core/framework/bfc_arena.h index 0c93eb8695..ef2a8f09aa 100644 --- a/onnxruntime/core/framework/bfc_arena.h +++ b/onnxruntime/core/framework/bfc_arena.h @@ -25,6 +25,7 @@ limitations under the License. #include "core/common/logging/severity.h" #include "core/platform/ort_mutex.h" #include "core/framework/arena.h" +#include "core/framework/arena_extend_strategy.h" #include "core/common/safeint.h" #include "onnxruntime_config.h" @@ -39,11 +40,6 @@ namespace onnxruntime { #endif #endif -enum class ArenaExtendStrategy : int32_t { - kNextPowerOfTwo = 0, - kSameAsRequested, -}; - // A memory allocator that implements a 'best-fit with coalescing' // algorithm. This is essentially a very simple version of Doug Lea's // malloc (dlmalloc). diff --git a/onnxruntime/core/framework/execution_provider.cc b/onnxruntime/core/framework/execution_provider.cc index 3812abb473..9725b23918 100644 --- a/onnxruntime/core/framework/execution_provider.cc +++ b/onnxruntime/core/framework/execution_provider.cc @@ -51,6 +51,8 @@ IExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph, #endif } +ProviderOptions IExecutionProvider::GetProviderOptions() const { return {}; } + common::Status IExecutionProvider::Sync() const { return Status::OK(); }; common::Status IExecutionProvider::OnRunStart() { return Status::OK(); } diff --git a/onnxruntime/core/platform/env_var_utils.h b/onnxruntime/core/platform/env_var_utils.h index e3c0e6d85a..b4d9569857 100644 --- a/onnxruntime/core/platform/env_var_utils.h +++ b/onnxruntime/core/platform/env_var_utils.h @@ -3,8 +3,6 @@ #pragma once -#include - #include "core/common/common.h" #include "core/common/optional.h" #include "core/platform/env.h" @@ -20,10 +18,9 @@ optional ParseEnvironmentVariable(const std::string& name) { return {}; } - std::istringstream is{value_str}; T parsed_value; ORT_ENFORCE( - is >> std::noskipws >> parsed_value && is.eof(), + TryParse(value_str, parsed_value), "Failed to parse environment variable - name: \"", name, "\", value: \"", value_str, "\""); return parsed_value; diff --git a/onnxruntime/core/providers/cpu/cpu_provider_factory.cc b/onnxruntime/core/providers/cpu/cpu_provider_factory.cc index 28445122c9..df2778f208 100644 --- a/onnxruntime/core/providers/cpu/cpu_provider_factory.cc +++ b/onnxruntime/core/providers/cpu/cpu_provider_factory.cc @@ -1,9 +1,13 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +#include "core/providers/cpu/cpu_provider_factory_creator.h" #include "core/providers/cpu/cpu_provider_factory.h" -#include -#include "cpu_execution_provider.h" + +#include + +#include "core/common/make_unique.h" +#include "core/providers/cpu/cpu_execution_provider.h" #include "core/session/abi_session_options_impl.h" #include "core/session/ort_apis.h" diff --git a/onnxruntime/core/providers/cpu/cpu_provider_factory_creator.h b/onnxruntime/core/providers/cpu/cpu_provider_factory_creator.h new file mode 100644 index 0000000000..637cecbf8e --- /dev/null +++ b/onnxruntime/core/providers/cpu/cpu_provider_factory_creator.h @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include + +#include "core/providers/providers.h" + +namespace onnxruntime { + +std::shared_ptr CreateExecutionProviderFactory_CPU(int use_arena); + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/cuda_execution_provider.cc b/onnxruntime/core/providers/cuda/cuda_execution_provider.cc index 2bb5f19e4b..835555d07d 100644 --- a/onnxruntime/core/providers/cuda/cuda_execution_provider.cc +++ b/onnxruntime/core/providers/cuda/cuda_execution_provider.cc @@ -7,6 +7,7 @@ #include "core/framework/fallback_cpu_capability.h" #include "core/framework/kernel_registry.h" #include "core/framework/memcpy.h" +#include "core/framework/provider_options_utils.h" #include "core/graph/graph_utils.h" #include "core/providers/cuda/cuda_allocator.h" #include "core/providers/cuda/cuda_fence.h" @@ -94,41 +95,14 @@ CUDAExecutionProvider::PerThreadContext::~PerThreadContext() { } } -/* - * This method should be called within the constructor, - * so that the configuration of provider related setting can be updated - * and kept at IExecutionProvider level. - */ -void CUDAExecutionProvider::UpdateProviderOptionsInfo() { - UnorderedMapStringToString options; - - options["device_id"] = std::to_string(device_id_); - options["cuda_mem_limit"] = std::to_string(cuda_mem_limit_); - std::string strategy; - if (arena_extend_strategy_ == ArenaExtendStrategy::kNextPowerOfTwo) { - strategy = "kNextPowerOfTwo"; - } else if (arena_extend_strategy_ == ArenaExtendStrategy::kSameAsRequested) { - strategy = "kSameAsRequested"; - } else { - strategy = "unknown"; - } - options["arena_extend_strategy"] = strategy; - - IExecutionProvider::SetProviderOptions(options); -} - CUDAExecutionProvider::CUDAExecutionProvider(const CUDAExecutionProviderInfo& info) : IExecutionProvider{onnxruntime::kCudaExecutionProvider}, - device_id_(info.device_id), - cuda_mem_limit_(info.cuda_mem_limit), - arena_extend_strategy_(info.arena_extend_strategy), - cudnn_conv_algo_(info.cudnn_conv_algo), - do_copy_in_default_stream_(info.do_copy_in_default_stream) { - CUDA_CALL_THROW(cudaSetDevice(device_id_)); + info_{info} { + CUDA_CALL_THROW(cudaSetDevice(info_.device_id)); // must wait GPU idle, otherwise cudaGetDeviceProperties might fail CUDA_CALL_THROW(cudaDeviceSynchronize()); - CUDA_CALL_THROW(cudaGetDeviceProperties(&device_prop_, device_id_)); + CUDA_CALL_THROW(cudaGetDeviceProperties(&device_prop_, info_.device_id)); size_t free = 0; size_t total = 0; @@ -138,10 +112,10 @@ CUDAExecutionProvider::CUDAExecutionProvider(const CUDAExecutionProviderInfo& in [](OrtDevice::DeviceId device_id) { return onnxruntime::make_unique(device_id, CUDA); }, - device_id_, + info_.device_id, true, - {cuda_mem_limit_, - static_cast(arena_extend_strategy_), + {info_.cuda_mem_limit, + static_cast(info_.arena_extend_strategy), -1, -1}); InsertAllocator(CreateAllocator(default_memory_info)); @@ -165,8 +139,6 @@ CUDAExecutionProvider::CUDAExecutionProvider(const CUDAExecutionProviderInfo& in CPU_ALLOCATOR_DEVICE_ID); InsertAllocator(CreateAllocator(cpu_memory_info)); - - UpdateProviderOptionsInfo(); } CUDAExecutionProvider::~CUDAExecutionProvider() { @@ -216,7 +188,7 @@ CUDAExecutionProvider::PerThreadContext& CUDAExecutionProvider::GetPerThreadCont // get or create a context if (context_state_.retired_context_pool.empty()) { - context = std::make_shared(device_id_, cuda_mem_limit_, arena_extend_strategy_); + context = std::make_shared(info_.device_id, info_.cuda_mem_limit, info_.arena_extend_strategy); } else { context = context_state_.retired_context_pool.back(); context_state_.retired_context_pool.pop_back(); @@ -1880,7 +1852,7 @@ static bool CastNeedFallbackToCPU(const onnxruntime::Node& node) { } std::unique_ptr CUDAExecutionProvider::GetDataTransfer() const { - return onnxruntime::make_unique(do_copy_in_default_stream_); + return onnxruntime::make_unique(info_.do_copy_in_default_stream); } std::vector> diff --git a/onnxruntime/core/providers/cuda/cuda_execution_provider.h b/onnxruntime/core/providers/cuda/cuda_execution_provider.h index 2f915477af..5bb2e8f8ba 100644 --- a/onnxruntime/core/providers/cuda/cuda_execution_provider.h +++ b/onnxruntime/core/providers/cuda/cuda_execution_provider.h @@ -8,9 +8,10 @@ #include "core/graph/constants.h" #include "core/framework/allocatormgr.h" -#include "core/framework/bfc_arena.h" +#include "core/framework/arena_extend_strategy.h" #include "core/framework/execution_provider.h" #include "core/platform/ort_mutex.h" +#include "core/providers/cuda/cuda_execution_provider_info.h" #include "core/providers/cuda/cuda_pch.h" #include "core/providers/cuda/gpu_data_transfer.h" #include "core/providers/cuda/shared_inc/cuda_utils.h" @@ -19,15 +20,6 @@ namespace onnxruntime { const int CPU_ALLOCATOR_DEVICE_ID = 0; -// Information needed to construct CUDA execution providers. -struct CUDAExecutionProviderInfo { - OrtDevice::DeviceId device_id{0}; - size_t cuda_mem_limit{std::numeric_limits::max()}; - ArenaExtendStrategy arena_extend_strategy{ArenaExtendStrategy::kNextPowerOfTwo}; - OrtCudnnConvAlgoSearch cudnn_conv_algo{OrtCudnnConvAlgoSearch::EXHAUSTIVE}; - bool do_copy_in_default_stream{true}; -}; - // Logical device representation. class CUDAExecutionProvider : public IExecutionProvider { public: @@ -67,7 +59,7 @@ class CUDAExecutionProvider : public IExecutionProvider { if (count_or_bytes == 0) return nullptr; - return IAllocator::MakeUniquePtr(GetAllocator(device_id_, OrtMemTypeDefault), count_or_bytes); + return IAllocator::MakeUniquePtr(GetAllocator(info_.device_id, OrtMemTypeDefault), count_or_bytes); } std::shared_ptr GetKernelRegistry() const override; @@ -77,19 +69,17 @@ class CUDAExecutionProvider : public IExecutionProvider { const onnxruntime::GraphViewer& graph, const std::vector& kernel_registries) const override; - int GetDeviceId() const { return device_id_; } + int GetDeviceId() const override { return info_.device_id; } const cudaDeviceProp& GetDeviceProp() const { return device_prop_; }; - int GetCudnnConvAlgo() const { return cudnn_conv_algo_; } - void UpdateProviderOptionsInfo(); + int GetCudnnConvAlgo() const { return info_.cudnn_conv_algo; } -private: - OrtDevice::DeviceId device_id_; + ProviderOptions GetProviderOptions() const override { + return CUDAExecutionProviderInfo::ToProviderOptions(info_); + } + + private: + CUDAExecutionProviderInfo info_; cudaDeviceProp device_prop_; - size_t cuda_mem_limit_; - ArenaExtendStrategy arena_extend_strategy_; - int cudnn_conv_algo_; - bool do_copy_in_default_stream_; - struct DeferredReleaseCPUPtrs { bool recorded = false; std::vector cpu_ptrs; diff --git a/onnxruntime/core/providers/cuda/cuda_execution_provider_info.cc b/onnxruntime/core/providers/cuda/cuda_execution_provider_info.cc new file mode 100644 index 0000000000..7fa4d5a046 --- /dev/null +++ b/onnxruntime/core/providers/cuda/cuda_execution_provider_info.cc @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cuda/cuda_execution_provider_info.h" + +#include "core/common/string_utils.h" +#include "core/framework/provider_options_utils.h" +#include "core/providers/cuda/cuda_common.h" + +namespace onnxruntime { +namespace provider_option_names { +constexpr const char* kDeviceId = "device_id"; +constexpr const char* kMemLimit = "cuda_mem_limit"; +constexpr const char* kArenaExtendStrategy = "arena_extend_strategy"; +constexpr const char* kCudnnConvAlgo = "cudnn_conv_algo"; +constexpr const char* kDoCopyInDefaultStream = "do_copy_in_default_stream"; +} // namespace provider_option_names + +CUDAExecutionProviderInfo CUDAExecutionProviderInfo::FromProviderOptions(const ProviderOptions& options) { + CUDAExecutionProviderInfo info{}; + + if (ReadProviderOption(options, provider_option_names::kDeviceId, info.device_id)) { + int num_devices = 0; + CUDA_CALL_THROW(cudaGetDeviceCount(&num_devices)); + ORT_ENFORCE( + 0 <= info.device_id && info.device_id < num_devices, + "Invalid ", provider_option_names::kDeviceId, " value: ", info.device_id, + ", must be between 0 (inclusive) and ", num_devices, " (exclusive)."); + } + ReadProviderOption(options, provider_option_names::kMemLimit, info.cuda_mem_limit); + ReadProviderOption(options, provider_option_names::kArenaExtendStrategy, info.arena_extend_strategy); + { + int cudnn_conv_algo_val; + if (ReadProviderOption(options, provider_option_names::kCudnnConvAlgo, cudnn_conv_algo_val)) { + switch (cudnn_conv_algo_val) { + case OrtCudnnConvAlgoSearch::EXHAUSTIVE: + case OrtCudnnConvAlgoSearch::HEURISTIC: + case OrtCudnnConvAlgoSearch::DEFAULT: + break; + default: + ORT_THROW("Invalid ", provider_option_names::kCudnnConvAlgo, " value: ", cudnn_conv_algo_val); + } + info.cudnn_conv_algo = static_cast(cudnn_conv_algo_val); + } + } + ReadProviderOption(options, provider_option_names::kDoCopyInDefaultStream, info.do_copy_in_default_stream); + + return info; +} + +ProviderOptions CUDAExecutionProviderInfo::ToProviderOptions(const CUDAExecutionProviderInfo& info) { + const ProviderOptions options{ + {provider_option_names::kDeviceId, MakeString(info.device_id)}, + {provider_option_names::kMemLimit, MakeString(info.cuda_mem_limit)}, + {provider_option_names::kArenaExtendStrategy, MakeString(info.arena_extend_strategy)}, + {provider_option_names::kCudnnConvAlgo, MakeString(static_cast(info.cudnn_conv_algo))}, + {provider_option_names::kDoCopyInDefaultStream, MakeString(info.do_copy_in_default_stream)}, + }; + + return options; +} +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/cuda_execution_provider_info.h b/onnxruntime/core/providers/cuda/cuda_execution_provider_info.h new file mode 100644 index 0000000000..b6c763c283 --- /dev/null +++ b/onnxruntime/core/providers/cuda/cuda_execution_provider_info.h @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include + +#include "core/framework/arena_extend_strategy.h" +#include "core/framework/ortdevice.h" +#include "core/framework/provider_options.h" +#include "core/session/onnxruntime_c_api.h" + +namespace onnxruntime { +// Information needed to construct CUDA execution providers. +struct CUDAExecutionProviderInfo { + OrtDevice::DeviceId device_id{0}; + size_t cuda_mem_limit{std::numeric_limits::max()}; + ArenaExtendStrategy arena_extend_strategy{ArenaExtendStrategy::kNextPowerOfTwo}; + OrtCudnnConvAlgoSearch cudnn_conv_algo{OrtCudnnConvAlgoSearch::EXHAUSTIVE}; + bool do_copy_in_default_stream{true}; + + static CUDAExecutionProviderInfo FromProviderOptions(const ProviderOptions& options); + static ProviderOptions ToProviderOptions(const CUDAExecutionProviderInfo& info); +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/cuda_provider_factory.cc b/onnxruntime/core/providers/cuda/cuda_provider_factory.cc index 5f22e4141a..0b1495c232 100644 --- a/onnxruntime/core/providers/cuda/cuda_provider_factory.cc +++ b/onnxruntime/core/providers/cuda/cuda_provider_factory.cc @@ -1,73 +1,64 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +#include "core/providers/cuda/cuda_provider_factory_creator.h" #include "core/providers/cuda/cuda_provider_factory.h" -#include -#include "core/graph/onnx_protobuf.h" -#include "cuda_execution_provider.h" + +#include + +#include "gsl/gsl" + +#include "core/common/make_unique.h" +#include "core/providers/cuda/cuda_execution_provider.h" +#include "core/providers/cuda/cuda_execution_provider_info.h" #include "core/session/abi_session_options_impl.h" #include "core/session/onnxruntime_c_api.h" #include "core/session/ort_apis.h" -#include "core/framework/bfc_arena.h" using namespace onnxruntime; namespace onnxruntime { struct CUDAProviderFactory : IExecutionProviderFactory { - CUDAProviderFactory(OrtDevice::DeviceId device_id, - size_t cuda_mem_limit = std::numeric_limits::max(), - ArenaExtendStrategy arena_extend_strategy = ArenaExtendStrategy::kNextPowerOfTwo, - OrtCudnnConvAlgoSearch cudnn_conv_algo_search = OrtCudnnConvAlgoSearch::EXHAUSTIVE, - bool do_copy_in_default_stream = true) - : device_id_(device_id), - cuda_mem_limit_(cuda_mem_limit), - arena_extend_strategy_(arena_extend_strategy), - cudnn_conv_algo_search_(cudnn_conv_algo_search), - do_copy_in_default_stream_(do_copy_in_default_stream) {} + CUDAProviderFactory(const CUDAExecutionProviderInfo& info) + : info_{info} {} ~CUDAProviderFactory() override {} std::unique_ptr CreateProvider() override; private: - OrtDevice::DeviceId device_id_; - size_t cuda_mem_limit_; - ArenaExtendStrategy arena_extend_strategy_; - OrtCudnnConvAlgoSearch cudnn_conv_algo_search_; - bool do_copy_in_default_stream_; + CUDAExecutionProviderInfo info_; }; std::unique_ptr CUDAProviderFactory::CreateProvider() { - CUDAExecutionProviderInfo info; - info.device_id = device_id_; - info.cuda_mem_limit = cuda_mem_limit_; - info.arena_extend_strategy = arena_extend_strategy_; - info.cudnn_conv_algo = cudnn_conv_algo_search_; - info.do_copy_in_default_stream = do_copy_in_default_stream_; - return onnxruntime::make_unique(info); + return onnxruntime::make_unique(info_); } -std::shared_ptr CreateExecutionProviderFactory_CUDA(OrtDevice::DeviceId device_id, - OrtCudnnConvAlgoSearch cudnn_conv_algo_search = OrtCudnnConvAlgoSearch::EXHAUSTIVE, - size_t cuda_mem_limit = std::numeric_limits::max(), - ArenaExtendStrategy arena_extend_strategy = ArenaExtendStrategy::kNextPowerOfTwo, - bool do_copy_in_default_stream = true) { - return std::make_shared(device_id, cuda_mem_limit, arena_extend_strategy, cudnn_conv_algo_search, do_copy_in_default_stream); +std::shared_ptr CreateExecutionProviderFactory_CUDA(const CUDAExecutionProviderInfo& info) { + return std::make_shared(info); } } // namespace onnxruntime ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProvider_CUDA, _In_ OrtSessionOptions* options, int device_id) { - options->provider_factories.push_back(onnxruntime::CreateExecutionProviderFactory_CUDA(static_cast(device_id))); + CUDAExecutionProviderInfo info{}; + info.device_id = gsl::narrow(device_id); + + options->provider_factories.push_back(onnxruntime::CreateExecutionProviderFactory_CUDA(info)); + return nullptr; } ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider_CUDA, _In_ OrtSessionOptions* options, _In_ const OrtCUDAProviderOptions* cuda_options) { - options->provider_factories.push_back(onnxruntime::CreateExecutionProviderFactory_CUDA(static_cast(cuda_options->device_id), - cuda_options->cudnn_conv_algo_search, cuda_options->cuda_mem_limit, - static_cast(cuda_options->arena_extend_strategy), - cuda_options->do_copy_in_default_stream)); + CUDAExecutionProviderInfo info{}; + info.device_id = gsl::narrow(cuda_options->device_id); + info.cuda_mem_limit = cuda_options->cuda_mem_limit; + info.arena_extend_strategy = static_cast(cuda_options->arena_extend_strategy); + info.cudnn_conv_algo = cuda_options->cudnn_conv_algo_search; + info.do_copy_in_default_stream = cuda_options->do_copy_in_default_stream; + + options->provider_factories.push_back(onnxruntime::CreateExecutionProviderFactory_CUDA(info)); return nullptr; } diff --git a/onnxruntime/core/providers/cuda/cuda_provider_factory_creator.h b/onnxruntime/core/providers/cuda/cuda_provider_factory_creator.h new file mode 100644 index 0000000000..bdc8f61ebd --- /dev/null +++ b/onnxruntime/core/providers/cuda/cuda_provider_factory_creator.h @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include + +#include "core/providers/providers.h" + +#include "core/providers/cuda/cuda_execution_provider_info.h" + +namespace onnxruntime { + +std::shared_ptr CreateExecutionProviderFactory_CUDA(const CUDAExecutionProviderInfo& info); + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/rocm/rocm_execution_provider.cc b/onnxruntime/core/providers/rocm/rocm_execution_provider.cc index 4732bfcf6f..27efe52a45 100644 --- a/onnxruntime/core/providers/rocm/rocm_execution_provider.cc +++ b/onnxruntime/core/providers/rocm/rocm_execution_provider.cc @@ -95,39 +95,14 @@ ROCMExecutionProvider::PerThreadContext::~PerThreadContext() { } } -/* - * This method should be called within the constructor, - * so that the configuration of provider related setting can be updated - * and kept at IExecutionProvider level. - */ -void ROCMExecutionProvider::UpdateProviderOptionsInfo() { - UnorderedMapStringToString options; - - options["device_id"] = std::to_string(device_id_); - options["hip_mem_limit"] = std::to_string(hip_mem_limit_); - std::string strategy; - if (arena_extend_strategy_ == ArenaExtendStrategy::kNextPowerOfTwo) { - strategy = "kNextPowerOfTwo"; - } else if (arena_extend_strategy_ == ArenaExtendStrategy::kSameAsRequested) { - strategy = "kSameAsRequested"; - } else { - strategy = "unknown"; - } - options["arena_extend_strategy"] = strategy; - - IExecutionProvider::SetProviderOptions(options); -} - ROCMExecutionProvider::ROCMExecutionProvider(const ROCMExecutionProviderInfo& info) : IExecutionProvider{onnxruntime::kRocmExecutionProvider}, - device_id_(info.device_id), - hip_mem_limit_(info.hip_mem_limit), - arena_extend_strategy_(info.arena_extend_strategy) { - HIP_CALL_THROW(hipSetDevice(device_id_)); + info_{info} { + HIP_CALL_THROW(hipSetDevice(info_.device_id)); // must wait GPU idle, otherwise hipGetDeviceProperties might fail HIP_CALL_THROW(hipDeviceSynchronize()); - HIP_CALL_THROW(hipGetDeviceProperties(&device_prop_, device_id_)); + HIP_CALL_THROW(hipGetDeviceProperties(&device_prop_, info_.device_id)); size_t free = 0; size_t total = 0; @@ -137,10 +112,10 @@ ROCMExecutionProvider::ROCMExecutionProvider(const ROCMExecutionProviderInfo& in [](OrtDevice::DeviceId device_id) { return onnxruntime::make_unique(device_id, CUDA); }, - device_id_, + info_.device_id, true, - {hip_mem_limit_, - static_cast(arena_extend_strategy_), + {info_.hip_mem_limit, + static_cast(info_.arena_extend_strategy), -1, -1}); InsertAllocator(CreateAllocator(default_memory_info)); @@ -164,8 +139,6 @@ ROCMExecutionProvider::ROCMExecutionProvider(const ROCMExecutionProviderInfo& in CPU_ALLOCATOR_DEVICE_ID); InsertAllocator(CreateAllocator(cpu_memory_info)); - - UpdateProviderOptionsInfo(); } ROCMExecutionProvider::~ROCMExecutionProvider() { @@ -215,7 +188,7 @@ ROCMExecutionProvider::PerThreadContext& ROCMExecutionProvider::GetPerThreadCont // get or create a context if (context_state_.retired_context_pool.empty()) { - context = std::make_shared(device_id_, hip_mem_limit_, arena_extend_strategy_); + context = std::make_shared(info_.device_id, info_.hip_mem_limit, info_.arena_extend_strategy); } else { context = context_state_.retired_context_pool.back(); context_state_.retired_context_pool.pop_back(); diff --git a/onnxruntime/core/providers/rocm/rocm_execution_provider.h b/onnxruntime/core/providers/rocm/rocm_execution_provider.h index e026c0dccd..9352c3144f 100644 --- a/onnxruntime/core/providers/rocm/rocm_execution_provider.h +++ b/onnxruntime/core/providers/rocm/rocm_execution_provider.h @@ -11,21 +11,15 @@ #include "core/framework/bfc_arena.h" #include "core/framework/execution_provider.h" #include "core/platform/ort_mutex.h" -#include "core/providers/rocm/rocm_pch.h" #include "core/providers/rocm/gpu_data_transfer.h" +#include "core/providers/rocm/rocm_execution_provider_info.h" +#include "core/providers/rocm/rocm_pch.h" #include "core/providers/rocm/shared_inc/rocm_utils.h" namespace onnxruntime { const int CPU_ALLOCATOR_DEVICE_ID = 0; -// Information needed to construct HIP execution providers. -struct ROCMExecutionProviderInfo { - OrtDevice::DeviceId device_id{0}; - size_t hip_mem_limit{std::numeric_limits::max()}; - ArenaExtendStrategy arena_extend_strategy{ArenaExtendStrategy::kNextPowerOfTwo}; -}; - // Logical device representation. class ROCMExecutionProvider : public IExecutionProvider { public: @@ -65,7 +59,7 @@ class ROCMExecutionProvider : public IExecutionProvider { if (count_or_bytes == 0) return nullptr; - return IAllocator::MakeUniquePtr(GetAllocator(device_id_, OrtMemTypeDefault), count_or_bytes); + return IAllocator::MakeUniquePtr(GetAllocator(info_.device_id, OrtMemTypeDefault), count_or_bytes); } std::shared_ptr GetKernelRegistry() const override; @@ -75,15 +69,16 @@ class ROCMExecutionProvider : public IExecutionProvider { const onnxruntime::GraphViewer& graph, const std::vector& kernel_registries) const override; - int GetDeviceId() const override { return device_id_; } + int GetDeviceId() const override { return info_.device_id; } const hipDeviceProp_t& GetDeviceProp() const { return device_prop_; }; - void UpdateProviderOptionsInfo(); + + ProviderOptions GetProviderOptions() const override { + return ROCMExecutionProviderInfo::ToProviderOptions(info_); + } private: - OrtDevice::DeviceId device_id_; + ROCMExecutionProviderInfo info_; hipDeviceProp_t device_prop_; - size_t hip_mem_limit_; - ArenaExtendStrategy arena_extend_strategy_; struct DeferredReleaseCPUPtrs { bool recorded = false; diff --git a/onnxruntime/core/providers/rocm/rocm_execution_provider_info.cc b/onnxruntime/core/providers/rocm/rocm_execution_provider_info.cc new file mode 100644 index 0000000000..46333b8109 --- /dev/null +++ b/onnxruntime/core/providers/rocm/rocm_execution_provider_info.cc @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/rocm/rocm_execution_provider_info.h" + +#include "core/common/string_utils.h" +#include "core/framework/provider_options_utils.h" + +namespace onnxruntime { +namespace provider_option_names { +constexpr const char* kDeviceId = "device_id"; +constexpr const char* kMemLimit = "hip_mem_limit"; +constexpr const char* kArenaExtendStrategy = "arena_extend_strategy"; +} // namespace provider_option_names + +ROCMExecutionProviderInfo ROCMExecutionProviderInfo::FromProviderOptions(const ProviderOptions& options) { + ROCMExecutionProviderInfo info{}; + + // TODO validate info.device_id + ReadProviderOption(options, provider_option_names::kDeviceId, info.device_id); + ReadProviderOption(options, provider_option_names::kMemLimit, info.hip_mem_limit); + ReadProviderOption(options, provider_option_names::kArenaExtendStrategy, info.arena_extend_strategy); + + return info; +} + +ProviderOptions ROCMExecutionProviderInfo::ToProviderOptions(const ROCMExecutionProviderInfo& info) { + const ProviderOptions options{ + {provider_option_names::kDeviceId, MakeString(info.device_id)}, + {provider_option_names::kMemLimit, MakeString(info.hip_mem_limit)}, + {provider_option_names::kArenaExtendStrategy, MakeString(info.arena_extend_strategy)}, + }; + + return options; +} +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/rocm/rocm_execution_provider_info.h b/onnxruntime/core/providers/rocm/rocm_execution_provider_info.h new file mode 100644 index 0000000000..4bf12499d9 --- /dev/null +++ b/onnxruntime/core/providers/rocm/rocm_execution_provider_info.h @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include + +#include "core/framework/arena_extend_strategy.h" +#include "core/framework/ortdevice.h" +#include "core/framework/provider_options.h" + +namespace onnxruntime { +// Information needed to construct HIP execution providers. +struct ROCMExecutionProviderInfo { + OrtDevice::DeviceId device_id{0}; + size_t hip_mem_limit{std::numeric_limits::max()}; + ArenaExtendStrategy arena_extend_strategy{ArenaExtendStrategy::kNextPowerOfTwo}; + + static ROCMExecutionProviderInfo FromProviderOptions(const ProviderOptions& options); + static ProviderOptions ToProviderOptions(const ROCMExecutionProviderInfo& info); +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/rocm/rocm_provider_factory.cc b/onnxruntime/core/providers/rocm/rocm_provider_factory.cc index 00fa64d297..1e5ba9f46c 100644 --- a/onnxruntime/core/providers/rocm/rocm_provider_factory.cc +++ b/onnxruntime/core/providers/rocm/rocm_provider_factory.cc @@ -1,51 +1,48 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +#include "core/providers/rocm/rocm_provider_factory_creator.h" #include "core/providers/rocm/rocm_provider_factory.h" -#include -#include "core/graph/onnx_protobuf.h" -#include "rocm_execution_provider.h" + +#include + +#include "gsl/gsl" + +#include "core/common/make_unique.h" +#include "core/providers/rocm/rocm_execution_provider.h" +#include "core/providers/rocm/rocm_execution_provider_info.h" #include "core/session/abi_session_options_impl.h" -#include "core/framework/bfc_arena.h" using namespace onnxruntime; namespace onnxruntime { struct HIPProviderFactory : IExecutionProviderFactory { - HIPProviderFactory(OrtDevice::DeviceId device_id, - size_t hip_mem_limit = std::numeric_limits::max(), - ArenaExtendStrategy arena_extend_strategy = ArenaExtendStrategy::kNextPowerOfTwo) - : device_id_(device_id), - hip_mem_limit_(hip_mem_limit), - arena_extend_strategy_(arena_extend_strategy) {} + HIPProviderFactory(const ROCMExecutionProviderInfo& info) + : info_{info} {} ~HIPProviderFactory() override {} std::unique_ptr CreateProvider() override; private: - OrtDevice::DeviceId device_id_; - size_t hip_mem_limit_; - ArenaExtendStrategy arena_extend_strategy_; + ROCMExecutionProviderInfo info_; }; std::unique_ptr HIPProviderFactory::CreateProvider() { - ROCMExecutionProviderInfo info; - info.device_id = device_id_; - info.hip_mem_limit = hip_mem_limit_; - info.arena_extend_strategy = arena_extend_strategy_; - return onnxruntime::make_unique(info); + return onnxruntime::make_unique(info_); } -std::shared_ptr CreateExecutionProviderFactory_ROCM(OrtDevice::DeviceId device_id, - size_t hip_mem_limit = std::numeric_limits::max(), - ArenaExtendStrategy arena_extend_strategy = ArenaExtendStrategy::kNextPowerOfTwo) { - return std::make_shared(device_id, hip_mem_limit, arena_extend_strategy); +std::shared_ptr CreateExecutionProviderFactory_ROCM(const ROCMExecutionProviderInfo& info) { + return std::make_shared(info); } } // namespace onnxruntime ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProvider_ROCM, _In_ OrtSessionOptions* options, int device_id) { - options->provider_factories.push_back(onnxruntime::CreateExecutionProviderFactory_ROCM(static_cast(device_id))); + ROCMExecutionProviderInfo info{}; + info.device_id = gsl::narrow(device_id); + + options->provider_factories.push_back(onnxruntime::CreateExecutionProviderFactory_ROCM(info)); + return nullptr; } diff --git a/onnxruntime/core/providers/rocm/rocm_provider_factory_creator.h b/onnxruntime/core/providers/rocm/rocm_provider_factory_creator.h new file mode 100644 index 0000000000..8fa67f72b3 --- /dev/null +++ b/onnxruntime/core/providers/rocm/rocm_provider_factory_creator.h @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include + +#include "core/providers/providers.h" + +#include "core/providers/rocm/rocm_execution_provider_info.h" + +namespace onnxruntime { + +std::shared_ptr CreateExecutionProviderFactory_ROCM(const ROCMExecutionProviderInfo& info); + +} // namespace onnxruntime diff --git a/onnxruntime/python/onnxruntime_pybind_state.cc b/onnxruntime/python/onnxruntime_pybind_state.cc index efa1e75460..3935a982fb 100644 --- a/onnxruntime/python/onnxruntime_pybind_state.cc +++ b/onnxruntime/python/onnxruntime_pybind_state.cc @@ -11,17 +11,28 @@ #include "core/common/logging/logging.h" #include "core/common/logging/severity.h" -#include "core/framework/bfc_arena.h" +#include "core/common/optional.h" +#include "core/framework/arena_extend_strategy.h" #include "core/framework/data_transfer_utils.h" #include "core/framework/data_types_internal.h" #include "core/framework/kernel_registry.h" +#include "core/framework/provider_options_utils.h" #include "core/framework/random_seed.h" #include "core/framework/tensorprotoutils.h" #include "core/framework/TensorSeq.h" #include "core/graph/graph_viewer.h" +#include "core/platform/env.h" #include "core/session/IOBinding.h" #include "core/session/abi_session_options_impl.h" -#include "core/platform/env.h" + +// execution provider factory creator headers +#include "core/providers/cpu/cpu_provider_factory_creator.h" +#ifdef USE_CUDA +#include "core/providers/cuda/cuda_provider_factory_creator.h" +#endif +#ifdef USE_ROCM +#include "core/providers/rocm/rocm_provider_factory_creator.h" +#endif struct OrtStatus { OrtErrorCode code; @@ -128,21 +139,21 @@ struct OrtStatus { #include "core/session/onnxruntime_cxx_api.h" #include "core/providers/providers.h" #include "core/providers/cpu/cpu_execution_provider.h" -#include "core/providers/cpu/cpu_provider_factory.h" #if defined(USE_CUDA) || defined(USE_ROCM) #ifdef USE_CUDA -#include "core/providers/cuda/cuda_provider_factory.h" #include "core/providers/cuda/shared_inc/cuda_call.h" -#include "core/providers/cuda/cuda_execution_provider.h" #include "core/providers/cuda/cuda_allocator.h" +// TODO remove deprecated global config OrtCudnnConvAlgoSearch cudnn_conv_algo_search = OrtCudnnConvAlgoSearch::EXHAUSTIVE; +// TODO remove deprecated global config bool do_copy_in_default_stream = true; -#elif USE_ROCM -#include "core/providers/rocm/rocm_provider_factory.h" #endif +// TODO remove deprecated global config OrtDevice::DeviceId cuda_device_id = 0; +// TODO remove deprecated global config size_t cuda_mem_limit = std::numeric_limits::max(); +// TODO remove deprecated global config onnxruntime::ArenaExtendStrategy arena_extend_strategy = onnxruntime::ArenaExtendStrategy::kNextPowerOfTwo; #endif @@ -154,10 +165,12 @@ onnxruntime::ArenaExtendStrategy arena_extend_strategy = onnxruntime::ArenaExten #endif #ifdef USE_OPENVINO #include "core/providers/openvino/openvino_provider_factory.h" +// TODO remove deprecated global config std::string openvino_device_type; #endif #ifdef USE_NUPHAR #include "core/providers/nuphar/nuphar_provider_factory.h" +// TODO remove deprecated global config std::string nuphar_settings; #endif #ifdef USE_VITISAI @@ -173,8 +186,6 @@ std::string nuphar_settings; #include "core/providers/dml/dml_provider_factory.h" #endif -#define PYBIND_UNREFERENCED_PARAMETER(parameter) ((void)(parameter)) - // Explicitly provide a definition for the static const var 'GPU' in the OrtDevice struct, // GCC 4.x doesn't seem to define this and it breaks the pipelines based on CentOS as it uses // GCC 4.x. @@ -182,15 +193,6 @@ std::string nuphar_settings; const OrtDevice::DeviceType OrtDevice::GPU; namespace onnxruntime { -std::shared_ptr CreateExecutionProviderFactory_CPU(int use_arena); -std::shared_ptr CreateExecutionProviderFactory_CUDA(OrtDevice::DeviceId device_id, - OrtCudnnConvAlgoSearch cudnn_conv_algo_search, - size_t cuda_mem_limit, - onnxruntime::ArenaExtendStrategy arena_extend_strategy, - bool do_copy_in_default_stream); -std::shared_ptr CreateExecutionProviderFactory_ROCM(OrtDevice::DeviceId device_id, - size_t cuda_mem_limit, - onnxruntime::ArenaExtendStrategy arena_extend_strategy); std::shared_ptr CreateExecutionProviderFactory_Tensorrt(int device_id); std::shared_ptr CreateExecutionProviderFactory_MIGraphX(int device_id); std::shared_ptr CreateExecutionProviderFactory_Dnnl(int use_arena); @@ -452,30 +454,6 @@ static const std::vector& GetAvailableProviders() { #ifdef USE_CUDA -/* - * Validate a string that is positive integer or zero. - * - * (-1234, 43.21, +43.21 ... are not valid, - * 1234, +4321, 12 ... are valid) - */ -static bool IsPositiveInteger(const std::string& s) { - if (s.length() == 0) { - return false; - } - - for (size_t i = 0; i < s.length(); i++) { - if (i == 0 && s[i] == '+' && s.length() > 1) { - continue; - } - - if (!isdigit(s[i])) { - return false; - } - } - - return true; -} - static bool IsCudaDeviceIdValid(const onnxruntime::logging::Logger& logger, int id) { int num_devices = 0; CUDA_CALL_THROW(cudaGetDeviceCount(&num_devices)); @@ -493,70 +471,6 @@ static bool IsCudaDeviceIdValid(const onnxruntime::logging::Logger& logger, int return true; } -static void UpdateCudaProviderOptions(InferenceSession* sess, onnxruntime::CUDAExecutionProviderInfo& options, - std::unordered_map options_map) { - std::unordered_map::iterator it; - - it = options_map.find("device_id"); - if (it != options_map.end()) { - OrtDevice::DeviceId device_id; - try { - int id = std::stoi(it->second); - device_id = static_cast(id); - } catch (...) { - throw std::runtime_error("Please provide device id with integer."); - } - - if (!IsCudaDeviceIdValid(*sess->GetLogger(), device_id)) { - throw std::runtime_error("Please provide available device id."); - } - options.device_id = device_id; - LOGS(*(sess->GetLogger()), INFO) << "cuda device id is set to " << device_id; - } - - it = options_map.find("cuda_mem_limit"); - if (it != options_map.end()) { - // The reason to check whether the string is positive integer upfront is that - // when calling stoull(), if the minus sign was part of the input sequence, - // the numeric value calculated from the sequence of digits is negated. - // In other words, it will cause wraparound. - // So, we rule out negative integer string beforehand. - if (!IsPositiveInteger(it->second)) { - throw std::runtime_error("Please provide cuda memory limitation size with positive integer."); - } - - size_t size; - try { -#if (defined(__amd64__) || defined(_M_AMD64) || defined(__aarch64__)) - size = std::stoull(it->second, nullptr, 0); -#else - size = std::stoul(it->second, nullptr, 0); -#endif - } catch (...) { - throw std::runtime_error("Please provide cuda memory limitation size with positive integer and within range."); - } - - options.cuda_mem_limit = size; - LOGS(*(sess->GetLogger()), INFO) << "cuda memory limitation is set to " << size; - } - - it = options_map.find("arena_extend_strategy"); - if (it != options_map.end()) { - onnxruntime::ArenaExtendStrategy strategy; - - if (it->second.compare("kNextPowerOfTwo") == 0) { - strategy = onnxruntime::ArenaExtendStrategy::kNextPowerOfTwo; - } else if (it->second.compare("kSameAsRequested") == 0) { - strategy = onnxruntime::ArenaExtendStrategy::kSameAsRequested; - } else { - throw std::runtime_error("Please provide proper cuda arena extend strategy."); - } - - options.arena_extend_strategy = strategy; - LOGS(*(sess->GetLogger()), INFO) << "cuda arean extend strategy is set to " << it->second; - } -} - static AllocatorPtr GetCudaAllocator(OrtDevice::DeviceId id) { // Current approach is not thread-safe, but there are some bigger infra pieces to put together in order to make // multi-threaded CUDA allocation work we need to maintain a per-thread CUDA allocator @@ -601,12 +515,10 @@ static const std::unordered_map* GetCudaToHos /* * Register execution provider with options. - * - * (note: currently only cuda EP supports this feature and rest of EPs use default options) */ static void RegisterExecutionProviders(InferenceSession* sess, const std::vector& provider_types, - ProviderOptionsMap& provider_options_map) { - PYBIND_UNREFERENCED_PARAMETER(provider_options_map); + const ProviderOptionsMap& provider_options_map) { + ORT_UNUSED_PARAMETER(provider_options_map); for (const std::string& type : provider_types) { if (type == kCpuExecutionProvider) { @@ -622,33 +534,39 @@ static void RegisterExecutionProviders(InferenceSession* sess, const std::vector #endif } else if (type == kCudaExecutionProvider) { #ifdef USE_CUDA + const auto it = provider_options_map.find(type); + const CUDAExecutionProviderInfo info = + it != provider_options_map.end() + ? CUDAExecutionProviderInfo::FromProviderOptions(it->second) + : [&]() { + CUDAExecutionProviderInfo info{}; + info.device_id = cuda_device_id; + info.cuda_mem_limit = cuda_mem_limit; + info.arena_extend_strategy = arena_extend_strategy; + info.cudnn_conv_algo = cudnn_conv_algo_search; + info.do_copy_in_default_stream = do_copy_in_default_stream; + return info; + }(); - auto it = provider_options_map.find(type); - if (it != provider_options_map.end()) { - onnxruntime::CUDAExecutionProviderInfo cuda_provider_options; - UpdateCudaProviderOptions(sess, cuda_provider_options, it->second); - - RegisterExecutionProvider( - sess, *onnxruntime::CreateExecutionProviderFactory_CUDA(cuda_provider_options.device_id, - cuda_provider_options.cudnn_conv_algo, - cuda_provider_options.cuda_mem_limit, - cuda_provider_options.arena_extend_strategy, - cuda_provider_options.do_copy_in_default_stream)); - } else { - RegisterExecutionProvider( - sess, *onnxruntime::CreateExecutionProviderFactory_CUDA(cuda_device_id, - cudnn_conv_algo_search, - cuda_mem_limit, - arena_extend_strategy, - do_copy_in_default_stream)); - } + RegisterExecutionProvider( + sess, *onnxruntime::CreateExecutionProviderFactory_CUDA(info)); #endif } else if (type == kRocmExecutionProvider) { #ifdef USE_ROCM + const auto it = provider_options_map.find(type); + const ROCMExecutionProviderInfo info = + it != provider_options_map.end() + ? ROCMExecutionProviderInfo::FromProviderOptions(it->second) + : [&]() { + ROCMExecutionProviderInfo info{}; + info.device_id = cuda_device_id; + info.hip_mem_limit = cuda_mem_limit; + info.arena_extend_strategy = arena_extend_strategy; + return info; + }(); + RegisterExecutionProvider( - sess, *onnxruntime::CreateExecutionProviderFactory_ROCM(cuda_device_id, - cuda_mem_limit, - arena_extend_strategy)); + sess, *onnxruntime::CreateExecutionProviderFactory_ROCM(info)); #endif } else if (type == kDnnlExecutionProvider) { #ifdef USE_DNNL @@ -666,8 +584,7 @@ static void RegisterExecutionProviders(InferenceSession* sess, const std::vector if (option.first == "device_type") { openvino_device_type = option.second; params.device_type = openvino_device_type.c_str(); - } - else if (option.first == "enable_vpu_fast_compile") { + } else if (option.first == "enable_vpu_fast_compile") { if (option.second == "True") { params.enable_vpu_fast_compile = true; } else if (option.second == "False") { @@ -692,6 +609,11 @@ static void RegisterExecutionProviders(InferenceSession* sess, const std::vector #endif } else if (type == kNupharExecutionProvider) { #if USE_NUPHAR + const auto it = provider_options_map.find(type); + if (it != provider_options_map.end()) { + ReadProviderOption(it->second, "nuphar_settings", nuphar_settings); + } + RegisterExecutionProvider( sess, *onnxruntime::CreateExecutionProviderFactory_Nuphar(true, nuphar_settings.c_str())); @@ -807,6 +729,17 @@ static bool CheckIfTensor(const std::vector& def_list, return type_proto.has_tensor_type(); } +#if defined(USE_NUPHAR) || \ + defined(USE_OPENVINO) || \ + defined(USE_CUDA) || \ + defined(USE_ROCM) +static void LogDeprecationWarning( + const std::string& deprecated, const optional& alternative = nullopt) { + LOGS_DEFAULT(WARNING) << "This is DEPRECATED and will be removed in the future: " << deprecated; + LOGS_DEFAULT_IF(alternative.has_value(), WARNING) << "As an alternative, use: " << *alternative; +} +#endif + void addGlobalMethods(py::module& m, Environment& env) { m.def("get_default_session_options", &GetDefaultCPUSessionOptions, "Return a default session_options instance."); m.def("get_session_initializer", &SessionObjectInitializer::Get, "Return a default session object initializer."); @@ -847,10 +780,14 @@ void addGlobalMethods(py::module& m, Environment& env) { }); #ifdef USE_NUPHAR + // TODO remove deprecated global config m.def("set_nuphar_settings", [](const std::string& str) { + LogDeprecationWarning("set_nuphar_settings", "Nuphar execution provider option \"nuphar_settings\""); nuphar_settings = str; }); + // TODO remove deprecated global config m.def("get_nuphar_settings", []() -> std::string { + LogDeprecationWarning("get_nuphar_settings"); return nuphar_settings; }); #endif @@ -865,13 +802,19 @@ void addGlobalMethods(py::module& m, Environment& env) { }, "Lists all OpenVINO device ids available."); /* -* The following APIs to set config options are deprecated. Use Session.set_providers() instead. -*/ + * The following APIs to set config options are deprecated. Use Session.set_providers() instead. + */ + // TODO remove deprecated global config m.def( - "set_openvino_device", [](const std::string& device_type) { openvino_device_type = device_type; }, + "set_openvino_device", [](const std::string& device_type) { + LogDeprecationWarning("set_openvino_device", "OpenVINO execution provider option \"device_type\""); + openvino_device_type = device_type; + }, "Set the prefered OpenVINO device type to be used. If left unset, the device type selected during build time will be used."); + // TODO remove deprecated global config m.def( "get_openvino_device", []() -> std::string { + LogDeprecationWarning("get_openvino_device"); return openvino_device_type; }, "Gets the dynamically selected OpenVINO device type for inference."); @@ -890,10 +833,26 @@ void addGlobalMethods(py::module& m, Environment& env) { std::vector> factories = { onnxruntime::CreateExecutionProviderFactory_CPU(0), #ifdef USE_CUDA - onnxruntime::CreateExecutionProviderFactory_CUDA(cuda_device_id, cudnn_conv_algo_search, cuda_mem_limit, arena_extend_strategy, do_copy_in_default_stream), + onnxruntime::CreateExecutionProviderFactory_CUDA( + [&]() { + CUDAExecutionProviderInfo info{}; + info.device_id = cuda_device_id; + info.cuda_mem_limit = cuda_mem_limit; + info.arena_extend_strategy = arena_extend_strategy; + info.cudnn_conv_algo = cudnn_conv_algo_search; + info.do_copy_in_default_stream = do_copy_in_default_stream; + return info; + }()), #endif #ifdef USE_ROCM - onnxruntime::CreateExecutionProviderFactory_ROCM(cuda_device_id, cuda_mem_limit, arena_extend_strategy), + onnxruntime::CreateExecutionProviderFactory_ROCM( + [&]() { + ROCMExecutionProviderInfo info{}; + info.device_id = cuda_device_id; + info.hip_mem_limit = cuda_mem_limit; + info.arena_extend_strategy = arena_extend_strategy; + return info; + }()), #endif #ifdef USE_DNNL onnxruntime::CreateExecutionProviderFactory_Dnnl(1), @@ -945,8 +904,14 @@ void addGlobalMethods(py::module& m, Environment& env) { * InferenceSession.set_providers(list_of_providers, list_of_provider_option_dicts) * */ - m.def("set_cuda_device_id", [](const int id) { cuda_device_id = static_cast(id); }); + // TODO remove deprecated global config + m.def("set_cuda_device_id", [](const int id) { + LogDeprecationWarning("set_cuda_device_id", "CUDA/ROCM execution provider option \"device_id\""); + cuda_device_id = static_cast(id); + }); + // TODO remove deprecated global config m.def("set_cudnn_conv_algo_search", [](const OrtCudnnConvAlgoSearch algo) { + LogDeprecationWarning("set_cudnn_conv_algo_search", "CUDA execution provider option \"cudnn_conv_algo\""); #ifdef USE_ROCM ORT_UNUSED_PARAMETER(algo); ORT_THROW("set_cudnn_conv_algo_search is not supported in ROCM"); @@ -954,7 +919,10 @@ void addGlobalMethods(py::module& m, Environment& env) { cudnn_conv_algo_search = algo; #endif }); + // TODO remove deprecated global config m.def("set_do_copy_in_default_stream", [](const bool use_single_stream) { + LogDeprecationWarning( + "set_do_copy_in_default_stream", "CUDA execution provider option \"do_copy_in_default_stream\""); #ifdef USE_ROCM ORT_UNUSED_PARAMETER(use_single_stream); ORT_THROW("set_do_copy_in_default_stream is not supported in ROCM"); @@ -962,8 +930,18 @@ void addGlobalMethods(py::module& m, Environment& env) { do_copy_in_default_stream = use_single_stream; #endif }); - m.def("set_cuda_mem_limit", [](const int64_t limit) { cuda_mem_limit = static_cast(limit); }); - m.def("set_arena_extend_strategy", [](const onnxruntime::ArenaExtendStrategy strategy) { arena_extend_strategy = strategy; }); + // TODO remove deprecated global config + m.def("set_cuda_mem_limit", [](const int64_t limit) { + LogDeprecationWarning( + "set_cuda_mem_limit", + "CUDA execution provider option \"cuda_mem_limit\", ROCM execution provider option \"hip_mem_limit\""); + cuda_mem_limit = gsl::narrow(limit); + }); + // TODO remove deprecated global config + m.def("set_arena_extend_strategy", [](const onnxruntime::ArenaExtendStrategy strategy) { + LogDeprecationWarning("set_arena_extend_strategy", "CUDA/ROCM execution provider option \"arena_extend_strategy\""); + arena_extend_strategy = strategy; + }); #endif } @@ -1112,6 +1090,11 @@ void addObjectMethods(py::module& m, Environment& env) { .value("CPU", OrtMemType::OrtMemTypeCPU) .value("DEFAULT", OrtMemType::OrtMemTypeDefault); + py::enum_(m, "OrtCudnnConvAlgoSearch") + .value("EXHAUSTIVE", OrtCudnnConvAlgoSearch::EXHAUSTIVE) + .value("HEURISTIC", OrtCudnnConvAlgoSearch::HEURISTIC) + .value("DEFAULT", OrtCudnnConvAlgoSearch::DEFAULT); + py::class_ device(m, "OrtDevice", R"pbdoc(ONNXRuntime device informaion.)pbdoc"); device.def(py::init()) .def("device_id", &OrtDevice::Id, R"pbdoc(Device Id.)pbdoc") @@ -1533,7 +1516,7 @@ Applies to session load, initialization, etc. Default is 0.)pbdoc") dim_name, onnxruntime::FreeDimensionOverrideType::Denotation, dim_value}); }, - "Rpbdoc(Specify the dimension size for each denotation associated with an input's free dimension.)pbdoc") + R"pbdoc(Specify the dimension size for each denotation associated with an input's free dimension.)pbdoc") .def( "add_free_dimension_override_by_name", [](PySessionOptions* options, const char* dim_name, int64_t dim_value) @@ -1542,7 +1525,7 @@ Applies to session load, initialization, etc. Default is 0.)pbdoc") dim_name, onnxruntime::FreeDimensionOverrideType::Name, dim_value}); }, - "Rpbdoc(Specify values of named dimensions within model inputs.)pbdoc") + R"pbdoc(Specify values of named dimensions within model inputs.)pbdoc") .def( "add_session_config_entry", [](PySessionOptions* options, const char* config_key, const char* config_value) -> void { @@ -1550,7 +1533,7 @@ Applies to session load, initialization, etc. Default is 0.)pbdoc") if (!status.IsOK()) throw std::runtime_error(status.ErrorMessage()); }, - "Rpbdoc(Set a single session configuration entry as a pair of strings.)pbdoc") + R"pbdoc(Set a single session configuration entry as a pair of strings.)pbdoc") .def( "get_session_config_entry", [](PySessionOptions* options, const char* config_key) -> std::string { @@ -1561,7 +1544,7 @@ Applies to session load, initialization, etc. Default is 0.)pbdoc") return value; }, - "Rpbdoc(Get a single session configuration value using the given configuration key.)pbdoc") + R"pbdoc(Get a single session configuration value using the given configuration key.)pbdoc") .def( "register_custom_ops_library", [](PySessionOptions* options, const char* library_path) @@ -1585,7 +1568,7 @@ Applies to session load, initialization, etc. Default is 0.)pbdoc") ORT_THROW("Custom Ops are not supported in this build."); #endif }, - "Rpbdoc(Specify the path to the shared library containing the custom op kernels required to run a model.)pbdoc") + R"pbdoc(Specify the path to the shared library containing the custom op kernels required to run a model.)pbdoc") .def( "add_initializer", [](PySessionOptions* options, const char* name, py::object& ml_value_pyobject) -> void { ORT_ENFORCE(strcmp(Py_TYPE(ml_value_pyobject.ptr())->tp_name, PYTHON_ORTVALUE_OBJECT_NAME) == 0, "The provided Python object must be an OrtValue"); diff --git a/onnxruntime/test/common/string_utils_test.cc b/onnxruntime/test/common/string_utils_test.cc new file mode 100644 index 0000000000..ca6688c6c5 --- /dev/null +++ b/onnxruntime/test/common/string_utils_test.cc @@ -0,0 +1,86 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/string_utils.h" + +#include "gtest/gtest.h" + +namespace onnxruntime { +namespace test { + +namespace { +template +void TestSuccessfulParse(const std::string& input, const T& expected_value) { + T value; + ASSERT_TRUE(TryParse(input, value)); + EXPECT_EQ(value, expected_value); +} + +template +void TestFailedParse(const std::string& input) { + T value; + EXPECT_FALSE(TryParse(input, value)); +} +} // namespace + +TEST(StringUtilsTest, TryParse) { + TestSuccessfulParse("-1", -1); + TestSuccessfulParse("42", 42u); + TestSuccessfulParse("2.5", 2.5f); + TestSuccessfulParse("1", true); + TestSuccessfulParse("0", false); + + // out of range + TestFailedParse("32768"); + TestFailedParse("-1"); + TestFailedParse("1e100"); + TestFailedParse("2"); + // invalid representation + TestFailedParse("1.2"); + TestFailedParse("one"); + // leading or trailing characters + TestFailedParse(" 1"); + TestFailedParse("1 "); +} + +TEST(StringUtilsTest, TryParseString) { + // when parsing a string as a string, allow leading and trailing whitespace + const std::string s = " this is a string! "; + TestSuccessfulParse(s, s); +} + +namespace { +struct S { + int i{}; + + bool operator==(const S& other) const { + return i == other.i; + } +}; + +std::ostream& operator<<(std::ostream& os, const S& s) { + os << "S " << s.i; + return os; +} + +std::istream& operator>>(std::istream& is, S& s) { + std::string struct_name; + is >> struct_name >> s.i; + if (struct_name != "S") { + is.setstate(std::ios_base::failbit); + } + return is; +} +} // namespace + +TEST(StringUtilsTest, MakeStringAndTryParseCustomType) { + S s; + s.i = 42; + const auto str = MakeString(s); + S parsed_s; + ASSERT_TRUE(TryParse(str, parsed_s)); + ASSERT_EQ(parsed_s, s); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/onnx/tensorprotoutils.cc b/onnxruntime/test/onnx/tensorprotoutils.cc index f9369e9be3..ba081d6af1 100644 --- a/onnxruntime/test/onnx/tensorprotoutils.cc +++ b/onnxruntime/test/onnx/tensorprotoutils.cc @@ -11,6 +11,7 @@ #include "mem_buffer.h" #include "core/common/safeint.h" #include "core/common/status.h" +#include "core/common/string_utils.h" #include "core/framework/data_types.h" #include "core/framework/endian.h" #include "core/framework/allocator.h" @@ -26,37 +27,6 @@ struct OrtStatus { namespace onnxruntime { namespace test { -//From core common -inline void MakeStringInternal(std::ostringstream& /*ss*/) noexcept { -} - -template -inline void MakeStringInternal(std::ostringstream& ss, const T& t) noexcept { - ss << t; -} - -template -inline void MakeStringInternal(std::ostringstream& ss, const T& t, const Args&... args) noexcept { - ::onnxruntime::MakeStringInternal(ss, t); - ::onnxruntime::MakeStringInternal(ss, args...); -} - -template -std::string MakeString(const Args&... args) { - std::ostringstream ss; - ::onnxruntime::MakeStringInternal(ss, args...); - return std::string(ss.str()); -} - -// Specializations for already-a-string types. -template <> -inline std::string MakeString(const std::string& str) { - return str; -} -inline std::string MakeString(const char* p_str) { - return p_str; -} - std::vector GetTensorShapeFromTensorProto(const onnx::TensorProto& tensor_proto) { const auto& dims = tensor_proto.dims(); std::vector tensor_shape_vec(static_cast(dims.size())); diff --git a/onnxruntime/test/util/default_providers.cc b/onnxruntime/test/util/default_providers.cc index 1d5c635d34..517affa862 100644 --- a/onnxruntime/test/util/default_providers.cc +++ b/onnxruntime/test/util/default_providers.cc @@ -3,21 +3,17 @@ #include "default_providers.h" #include "providers.h" +#include "core/providers/cpu/cpu_provider_factory_creator.h" +#ifdef USE_CUDA +#include "core/providers/cuda/cuda_provider_factory_creator.h" +#endif +#ifdef USE_ROCM +#include "core/providers/rocm/rocm_provider_factory_creator.h" +#endif #include "core/session/onnxruntime_cxx_api.h" -#include "core/providers/providers.h" -#include "core/framework/bfc_arena.h" namespace onnxruntime { -std::shared_ptr CreateExecutionProviderFactory_CPU(int use_arena); - -std::shared_ptr CreateExecutionProviderFactory_CUDA( - OrtDevice::DeviceId device_id, - OrtCudnnConvAlgoSearch cudnn_conv_algo = OrtCudnnConvAlgoSearch::EXHAUSTIVE, - size_t cuda_mem_limit = std::numeric_limits::max(), - ArenaExtendStrategy arena_extend_strategy = ArenaExtendStrategy::kNextPowerOfTwo, - bool do_copy_in_default_stream = true); - std::shared_ptr CreateExecutionProviderFactory_OpenVINO( const char* device_type, bool enable_vpu_fast_compile, const char* device_id, size_t num_of_threads); @@ -30,9 +26,6 @@ std::shared_ptr CreateExecutionProviderFactory_Tensor std::shared_ptr CreateExecutionProviderFactory_MIGraphX(int device_id); std::shared_ptr CreateExecutionProviderFactory_ACL(int use_arena); std::shared_ptr CreateExecutionProviderFactory_ArmNN(int use_arena); -std::shared_ptr CreateExecutionProviderFactory_ROCM(OrtDevice::DeviceId device_id, - size_t hip_mem_limit = std::numeric_limits::max(), - ArenaExtendStrategy arena_extend_strategy = ArenaExtendStrategy::kNextPowerOfTwo); // EP for internal testing std::shared_ptr CreateExecutionProviderFactory_InternalTesting( @@ -71,7 +64,7 @@ std::unique_ptr DefaultOpenVINOExecutionProvider() { std::unique_ptr DefaultCudaExecutionProvider() { #ifdef USE_CUDA - return CreateExecutionProviderFactory_CUDA(0)->CreateProvider(); + return CreateExecutionProviderFactory_CUDA(CUDAExecutionProviderInfo{})->CreateProvider(); #else return nullptr; #endif @@ -134,7 +127,7 @@ std::unique_ptr DefaultArmNNExecutionProvider(bool enable_ar std::unique_ptr DefaultRocmExecutionProvider() { #ifdef USE_ROCM - return CreateExecutionProviderFactory_ROCM(0)->CreateProvider(); + return CreateExecutionProviderFactory_ROCM(ROCMExecutionProviderInfo{})->CreateProvider(); #else return nullptr; #endif diff --git a/orttraining/orttraining/models/bert/main.cc b/orttraining/orttraining/models/bert/main.cc index c027ef4ecb..10f17ef328 100644 --- a/orttraining/orttraining/models/bert/main.cc +++ b/orttraining/orttraining/models/bert/main.cc @@ -10,11 +10,14 @@ #include "core/session/environment.h" #include "core/framework/bfc_arena.h" #include "core/framework/random_seed.h" +#include "core/providers/cpu/cpu_provider_factory_creator.h" #ifdef USE_CUDA #include "core/providers/cuda/cuda_allocator.h" +#include "core/providers/cuda/cuda_provider_factory_creator.h" #endif #ifdef USE_ROCM #include "core/providers/rocm/rocm_allocator.h" +#include "core/providers/rocm/rocm_provider_factory_creator.h" #endif #include "orttraining/core/session/training_session.h" #include "orttraining/core/framework/tensorboard/event_writer.h" @@ -24,21 +27,6 @@ #include "orttraining/models/runner/training_util.h" #include "orttraining/models/runner/data_loader.h" -namespace onnxruntime { -#ifdef USE_CUDA -std::shared_ptr CreateExecutionProviderFactory_CUDA(OrtDevice::DeviceId device_id, - OrtCudnnConvAlgoSearch cudnn_conv_algo_search = OrtCudnnConvAlgoSearch::EXHAUSTIVE, - size_t cuda_mem_limit = std::numeric_limits::max(), - onnxruntime::ArenaExtendStrategy arena_extend_strategy = ArenaExtendStrategy::kNextPowerOfTwo, - bool do_copy_in_default_stream = true); -#endif -#ifdef USE_ROCM -std::shared_ptr CreateExecutionProviderFactory_ROCM(OrtDevice::DeviceId device_id, - size_t hip_mem_limit = std::numeric_limits::max(), - onnxruntime::ArenaExtendStrategy arena_extend_strategy = ArenaExtendStrategy::kNextPowerOfTwo); -#endif -} - using namespace onnxruntime; using namespace onnxruntime::common; using namespace onnxruntime::training; @@ -582,20 +570,27 @@ void setup_training_params(BertParameters& params) { #endif #ifdef USE_CUDA - OrtDevice::DeviceId device_id = static_cast(MPIContext::GetInstance().GetLocalRank()); - size_t cuda_mem_limit = std::numeric_limits::max(); - if (params.cuda_mem_limit_in_gb > 0) - cuda_mem_limit = static_cast(params.cuda_mem_limit_in_gb * 1024 * 1024 * 1024); - params.providers.emplace(kCudaExecutionProvider, CreateExecutionProviderFactory_CUDA(device_id, OrtCudnnConvAlgoSearch::EXHAUSTIVE, - cuda_mem_limit)); - params.input_allocator = std::make_shared(device_id, CUDA_PINNED); + { + CUDAExecutionProviderInfo info{}; + info.device_id = gsl::narrow(MPIContext::GetInstance().GetLocalRank()); + if (params.cuda_mem_limit_in_gb > 0) { + info.cuda_mem_limit = gsl::narrow(params.cuda_mem_limit_in_gb * 1024 * 1024 * 1024); + } + info.cudnn_conv_algo = OrtCudnnConvAlgoSearch::EXHAUSTIVE; + + params.providers.emplace(kCudaExecutionProvider, CreateExecutionProviderFactory_CUDA(info)); + params.input_allocator = std::make_shared(info.device_id, CUDA_PINNED); + } #endif #ifdef USE_ROCM - OrtDevice::DeviceId device_id = static_cast(MPIContext::GetInstance().GetLocalRank()); - size_t hip_mem_limit = std::numeric_limits::max(); - params.providers.emplace(kRocmExecutionProvider, CreateExecutionProviderFactory_ROCM(device_id, hip_mem_limit)); - params.input_allocator = std::make_shared(device_id, CUDA_PINNED); + { + ROCMExecutionProviderInfo info{}; + info.device_id = gsl::narrow(MPIContext::GetInstance().GetLocalRank()); + + params.providers.emplace(kRocmExecutionProvider, CreateExecutionProviderFactory_ROCM(info)); + params.input_allocator = std::make_shared(info.device_id, CUDA_PINNED); + } #endif params.loss_func_info = LossFunctionInfo(OpDef("BertLoss", kOnnxDomain), diff --git a/orttraining/orttraining/models/gpt2/main.cc b/orttraining/orttraining/models/gpt2/main.cc index 5adc35e098..538657ac0c 100644 --- a/orttraining/orttraining/models/gpt2/main.cc +++ b/orttraining/orttraining/models/gpt2/main.cc @@ -9,7 +9,10 @@ #include "core/session/environment.h" #include "core/framework/random_seed.h" #include "core/framework/bfc_arena.h" +#ifdef USE_CUDA #include "core/providers/cuda/cuda_allocator.h" +#include "core/providers/cuda/cuda_provider_factory_creator.h" +#endif #include "orttraining/core/framework/mpi_context.h" #include "orttraining/core/framework/tensorboard/event_writer.h" #include "orttraining/core/session/training_session.h" @@ -18,14 +21,6 @@ #include "orttraining/models/runner/training_util.h" #include "orttraining/models/runner/data_loader.h" -namespace onnxruntime { -std::shared_ptr CreateExecutionProviderFactory_CUDA(OrtDevice::DeviceId device_id, - OrtCudnnConvAlgoSearch cudnn_conv_algo_search = OrtCudnnConvAlgoSearch::EXHAUSTIVE, - size_t cuda_mem_limit = std::numeric_limits::max(), - onnxruntime::ArenaExtendStrategy arena_extend_strategy = ArenaExtendStrategy::kNextPowerOfTwo, - bool do_copy_in_default_stream = true); -} - using namespace onnxruntime; using namespace onnxruntime::common; using namespace onnxruntime::training; @@ -344,9 +339,13 @@ void setup_training_params(GPT2Parameters& params) { params.model_type = "gpt2"; #ifdef USE_CUDA - OrtDevice::DeviceId device_id = static_cast(MPIContext::GetInstance().GetLocalRank()); - params.providers.emplace(kCudaExecutionProvider, CreateExecutionProviderFactory_CUDA(device_id)); - params.input_allocator = std::make_shared(device_id, CUDA_PINNED); + { + CUDAExecutionProviderInfo info{}; + info.device_id = gsl::narrow(MPIContext::GetInstance().GetLocalRank()); + + params.providers.emplace(kCudaExecutionProvider, CreateExecutionProviderFactory_CUDA(info)); + params.input_allocator = std::make_shared(info.device_id, CUDA_PINNED); + } #endif params.use_nccl = true; diff --git a/orttraining/orttraining/models/mnist/main.cc b/orttraining/orttraining/models/mnist/main.cc index 88fc7660a4..5a59343de3 100644 --- a/orttraining/orttraining/models/mnist/main.cc +++ b/orttraining/orttraining/models/mnist/main.cc @@ -6,6 +6,9 @@ #include "core/common/logging/sinks/clog_sink.h" #include "core/framework/bfc_arena.h" #include "core/platform/env.h" +#ifdef USE_CUDA +#include "core/providers/cuda/cuda_provider_factory_creator.h" +#endif #include "core/session/environment.h" #include "orttraining/core/session/training_session.h" #include "orttraining/core/framework/tensorboard/event_writer.h" @@ -18,14 +21,6 @@ #include #include -namespace onnxruntime { -std::shared_ptr CreateExecutionProviderFactory_CUDA(OrtDevice::DeviceId device_id, - OrtCudnnConvAlgoSearch cudnn_conv_algo_search = OrtCudnnConvAlgoSearch::EXHAUSTIVE, - size_t cuda_mem_limit = std::numeric_limits::max(), - onnxruntime::ArenaExtendStrategy arena_extend_strategy = ArenaExtendStrategy::kNextPowerOfTwo, - bool do_copy_in_default_stream = true); -} - using namespace onnxruntime; using namespace onnxruntime::common; using namespace onnxruntime::training; @@ -156,7 +151,7 @@ Status ParseArguments(int argc, char* argv[], TrainingRunner::Parameters& params #ifdef USE_CUDA bool use_cuda = flags.count("use_cuda") > 0; if (use_cuda) { - params.providers.emplace(kCudaExecutionProvider, CreateExecutionProviderFactory_CUDA(0)); + params.providers.emplace(kCudaExecutionProvider, CreateExecutionProviderFactory_CUDA(CUDAExecutionProviderInfo{})); } #endif } catch (const exception& e) { diff --git a/tools/ci_build/amd_hipify.py b/tools/ci_build/amd_hipify.py index c28f6b5e84..c5c317ba00 100644 --- a/tools/ci_build/amd_hipify.py +++ b/tools/ci_build/amd_hipify.py @@ -1,13 +1,17 @@ -#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. import os import subprocess +from logger import get_logger + +log = get_logger("amd_hipify") contrib_ops_path = 'onnxruntime/contrib_ops' -core_ops_path = 'onnxruntime/core/providers' +providers_path = 'onnxruntime/core/providers' training_ops_path = 'orttraining/orttraining/training_ops' -contrib_ops_files = [ +contrib_ops_excluded_files = [ 'bert/attention.cc', 'bert/attention.h', 'bert/attention_impl.cu', @@ -65,7 +69,7 @@ contrib_ops_files = [ 'inverse.cc' ] -core_ops_files = [ +provider_excluded_files = [ 'atomic/common.cuh', 'controlflow/if.cc', 'controlflow/if.h', @@ -208,6 +212,8 @@ core_ops_files = [ 'cuda_allocator.h', 'cuda_call.cc', 'cuda_common.h', + 'cuda_execution_provider_info.cc', + 'cuda_execution_provider_info.h', 'cuda_execution_provider.cc', 'cuda_execution_provider.h', 'cuda_fence.cc', @@ -227,7 +233,7 @@ core_ops_files = [ 'symbols.txt', ] -training_ops_files = [ +training_ops_excluded_files = [ 'activation/activations_grad.cc', 'collective/horovod_kernels.cc', 'collective/horovod_kernels.h', @@ -274,10 +280,11 @@ training_ops_files = [ ] HIPIFY_PERL = '/opt/rocm/bin/hipify-perl' -FINDCODE = '/opt/rocm/bin/findcode.sh' def hipify(src_file_path, dst_file_path): + log.debug('Hipifying: "{}" -> "{}"'.format(src_file_path, dst_file_path)) + dst_file_path = dst_file_path.replace('cuda', 'rocm') dir_name = os.path.dirname(dst_file_path) if not os.path.exists(dir_name): @@ -353,29 +360,25 @@ def amd_hipify(config_build_dir): rocm_contrib_path = os.path.join(config_build_dir, 'amdgpu', contrib_ops_path, 'rocm') contrib_files = list_files(cuda_contrib_path, '') for file in contrib_files: - if file not in contrib_ops_files: + if file not in contrib_ops_excluded_files: src_file_path = os.path.join(cuda_contrib_path, file) dst_file_path = os.path.join(rocm_contrib_path, file) hipify(src_file_path, dst_file_path) - cuda_core_path = os.path.join(core_ops_path, 'cuda') - rocm_core_path = os.path.join(config_build_dir, 'amdgpu', core_ops_path, 'rocm') - core_files = list_files(cuda_core_path, '') - for file in core_files: - if file not in core_ops_files: - src_file_path = os.path.join(cuda_core_path, file) - dst_file_path = os.path.join(rocm_core_path, file) + cuda_provider_path = os.path.join(providers_path, 'cuda') + rocm_provider_path = os.path.join(config_build_dir, 'amdgpu', providers_path, 'rocm') + provider_files = list_files(cuda_provider_path, '') + for file in provider_files: + if file not in provider_excluded_files: + src_file_path = os.path.join(cuda_provider_path, file) + dst_file_path = os.path.join(rocm_provider_path, file) hipify(src_file_path, dst_file_path) cuda_training_path = os.path.join(training_ops_path, 'cuda') rocm_training_path = os.path.join(config_build_dir, 'amdgpu', training_ops_path, 'rocm') training_files = list_files(cuda_training_path, '') for file in training_files: - if file not in training_ops_files: + if file not in training_ops_excluded_files: src_file_path = os.path.join(cuda_training_path, file) dst_file_path = os.path.join(rocm_training_path, file) hipify(src_file_path, dst_file_path) - - -if __name__ == '__main__': - amd_hipify() diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py old mode 100644 new mode 100755