From 74ee24cf7f4810b056d04ba19bc5acc48e939f1e Mon Sep 17 00:00:00 2001 From: Weixing Zhang Date: Mon, 5 Apr 2021 09:04:04 -0700 Subject: [PATCH] rename cuda_mem_limit and hip_mem_limit to gpu_mem_limit for both CUDA EP and ROCm EP (#7226) With this change, differentiating CUDA EP and ROCm EP is not needed in training script when mem_limit option needs to be set. Co-authored-by: Weixing Zhang --- .../providers/rocm/rocm_provider_factory.h | 2 +- .../core/session/onnxruntime_c_api.h | 4 ++-- .../providers/cuda/cuda_execution_provider.cc | 12 +++++------ .../providers/cuda/cuda_execution_provider.h | 4 ++-- .../cuda/cuda_execution_provider_info.cc | 6 +++--- .../cuda/cuda_execution_provider_info.h | 2 +- .../providers/cuda/cuda_provider_factory.cc | 2 +- .../providers/rocm/rocm_execution_provider.cc | 8 ++++---- .../providers/rocm/rocm_execution_provider.h | 2 +- .../rocm/rocm_execution_provider_info.cc | 6 +++--- .../rocm/rocm_execution_provider_info.h | 2 +- .../providers/rocm/rocm_provider_factory.cc | 2 +- .../python/onnxruntime_pybind_state.cc | 16 +++++++-------- .../test/python/onnxruntime_test_python.py | 20 +++++++++---------- orttraining/orttraining/models/bert/main.cc | 10 +++++----- orttraining/orttraining/models/mnist/main.cc | 2 +- .../models/runner/training_runner.h | 4 ++-- .../orttraining/python/training/orttrainer.py | 14 +++++-------- .../test/gradient/allreduce_op_test.cc | 6 +++--- 19 files changed, 60 insertions(+), 64 deletions(-) diff --git a/include/onnxruntime/core/providers/rocm/rocm_provider_factory.h b/include/onnxruntime/core/providers/rocm/rocm_provider_factory.h index 6152883bb5..cf8a1c8a75 100644 --- a/include/onnxruntime/core/providers/rocm/rocm_provider_factory.h +++ b/include/onnxruntime/core/providers/rocm/rocm_provider_factory.h @@ -10,7 +10,7 @@ extern "C" { /** * \param device_id hip device id, starts from zero. */ -ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_ROCM, _In_ OrtSessionOptions* options, int device_id, size_t hip_mem_limit); +ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_ROCM, _In_ OrtSessionOptions* options, int device_id, size_t gpu_mem_limit); #ifdef __cplusplus } diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index df0b1c221a..4cb1b2c746 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -268,7 +268,7 @@ typedef enum OrtCudnnConvAlgoSearch { typedef struct OrtCUDAProviderOptions { int device_id; // cuda device with id=0 as default device. OrtCudnnConvAlgoSearch cudnn_conv_algo_search; // cudnn conv algo search option - size_t cuda_mem_limit; // default cuda memory limitation to maximum finite value of size_t. + size_t gpu_mem_limit; // default cuda memory limitation to maximum finite value of size_t. int arena_extend_strategy; // default area extend strategy to KNextPowerOfTwo. int do_copy_in_default_stream; int has_user_compute_stream; @@ -281,7 +281,7 @@ typedef struct OrtCUDAProviderOptions { typedef struct OrtROCMProviderOptions { int device_id; // hip device with id=0 as default device. int miopen_conv_exhaustive_search; // miopen conv algo exhaustive search option - size_t hip_mem_limit; // default hip memory limitation to maximum finite value of size_t. + size_t gpu_mem_limit; // default hip memory limitation to maximum finite value of size_t. int arena_extend_strategy; // default area extend strategy to KNextPowerOfTwo. } OrtROCMProviderOptions; diff --git a/onnxruntime/core/providers/cuda/cuda_execution_provider.cc b/onnxruntime/core/providers/cuda/cuda_execution_provider.cc index 1ab5ebf21c..5e6b0a3be0 100644 --- a/onnxruntime/core/providers/cuda/cuda_execution_provider.cc +++ b/onnxruntime/core/providers/cuda/cuda_execution_provider.cc @@ -59,7 +59,7 @@ ONNX_OPERATOR_KERNEL_EX( } // namespace cuda -AllocatorPtr CUDAExecutionProvider::CreateCudaAllocator(OrtDevice::DeviceId device_id, size_t cuda_mem_limit, ArenaExtendStrategy arena_extend_strategy, +AllocatorPtr CUDAExecutionProvider::CreateCudaAllocator(OrtDevice::DeviceId device_id, size_t gpu_mem_limit, ArenaExtendStrategy arena_extend_strategy, CUDAExecutionProviderExternalAllocatorInfo external_allocator_info) { if (external_allocator_info.UseExternalAllocator()) { AllocatorCreationInfo default_memory_info( @@ -78,7 +78,7 @@ AllocatorPtr CUDAExecutionProvider::CreateCudaAllocator(OrtDevice::DeviceId devi }, device_id, true, - {cuda_mem_limit, + {gpu_mem_limit, static_cast(arena_extend_strategy), -1, -1}); @@ -87,7 +87,7 @@ AllocatorPtr CUDAExecutionProvider::CreateCudaAllocator(OrtDevice::DeviceId devi } } -CUDAExecutionProvider::PerThreadContext::PerThreadContext(OrtDevice::DeviceId device_id, cudaStream_t stream, size_t cuda_mem_limit, +CUDAExecutionProvider::PerThreadContext::PerThreadContext(OrtDevice::DeviceId device_id, cudaStream_t stream, size_t gpu_mem_limit, ArenaExtendStrategy arena_extend_strategy, CUDAExecutionProviderExternalAllocatorInfo external_allocator_info) { CUDA_CALL_THROW(cudaSetDevice(device_id)); stream_ = stream; @@ -99,7 +99,7 @@ CUDAExecutionProvider::PerThreadContext::PerThreadContext(OrtDevice::DeviceId de CUDNN_CALL_THROW(cudnnSetStream(cudnn_handle_, stream)); // CUDA malloc/free is expensive so always use an arena - allocator_ = CreateCudaAllocator(device_id, cuda_mem_limit, arena_extend_strategy, external_allocator_info); + allocator_ = CreateCudaAllocator(device_id, gpu_mem_limit, arena_extend_strategy, external_allocator_info); } CUDAExecutionProvider::PerThreadContext::~PerThreadContext() { @@ -198,7 +198,7 @@ CUDAExecutionProvider::PerThreadContext& CUDAExecutionProvider::GetPerThreadCont // get or create a context if (context_state_.retired_context_pool.empty()) { - context = std::make_shared(info_.device_id, static_cast(GetComputeStream()), info_.cuda_mem_limit, info_.arena_extend_strategy, info_.external_allocator_info); + context = std::make_shared(info_.device_id, static_cast(GetComputeStream()), info_.gpu_mem_limit, info_.arena_extend_strategy, info_.external_allocator_info); } else { context = context_state_.retired_context_pool.back(); context_state_.retired_context_pool.pop_back(); @@ -2046,7 +2046,7 @@ void CUDAExecutionProvider::RegisterAllocator(std::shared_ptr // Used to allocate CUDA device memory auto cuda_alloc = allocator_manager->GetAllocator(info_.device_id, OrtMemTypeDefault); if (nullptr == cuda_alloc) { - cuda_alloc = CreateCudaAllocator(info_.device_id, info_.cuda_mem_limit, info_.arena_extend_strategy, info_.external_allocator_info); + cuda_alloc = CreateCudaAllocator(info_.device_id, info_.gpu_mem_limit, info_.arena_extend_strategy, info_.external_allocator_info); allocator_manager->InsertAllocator(cuda_alloc); } TryInsertAllocator(cuda_alloc); diff --git a/onnxruntime/core/providers/cuda/cuda_execution_provider.h b/onnxruntime/core/providers/cuda/cuda_execution_provider.h index c37cd2d6cb..e85fb19807 100644 --- a/onnxruntime/core/providers/cuda/cuda_execution_provider.h +++ b/onnxruntime/core/providers/cuda/cuda_execution_provider.h @@ -80,7 +80,7 @@ class CUDAExecutionProvider : public IExecutionProvider { } void RegisterAllocator(std::shared_ptr allocator_manager) override; - static AllocatorPtr CreateCudaAllocator(OrtDevice::DeviceId device_id, size_t cuda_mem_limit, ArenaExtendStrategy arena_extend_strategy, + static AllocatorPtr CreateCudaAllocator(OrtDevice::DeviceId device_id, size_t gpu_mem_limit, ArenaExtendStrategy arena_extend_strategy, CUDAExecutionProviderExternalAllocatorInfo external_alloc_info); private: @@ -99,7 +99,7 @@ class CUDAExecutionProvider : public IExecutionProvider { class PerThreadContext final { public: - PerThreadContext(OrtDevice::DeviceId device_id, cudaStream_t stream, size_t cuda_mem_limit, ArenaExtendStrategy arena_extend_strategy, + PerThreadContext(OrtDevice::DeviceId device_id, cudaStream_t stream, size_t gpu_mem_limit, ArenaExtendStrategy arena_extend_strategy, CUDAExecutionProviderExternalAllocatorInfo external_alloc_info); ~PerThreadContext(); diff --git a/onnxruntime/core/providers/cuda/cuda_execution_provider_info.cc b/onnxruntime/core/providers/cuda/cuda_execution_provider_info.cc index 927e6687fd..4dc376a64a 100644 --- a/onnxruntime/core/providers/cuda/cuda_execution_provider_info.cc +++ b/onnxruntime/core/providers/cuda/cuda_execution_provider_info.cc @@ -12,7 +12,7 @@ namespace onnxruntime { namespace cuda { namespace provider_option_names { constexpr const char* kDeviceId = "device_id"; -constexpr const char* kMemLimit = "cuda_mem_limit"; +constexpr const char* kMemLimit = "gpu_mem_limit"; constexpr const char* kArenaExtendStrategy = "arena_extend_strategy"; constexpr const char* kCudnnConvAlgoSearch = "cudnn_conv_algo_search"; constexpr const char* kDoCopyInDefaultStream = "do_copy_in_default_stream"; @@ -70,7 +70,7 @@ CUDAExecutionProviderInfo CUDAExecutionProviderInfo::FromProviderOptions(const P free = reinterpret_cast(address); return Status::OK(); }) - .AddAssignmentToReference(cuda::provider_option_names::kMemLimit, info.cuda_mem_limit) + .AddAssignmentToReference(cuda::provider_option_names::kMemLimit, info.gpu_mem_limit) .AddAssignmentToEnumReference( cuda::provider_option_names::kArenaExtendStrategy, arena_extend_strategy_mapping, info.arena_extend_strategy) @@ -88,7 +88,7 @@ CUDAExecutionProviderInfo CUDAExecutionProviderInfo::FromProviderOptions(const P ProviderOptions CUDAExecutionProviderInfo::ToProviderOptions(const CUDAExecutionProviderInfo& info) { const ProviderOptions options{ {cuda::provider_option_names::kDeviceId, MakeStringWithClassicLocale(info.device_id)}, - {cuda::provider_option_names::kMemLimit, MakeStringWithClassicLocale(info.cuda_mem_limit)}, + {cuda::provider_option_names::kMemLimit, MakeStringWithClassicLocale(info.gpu_mem_limit)}, {cuda::provider_option_names::kcudaExternalAlloc, MakeStringWithClassicLocale(reinterpret_cast(info.external_allocator_info.alloc))}, {cuda::provider_option_names::kcudaExternalFree, MakeStringWithClassicLocale(reinterpret_cast(info.external_allocator_info.free))}, {cuda::provider_option_names::kArenaExtendStrategy, diff --git a/onnxruntime/core/providers/cuda/cuda_execution_provider_info.h b/onnxruntime/core/providers/cuda/cuda_execution_provider_info.h index d08fee602b..9ee0a4375a 100644 --- a/onnxruntime/core/providers/cuda/cuda_execution_provider_info.h +++ b/onnxruntime/core/providers/cuda/cuda_execution_provider_info.h @@ -33,7 +33,7 @@ struct CUDAExecutionProviderExternalAllocatorInfo { struct CUDAExecutionProviderInfo { OrtDevice::DeviceId device_id{0}; - size_t cuda_mem_limit{std::numeric_limits::max()}; + size_t gpu_mem_limit{std::numeric_limits::max()}; ArenaExtendStrategy arena_extend_strategy{ArenaExtendStrategy::kNextPowerOfTwo}; OrtCudnnConvAlgoSearch cudnn_conv_algo_search{OrtCudnnConvAlgoSearch::EXHAUSTIVE}; bool do_copy_in_default_stream{true}; diff --git a/onnxruntime/core/providers/cuda/cuda_provider_factory.cc b/onnxruntime/core/providers/cuda/cuda_provider_factory.cc index 1300703839..4c77ccd103 100644 --- a/onnxruntime/core/providers/cuda/cuda_provider_factory.cc +++ b/onnxruntime/core/providers/cuda/cuda_provider_factory.cc @@ -53,7 +53,7 @@ ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider_CUDA, _In_ OrtSessionOptions* options, _In_ const OrtCUDAProviderOptions* cuda_options) { CUDAExecutionProviderInfo info{}; info.device_id = gsl::narrow(cuda_options->device_id); - info.cuda_mem_limit = cuda_options->cuda_mem_limit; + info.gpu_mem_limit = cuda_options->gpu_mem_limit; info.arena_extend_strategy = static_cast(cuda_options->arena_extend_strategy); info.cudnn_conv_algo_search = cuda_options->cudnn_conv_algo_search; info.do_copy_in_default_stream = cuda_options->do_copy_in_default_stream; diff --git a/onnxruntime/core/providers/rocm/rocm_execution_provider.cc b/onnxruntime/core/providers/rocm/rocm_execution_provider.cc index 2e93fae664..db7cc065db 100644 --- a/onnxruntime/core/providers/rocm/rocm_execution_provider.cc +++ b/onnxruntime/core/providers/rocm/rocm_execution_provider.cc @@ -58,7 +58,7 @@ ONNX_OPERATOR_KERNEL_EX( } // namespace rocm -ROCMExecutionProvider::PerThreadContext::PerThreadContext(OrtDevice::DeviceId device_id, hipStream_t stream, size_t hip_mem_limit, ArenaExtendStrategy arena_extend_strategy) { +ROCMExecutionProvider::PerThreadContext::PerThreadContext(OrtDevice::DeviceId device_id, hipStream_t stream, size_t gpu_mem_limit, ArenaExtendStrategy arena_extend_strategy) { HIP_CALL_THROW(hipSetDevice(device_id)); stream_ = stream; @@ -73,7 +73,7 @@ ROCMExecutionProvider::PerThreadContext::PerThreadContext(OrtDevice::DeviceId de }, device_id, true, - {hip_mem_limit, + {gpu_mem_limit, static_cast(arena_extend_strategy), -1, -1}); @@ -128,7 +128,7 @@ ROCMExecutionProvider::ROCMExecutionProvider(const ROCMExecutionProviderInfo& in }, info_.device_id, true, - {info_.hip_mem_limit, + {info_.gpu_mem_limit, static_cast(info_.arena_extend_strategy), -1, -1}); @@ -206,7 +206,7 @@ ROCMExecutionProvider::PerThreadContext& ROCMExecutionProvider::GetPerThreadCont // get or create a context if (context_state_.retired_context_pool.empty()) { - context = std::make_shared(info_.device_id, static_cast(GetComputeStream()), info_.hip_mem_limit, info_.arena_extend_strategy); + context = std::make_shared(info_.device_id, static_cast(GetComputeStream()), info_.gpu_mem_limit, info_.arena_extend_strategy); } else { context = context_state_.retired_context_pool.back(); context_state_.retired_context_pool.pop_back(); diff --git a/onnxruntime/core/providers/rocm/rocm_execution_provider.h b/onnxruntime/core/providers/rocm/rocm_execution_provider.h index 5360f93444..9cb63fa6fd 100644 --- a/onnxruntime/core/providers/rocm/rocm_execution_provider.h +++ b/onnxruntime/core/providers/rocm/rocm_execution_provider.h @@ -105,7 +105,7 @@ class ROCMExecutionProvider : public IExecutionProvider { class PerThreadContext final { public: - PerThreadContext(OrtDevice::DeviceId device_id, hipStream_t stream, size_t hip_mem_limit, ArenaExtendStrategy arena_extend_strategy); + PerThreadContext(OrtDevice::DeviceId device_id, hipStream_t stream, size_t gpu_mem_limit, ArenaExtendStrategy arena_extend_strategy); ~PerThreadContext(); rocblas_handle RocblasHandle() const { diff --git a/onnxruntime/core/providers/rocm/rocm_execution_provider_info.cc b/onnxruntime/core/providers/rocm/rocm_execution_provider_info.cc index 78654953d8..9652eb28ce 100644 --- a/onnxruntime/core/providers/rocm/rocm_execution_provider_info.cc +++ b/onnxruntime/core/providers/rocm/rocm_execution_provider_info.cc @@ -10,7 +10,7 @@ namespace onnxruntime { namespace rocm { namespace provider_option_names { constexpr const char* kDeviceId = "device_id"; -constexpr const char* kMemLimit = "hip_mem_limit"; +constexpr const char* kMemLimit = "gpu_mem_limit"; constexpr const char* kArenaExtendStrategy = "arena_extend_strategy"; constexpr const char* kConvExhaustiveSearch = "conv_exhaustive_search"; } // namespace provider_option_names @@ -30,7 +30,7 @@ ROCMExecutionProviderInfo ROCMExecutionProviderInfo::FromProviderOptions(const P ProviderOptionsParser{} // TODO validate info.device_id .AddAssignmentToReference(rocm::provider_option_names::kDeviceId, info.device_id) - .AddAssignmentToReference(rocm::provider_option_names::kMemLimit, info.hip_mem_limit) + .AddAssignmentToReference(rocm::provider_option_names::kMemLimit, info.gpu_mem_limit) .AddAssignmentToReference(rocm::provider_option_names::kConvExhaustiveSearch, info.miopen_conv_exhaustive_search) .AddAssignmentToEnumReference( rocm::provider_option_names::kArenaExtendStrategy, @@ -43,7 +43,7 @@ ROCMExecutionProviderInfo ROCMExecutionProviderInfo::FromProviderOptions(const P ProviderOptions ROCMExecutionProviderInfo::ToProviderOptions(const ROCMExecutionProviderInfo& info) { const ProviderOptions options{ {rocm::provider_option_names::kDeviceId, MakeStringWithClassicLocale(info.device_id)}, - {rocm::provider_option_names::kMemLimit, MakeStringWithClassicLocale(info.hip_mem_limit)}, + {rocm::provider_option_names::kMemLimit, MakeStringWithClassicLocale(info.gpu_mem_limit)}, {rocm::provider_option_names::kConvExhaustiveSearch, MakeStringWithClassicLocale(info.miopen_conv_exhaustive_search)}, {rocm::provider_option_names::kArenaExtendStrategy, EnumToName(arena_extend_strategy_mapping, info.arena_extend_strategy)}, diff --git a/onnxruntime/core/providers/rocm/rocm_execution_provider_info.h b/onnxruntime/core/providers/rocm/rocm_execution_provider_info.h index 39e92a78ad..4641397ff6 100644 --- a/onnxruntime/core/providers/rocm/rocm_execution_provider_info.h +++ b/onnxruntime/core/providers/rocm/rocm_execution_provider_info.h @@ -13,7 +13,7 @@ namespace onnxruntime { // Information needed to construct HIP execution providers. struct ROCMExecutionProviderInfo { OrtDevice::DeviceId device_id{0}; - size_t hip_mem_limit{std::numeric_limits::max()}; + size_t gpu_mem_limit{std::numeric_limits::max()}; ArenaExtendStrategy arena_extend_strategy{ArenaExtendStrategy::kNextPowerOfTwo}; bool miopen_conv_exhaustive_search{false}; bool do_copy_in_default_stream{true}; diff --git a/onnxruntime/core/providers/rocm/rocm_provider_factory.cc b/onnxruntime/core/providers/rocm/rocm_provider_factory.cc index e8c439ce0d..d26adabfbb 100644 --- a/onnxruntime/core/providers/rocm/rocm_provider_factory.cc +++ b/onnxruntime/core/providers/rocm/rocm_provider_factory.cc @@ -52,7 +52,7 @@ ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider_ROCM, _In_ OrtSessionOptions* options, _In_ const OrtROCMProviderOptions* rocm_options) { ROCMExecutionProviderInfo info{}; info.device_id = gsl::narrow(rocm_options->device_id); - info.hip_mem_limit = rocm_options->hip_mem_limit; + info.gpu_mem_limit = rocm_options->gpu_mem_limit; info.arena_extend_strategy = static_cast(rocm_options->arena_extend_strategy); info.miopen_conv_exhaustive_search = rocm_options->miopen_conv_exhaustive_search; diff --git a/onnxruntime/python/onnxruntime_pybind_state.cc b/onnxruntime/python/onnxruntime_pybind_state.cc index 25bfc7611a..a69db7962a 100644 --- a/onnxruntime/python/onnxruntime_pybind_state.cc +++ b/onnxruntime/python/onnxruntime_pybind_state.cc @@ -153,7 +153,7 @@ onnxruntime::CUDAExecutionProviderExternalAllocatorInfo external_allocator_info{ // TODO remove deprecated global config OrtDevice::DeviceId cuda_device_id = 0; // TODO remove deprecated global config -size_t cuda_mem_limit = std::numeric_limits::max(); +size_t gpu_mem_limit = std::numeric_limits::max(); // TODO remove deprecated global config onnxruntime::ArenaExtendStrategy arena_extend_strategy = onnxruntime::ArenaExtendStrategy::kNextPowerOfTwo; #endif @@ -458,7 +458,7 @@ static AllocatorPtr GetCudaAllocator(OrtDevice::DeviceId id) { static std::unordered_map id_to_allocator_map; if (id_to_allocator_map.find(id) == id_to_allocator_map.end()) { - id_to_allocator_map.insert({id, CUDAExecutionProvider::CreateCudaAllocator(id, cuda_mem_limit, arena_extend_strategy, external_allocator_info)}); + id_to_allocator_map.insert({id, CUDAExecutionProvider::CreateCudaAllocator(id, gpu_mem_limit, arena_extend_strategy, external_allocator_info)}); } return id_to_allocator_map[id]; @@ -564,7 +564,7 @@ static void RegisterExecutionProviders(InferenceSession* sess, const std::vector : [&]() { CUDAExecutionProviderInfo info{}; info.device_id = cuda_device_id; - info.cuda_mem_limit = cuda_mem_limit; + info.gpu_mem_limit = gpu_mem_limit; info.arena_extend_strategy = arena_extend_strategy; info.cudnn_conv_algo_search = cudnn_conv_algo_search; info.do_copy_in_default_stream = do_copy_in_default_stream; @@ -588,7 +588,7 @@ static void RegisterExecutionProviders(InferenceSession* sess, const std::vector : [&]() { ROCMExecutionProviderInfo info{}; info.device_id = cuda_device_id; - info.hip_mem_limit = cuda_mem_limit; + info.gpu_mem_limit = gpu_mem_limit; info.arena_extend_strategy = arena_extend_strategy; return info; }(); @@ -884,7 +884,7 @@ void addGlobalMethods(py::module& m, Environment& env) { [&]() { CUDAExecutionProviderInfo info{}; info.device_id = cuda_device_id; - info.cuda_mem_limit = cuda_mem_limit; + info.gpu_mem_limit = gpu_mem_limit; info.arena_extend_strategy = arena_extend_strategy; info.cudnn_conv_algo_search = cudnn_conv_algo_search; info.do_copy_in_default_stream = do_copy_in_default_stream; @@ -897,7 +897,7 @@ void addGlobalMethods(py::module& m, Environment& env) { [&]() { ROCMExecutionProviderInfo info{}; info.device_id = cuda_device_id; - info.hip_mem_limit = cuda_mem_limit; + info.gpu_mem_limit = gpu_mem_limit; info.arena_extend_strategy = arena_extend_strategy; return info; }()), @@ -989,8 +989,8 @@ void addGlobalMethods(py::module& m, Environment& env) { m.def("set_cuda_mem_limit", [](const int64_t limit) { LogDeprecationWarning( "set_cuda_mem_limit", - "CUDA execution provider option \"cuda_mem_limit\", ROCM execution provider option \"hip_mem_limit\""); - cuda_mem_limit = gsl::narrow(limit); + "CUDA execution provider option \"gpu_mem_limit\", ROCM execution provider option \"gpu_mem_limit\""); + gpu_mem_limit = gsl::narrow(limit); }); // TODO remove deprecated global config m.def("set_arena_extend_strategy", [](const onnxruntime::ArenaExtendStrategy strategy) { diff --git a/onnxruntime/test/python/onnxruntime_test_python.py b/onnxruntime/test/python/onnxruntime_test_python.py index 545cd6e561..b413689897 100644 --- a/onnxruntime/test/python/onnxruntime_test_python.py +++ b/onnxruntime/test/python/onnxruntime_test_python.py @@ -82,22 +82,22 @@ class TestInferenceSession(unittest.TestCase): sess = onnxrt.InferenceSession(get_name("mul_1.onnx")) self.assertIn('CUDAExecutionProvider', sess.get_providers()) - # test get/set of "cuda_mem_limit" configuration. + # test get/set of "gpu_mem_limit" configuration. options = sess.get_provider_options() self.assertIn('CUDAExecutionProvider', options) option = options['CUDAExecutionProvider'] - self.assertIn('cuda_mem_limit', option) - ori_mem_limit = option['cuda_mem_limit'] + self.assertIn('gpu_mem_limit', option) + ori_mem_limit = option['gpu_mem_limit'] new_mem_limit = int(ori_mem_limit) // 2 - option['cuda_mem_limit'] = new_mem_limit + option['gpu_mem_limit'] = new_mem_limit sess.set_providers(['CUDAExecutionProvider'], [option]) options = sess.get_provider_options() - self.assertEqual(options['CUDAExecutionProvider']['cuda_mem_limit'], str(new_mem_limit)) + self.assertEqual(options['CUDAExecutionProvider']['gpu_mem_limit'], str(new_mem_limit)) - option['cuda_mem_limit'] = ori_mem_limit + option['gpu_mem_limit'] = ori_mem_limit sess.set_providers(['CUDAExecutionProvider'], [option]) options = sess.get_provider_options() - self.assertEqual(options['CUDAExecutionProvider']['cuda_mem_limit'], ori_mem_limit) + self.assertEqual(options['CUDAExecutionProvider']['gpu_mem_limit'], ori_mem_limit) def test_get_and_set_option_with_values(option_name, option_values): provider_options = sess.get_provider_options() @@ -138,15 +138,15 @@ class TestInferenceSession(unittest.TestCase): with self.assertRaises(RuntimeError): sess.set_providers(['CUDAExecutionProvider'], [option]) - option['cuda_mem_limit'] = -1024 + option['gpu_mem_limit'] = -1024 with self.assertRaises(RuntimeError): sess.set_providers(['CUDAExecutionProvider'], [option]) - option['cuda_mem_limit'] = 1024.1024 + option['gpu_mem_limit'] = 1024.1024 with self.assertRaises(RuntimeError): sess.set_providers(['CUDAExecutionProvider'], [option]) - option['cuda_mem_limit'] = 'wrong_value' + option['gpu_mem_limit'] = 'wrong_value' with self.assertRaises(RuntimeError): sess.set_providers(['CUDAExecutionProvider'], [option]) diff --git a/orttraining/orttraining/models/bert/main.cc b/orttraining/orttraining/models/bert/main.cc index 49d43554d1..dc4d3044c5 100644 --- a/orttraining/orttraining/models/bert/main.cc +++ b/orttraining/orttraining/models/bert/main.cc @@ -64,7 +64,7 @@ struct BertParameters : public TrainingRunner::Parameters { float initial_lr_phase2; size_t num_train_steps_phase2; float warmup_ratio_phase2; - float cuda_mem_limit_in_gb = -1; + float gpu_mem_limit_in_gb = -1; bool debug_break = false; PathString train_data_dir_phase2; PathString test_data_dir_phase2; @@ -181,7 +181,7 @@ Status ParseArguments(int argc, char* argv[], BertParameters& params, OrtParamet cxxopts::value()->default_value("0")) ("ratio_min", "Lamb min ratio parameter", cxxopts::value()->default_value("0.05")) ("ratio_max", "Lamb max ratio parameter", cxxopts::value()->default_value("5.0")) - ("cuda_mem_limit_in_gb", "Max cuda memory ort can use, in GB", cxxopts::value()->default_value("-1.0")) + ("gpu_mem_limit_in_gb", "Max cuda memory ort can use, in GB", cxxopts::value()->default_value("-1.0")) ("data_parallel_size", "Data parallel group size.", cxxopts::value()->default_value("1")) ("horizontal_parallel_size", "Horizontal model parallel group size.", cxxopts::value()->default_value("1")) ("pipeline_parallel_size", "Number of pipeline stages.", cxxopts::value()->default_value("1")) @@ -238,7 +238,7 @@ Status ParseArguments(int argc, char* argv[], BertParameters& params, OrtParamet } params.lr_params.warmup_ratio = ratio; - params.cuda_mem_limit_in_gb = flags["cuda_mem_limit_in_gb"].as(); + params.gpu_mem_limit_in_gb = flags["gpu_mem_limit_in_gb"].as(); float ratio_phase2 = flags["warmup_ratio_phase2"].as(); if (ratio_phase2 > 1.f || ratio_phase2 < 0.f) { @@ -601,8 +601,8 @@ void setup_training_params(BertParameters& params) { { CUDAExecutionProviderInfo info{}; info.device_id = gsl::narrow(MPIContext::GetInstance().GetLocalRank()); - if (params.cuda_mem_limit_in_gb > 0) { - info.cuda_mem_limit = gsl::narrow(params.cuda_mem_limit_in_gb * 1024 * 1024 * 1024); + if (params.gpu_mem_limit_in_gb > 0) { + info.gpu_mem_limit = gsl::narrow(params.gpu_mem_limit_in_gb * 1024 * 1024 * 1024); } info.cudnn_conv_algo_search = OrtCudnnConvAlgoSearch::EXHAUSTIVE; diff --git a/orttraining/orttraining/models/mnist/main.cc b/orttraining/orttraining/models/mnist/main.cc index 3e42692b19..690fce69fe 100644 --- a/orttraining/orttraining/models/mnist/main.cc +++ b/orttraining/orttraining/models/mnist/main.cc @@ -24,7 +24,7 @@ namespace onnxruntime { std::shared_ptr CreateExecutionProviderFactory_CUDA(OrtDevice::DeviceId device_id, OrtCudnnConvAlgoSearch cudnn_conv_algo_search = OrtCudnnConvAlgoSearch::EXHAUSTIVE, - size_t cuda_mem_limit = std::numeric_limits::max(), + size_t gpu_mem_limit = std::numeric_limits::max(), onnxruntime::ArenaExtendStrategy arena_extend_strategy = ArenaExtendStrategy::kNextPowerOfTwo, bool do_copy_in_default_stream = true); std::shared_ptr CreateExecutionProviderFactory_Dnnl(int use_arena); diff --git a/orttraining/orttraining/models/runner/training_runner.h b/orttraining/orttraining/models/runner/training_runner.h index d9d82dad87..84e2a4783e 100644 --- a/orttraining/orttraining/models/runner/training_runner.h +++ b/orttraining/orttraining/models/runner/training_runner.h @@ -120,8 +120,8 @@ class TrainingRunner { VectorString histogram_names; VectorString norm_names; - //Default value is -1.0f. When cuda_mem_limit_in_gb < 0, ORT can use all cuda memory available. - float cuda_mem_limit_in_gb = -1.0f; + //Default value is -1.0f. When gpu_mem_limit_in_gb < 0, ORT can use all cuda memory available. + float gpu_mem_limit_in_gb = -1.0f; bool EnableTensorboard() const { return !is_perf_test && !log_dir.empty() && MPIContext::GetInstance().GetWorldRank() == 0; diff --git a/orttraining/orttraining/python/training/orttrainer.py b/orttraining/orttraining/python/training/orttrainer.py index 3965c32523..1fffbc9f21 100644 --- a/orttraining/orttraining/python/training/orttrainer.py +++ b/orttraining/orttraining/python/training/orttrainer.py @@ -689,21 +689,17 @@ class ORTTrainer(object): providers[providers.index(provider_name)] = (provider_name, provider_options[provider_name]) #default: using cuda elif 'cuda' in self.options.device.id.lower(): - cuda_ep_options = {"device_id": _utils.get_device_index(self.options.device.id)} - - cuda_ep_name = ("ROCMExecutionProvider" if self.is_rocm_pytorch else "CUDAExecutionProvider") + gpu_ep_options = {"device_id": _utils.get_device_index(self.options.device.id)} + gpu_ep_name = ("ROCMExecutionProvider" if self.is_rocm_pytorch else "CUDAExecutionProvider") if self.options.device.mem_limit > 0: - if not self.is_rocm_pytorch: - cuda_ep_options["cuda_mem_limit"] = self.options.device.mem_limit - else: - warnings.warn("Ignoring 'mem_limit' for {}".format(cuda_ep_name)) + gpu_ep_options["gpu_mem_limit"] = self.options.device.mem_limit - if cuda_ep_name not in providers: + if gpu_ep_name not in providers: raise RuntimeError( "ORTTrainer options specify a CUDA device but the {} provider is unavailable.".format( cuda_ep_name)) - providers[providers.index(cuda_ep_name)] = (cuda_ep_name, cuda_ep_options) + providers[providers.index(gpu_ep_name)] = (gpu_ep_name, gpu_ep_options) return providers diff --git a/orttraining/orttraining/test/gradient/allreduce_op_test.cc b/orttraining/orttraining/test/gradient/allreduce_op_test.cc index f18c858ced..1cd22e0566 100644 --- a/orttraining/orttraining/test/gradient/allreduce_op_test.cc +++ b/orttraining/orttraining/test/gradient/allreduce_op_test.cc @@ -434,11 +434,11 @@ void build_allreduce_graph(Graph& graph, AllreduceGraphConfigVector& config, std::unique_ptr create_cuda_execution_provider() { CUDAExecutionProviderInfo info; OrtDevice::DeviceId device_id = static_cast(training::MPIContext::GetInstance().GetLocalRank()); - size_t cuda_mem_limit = std::numeric_limits::max(); - cuda_mem_limit = static_cast(1 * 1024 * 1024 * 1024); + size_t gpu_mem_limit = std::numeric_limits::max(); + gpu_mem_limit = static_cast(1 * 1024 * 1024 * 1024); info.device_id = device_id; - info.cuda_mem_limit = cuda_mem_limit; + info.gpu_mem_limit = gpu_mem_limit; info.arena_extend_strategy = ArenaExtendStrategy::kNextPowerOfTwo; return onnxruntime::make_unique(info); }