saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-30 20:18:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Deprecate Python global configuration functions [Part 1] (#5923)

Browse source 
Enable options to be set via execution provider (EP)-specific options and log deprecation warning from current global configuration functions.
This commit is contained in:
Edward Chen 2020-12-15 11:32:43 -08:00 committed by GitHub
parent a8d549e181
commit 64709b1335
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
33 changed files with 778 additions and 487 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
include/onnxruntime/core/common/common.h
View file

@ -35,6 +35,7 @@
#include "core/common/exceptions.h"
#include "core/common/make_unique.h"
#include "core/common/status.h"
#include "core/common/string_utils.h"
#ifdef USE_MIMALLOC_ARENA_ALLOCATOR
#include <mimalloc.h>
@ -261,36 +262,6 @@ void LogRuntimeError(uint32_t session_id, const common::Status& status, const ch
#define GSL_SUPPRESS(tag)
#endif
inline void MakeStringInternal(std::ostringstream& /*ss*/) noexcept {
}
template <typename T>
inline void MakeStringInternal(std::ostringstream& ss, const T& t) noexcept {
ss << t;
}
template <typename T, typename... Args>
inline void MakeStringInternal(std::ostringstream& ss, const T& t, const Args&... args) noexcept {
::onnxruntime::MakeStringInternal(ss, t);
::onnxruntime::MakeStringInternal(ss, args...);
}
template <typename... Args>
std::string MakeString(const Args&... args) {
std::ostringstream ss;
::onnxruntime::MakeStringInternal(ss, args...);
return std::string(ss.str());
}
// Specializations for already-a-string types.
template <>
inline std::string MakeString(const std::string& str) {
return str;
}
inline std::string MakeString(const char* p_str) {
return p_str;
}
inline long long TimeDiffMicroSeconds(TimePoint start_time) {
auto end_time = std::chrono::high_resolution_clock::now();
return std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time).count();

100
include/onnxruntime/core/common/string_utils.h Normal file
View file

@ -0,0 +1,100 @@
/**
* Copyright (c) 2016-present, Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Portions Copyright (c) Microsoft Corporation
#pragma once
#include <locale>
#include <sstream>
#include <type_traits>
namespace onnxruntime {
namespace detail {
inline void MakeStringImpl(std::ostringstream& /*ss*/) noexcept {
}
template <typename T>
inline void MakeStringImpl(std::ostringstream& ss, const T& t) noexcept {
ss << t;
}
template <typename T, typename... Args>
inline void MakeStringImpl(std::ostringstream& ss, const T& t, const Args&... args) noexcept {
MakeStringImpl(ss, t);
MakeStringImpl(ss, args...);
}
} // namespace detail
/**
* Makes a string by concatenating string representations of the arguments.
*/
template <typename... Args>
std::string MakeString(const Args&... args) {
std::ostringstream ss;
ss.imbue(std::locale::classic());
detail::MakeStringImpl(ss, args...);
return ss.str();
}
// MakeString versions for already-a-string types.
inline std::string MakeString(const std::string& str) {
return str;
}
inline std::string MakeString(const char* cstr) {
return cstr;
}
/**
* Tries to parse a value from an entire string.
*/
template <typename T>
bool TryParse(const std::string& str, T& value) {
if (std::is_integral<T>::value && std::is_unsigned<T>::value) {
// if T is unsigned integral type, reject negative values which will wrap
if (!str.empty() && str[0] == '-') {
return false;
}
}
// don't allow leading whitespace
if (!str.empty() && std::isspace(str[0], std::locale::classic())) {
return false;
}
std::istringstream is{str};
is.imbue(std::locale::classic());
T parsed_value{};
const bool parse_successful =
is >> parsed_value &&
is.get() == std::istringstream::traits_type::eof(); // don't allow trailing characters
if (!parse_successful) {
return false;
}
value = std::move(parsed_value);
return true;
}
inline bool TryParse(const std::string& str, std::string& value) {
value = str;
return true;
}
} // namespace onnxruntime

27
include/onnxruntime/core/framework/execution_provider.h
View file

@ -6,11 +6,12 @@
#include <unordered_map>
#include "gsl/gsl"
#include "core/common/status.h"
#include "core/common/logging/logging.h"
#include "core/framework/tensor.h"
#include "core/framework/func_api.h"
#include "core/common/status.h"
#include "core/framework/data_transfer.h"
#include "core/framework/func_api.h"
#include "core/framework/provider_options.h"
#include "core/framework/tensor.h"
namespace onnxruntime {
class GraphViewer;
@ -31,13 +32,6 @@ using CreateFunctionStateFunc = std::function<int(ComputeContext*, FunctionState
using ComputeFunc = std::function<Status(FunctionState, const OrtApi*, OrtKernelContext*)>;
using DestroyFunctionStateFunc = std::function<void(FunctionState)>;
//unordered maps
using UnorderedMapStringToString = std::unordered_map<std::string, std::string>;
//data types for execution provider options
using ProviderOptionsVector = std::vector<UnorderedMapStringToString>;
using ProviderOptionsMap = std::unordered_map<std::string, UnorderedMapStringToString>;
struct NodeComputeInfo {
CreateFunctionStateFunc create_state_func;
ComputeFunc compute_func;
@ -107,16 +101,9 @@ class IExecutionProvider {
virtual int GetDeviceId() const { return -1; };
/**
Get execution provider's configurations.
Get execution provider's configuration options.
*/
const UnorderedMapStringToString& GetProviderOptions() const { return provider_options_; }
/**
Store execution provider's configurations.
*/
void SetProviderOptions(UnorderedMapStringToString& options) {
provider_options_ = options;
}
virtual ProviderOptions GetProviderOptions() const;
/**
Returns an opaque handle whose exact type varies based on the provider
@ -248,7 +235,5 @@ class IExecutionProvider {
// convenience list of the allocators so GetAllocatorList doesn't have to build a new vector each time
// contains the same instances as allocators_
std::vector<AllocatorPtr> allocator_list_;
// It will be set when constructor is being called
UnorderedMapStringToString provider_options_;
};
} // namespace onnxruntime

2
include/onnxruntime/core/framework/ortdevice.h
View file

@ -3,6 +3,8 @@
#pragma once
#include <sstream>
// Struct to represent a physical device.
struct OrtDevice {
using DeviceType = int8_t;

18
include/onnxruntime/core/framework/provider_options.h Normal file
View file

@ -0,0 +1,18 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#pragma once
#include <string>
#include <unordered_map>
#include <vector>
namespace onnxruntime {
// data types for execution provider options
using ProviderOptions = std::unordered_map<std::string, std::string>;
using ProviderOptionsVector = std::vector<ProviderOptions>;
using ProviderOptionsMap = std::unordered_map<std::string, ProviderOptions>;
} // namespace onnxruntime

39
include/onnxruntime/core/framework/provider_options_utils.h Normal file
View file

@ -0,0 +1,39 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#pragma once
#include "core/common/common.h"
#include "core/framework/provider_options.h"
namespace onnxruntime {
/**
* Reads the named provider option.
* Returns true if the option is present, false otherwise.
*/
template <typename T>
bool ReadProviderOption(const ProviderOptions& options, const std::string& key, T& value) {
auto it = options.find(key);
if (it != options.end()) {
ORT_ENFORCE(
TryParse(it->second, value),
"Failed to parse provider option \"", key, "\" with value \"", it->second, "\".");
return true;
}
return false;
}
/**
* Reads the named provider option.
* Returns the value if the option is present or the specified default value otherwise.
*/
template <typename T>
T ReadProviderOptionOrDefault(
const ProviderOptions& options, const std::string& key, const T& default_value) {
T value{};
if (ReadProviderOption(options, key, value)) {
return value;
}
return default_value;
}
} // namespace onnxruntime

44
onnxruntime/core/framework/arena_extend_strategy.h Normal file
View file

@ -0,0 +1,44 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#pragma once
#include <iostream>
namespace onnxruntime {
enum class ArenaExtendStrategy : int32_t {
kNextPowerOfTwo = 0,
kSameAsRequested,
};
inline std::istream& operator>>(std::istream& is, ArenaExtendStrategy& value) {
std::string value_str;
if (is >> value_str) {
if (value_str == "kNextPowerOfTwo") {
value = ArenaExtendStrategy::kNextPowerOfTwo;
} else if (value_str == "kSameAsRequested") {
value = ArenaExtendStrategy::kSameAsRequested;
} else {
is.setstate(std::ios_base::failbit);
}
}
return is;
}
inline std::ostream& operator<<(std::ostream& os, ArenaExtendStrategy value) {
switch (value) {
case ArenaExtendStrategy::kNextPowerOfTwo:
os << "kNextPowerOfTwo";
break;
case ArenaExtendStrategy::kSameAsRequested:
os << "kSameAsRequested";
break;
default:
os << "unknown";
break;
}
return os;
}
} // namespace onnxruntime

6
onnxruntime/core/framework/bfc_arena.h
View file

@ -25,6 +25,7 @@ limitations under the License.
#include "core/common/logging/severity.h"
#include "core/platform/ort_mutex.h"
#include "core/framework/arena.h"
#include "core/framework/arena_extend_strategy.h"
#include "core/common/safeint.h"
#include "onnxruntime_config.h"
@ -39,11 +40,6 @@ namespace onnxruntime {
#endif
#endif
enum class ArenaExtendStrategy : int32_t {
kNextPowerOfTwo = 0,
kSameAsRequested,
};
// A memory allocator that implements a 'best-fit with coalescing'
// algorithm. This is essentially a very simple version of Doug Lea's
// malloc (dlmalloc).

2
onnxruntime/core/framework/execution_provider.cc
View file

@ -51,6 +51,8 @@ IExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph,
#endif
}
ProviderOptions IExecutionProvider::GetProviderOptions() const { return {}; }
common::Status IExecutionProvider::Sync() const { return Status::OK(); };
common::Status IExecutionProvider::OnRunStart() { return Status::OK(); }

5
onnxruntime/core/platform/env_var_utils.h
View file

@ -3,8 +3,6 @@
#pragma once
#include <sstream>
#include "core/common/common.h"
#include "core/common/optional.h"
#include "core/platform/env.h"
@ -20,10 +18,9 @@ optional<T> ParseEnvironmentVariable(const std::string& name) {
return {};
}
std::istringstream is{value_str};
T parsed_value;
ORT_ENFORCE(
is >> std::noskipws >> parsed_value && is.eof(),
TryParse(value_str, parsed_value),
"Failed to parse environment variable - name: \"", name, "\", value: \"", value_str, "\"");
return parsed_value;

8
onnxruntime/core/providers/cpu/cpu_provider_factory.cc
View file

@ -1,9 +1,13 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "core/providers/cpu/cpu_provider_factory_creator.h"
#include "core/providers/cpu/cpu_provider_factory.h"
#include <atomic>
#include "cpu_execution_provider.h"
#include <memory>
#include "core/common/make_unique.h"
#include "core/providers/cpu/cpu_execution_provider.h"
#include "core/session/abi_session_options_impl.h"
#include "core/session/ort_apis.h"

14
onnxruntime/core/providers/cpu/cpu_provider_factory_creator.h Normal file
View file

@ -0,0 +1,14 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#pragma once
#include <memory>
#include "core/providers/providers.h"
namespace onnxruntime {
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_CPU(int use_arena);
} // namespace onnxruntime

46
onnxruntime/core/providers/cuda/cuda_execution_provider.cc
View file

@ -7,6 +7,7 @@
#include "core/framework/fallback_cpu_capability.h"
#include "core/framework/kernel_registry.h"
#include "core/framework/memcpy.h"
#include "core/framework/provider_options_utils.h"
#include "core/graph/graph_utils.h"
#include "core/providers/cuda/cuda_allocator.h"
#include "core/providers/cuda/cuda_fence.h"
@ -94,41 +95,14 @@ CUDAExecutionProvider::PerThreadContext::~PerThreadContext() {
}
}
/*
* This method should be called within the constructor,
* so that the configuration of provider related setting can be updated
* and kept at IExecutionProvider level.
*/
void CUDAExecutionProvider::UpdateProviderOptionsInfo() {
UnorderedMapStringToString options;
options["device_id"] = std::to_string(device_id_);
options["cuda_mem_limit"] = std::to_string(cuda_mem_limit_);
std::string strategy;
if (arena_extend_strategy_ == ArenaExtendStrategy::kNextPowerOfTwo) {
strategy = "kNextPowerOfTwo";
} else if (arena_extend_strategy_ == ArenaExtendStrategy::kSameAsRequested) {
strategy = "kSameAsRequested";
} else {
strategy = "unknown";
}
options["arena_extend_strategy"] = strategy;
IExecutionProvider::SetProviderOptions(options);
}
CUDAExecutionProvider::CUDAExecutionProvider(const CUDAExecutionProviderInfo& info)
: IExecutionProvider{onnxruntime::kCudaExecutionProvider},
device_id_(info.device_id),
cuda_mem_limit_(info.cuda_mem_limit),
arena_extend_strategy_(info.arena_extend_strategy),
cudnn_conv_algo_(info.cudnn_conv_algo),
do_copy_in_default_stream_(info.do_copy_in_default_stream) {
CUDA_CALL_THROW(cudaSetDevice(device_id_));
info_{info} {
CUDA_CALL_THROW(cudaSetDevice(info_.device_id));
// must wait GPU idle, otherwise cudaGetDeviceProperties might fail
CUDA_CALL_THROW(cudaDeviceSynchronize());
CUDA_CALL_THROW(cudaGetDeviceProperties(&device_prop_, device_id_));
CUDA_CALL_THROW(cudaGetDeviceProperties(&device_prop_, info_.device_id));
size_t free = 0;
size_t total = 0;
@ -138,10 +112,10 @@ CUDAExecutionProvider::CUDAExecutionProvider(const CUDAExecutionProviderInfo& in
[](OrtDevice::DeviceId device_id) {
return onnxruntime::make_unique<CUDAAllocator>(device_id, CUDA);
},
device_id_,
info_.device_id,
true,
{cuda_mem_limit_,
static_cast<int>(arena_extend_strategy_),
{info_.cuda_mem_limit,
static_cast<int>(info_.arena_extend_strategy),
-1, -1});
InsertAllocator(CreateAllocator(default_memory_info));
@ -165,8 +139,6 @@ CUDAExecutionProvider::CUDAExecutionProvider(const CUDAExecutionProviderInfo& in
CPU_ALLOCATOR_DEVICE_ID);
InsertAllocator(CreateAllocator(cpu_memory_info));
UpdateProviderOptionsInfo();
}
CUDAExecutionProvider::~CUDAExecutionProvider() {
@ -216,7 +188,7 @@ CUDAExecutionProvider::PerThreadContext& CUDAExecutionProvider::GetPerThreadCont
// get or create a context
if (context_state_.retired_context_pool.empty()) {
context = std::make_shared<PerThreadContext>(device_id_, cuda_mem_limit_, arena_extend_strategy_);
context = std::make_shared<PerThreadContext>(info_.device_id, info_.cuda_mem_limit, info_.arena_extend_strategy);
} else {
context = context_state_.retired_context_pool.back();
context_state_.retired_context_pool.pop_back();
@ -1880,7 +1852,7 @@ static bool CastNeedFallbackToCPU(const onnxruntime::Node& node) {
}
std::unique_ptr<onnxruntime::IDataTransfer> CUDAExecutionProvider::GetDataTransfer() const {
return onnxruntime::make_unique<onnxruntime::GPUDataTransfer>(do_copy_in_default_stream_);
return onnxruntime::make_unique<onnxruntime::GPUDataTransfer>(info_.do_copy_in_default_stream);
}
std::vector<std::unique_ptr<ComputeCapability>>

32
onnxruntime/core/providers/cuda/cuda_execution_provider.h
View file

@ -8,9 +8,10 @@
#include "core/graph/constants.h"
#include "core/framework/allocatormgr.h"
#include "core/framework/bfc_arena.h"
#include "core/framework/arena_extend_strategy.h"
#include "core/framework/execution_provider.h"
#include "core/platform/ort_mutex.h"
#include "core/providers/cuda/cuda_execution_provider_info.h"
#include "core/providers/cuda/cuda_pch.h"
#include "core/providers/cuda/gpu_data_transfer.h"
#include "core/providers/cuda/shared_inc/cuda_utils.h"
@ -19,15 +20,6 @@ namespace onnxruntime {
const int CPU_ALLOCATOR_DEVICE_ID = 0;
// Information needed to construct CUDA execution providers.
struct CUDAExecutionProviderInfo {
OrtDevice::DeviceId device_id{0};
size_t cuda_mem_limit{std::numeric_limits<size_t>::max()};
ArenaExtendStrategy arena_extend_strategy{ArenaExtendStrategy::kNextPowerOfTwo};
OrtCudnnConvAlgoSearch cudnn_conv_algo{OrtCudnnConvAlgoSearch::EXHAUSTIVE};
bool do_copy_in_default_stream{true};
};
// Logical device representation.
class CUDAExecutionProvider : public IExecutionProvider {
public:
@ -67,7 +59,7 @@ class CUDAExecutionProvider : public IExecutionProvider {
if (count_or_bytes == 0)
return nullptr;
return IAllocator::MakeUniquePtr<T>(GetAllocator(device_id_, OrtMemTypeDefault), count_or_bytes);
return IAllocator::MakeUniquePtr<T>(GetAllocator(info_.device_id, OrtMemTypeDefault), count_or_bytes);
}
std::shared_ptr<KernelRegistry> GetKernelRegistry() const override;
@ -77,19 +69,17 @@ class CUDAExecutionProvider : public IExecutionProvider {
const onnxruntime::GraphViewer& graph,
const std::vector<const KernelRegistry*>& kernel_registries) const override;
int GetDeviceId() const { return device_id_; }
int GetDeviceId() const override { return info_.device_id; }
const cudaDeviceProp& GetDeviceProp() const { return device_prop_; };
int GetCudnnConvAlgo() const { return cudnn_conv_algo_; }
void UpdateProviderOptionsInfo();
int GetCudnnConvAlgo() const { return info_.cudnn_conv_algo; }
private:
OrtDevice::DeviceId device_id_;
ProviderOptions GetProviderOptions() const override {
return CUDAExecutionProviderInfo::ToProviderOptions(info_);
}
private:
CUDAExecutionProviderInfo info_;
cudaDeviceProp device_prop_;
size_t cuda_mem_limit_;
ArenaExtendStrategy arena_extend_strategy_;
int cudnn_conv_algo_;
bool do_copy_in_default_stream_;
struct DeferredReleaseCPUPtrs {
bool recorded = false;
std::vector<void*> cpu_ptrs;

62
onnxruntime/core/providers/cuda/cuda_execution_provider_info.cc Normal file
View file

@ -0,0 +1,62 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "core/providers/cuda/cuda_execution_provider_info.h"
#include "core/common/string_utils.h"
#include "core/framework/provider_options_utils.h"
#include "core/providers/cuda/cuda_common.h"
namespace onnxruntime {
namespace provider_option_names {
constexpr const char* kDeviceId = "device_id";
constexpr const char* kMemLimit = "cuda_mem_limit";
constexpr const char* kArenaExtendStrategy = "arena_extend_strategy";
constexpr const char* kCudnnConvAlgo = "cudnn_conv_algo";
constexpr const char* kDoCopyInDefaultStream = "do_copy_in_default_stream";
} // namespace provider_option_names
CUDAExecutionProviderInfo CUDAExecutionProviderInfo::FromProviderOptions(const ProviderOptions& options) {
CUDAExecutionProviderInfo info{};
if (ReadProviderOption(options, provider_option_names::kDeviceId, info.device_id)) {
int num_devices = 0;
CUDA_CALL_THROW(cudaGetDeviceCount(&num_devices));
ORT_ENFORCE(
0 <= info.device_id && info.device_id < num_devices,
"Invalid ", provider_option_names::kDeviceId, " value: ", info.device_id,
", must be between 0 (inclusive) and ", num_devices, " (exclusive).");
}
ReadProviderOption(options, provider_option_names::kMemLimit, info.cuda_mem_limit);
ReadProviderOption(options, provider_option_names::kArenaExtendStrategy, info.arena_extend_strategy);
{
int cudnn_conv_algo_val;
if (ReadProviderOption(options, provider_option_names::kCudnnConvAlgo, cudnn_conv_algo_val)) {
switch (cudnn_conv_algo_val) {
case OrtCudnnConvAlgoSearch::EXHAUSTIVE:
case OrtCudnnConvAlgoSearch::HEURISTIC:
case OrtCudnnConvAlgoSearch::DEFAULT:
break;
default:
ORT_THROW("Invalid ", provider_option_names::kCudnnConvAlgo, " value: ", cudnn_conv_algo_val);
}
info.cudnn_conv_algo = static_cast<OrtCudnnConvAlgoSearch>(cudnn_conv_algo_val);
}
}
ReadProviderOption(options, provider_option_names::kDoCopyInDefaultStream, info.do_copy_in_default_stream);
return info;
}
ProviderOptions CUDAExecutionProviderInfo::ToProviderOptions(const CUDAExecutionProviderInfo& info) {
const ProviderOptions options{
{provider_option_names::kDeviceId, MakeString(info.device_id)},
{provider_option_names::kMemLimit, MakeString(info.cuda_mem_limit)},
{provider_option_names::kArenaExtendStrategy, MakeString(info.arena_extend_strategy)},
{provider_option_names::kCudnnConvAlgo, MakeString(static_cast<int>(info.cudnn_conv_algo))},
{provider_option_names::kDoCopyInDefaultStream, MakeString(info.do_copy_in_default_stream)},
};
return options;
}
} // namespace onnxruntime

25
onnxruntime/core/providers/cuda/cuda_execution_provider_info.h Normal file
View file

@ -0,0 +1,25 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#pragma once
#include <limits>
#include "core/framework/arena_extend_strategy.h"
#include "core/framework/ortdevice.h"
#include "core/framework/provider_options.h"
#include "core/session/onnxruntime_c_api.h"
namespace onnxruntime {
// Information needed to construct CUDA execution providers.
struct CUDAExecutionProviderInfo {
OrtDevice::DeviceId device_id{0};
size_t cuda_mem_limit{std::numeric_limits<size_t>::max()};
ArenaExtendStrategy arena_extend_strategy{ArenaExtendStrategy::kNextPowerOfTwo};
OrtCudnnConvAlgoSearch cudnn_conv_algo{OrtCudnnConvAlgoSearch::EXHAUSTIVE};
bool do_copy_in_default_stream{true};
static CUDAExecutionProviderInfo FromProviderOptions(const ProviderOptions& options);
static ProviderOptions ToProviderOptions(const CUDAExecutionProviderInfo& info);
};
} // namespace onnxruntime

65
onnxruntime/core/providers/cuda/cuda_provider_factory.cc
View file

@ -1,73 +1,64 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "core/providers/cuda/cuda_provider_factory_creator.h"
#include "core/providers/cuda/cuda_provider_factory.h"
#include <atomic>
#include "core/graph/onnx_protobuf.h"
#include "cuda_execution_provider.h"
#include <memory>
#include "gsl/gsl"
#include "core/common/make_unique.h"
#include "core/providers/cuda/cuda_execution_provider.h"
#include "core/providers/cuda/cuda_execution_provider_info.h"
#include "core/session/abi_session_options_impl.h"
#include "core/session/onnxruntime_c_api.h"
#include "core/session/ort_apis.h"
#include "core/framework/bfc_arena.h"
using namespace onnxruntime;
namespace onnxruntime {
struct CUDAProviderFactory : IExecutionProviderFactory {
CUDAProviderFactory(OrtDevice::DeviceId device_id,
size_t cuda_mem_limit = std::numeric_limits<size_t>::max(),
ArenaExtendStrategy arena_extend_strategy = ArenaExtendStrategy::kNextPowerOfTwo,
OrtCudnnConvAlgoSearch cudnn_conv_algo_search = OrtCudnnConvAlgoSearch::EXHAUSTIVE,
bool do_copy_in_default_stream = true)
: device_id_(device_id),
cuda_mem_limit_(cuda_mem_limit),
arena_extend_strategy_(arena_extend_strategy),
cudnn_conv_algo_search_(cudnn_conv_algo_search),
do_copy_in_default_stream_(do_copy_in_default_stream) {}
CUDAProviderFactory(const CUDAExecutionProviderInfo& info)
: info_{info} {}
~CUDAProviderFactory() override {}
std::unique_ptr<IExecutionProvider> CreateProvider() override;
private:
OrtDevice::DeviceId device_id_;
size_t cuda_mem_limit_;
ArenaExtendStrategy arena_extend_strategy_;
OrtCudnnConvAlgoSearch cudnn_conv_algo_search_;
bool do_copy_in_default_stream_;
CUDAExecutionProviderInfo info_;
};
std::unique_ptr<IExecutionProvider> CUDAProviderFactory::CreateProvider() {
CUDAExecutionProviderInfo info;
info.device_id = device_id_;
info.cuda_mem_limit = cuda_mem_limit_;
info.arena_extend_strategy = arena_extend_strategy_;
info.cudnn_conv_algo = cudnn_conv_algo_search_;
info.do_copy_in_default_stream = do_copy_in_default_stream_;
return onnxruntime::make_unique<CUDAExecutionProvider>(info);
return onnxruntime::make_unique<CUDAExecutionProvider>(info_);
}
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_CUDA(OrtDevice::DeviceId device_id,
OrtCudnnConvAlgoSearch cudnn_conv_algo_search = OrtCudnnConvAlgoSearch::EXHAUSTIVE,
size_t cuda_mem_limit = std::numeric_limits<size_t>::max(),
ArenaExtendStrategy arena_extend_strategy = ArenaExtendStrategy::kNextPowerOfTwo,
bool do_copy_in_default_stream = true) {
return std::make_shared<onnxruntime::CUDAProviderFactory>(device_id, cuda_mem_limit, arena_extend_strategy, cudnn_conv_algo_search, do_copy_in_default_stream);
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_CUDA(const CUDAExecutionProviderInfo& info) {
return std::make_shared<onnxruntime::CUDAProviderFactory>(info);
}
} // namespace onnxruntime
ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProvider_CUDA, _In_ OrtSessionOptions* options, int device_id) {
options->provider_factories.push_back(onnxruntime::CreateExecutionProviderFactory_CUDA(static_cast<OrtDevice::DeviceId>(device_id)));
CUDAExecutionProviderInfo info{};
info.device_id = gsl::narrow<OrtDevice::DeviceId>(device_id);
options->provider_factories.push_back(onnxruntime::CreateExecutionProviderFactory_CUDA(info));
return nullptr;
}
ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider_CUDA,
_In_ OrtSessionOptions* options, _In_ const OrtCUDAProviderOptions* cuda_options) {
options->provider_factories.push_back(onnxruntime::CreateExecutionProviderFactory_CUDA(static_cast<OrtDevice::DeviceId>(cuda_options->device_id),
cuda_options->cudnn_conv_algo_search, cuda_options->cuda_mem_limit,
static_cast<onnxruntime::ArenaExtendStrategy>(cuda_options->arena_extend_strategy),
cuda_options->do_copy_in_default_stream));
CUDAExecutionProviderInfo info{};
info.device_id = gsl::narrow<OrtDevice::DeviceId>(cuda_options->device_id);
info.cuda_mem_limit = cuda_options->cuda_mem_limit;
info.arena_extend_strategy = static_cast<onnxruntime::ArenaExtendStrategy>(cuda_options->arena_extend_strategy);
info.cudnn_conv_algo = cuda_options->cudnn_conv_algo_search;
info.do_copy_in_default_stream = cuda_options->do_copy_in_default_stream;
options->provider_factories.push_back(onnxruntime::CreateExecutionProviderFactory_CUDA(info));
return nullptr;
}

16
onnxruntime/core/providers/cuda/cuda_provider_factory_creator.h Normal file
View file

@ -0,0 +1,16 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#pragma once
#include <memory>
#include "core/providers/providers.h"
#include "core/providers/cuda/cuda_execution_provider_info.h"
namespace onnxruntime {
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_CUDA(const CUDAExecutionProviderInfo& info);
} // namespace onnxruntime

41
onnxruntime/core/providers/rocm/rocm_execution_provider.cc
View file

@ -95,39 +95,14 @@ ROCMExecutionProvider::PerThreadContext::~PerThreadContext() {
}
}
/*
* This method should be called within the constructor,
* so that the configuration of provider related setting can be updated
* and kept at IExecutionProvider level.
*/
void ROCMExecutionProvider::UpdateProviderOptionsInfo() {
UnorderedMapStringToString options;
options["device_id"] = std::to_string(device_id_);
options["hip_mem_limit"] = std::to_string(hip_mem_limit_);
std::string strategy;
if (arena_extend_strategy_ == ArenaExtendStrategy::kNextPowerOfTwo) {
strategy = "kNextPowerOfTwo";
} else if (arena_extend_strategy_ == ArenaExtendStrategy::kSameAsRequested) {
strategy = "kSameAsRequested";
} else {
strategy = "unknown";
}
options["arena_extend_strategy"] = strategy;
IExecutionProvider::SetProviderOptions(options);
}
ROCMExecutionProvider::ROCMExecutionProvider(const ROCMExecutionProviderInfo& info)
: IExecutionProvider{onnxruntime::kRocmExecutionProvider},
device_id_(info.device_id),
hip_mem_limit_(info.hip_mem_limit),
arena_extend_strategy_(info.arena_extend_strategy) {
HIP_CALL_THROW(hipSetDevice(device_id_));
info_{info} {
HIP_CALL_THROW(hipSetDevice(info_.device_id));
// must wait GPU idle, otherwise hipGetDeviceProperties might fail
HIP_CALL_THROW(hipDeviceSynchronize());
HIP_CALL_THROW(hipGetDeviceProperties(&device_prop_, device_id_));
HIP_CALL_THROW(hipGetDeviceProperties(&device_prop_, info_.device_id));
size_t free = 0;
size_t total = 0;
@ -137,10 +112,10 @@ ROCMExecutionProvider::ROCMExecutionProvider(const ROCMExecutionProviderInfo& in
[](OrtDevice::DeviceId device_id) {
return onnxruntime::make_unique<ROCMAllocator>(device_id, CUDA);
},
device_id_,
info_.device_id,
true,
{hip_mem_limit_,
static_cast<int>(arena_extend_strategy_),
{info_.hip_mem_limit,
static_cast<int>(info_.arena_extend_strategy),
-1, -1});
InsertAllocator(CreateAllocator(default_memory_info));
@ -164,8 +139,6 @@ ROCMExecutionProvider::ROCMExecutionProvider(const ROCMExecutionProviderInfo& in
CPU_ALLOCATOR_DEVICE_ID);
InsertAllocator(CreateAllocator(cpu_memory_info));
UpdateProviderOptionsInfo();
}
ROCMExecutionProvider::~ROCMExecutionProvider() {
@ -215,7 +188,7 @@ ROCMExecutionProvider::PerThreadContext& ROCMExecutionProvider::GetPerThreadCont
// get or create a context
if (context_state_.retired_context_pool.empty()) {
context = std::make_shared<PerThreadContext>(device_id_, hip_mem_limit_, arena_extend_strategy_);
context = std::make_shared<PerThreadContext>(info_.device_id, info_.hip_mem_limit, info_.arena_extend_strategy);
} else {
context = context_state_.retired_context_pool.back();
context_state_.retired_context_pool.pop_back();

23
onnxruntime/core/providers/rocm/rocm_execution_provider.h
View file

@ -11,21 +11,15 @@
#include "core/framework/bfc_arena.h"
#include "core/framework/execution_provider.h"
#include "core/platform/ort_mutex.h"
#include "core/providers/rocm/rocm_pch.h"
#include "core/providers/rocm/gpu_data_transfer.h"
#include "core/providers/rocm/rocm_execution_provider_info.h"
#include "core/providers/rocm/rocm_pch.h"
#include "core/providers/rocm/shared_inc/rocm_utils.h"
namespace onnxruntime {
const int CPU_ALLOCATOR_DEVICE_ID = 0;
// Information needed to construct HIP execution providers.
struct ROCMExecutionProviderInfo {
OrtDevice::DeviceId device_id{0};
size_t hip_mem_limit{std::numeric_limits<size_t>::max()};
ArenaExtendStrategy arena_extend_strategy{ArenaExtendStrategy::kNextPowerOfTwo};
};
// Logical device representation.
class ROCMExecutionProvider : public IExecutionProvider {
public:
@ -65,7 +59,7 @@ class ROCMExecutionProvider : public IExecutionProvider {
if (count_or_bytes == 0)
return nullptr;
return IAllocator::MakeUniquePtr<T>(GetAllocator(device_id_, OrtMemTypeDefault), count_or_bytes);
return IAllocator::MakeUniquePtr<T>(GetAllocator(info_.device_id, OrtMemTypeDefault), count_or_bytes);
}
std::shared_ptr<KernelRegistry> GetKernelRegistry() const override;
@ -75,15 +69,16 @@ class ROCMExecutionProvider : public IExecutionProvider {
const onnxruntime::GraphViewer& graph,
const std::vector<const KernelRegistry*>& kernel_registries) const override;
int GetDeviceId() const override { return device_id_; }
int GetDeviceId() const override { return info_.device_id; }
const hipDeviceProp_t& GetDeviceProp() const { return device_prop_; };
void UpdateProviderOptionsInfo();
ProviderOptions GetProviderOptions() const override {
return ROCMExecutionProviderInfo::ToProviderOptions(info_);
}
private:
OrtDevice::DeviceId device_id_;
ROCMExecutionProviderInfo info_;
hipDeviceProp_t device_prop_;
size_t hip_mem_limit_;
ArenaExtendStrategy arena_extend_strategy_;
struct DeferredReleaseCPUPtrs {
bool recorded = false;

36
onnxruntime/core/providers/rocm/rocm_execution_provider_info.cc Normal file
View file

@ -0,0 +1,36 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "core/providers/rocm/rocm_execution_provider_info.h"
#include "core/common/string_utils.h"
#include "core/framework/provider_options_utils.h"
namespace onnxruntime {
namespace provider_option_names {
constexpr const char* kDeviceId = "device_id";
constexpr const char* kMemLimit = "hip_mem_limit";
constexpr const char* kArenaExtendStrategy = "arena_extend_strategy";
} // namespace provider_option_names
ROCMExecutionProviderInfo ROCMExecutionProviderInfo::FromProviderOptions(const ProviderOptions& options) {
ROCMExecutionProviderInfo info{};
// TODO validate info.device_id
ReadProviderOption(options, provider_option_names::kDeviceId, info.device_id);
ReadProviderOption(options, provider_option_names::kMemLimit, info.hip_mem_limit);
ReadProviderOption(options, provider_option_names::kArenaExtendStrategy, info.arena_extend_strategy);
return info;
}
ProviderOptions ROCMExecutionProviderInfo::ToProviderOptions(const ROCMExecutionProviderInfo& info) {
const ProviderOptions options{
{provider_option_names::kDeviceId, MakeString(info.device_id)},
{provider_option_names::kMemLimit, MakeString(info.hip_mem_limit)},
{provider_option_names::kArenaExtendStrategy, MakeString(info.arena_extend_strategy)},
};
return options;
}
} // namespace onnxruntime

22
onnxruntime/core/providers/rocm/rocm_execution_provider_info.h Normal file
View file

@ -0,0 +1,22 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#pragma once
#include <limits>
#include "core/framework/arena_extend_strategy.h"
#include "core/framework/ortdevice.h"
#include "core/framework/provider_options.h"
namespace onnxruntime {
// Information needed to construct HIP execution providers.
struct ROCMExecutionProviderInfo {
OrtDevice::DeviceId device_id{0};
size_t hip_mem_limit{std::numeric_limits<size_t>::max()};
ArenaExtendStrategy arena_extend_strategy{ArenaExtendStrategy::kNextPowerOfTwo};
static ROCMExecutionProviderInfo FromProviderOptions(const ProviderOptions& options);
static ProviderOptions ToProviderOptions(const ROCMExecutionProviderInfo& info);
};
} // namespace onnxruntime

43
onnxruntime/core/providers/rocm/rocm_provider_factory.cc
View file

@ -1,51 +1,48 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "core/providers/rocm/rocm_provider_factory_creator.h"
#include "core/providers/rocm/rocm_provider_factory.h"
#include <atomic>
#include "core/graph/onnx_protobuf.h"
#include "rocm_execution_provider.h"
#include <memory>
#include "gsl/gsl"
#include "core/common/make_unique.h"
#include "core/providers/rocm/rocm_execution_provider.h"
#include "core/providers/rocm/rocm_execution_provider_info.h"
#include "core/session/abi_session_options_impl.h"
#include "core/framework/bfc_arena.h"
using namespace onnxruntime;
namespace onnxruntime {
struct HIPProviderFactory : IExecutionProviderFactory {
HIPProviderFactory(OrtDevice::DeviceId device_id,
size_t hip_mem_limit = std::numeric_limits<size_t>::max(),
ArenaExtendStrategy arena_extend_strategy = ArenaExtendStrategy::kNextPowerOfTwo)
: device_id_(device_id),
hip_mem_limit_(hip_mem_limit),
arena_extend_strategy_(arena_extend_strategy) {}
HIPProviderFactory(const ROCMExecutionProviderInfo& info)
: info_{info} {}
~HIPProviderFactory() override {}
std::unique_ptr<IExecutionProvider> CreateProvider() override;
private:
OrtDevice::DeviceId device_id_;
size_t hip_mem_limit_;
ArenaExtendStrategy arena_extend_strategy_;
ROCMExecutionProviderInfo info_;
};
std::unique_ptr<IExecutionProvider> HIPProviderFactory::CreateProvider() {
ROCMExecutionProviderInfo info;
info.device_id = device_id_;
info.hip_mem_limit = hip_mem_limit_;
info.arena_extend_strategy = arena_extend_strategy_;
return onnxruntime::make_unique<ROCMExecutionProvider>(info);
return onnxruntime::make_unique<ROCMExecutionProvider>(info_);
}
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_ROCM(OrtDevice::DeviceId device_id,
size_t hip_mem_limit = std::numeric_limits<size_t>::max(),
ArenaExtendStrategy arena_extend_strategy = ArenaExtendStrategy::kNextPowerOfTwo) {
return std::make_shared<HIPProviderFactory>(device_id, hip_mem_limit, arena_extend_strategy);
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_ROCM(const ROCMExecutionProviderInfo& info) {
return std::make_shared<HIPProviderFactory>(info);
}
} // namespace onnxruntime
ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProvider_ROCM, _In_ OrtSessionOptions* options, int device_id) {
options->provider_factories.push_back(onnxruntime::CreateExecutionProviderFactory_ROCM(static_cast<OrtDevice::DeviceId>(device_id)));
ROCMExecutionProviderInfo info{};
info.device_id = gsl::narrow<OrtDevice::DeviceId>(device_id);
options->provider_factories.push_back(onnxruntime::CreateExecutionProviderFactory_ROCM(info));
return nullptr;
}

16
onnxruntime/core/providers/rocm/rocm_provider_factory_creator.h Normal file
View file

@ -0,0 +1,16 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#pragma once
#include <memory>
#include "core/providers/providers.h"
#include "core/providers/rocm/rocm_execution_provider_info.h"
namespace onnxruntime {
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_ROCM(const ROCMExecutionProviderInfo& info);
} // namespace onnxruntime

277
onnxruntime/python/onnxruntime_pybind_state.cc
View file

@ -11,17 +11,28 @@
#include "core/common/logging/logging.h"
#include "core/common/logging/severity.h"
#include "core/framework/bfc_arena.h"
#include "core/common/optional.h"
#include "core/framework/arena_extend_strategy.h"
#include "core/framework/data_transfer_utils.h"
#include "core/framework/data_types_internal.h"
#include "core/framework/kernel_registry.h"
#include "core/framework/provider_options_utils.h"
#include "core/framework/random_seed.h"
#include "core/framework/tensorprotoutils.h"
#include "core/framework/TensorSeq.h"
#include "core/graph/graph_viewer.h"
#include "core/platform/env.h"
#include "core/session/IOBinding.h"
#include "core/session/abi_session_options_impl.h"
#include "core/platform/env.h"
// execution provider factory creator headers
#include "core/providers/cpu/cpu_provider_factory_creator.h"
#ifdef USE_CUDA
#include "core/providers/cuda/cuda_provider_factory_creator.h"
#endif
#ifdef USE_ROCM
#include "core/providers/rocm/rocm_provider_factory_creator.h"
#endif
struct OrtStatus {
OrtErrorCode code;
@ -128,21 +139,21 @@ struct OrtStatus {
#include "core/session/onnxruntime_cxx_api.h"
#include "core/providers/providers.h"
#include "core/providers/cpu/cpu_execution_provider.h"
#include "core/providers/cpu/cpu_provider_factory.h"
#if defined(USE_CUDA) || defined(USE_ROCM)
#ifdef USE_CUDA
#include "core/providers/cuda/cuda_provider_factory.h"
#include "core/providers/cuda/shared_inc/cuda_call.h"
#include "core/providers/cuda/cuda_execution_provider.h"
#include "core/providers/cuda/cuda_allocator.h"
// TODO remove deprecated global config
OrtCudnnConvAlgoSearch cudnn_conv_algo_search = OrtCudnnConvAlgoSearch::EXHAUSTIVE;
// TODO remove deprecated global config
bool do_copy_in_default_stream = true;
#elif USE_ROCM
#include "core/providers/rocm/rocm_provider_factory.h"
#endif
// TODO remove deprecated global config
OrtDevice::DeviceId cuda_device_id = 0;
// TODO remove deprecated global config
size_t cuda_mem_limit = std::numeric_limits<size_t>::max();
// TODO remove deprecated global config
onnxruntime::ArenaExtendStrategy arena_extend_strategy = onnxruntime::ArenaExtendStrategy::kNextPowerOfTwo;
#endif
@ -154,10 +165,12 @@ onnxruntime::ArenaExtendStrategy arena_extend_strategy = onnxruntime::ArenaExten
#endif
#ifdef USE_OPENVINO
#include "core/providers/openvino/openvino_provider_factory.h"
// TODO remove deprecated global config
std::string openvino_device_type;
#endif
#ifdef USE_NUPHAR
#include "core/providers/nuphar/nuphar_provider_factory.h"
// TODO remove deprecated global config
std::string nuphar_settings;
#endif
#ifdef USE_VITISAI
@ -173,8 +186,6 @@ std::string nuphar_settings;
#include "core/providers/dml/dml_provider_factory.h"
#endif
#define PYBIND_UNREFERENCED_PARAMETER(parameter) ((void)(parameter))
// Explicitly provide a definition for the static const var 'GPU' in the OrtDevice struct,
// GCC 4.x doesn't seem to define this and it breaks the pipelines based on CentOS as it uses
// GCC 4.x.
@ -182,15 +193,6 @@ std::string nuphar_settings;
const OrtDevice::DeviceType OrtDevice::GPU;
namespace onnxruntime {
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_CPU(int use_arena);
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_CUDA(OrtDevice::DeviceId device_id,
OrtCudnnConvAlgoSearch cudnn_conv_algo_search,
size_t cuda_mem_limit,
onnxruntime::ArenaExtendStrategy arena_extend_strategy,
bool do_copy_in_default_stream);
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_ROCM(OrtDevice::DeviceId device_id,
size_t cuda_mem_limit,
onnxruntime::ArenaExtendStrategy arena_extend_strategy);
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_Tensorrt(int device_id);
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_MIGraphX(int device_id);
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_Dnnl(int use_arena);
@ -452,30 +454,6 @@ static const std::vector<std::string>& GetAvailableProviders() {
#ifdef USE_CUDA
/*
* Validate a string that is positive integer or zero.
*
* (-1234, 43.21, +43.21 ... are not valid,
* 1234, +4321, 12 ... are valid)
*/
static bool IsPositiveInteger(const std::string& s) {
if (s.length() == 0) {
return false;
}
for (size_t i = 0; i < s.length(); i++) {
if (i == 0 && s[i] == '+' && s.length() > 1) {
continue;
}
if (!isdigit(s[i])) {
return false;
}
}
return true;
}
static bool IsCudaDeviceIdValid(const onnxruntime::logging::Logger& logger, int id) {
int num_devices = 0;
CUDA_CALL_THROW(cudaGetDeviceCount(&num_devices));
@ -493,70 +471,6 @@ static bool IsCudaDeviceIdValid(const onnxruntime::logging::Logger& logger, int
return true;
}
static void UpdateCudaProviderOptions(InferenceSession* sess, onnxruntime::CUDAExecutionProviderInfo& options,
std::unordered_map<std::string, std::string> options_map) {
std::unordered_map<std::string, std::string>::iterator it;
it = options_map.find("device_id");
if (it != options_map.end()) {
OrtDevice::DeviceId device_id;
try {
int id = std::stoi(it->second);
device_id = static_cast<int8_t>(id);
} catch (...) {
throw std::runtime_error("Please provide device id with integer.");
}
if (!IsCudaDeviceIdValid(*sess->GetLogger(), device_id)) {
throw std::runtime_error("Please provide available device id.");
}
options.device_id = device_id;
LOGS(*(sess->GetLogger()), INFO) << "cuda device id is set to " << device_id;
}
it = options_map.find("cuda_mem_limit");
if (it != options_map.end()) {
// The reason to check whether the string is positive integer upfront is that
// when calling stoull(), if the minus sign was part of the input sequence,
// the numeric value calculated from the sequence of digits is negated.
// In other words, it will cause wraparound.
// So, we rule out negative integer string beforehand.
if (!IsPositiveInteger(it->second)) {
throw std::runtime_error("Please provide cuda memory limitation size with positive integer.");
}
size_t size;
try {
#if (defined(__amd64__) || defined(_M_AMD64) || defined(__aarch64__))
size = std::stoull(it->second, nullptr, 0);
#else
size = std::stoul(it->second, nullptr, 0);
#endif
} catch (...) {
throw std::runtime_error("Please provide cuda memory limitation size with positive integer and within range.");
}
options.cuda_mem_limit = size;
LOGS(*(sess->GetLogger()), INFO) << "cuda memory limitation is set to " << size;
}
it = options_map.find("arena_extend_strategy");
if (it != options_map.end()) {
onnxruntime::ArenaExtendStrategy strategy;
if (it->second.compare("kNextPowerOfTwo") == 0) {
strategy = onnxruntime::ArenaExtendStrategy::kNextPowerOfTwo;
} else if (it->second.compare("kSameAsRequested") == 0) {
strategy = onnxruntime::ArenaExtendStrategy::kSameAsRequested;
} else {
throw std::runtime_error("Please provide proper cuda arena extend strategy.");
}
options.arena_extend_strategy = strategy;
LOGS(*(sess->GetLogger()), INFO) << "cuda arean extend strategy is set to " << it->second;
}
}
static AllocatorPtr GetCudaAllocator(OrtDevice::DeviceId id) {
// Current approach is not thread-safe, but there are some bigger infra pieces to put together in order to make
// multi-threaded CUDA allocation work we need to maintain a per-thread CUDA allocator
@ -601,12 +515,10 @@ static const std::unordered_map<OrtDevice::DeviceType, MemCpyFunc>* GetCudaToHos
/*
* Register execution provider with options.
*
* (note: currently only cuda EP supports this feature and rest of EPs use default options)
*/
static void RegisterExecutionProviders(InferenceSession* sess, const std::vector<std::string>& provider_types,
ProviderOptionsMap& provider_options_map) {
PYBIND_UNREFERENCED_PARAMETER(provider_options_map);
const ProviderOptionsMap& provider_options_map) {
ORT_UNUSED_PARAMETER(provider_options_map);
for (const std::string& type : provider_types) {
if (type == kCpuExecutionProvider) {
@ -622,33 +534,39 @@ static void RegisterExecutionProviders(InferenceSession* sess, const std::vector
#endif
} else if (type == kCudaExecutionProvider) {
#ifdef USE_CUDA
const auto it = provider_options_map.find(type);
const CUDAExecutionProviderInfo info =
it != provider_options_map.end()
? CUDAExecutionProviderInfo::FromProviderOptions(it->second)
: [&]() {
CUDAExecutionProviderInfo info{};
info.device_id = cuda_device_id;
info.cuda_mem_limit = cuda_mem_limit;
info.arena_extend_strategy = arena_extend_strategy;
info.cudnn_conv_algo = cudnn_conv_algo_search;
info.do_copy_in_default_stream = do_copy_in_default_stream;
return info;
}();
auto it = provider_options_map.find(type);
if (it != provider_options_map.end()) {
onnxruntime::CUDAExecutionProviderInfo cuda_provider_options;
UpdateCudaProviderOptions(sess, cuda_provider_options, it->second);
RegisterExecutionProvider(
sess, *onnxruntime::CreateExecutionProviderFactory_CUDA(cuda_provider_options.device_id,
cuda_provider_options.cudnn_conv_algo,
cuda_provider_options.cuda_mem_limit,
cuda_provider_options.arena_extend_strategy,
cuda_provider_options.do_copy_in_default_stream));
} else {
RegisterExecutionProvider(
sess, *onnxruntime::CreateExecutionProviderFactory_CUDA(cuda_device_id,
cudnn_conv_algo_search,
cuda_mem_limit,
arena_extend_strategy,
do_copy_in_default_stream));
}
RegisterExecutionProvider(
sess, *onnxruntime::CreateExecutionProviderFactory_CUDA(info));
#endif
} else if (type == kRocmExecutionProvider) {
#ifdef USE_ROCM
const auto it = provider_options_map.find(type);
const ROCMExecutionProviderInfo info =
it != provider_options_map.end()
? ROCMExecutionProviderInfo::FromProviderOptions(it->second)
: [&]() {
ROCMExecutionProviderInfo info{};
info.device_id = cuda_device_id;
info.hip_mem_limit = cuda_mem_limit;
info.arena_extend_strategy = arena_extend_strategy;
return info;
}();
RegisterExecutionProvider(
sess, *onnxruntime::CreateExecutionProviderFactory_ROCM(cuda_device_id,
cuda_mem_limit,
arena_extend_strategy));
sess, *onnxruntime::CreateExecutionProviderFactory_ROCM(info));
#endif
} else if (type == kDnnlExecutionProvider) {
#ifdef USE_DNNL
@ -666,8 +584,7 @@ static void RegisterExecutionProviders(InferenceSession* sess, const std::vector
if (option.first == "device_type") {
openvino_device_type = option.second;
params.device_type = openvino_device_type.c_str();
}
else if (option.first == "enable_vpu_fast_compile") {
} else if (option.first == "enable_vpu_fast_compile") {
if (option.second == "True") {
params.enable_vpu_fast_compile = true;
} else if (option.second == "False") {
@ -692,6 +609,11 @@ static void RegisterExecutionProviders(InferenceSession* sess, const std::vector
#endif
} else if (type == kNupharExecutionProvider) {
#if USE_NUPHAR
const auto it = provider_options_map.find(type);
if (it != provider_options_map.end()) {
ReadProviderOption(it->second, "nuphar_settings", nuphar_settings);
}
RegisterExecutionProvider(
sess, *onnxruntime::CreateExecutionProviderFactory_Nuphar(true, nuphar_settings.c_str()));
@ -807,6 +729,17 @@ static bool CheckIfTensor(const std::vector<const NodeArg*>& def_list,
return type_proto.has_tensor_type();
}
#if defined(USE_NUPHAR) || \
defined(USE_OPENVINO) || \
defined(USE_CUDA) || \
defined(USE_ROCM)
static void LogDeprecationWarning(
const std::string& deprecated, const optional<std::string>& alternative = nullopt) {
LOGS_DEFAULT(WARNING) << "This is DEPRECATED and will be removed in the future: " << deprecated;
LOGS_DEFAULT_IF(alternative.has_value(), WARNING) << "As an alternative, use: " << *alternative;
}
#endif
void addGlobalMethods(py::module& m, Environment& env) {
m.def("get_default_session_options", &GetDefaultCPUSessionOptions, "Return a default session_options instance.");
m.def("get_session_initializer", &SessionObjectInitializer::Get, "Return a default session object initializer.");
@ -847,10 +780,14 @@ void addGlobalMethods(py::module& m, Environment& env) {
});
#ifdef USE_NUPHAR
// TODO remove deprecated global config
m.def("set_nuphar_settings", [](const std::string& str) {
LogDeprecationWarning("set_nuphar_settings", "Nuphar execution provider option \"nuphar_settings\"");
nuphar_settings = str;
});
// TODO remove deprecated global config
m.def("get_nuphar_settings", []() -> std::string {
LogDeprecationWarning("get_nuphar_settings");
return nuphar_settings;
});
#endif
@ -865,13 +802,19 @@ void addGlobalMethods(py::module& m, Environment& env) {
},
"Lists all OpenVINO device ids available.");
/*
* The following APIs to set config options are deprecated. Use Session.set_providers() instead.
*/
* The following APIs to set config options are deprecated. Use Session.set_providers() instead.
*/
// TODO remove deprecated global config
m.def(
"set_openvino_device", [](const std::string& device_type) { openvino_device_type = device_type; },
"set_openvino_device", [](const std::string& device_type) {
LogDeprecationWarning("set_openvino_device", "OpenVINO execution provider option \"device_type\"");
openvino_device_type = device_type;
},
"Set the prefered OpenVINO device type to be used. If left unset, the device type selected during build time will be used.");
// TODO remove deprecated global config
m.def(
"get_openvino_device", []() -> std::string {
LogDeprecationWarning("get_openvino_device");
return openvino_device_type;
},
"Gets the dynamically selected OpenVINO device type for inference.");
@ -890,10 +833,26 @@ void addGlobalMethods(py::module& m, Environment& env) {
std::vector<std::shared_ptr<onnxruntime::IExecutionProviderFactory>> factories = {
onnxruntime::CreateExecutionProviderFactory_CPU(0),
#ifdef USE_CUDA
onnxruntime::CreateExecutionProviderFactory_CUDA(cuda_device_id, cudnn_conv_algo_search, cuda_mem_limit, arena_extend_strategy, do_copy_in_default_stream),
onnxruntime::CreateExecutionProviderFactory_CUDA(
[&]() {
CUDAExecutionProviderInfo info{};
info.device_id = cuda_device_id;
info.cuda_mem_limit = cuda_mem_limit;
info.arena_extend_strategy = arena_extend_strategy;
info.cudnn_conv_algo = cudnn_conv_algo_search;
info.do_copy_in_default_stream = do_copy_in_default_stream;
return info;
}()),
#endif
#ifdef USE_ROCM
onnxruntime::CreateExecutionProviderFactory_ROCM(cuda_device_id, cuda_mem_limit, arena_extend_strategy),
onnxruntime::CreateExecutionProviderFactory_ROCM(
[&]() {
ROCMExecutionProviderInfo info{};
info.device_id = cuda_device_id;
info.hip_mem_limit = cuda_mem_limit;
info.arena_extend_strategy = arena_extend_strategy;
return info;
}()),
#endif
#ifdef USE_DNNL
onnxruntime::CreateExecutionProviderFactory_Dnnl(1),
@ -945,8 +904,14 @@ void addGlobalMethods(py::module& m, Environment& env) {
* InferenceSession.set_providers(list_of_providers, list_of_provider_option_dicts)
*
*/
m.def("set_cuda_device_id", [](const int id) { cuda_device_id = static_cast<OrtDevice::DeviceId>(id); });
// TODO remove deprecated global config
m.def("set_cuda_device_id", [](const int id) {
LogDeprecationWarning("set_cuda_device_id", "CUDA/ROCM execution provider option \"device_id\"");
cuda_device_id = static_cast<OrtDevice::DeviceId>(id);
});
// TODO remove deprecated global config
m.def("set_cudnn_conv_algo_search", [](const OrtCudnnConvAlgoSearch algo) {
LogDeprecationWarning("set_cudnn_conv_algo_search", "CUDA execution provider option \"cudnn_conv_algo\"");
#ifdef USE_ROCM
ORT_UNUSED_PARAMETER(algo);
ORT_THROW("set_cudnn_conv_algo_search is not supported in ROCM");
@ -954,7 +919,10 @@ void addGlobalMethods(py::module& m, Environment& env) {
cudnn_conv_algo_search = algo;
#endif
});
// TODO remove deprecated global config
m.def("set_do_copy_in_default_stream", [](const bool use_single_stream) {
LogDeprecationWarning(
"set_do_copy_in_default_stream", "CUDA execution provider option \"do_copy_in_default_stream\"");
#ifdef USE_ROCM
ORT_UNUSED_PARAMETER(use_single_stream);
ORT_THROW("set_do_copy_in_default_stream is not supported in ROCM");
@ -962,8 +930,18 @@ void addGlobalMethods(py::module& m, Environment& env) {
do_copy_in_default_stream = use_single_stream;
#endif
});
m.def("set_cuda_mem_limit", [](const int64_t limit) { cuda_mem_limit = static_cast<size_t>(limit); });
m.def("set_arena_extend_strategy", [](const onnxruntime::ArenaExtendStrategy strategy) { arena_extend_strategy = strategy; });
// TODO remove deprecated global config
m.def("set_cuda_mem_limit", [](const int64_t limit) {
LogDeprecationWarning(
"set_cuda_mem_limit",
"CUDA execution provider option \"cuda_mem_limit\", ROCM execution provider option \"hip_mem_limit\"");
cuda_mem_limit = gsl::narrow<size_t>(limit);
});
// TODO remove deprecated global config
m.def("set_arena_extend_strategy", [](const onnxruntime::ArenaExtendStrategy strategy) {
LogDeprecationWarning("set_arena_extend_strategy", "CUDA/ROCM execution provider option \"arena_extend_strategy\"");
arena_extend_strategy = strategy;
});
#endif
}
@ -1112,6 +1090,11 @@ void addObjectMethods(py::module& m, Environment& env) {
.value("CPU", OrtMemType::OrtMemTypeCPU)
.value("DEFAULT", OrtMemType::OrtMemTypeDefault);
py::enum_<OrtCudnnConvAlgoSearch>(m, "OrtCudnnConvAlgoSearch")
.value("EXHAUSTIVE", OrtCudnnConvAlgoSearch::EXHAUSTIVE)
.value("HEURISTIC", OrtCudnnConvAlgoSearch::HEURISTIC)
.value("DEFAULT", OrtCudnnConvAlgoSearch::DEFAULT);
py::class_<OrtDevice> device(m, "OrtDevice", R"pbdoc(ONNXRuntime device informaion.)pbdoc");
device.def(py::init<OrtDevice::DeviceType, OrtDevice::MemoryType, OrtDevice::DeviceId>())
.def("device_id", &OrtDevice::Id, R"pbdoc(Device Id.)pbdoc")
@ -1533,7 +1516,7 @@ Applies to session load, initialization, etc. Default is 0.)pbdoc")
dim_name,
onnxruntime::FreeDimensionOverrideType::Denotation,
dim_value}); },
"Rpbdoc(Specify the dimension size for each denotation associated with an input's free dimension.)pbdoc")
R"pbdoc(Specify the dimension size for each denotation associated with an input's free dimension.)pbdoc")
.def(
"add_free_dimension_override_by_name",
[](PySessionOptions* options, const char* dim_name, int64_t dim_value)
@ -1542,7 +1525,7 @@ Applies to session load, initialization, etc. Default is 0.)pbdoc")
dim_name,
onnxruntime::FreeDimensionOverrideType::Name,
dim_value}); },
"Rpbdoc(Specify values of named dimensions within model inputs.)pbdoc")
R"pbdoc(Specify values of named dimensions within model inputs.)pbdoc")
.def(
"add_session_config_entry",
[](PySessionOptions* options, const char* config_key, const char* config_value) -> void {
@ -1550,7 +1533,7 @@ Applies to session load, initialization, etc. Default is 0.)pbdoc")
if (!status.IsOK())
throw std::runtime_error(status.ErrorMessage());
},
"Rpbdoc(Set a single session configuration entry as a pair of strings.)pbdoc")
R"pbdoc(Set a single session configuration entry as a pair of strings.)pbdoc")
.def(
"get_session_config_entry",
[](PySessionOptions* options, const char* config_key) -> std::string {
@ -1561,7 +1544,7 @@ Applies to session load, initialization, etc. Default is 0.)pbdoc")
return value;
},
"Rpbdoc(Get a single session configuration value using the given configuration key.)pbdoc")
R"pbdoc(Get a single session configuration value using the given configuration key.)pbdoc")
.def(
"register_custom_ops_library",
[](PySessionOptions* options, const char* library_path)
@ -1585,7 +1568,7 @@ Applies to session load, initialization, etc. Default is 0.)pbdoc")
ORT_THROW("Custom Ops are not supported in this build.");
#endif
},
"Rpbdoc(Specify the path to the shared library containing the custom op kernels required to run a model.)pbdoc")
R"pbdoc(Specify the path to the shared library containing the custom op kernels required to run a model.)pbdoc")
.def(
"add_initializer", [](PySessionOptions* options, const char* name, py::object& ml_value_pyobject) -> void {
ORT_ENFORCE(strcmp(Py_TYPE(ml_value_pyobject.ptr())->tp_name, PYTHON_ORTVALUE_OBJECT_NAME) == 0, "The provided Python object must be an OrtValue");

86
onnxruntime/test/common/string_utils_test.cc Normal file
View file

@ -0,0 +1,86 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "core/common/string_utils.h"
#include "gtest/gtest.h"
namespace onnxruntime {
namespace test {
namespace {
template <typename T>
void TestSuccessfulParse(const std::string& input, const T& expected_value) {
T value;
ASSERT_TRUE(TryParse(input, value));
EXPECT_EQ(value, expected_value);
}
template <typename T>
void TestFailedParse(const std::string& input) {
T value;
EXPECT_FALSE(TryParse(input, value));
}
} // namespace
TEST(StringUtilsTest, TryParse) {
TestSuccessfulParse("-1", -1);
TestSuccessfulParse("42", 42u);
TestSuccessfulParse("2.5", 2.5f);
TestSuccessfulParse("1", true);
TestSuccessfulParse("0", false);
// out of range
TestFailedParse<int16_t>("32768");
TestFailedParse<uint32_t>("-1");
TestFailedParse<float>("1e100");
TestFailedParse<bool>("2");
// invalid representation
TestFailedParse<int32_t>("1.2");
TestFailedParse<int32_t>("one");
// leading or trailing characters
TestFailedParse<int32_t>(" 1");
TestFailedParse<int32_t>("1 ");
}
TEST(StringUtilsTest, TryParseString) {
// when parsing a string as a string, allow leading and trailing whitespace
const std::string s = " this is a string! ";
TestSuccessfulParse(s, s);
}
namespace {
struct S {
int i{};
bool operator==(const S& other) const {
return i == other.i;
}
};
std::ostream& operator<<(std::ostream& os, const S& s) {
os << "S " << s.i;
return os;
}
std::istream& operator>>(std::istream& is, S& s) {
std::string struct_name;
is >> struct_name >> s.i;
if (struct_name != "S") {
is.setstate(std::ios_base::failbit);
}
return is;
}
} // namespace
TEST(StringUtilsTest, MakeStringAndTryParseCustomType) {
S s;
s.i = 42;
const auto str = MakeString(s);
S parsed_s;
ASSERT_TRUE(TryParse(str, parsed_s));
ASSERT_EQ(parsed_s, s);
}
} // namespace test
} // namespace onnxruntime

32
onnxruntime/test/onnx/tensorprotoutils.cc
View file

@ -11,6 +11,7 @@
#include "mem_buffer.h"
#include "core/common/safeint.h"
#include "core/common/status.h"
#include "core/common/string_utils.h"
#include "core/framework/data_types.h"
#include "core/framework/endian.h"
#include "core/framework/allocator.h"
@ -26,37 +27,6 @@ struct OrtStatus {
namespace onnxruntime {
namespace test {
//From core common
inline void MakeStringInternal(std::ostringstream& /*ss*/) noexcept {
}
template <typename T>
inline void MakeStringInternal(std::ostringstream& ss, const T& t) noexcept {
ss << t;
}
template <typename T, typename... Args>
inline void MakeStringInternal(std::ostringstream& ss, const T& t, const Args&... args) noexcept {
::onnxruntime::MakeStringInternal(ss, t);
::onnxruntime::MakeStringInternal(ss, args...);
}
template <typename... Args>
std::string MakeString(const Args&... args) {
std::ostringstream ss;
::onnxruntime::MakeStringInternal(ss, args...);
return std::string(ss.str());
}
// Specializations for already-a-string types.
template <>
inline std::string MakeString(const std::string& str) {
return str;
}
inline std::string MakeString(const char* p_str) {
return p_str;
}
std::vector<int64_t> GetTensorShapeFromTensorProto(const onnx::TensorProto& tensor_proto) {
const auto& dims = tensor_proto.dims();
std::vector<int64_t> tensor_shape_vec(static_cast<size_t>(dims.size()));

25
onnxruntime/test/util/default_providers.cc
View file

@ -3,21 +3,17 @@
#include "default_providers.h"
#include "providers.h"
#include "core/providers/cpu/cpu_provider_factory_creator.h"
#ifdef USE_CUDA
#include "core/providers/cuda/cuda_provider_factory_creator.h"
#endif
#ifdef USE_ROCM
#include "core/providers/rocm/rocm_provider_factory_creator.h"
#endif
#include "core/session/onnxruntime_cxx_api.h"
#include "core/providers/providers.h"
#include "core/framework/bfc_arena.h"
namespace onnxruntime {
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_CPU(int use_arena);
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_CUDA(
OrtDevice::DeviceId device_id,
OrtCudnnConvAlgoSearch cudnn_conv_algo = OrtCudnnConvAlgoSearch::EXHAUSTIVE,
size_t cuda_mem_limit = std::numeric_limits<size_t>::max(),
ArenaExtendStrategy arena_extend_strategy = ArenaExtendStrategy::kNextPowerOfTwo,
bool do_copy_in_default_stream = true);
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_OpenVINO(
const char* device_type, bool enable_vpu_fast_compile, const char* device_id, size_t num_of_threads);
@ -30,9 +26,6 @@ std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_Tensor
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_MIGraphX(int device_id);
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_ACL(int use_arena);
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_ArmNN(int use_arena);
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_ROCM(OrtDevice::DeviceId device_id,
size_t hip_mem_limit = std::numeric_limits<size_t>::max(),
ArenaExtendStrategy arena_extend_strategy = ArenaExtendStrategy::kNextPowerOfTwo);
// EP for internal testing
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_InternalTesting(
@ -71,7 +64,7 @@ std::unique_ptr<IExecutionProvider> DefaultOpenVINOExecutionProvider() {
std::unique_ptr<IExecutionProvider> DefaultCudaExecutionProvider() {
#ifdef USE_CUDA
return CreateExecutionProviderFactory_CUDA(0)->CreateProvider();
return CreateExecutionProviderFactory_CUDA(CUDAExecutionProviderInfo{})->CreateProvider();
#else
return nullptr;
#endif
@ -134,7 +127,7 @@ std::unique_ptr<IExecutionProvider> DefaultArmNNExecutionProvider(bool enable_ar
std::unique_ptr<IExecutionProvider> DefaultRocmExecutionProvider() {
#ifdef USE_ROCM
return CreateExecutionProviderFactory_ROCM(0)->CreateProvider();
return CreateExecutionProviderFactory_ROCM(ROCMExecutionProviderInfo{})->CreateProvider();
#else
return nullptr;
#endif

47
orttraining/orttraining/models/bert/main.cc
View file

@ -10,11 +10,14 @@
#include "core/session/environment.h"
#include "core/framework/bfc_arena.h"
#include "core/framework/random_seed.h"
#include "core/providers/cpu/cpu_provider_factory_creator.h"
#ifdef USE_CUDA
#include "core/providers/cuda/cuda_allocator.h"
#include "core/providers/cuda/cuda_provider_factory_creator.h"
#endif
#ifdef USE_ROCM
#include "core/providers/rocm/rocm_allocator.h"
#include "core/providers/rocm/rocm_provider_factory_creator.h"
#endif
#include "orttraining/core/session/training_session.h"
#include "orttraining/core/framework/tensorboard/event_writer.h"
@ -24,21 +27,6 @@
#include "orttraining/models/runner/training_util.h"
#include "orttraining/models/runner/data_loader.h"
namespace onnxruntime {
#ifdef USE_CUDA
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_CUDA(OrtDevice::DeviceId device_id,
OrtCudnnConvAlgoSearch cudnn_conv_algo_search = OrtCudnnConvAlgoSearch::EXHAUSTIVE,
size_t cuda_mem_limit = std::numeric_limits<size_t>::max(),
onnxruntime::ArenaExtendStrategy arena_extend_strategy = ArenaExtendStrategy::kNextPowerOfTwo,
bool do_copy_in_default_stream = true);
#endif
#ifdef USE_ROCM
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_ROCM(OrtDevice::DeviceId device_id,
size_t hip_mem_limit = std::numeric_limits<size_t>::max(),
onnxruntime::ArenaExtendStrategy arena_extend_strategy = ArenaExtendStrategy::kNextPowerOfTwo);
#endif
}
using namespace onnxruntime;
using namespace onnxruntime::common;
using namespace onnxruntime::training;
@ -582,20 +570,27 @@ void setup_training_params(BertParameters& params) {
#endif
#ifdef USE_CUDA
OrtDevice::DeviceId device_id = static_cast<OrtDevice::DeviceId>(MPIContext::GetInstance().GetLocalRank());
size_t cuda_mem_limit = std::numeric_limits<size_t>::max();
if (params.cuda_mem_limit_in_gb > 0)
cuda_mem_limit = static_cast<size_t>(params.cuda_mem_limit_in_gb * 1024 * 1024 * 1024);
params.providers.emplace(kCudaExecutionProvider, CreateExecutionProviderFactory_CUDA(device_id, OrtCudnnConvAlgoSearch::EXHAUSTIVE,
cuda_mem_limit));
params.input_allocator = std::make_shared<CUDAPinnedAllocator>(device_id, CUDA_PINNED);
{
CUDAExecutionProviderInfo info{};
info.device_id = gsl::narrow<OrtDevice::DeviceId>(MPIContext::GetInstance().GetLocalRank());
if (params.cuda_mem_limit_in_gb > 0) {
info.cuda_mem_limit = gsl::narrow<size_t>(params.cuda_mem_limit_in_gb * 1024 * 1024 * 1024);
}
info.cudnn_conv_algo = OrtCudnnConvAlgoSearch::EXHAUSTIVE;
params.providers.emplace(kCudaExecutionProvider, CreateExecutionProviderFactory_CUDA(info));
params.input_allocator = std::make_shared<CUDAPinnedAllocator>(info.device_id, CUDA_PINNED);
}
#endif
#ifdef USE_ROCM
OrtDevice::DeviceId device_id = static_cast<OrtDevice::DeviceId>(MPIContext::GetInstance().GetLocalRank());
size_t hip_mem_limit = std::numeric_limits<size_t>::max();
params.providers.emplace(kRocmExecutionProvider, CreateExecutionProviderFactory_ROCM(device_id, hip_mem_limit));
params.input_allocator = std::make_shared<ROCMPinnedAllocator>(device_id, CUDA_PINNED);
{
ROCMExecutionProviderInfo info{};
info.device_id = gsl::narrow<OrtDevice::DeviceId>(MPIContext::GetInstance().GetLocalRank());
params.providers.emplace(kRocmExecutionProvider, CreateExecutionProviderFactory_ROCM(info));
params.input_allocator = std::make_shared<ROCMPinnedAllocator>(info.device_id, CUDA_PINNED);
}
#endif
params.loss_func_info = LossFunctionInfo(OpDef("BertLoss", kOnnxDomain),

21
orttraining/orttraining/models/gpt2/main.cc
View file

@ -9,7 +9,10 @@
#include "core/session/environment.h"
#include "core/framework/random_seed.h"
#include "core/framework/bfc_arena.h"
#ifdef USE_CUDA
#include "core/providers/cuda/cuda_allocator.h"
#include "core/providers/cuda/cuda_provider_factory_creator.h"
#endif
#include "orttraining/core/framework/mpi_context.h"
#include "orttraining/core/framework/tensorboard/event_writer.h"
#include "orttraining/core/session/training_session.h"
@ -18,14 +21,6 @@
#include "orttraining/models/runner/training_util.h"
#include "orttraining/models/runner/data_loader.h"
namespace onnxruntime {
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_CUDA(OrtDevice::DeviceId device_id,
OrtCudnnConvAlgoSearch cudnn_conv_algo_search = OrtCudnnConvAlgoSearch::EXHAUSTIVE,
size_t cuda_mem_limit = std::numeric_limits<size_t>::max(),
onnxruntime::ArenaExtendStrategy arena_extend_strategy = ArenaExtendStrategy::kNextPowerOfTwo,
bool do_copy_in_default_stream = true);
}
using namespace onnxruntime;
using namespace onnxruntime::common;
using namespace onnxruntime::training;
@ -344,9 +339,13 @@ void setup_training_params(GPT2Parameters& params) {
params.model_type = "gpt2";
#ifdef USE_CUDA
OrtDevice::DeviceId device_id = static_cast<OrtDevice::DeviceId>(MPIContext::GetInstance().GetLocalRank());
params.providers.emplace(kCudaExecutionProvider, CreateExecutionProviderFactory_CUDA(device_id));
params.input_allocator = std::make_shared<CUDAPinnedAllocator>(device_id, CUDA_PINNED);
{
CUDAExecutionProviderInfo info{};
info.device_id = gsl::narrow<OrtDevice::DeviceId>(MPIContext::GetInstance().GetLocalRank());
params.providers.emplace(kCudaExecutionProvider, CreateExecutionProviderFactory_CUDA(info));
params.input_allocator = std::make_shared<CUDAPinnedAllocator>(info.device_id, CUDA_PINNED);
}
#endif
params.use_nccl = true;

13
orttraining/orttraining/models/mnist/main.cc
View file

@ -6,6 +6,9 @@
#include "core/common/logging/sinks/clog_sink.h"
#include "core/framework/bfc_arena.h"
#include "core/platform/env.h"
#ifdef USE_CUDA
#include "core/providers/cuda/cuda_provider_factory_creator.h"
#endif
#include "core/session/environment.h"
#include "orttraining/core/session/training_session.h"
#include "orttraining/core/framework/tensorboard/event_writer.h"
@ -18,14 +21,6 @@
#include <mutex>
#include <tuple>
namespace onnxruntime {
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_CUDA(OrtDevice::DeviceId device_id,
OrtCudnnConvAlgoSearch cudnn_conv_algo_search = OrtCudnnConvAlgoSearch::EXHAUSTIVE,
size_t cuda_mem_limit = std::numeric_limits<size_t>::max(),
onnxruntime::ArenaExtendStrategy arena_extend_strategy = ArenaExtendStrategy::kNextPowerOfTwo,
bool do_copy_in_default_stream = true);
}
using namespace onnxruntime;
using namespace onnxruntime::common;
using namespace onnxruntime::training;
@ -156,7 +151,7 @@ Status ParseArguments(int argc, char* argv[], TrainingRunner::Parameters& params
#ifdef USE_CUDA
bool use_cuda = flags.count("use_cuda") > 0;
if (use_cuda) {
params.providers.emplace(kCudaExecutionProvider, CreateExecutionProviderFactory_CUDA(0));
params.providers.emplace(kCudaExecutionProvider, CreateExecutionProviderFactory_CUDA(CUDAExecutionProviderInfo{}));
}
#endif
} catch (const exception& e) {

41
tools/ci_build/amd_hipify.py
View file

@ -1,13 +1,17 @@
#!/usr/bin/env python3
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
import os
import subprocess
from logger import get_logger
log = get_logger("amd_hipify")
contrib_ops_path = 'onnxruntime/contrib_ops'
core_ops_path = 'onnxruntime/core/providers'
providers_path = 'onnxruntime/core/providers'
training_ops_path = 'orttraining/orttraining/training_ops'
contrib_ops_files = [
contrib_ops_excluded_files = [
'bert/attention.cc',
'bert/attention.h',
'bert/attention_impl.cu',
@ -65,7 +69,7 @@ contrib_ops_files = [
'inverse.cc'
]
core_ops_files = [
provider_excluded_files = [
'atomic/common.cuh',
'controlflow/if.cc',
'controlflow/if.h',
@ -208,6 +212,8 @@ core_ops_files = [
'cuda_allocator.h',
'cuda_call.cc',
'cuda_common.h',
'cuda_execution_provider_info.cc',
'cuda_execution_provider_info.h',
'cuda_execution_provider.cc',
'cuda_execution_provider.h',
'cuda_fence.cc',
@ -227,7 +233,7 @@ core_ops_files = [
'symbols.txt',
]
training_ops_files = [
training_ops_excluded_files = [
'activation/activations_grad.cc',
'collective/horovod_kernels.cc',
'collective/horovod_kernels.h',
@ -274,10 +280,11 @@ training_ops_files = [
]
HIPIFY_PERL = '/opt/rocm/bin/hipify-perl'
FINDCODE = '/opt/rocm/bin/findcode.sh'
def hipify(src_file_path, dst_file_path):
log.debug('Hipifying: "{}" -> "{}"'.format(src_file_path, dst_file_path))
dst_file_path = dst_file_path.replace('cuda', 'rocm')
dir_name = os.path.dirname(dst_file_path)
if not os.path.exists(dir_name):
@ -353,29 +360,25 @@ def amd_hipify(config_build_dir):
rocm_contrib_path = os.path.join(config_build_dir, 'amdgpu', contrib_ops_path, 'rocm')
contrib_files = list_files(cuda_contrib_path, '')
for file in contrib_files:
if file not in contrib_ops_files:
if file not in contrib_ops_excluded_files:
src_file_path = os.path.join(cuda_contrib_path, file)
dst_file_path = os.path.join(rocm_contrib_path, file)
hipify(src_file_path, dst_file_path)
cuda_core_path = os.path.join(core_ops_path, 'cuda')
rocm_core_path = os.path.join(config_build_dir, 'amdgpu', core_ops_path, 'rocm')
core_files = list_files(cuda_core_path, '')
for file in core_files:
if file not in core_ops_files:
src_file_path = os.path.join(cuda_core_path, file)
dst_file_path = os.path.join(rocm_core_path, file)
cuda_provider_path = os.path.join(providers_path, 'cuda')
rocm_provider_path = os.path.join(config_build_dir, 'amdgpu', providers_path, 'rocm')
provider_files = list_files(cuda_provider_path, '')
for file in provider_files:
if file not in provider_excluded_files:
src_file_path = os.path.join(cuda_provider_path, file)
dst_file_path = os.path.join(rocm_provider_path, file)
hipify(src_file_path, dst_file_path)
cuda_training_path = os.path.join(training_ops_path, 'cuda')
rocm_training_path = os.path.join(config_build_dir, 'amdgpu', training_ops_path, 'rocm')
training_files = list_files(cuda_training_path, '')
for file in training_files:
if file not in training_ops_files:
if file not in training_ops_excluded_files:
src_file_path = os.path.join(cuda_training_path, file)
dst_file_path = os.path.join(rocm_training_path, file)
hipify(src_file_path, dst_file_path)
if __name__ == '__main__':
amd_hipify()

0
tools/ci_build/build.py Normal file → Executable file
View file