saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-30 20:18:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

[ROCm] Remove tuning options on transformerOptions (#13689)

Browse source 
### Description
<!-- Describe your changes. -->

Remove tuning options on transformerOptions, use IsTunableOpEnabled from
provider in the future.

### Motivation and Context
<!-- - Why is this change required? What problem does it solve?
- If it fixes an open issue, please link to the issue here. -->

Co-authored-by: peixuanzuo <peixuanzuo@linmif39a000004.zvflicr54joexhdgnhvmxrxygg.phxx.internal.cloudapp.net>
This commit is contained in:
PeixuanZuo 2022-11-23 15:36:09 +08:00 committed by GitHub
parent c43ce64795
commit 977da6635b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 40 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
onnxruntime/contrib_ops/rocm/bert/fast_gelu.cc
View file

@ -33,12 +33,6 @@ REGISTER_KERNEL_TYPED(BFloat16)
using namespace ONNX_NAMESPACE;
template <typename T>
FastGelu<T>::FastGelu(const OpKernelInfo& op_kernel_info) : RocmKernel(op_kernel_info) {
const TransformerOptions* options = TransformerOptions::GetInstance();
tuning_ = options->IsTuningEnabled();
}
template <typename T>
Status FastGelu<T>::ComputeInternal(OpKernelContext* context) const {
ORT_RETURN_IF_ERROR(bias_gelu_helper::CheckInputs(context));
@ -54,13 +48,13 @@ Status FastGelu<T>::ComputeInternal(OpKernelContext* context) const {
int64_t bias_length = (nullptr == bias) ? 0 : bias->Shape().Size();
typedef typename ToHipType<T>::MappedType HipT;
return LaunchFastGeluKernel<HipT>(Stream(),
static_cast<int>(input_length),
static_cast<int>(bias_length),
reinterpret_cast<const HipT*>(input->Data<T>()),
(nullptr != bias) ? reinterpret_cast<const HipT*>(bias->Data<T>()) : nullptr,
reinterpret_cast<HipT*>(output->MutableData<T>()),
tuning_);
return LaunchFastGeluKernel<HipT>(IsTunableOpEnabled(),
Stream(),
static_cast<int>(input_length),
static_cast<int>(bias_length),
reinterpret_cast<const HipT*>(input->Data<T>()),
(nullptr != bias) ? reinterpret_cast<const HipT*>(bias->Data<T>()) : nullptr,
reinterpret_cast<HipT*>(output->MutableData<T>()));
}
} // namespace rocm

5
onnxruntime/contrib_ops/rocm/bert/fast_gelu.h
View file

@ -16,11 +16,8 @@ using namespace onnxruntime::rocm;
template <typename T>
class FastGelu final : public RocmKernel {
public:
FastGelu(const OpKernelInfo& op_kernel_info);
FastGelu(const OpKernelInfo& op_kernel_info) : RocmKernel(op_kernel_info) {}
Status ComputeInternal(OpKernelContext* ctx) const override;
private:
bool tuning_;
};
} // namespace rocm

17
onnxruntime/contrib_ops/rocm/bert/fast_gelu_impl.cu
View file

@ -40,10 +40,9 @@ namespace contrib {
namespace rocm {
template <typename T>
Status LaunchFastGeluKernel(hipStream_t stream, int input_length, int bias_length,
const T* input, const T* bias, T* output, bool tuning) {
Status LaunchFastGeluKernel(bool tuning, hipStream_t stream, int input_length, int bias_length,
const T* input, const T* bias, T* output) {
FastGeluParams<T> params(stream, input, bias, output, input_length, bias_length);
if (tuning) {
static FastGeluTunableOp<T> op;
op.EnableTuning();
@ -53,14 +52,14 @@ Status LaunchFastGeluKernel(hipStream_t stream, int input_length, int bias_lengt
return FastGeluStaticSelection<T>(&params);
}
template Status LaunchFastGeluKernel<float>(hipStream_t stream, int input_length, int bias_length,
const float* input, const float* bias, float* output, bool tuning);
template Status LaunchFastGeluKernel<float>(bool tuning, hipStream_t stream, int input_length, int bias_length,
const float* input, const float* bias, float* output);
template Status LaunchFastGeluKernel<BFloat16>(hipStream_t stream, int input_length, int bias_length,
const BFloat16* input, const BFloat16* bias, BFloat16* output, bool tuning);
template Status LaunchFastGeluKernel<BFloat16>(bool tuning, hipStream_t stream, int input_length, int bias_length,
const BFloat16* input, const BFloat16* bias, BFloat16* output);
template Status LaunchFastGeluKernel<half>(hipStream_t stream, int input_length, int bias_length,
const half* input, const half* bias, half* output, bool tuning);
template Status LaunchFastGeluKernel<half>(bool tuning, hipStream_t stream, int input_length, int bias_length,
const half* input, const half* bias, half* output);
} // namespace rocm
} // namespace contrib

4
onnxruntime/contrib_ops/rocm/bert/fast_gelu_impl.h
View file

@ -13,8 +13,8 @@ namespace contrib {
namespace rocm {
template <typename T>
Status LaunchFastGeluKernel(hipStream_t stream, int input_length, int bias_length,
const T* input, const T* bias, T* output, bool tuning);
Status LaunchFastGeluKernel(bool tuning, hipStream_t stream, int input_length, int bias_length,
const T* input, const T* bias, T* output);
} // namespace rocm
} // namespace contrib

6
onnxruntime/contrib_ops/rocm/bert/gemm_fast_gelu_tunable_op.h
View file

@ -57,13 +57,13 @@ Status GemmFastGeluUnfused(const GemmFastGeluParams<T>* params) {
int64_t bias_length = (params->bias != nullptr) ? params->n : 0;
// inplace computation
return LaunchFastGeluKernel<T>(params->stream,
return LaunchFastGeluKernel<T>(params->tuning,
params->stream,
static_cast<int>(fast_gelu_input_length),
static_cast<int>(bias_length),
params->c,
params->bias,
params->c,
params->tuning);
params->c);
}
template <typename T>

6
onnxruntime/contrib_ops/rocm/bert/skip_layer_norm.cc
View file

@ -31,8 +31,6 @@ template <typename T>
SkipLayerNorm<T>::SkipLayerNorm(const OpKernelInfo& op_kernel_info) : RocmKernel(op_kernel_info) {
ORT_ENFORCE(op_kernel_info.GetAttr<float>("epsilon", &epsilon_).IsOK());
ORT_ENFORCE(epsilon_ >= 0);
const TransformerOptions* options = TransformerOptions::GetInstance();
tuning_ = options->IsTuningEnabled();
}
template <typename T>
@ -100,6 +98,7 @@ Status SkipLayerNorm<T>::ComputeInternal(OpKernelContext* ctx) const {
typedef typename ToHipType<T>::MappedType HipT;
return LaunchSkipLayerNormKernel<HipT>(
IsTunableOpEnabled(),
Stream(),
reinterpret_cast<HipT*>(output->MutableData<T>()),
reinterpret_cast<const HipT*>(input->Data<T>()),
@ -109,8 +108,7 @@ Status SkipLayerNorm<T>::ComputeInternal(OpKernelContext* ctx) const {
(bias != nullptr) ? reinterpret_cast<const HipT*>(bias->Data<T>()) : nullptr,
epsilon_,
hidden_size,
static_cast<int>(element_count),
tuning_);
static_cast<int>(element_count));
}
} // namespace rocm

1
onnxruntime/contrib_ops/rocm/bert/skip_layer_norm.h
View file

@ -19,7 +19,6 @@ class SkipLayerNorm final : public RocmKernel {
private:
float epsilon_;
bool tuning_;
};
} // namespace rocm

12
onnxruntime/contrib_ops/rocm/bert/skip_layer_norm_impl.cu
View file

@ -41,8 +41,8 @@ namespace rocm {
template <typename T>
Status LaunchSkipLayerNormKernel(
hipStream_t stream, T* output, const T* input, const T* skip, const T* gamma,
const T* beta, const T* bias, float epsilon, int ld, int element_count, bool tuning) {
bool tuning, hipStream_t stream, T* output, const T* input, const T* skip, const T* gamma,
const T* beta, const T* bias, float epsilon, int ld, int element_count) {
// this must be true because element_count is the total size of the tensor
assert(element_count % ld == 0);
@ -57,15 +57,15 @@ Status LaunchSkipLayerNormKernel(
return SkipLayerNormStaticSelection<T>(&params);
}
template Status LaunchSkipLayerNormKernel<float>(hipStream_t stream, float* output, const float* input,
template Status LaunchSkipLayerNormKernel<float>(bool tuning, hipStream_t stream, float* output, const float* input,
const float* skip, const float* gamma, const float* beta,
const float* bias, float epsilon, int ld,
int element_count, bool tuning);
int element_count);
template Status LaunchSkipLayerNormKernel<half>(hipStream_t stream, half* output, const half* input,
template Status LaunchSkipLayerNormKernel<half>(bool tuning, hipStream_t stream, half* output, const half* input,
const half* skip, const half* gamma, const half* beta,
const half* bias, float epsilon, int ld,
int element_count, bool tuning);
int element_count);
} // namespace rocm
} // namespace contrib

21
onnxruntime/contrib_ops/rocm/bert/skip_layer_norm_impl.h
View file

@ -10,17 +10,18 @@ namespace rocm {
template <typename T>
Status LaunchSkipLayerNormKernel(
bool tuning,
hipStream_t stream,
T* output, // output tensor
const T* input, // input tensor
const T* skip, // skip tensor
const T* gamma, // Layer normalization gamma tensor
const T* beta, // Layer normalization beta tensor
const T* bias, // Layer normalization beta tensor
float epsilon, // Layer normalization epsilon
int hidden_size, // hidden size, it is the leading dimension (ld)
int element_count, // number of elements in input tensor
bool tuning);
T* output, // output tensor
const T* input, // input tensor
const T* skip, // skip tensor
const T* gamma, // Layer normalization gamma tensor
const T* beta, // Layer normalization beta tensor
const T* bias, // Layer normalization beta tensor
float epsilon, // Layer normalization epsilon
int hidden_size, // hidden size, it is the leading dimension (ld)
int element_count // number of elements in input tensor
);
} // namespace rocm
} // namespace contrib

5
onnxruntime/contrib_ops/rocm/bert/transformer_common.h
View file

@ -19,13 +19,10 @@ class TransformerOptions {
bool DisableHalf2() const { return disable_half2_; }
bool IsTuningEnabled() const { return tuning_; }
void Initialize(int value) {
is_precision_mode_ = (value & 0x01) > 0;
disable_persistent_softmax_ = (value & 0x02) > 0;
disable_half2_ = (value & 0x04) > 0;
tuning_ = (value & 0x08) > 0;
initialized_ = true;
}
@ -40,8 +37,6 @@ class TransformerOptions {
bool disable_half2_{false};
bool initialized_{false};
bool tuning_{false};
static TransformerOptions instance;
};