mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-28 20:11:22 +00:00
20c47ce91c
* t5 layer norm changes * add t5 layer norm kernel * use template for t5 layer norm * template definition changes * no build error * add CPU cuda kernel * first unit test * other forward unit tests * add T5LayerNormGrad * Add c++ transform and test for T5 LN * fix and some debug prints * fix cuda error * rename from t5 to simplified * PR comments * revert change on invertible LM code path * remove duplicate forward computation * add GradientCheckerTest.SimplifiedLayerNormGrad * change back macro * Fix SimplifiedLayerNorm Gradient * merge with Sherlockss changes * changed cuda kernel * reapply cpu kernel changes Co-authored-by: Jingyan Wang <jingywa@OrtTrainingDev3.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net> Co-authored-by: aishwarya bhandare <aibhanda@microsoft.com> Co-authored-by: Sherlock Huang <bahuang@OrtTrainingDev3.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
47 lines
1.2 KiB
C++
47 lines
1.2 KiB
C++
/**
|
|
* Copyright (c) 2016-present, Facebook, Inc.
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
//
|
|
// Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved.
|
|
// NVIDIA/apex is licensed under the
|
|
// BSD 3 - Clause "New" or "Revised" License
|
|
//
|
|
|
|
/* Modifications Copyright (c) Microsoft. */
|
|
|
|
#pragma once
|
|
#include "core/providers/cuda/cuda_common.h"
|
|
|
|
namespace onnxruntime {
|
|
namespace contrib {
|
|
namespace cuda {
|
|
|
|
template <typename T, typename U, bool simplified>
|
|
void HostApplyLayerNorm(
|
|
const cudaDeviceProp& prop,
|
|
T* output,
|
|
U* mean,
|
|
U* invvar,
|
|
const T* input,
|
|
int64_t n1,
|
|
int64_t n2,
|
|
double epsilon,
|
|
const T* gamma,
|
|
const T* beta);
|
|
|
|
} // namespace cuda
|
|
} // namespace contrib
|
|
} // namespace onnxruntime
|