From 2e566206110d2fc2a96e45a785d94b37bac9daf2 Mon Sep 17 00:00:00 2001 From: Tianlei Wu Date: Mon, 27 Mar 2023 11:04:19 -0700 Subject: [PATCH] Add file and line info in CudaCall and RocmCall macros (#15148) This PR add file and line information so that it is easy to trouble shoot the issue of cuda error. Update Rocm call as well for hipify. --- onnxruntime/core/providers/cuda/cuda_call.cc | 30 ++++++++-------- .../providers/cuda/cuda_provider_factory.cc | 4 +-- .../providers/cuda/cuda_provider_factory.h | 7 ++-- .../providers/cuda/shared_inc/cuda_call.h | 32 ++++++++--------- .../core/providers/migraphx/migraphx_call.cc | 18 +++++----- .../core/providers/migraphx/migraphx_call.h | 8 ++--- onnxruntime/core/providers/rocm/rocm_call.cc | 30 ++++++++-------- .../providers/rocm/rocm_provider_factory.cc | 4 +-- .../providers/rocm/rocm_provider_factory.h | 4 +-- .../providers/rocm/shared_inc/rocm_call.h | 34 +++++++++---------- .../shared_library/provider_interfaces.h | 8 ++--- .../tensorrt/tensorrt_execution_provider.cc | 30 ++++++++-------- .../core/session/provider_bridge_ort.cc | 10 +++--- 13 files changed, 108 insertions(+), 111 deletions(-) diff --git a/onnxruntime/core/providers/cuda/cuda_call.cc b/onnxruntime/core/providers/cuda/cuda_call.cc index e6546765d1..4f223041e0 100644 --- a/onnxruntime/core/providers/cuda/cuda_call.cc +++ b/onnxruntime/core/providers/cuda/cuda_call.cc @@ -87,7 +87,7 @@ const char* CudaErrString(ncclResult_t e) { template std::conditional_t CudaCall( - ERRTYPE retCode, const char* exprString, const char* libName, ERRTYPE successCode, const char* msg) { + ERRTYPE retCode, const char* exprString, const char* libName, ERRTYPE successCode, const char* msg, const char* file, const int line) { if (retCode != successCode) { try { #ifdef _WIN32 @@ -105,10 +105,10 @@ std::conditional_t CudaCall( cudaGetDevice(¤tCudaDevice); cudaGetLastError(); // clear last CUDA error static char str[1024]; - snprintf(str, 1024, "%s failure %d: %s ; GPU=%d ; hostname=%s ; expr=%s; %s", + snprintf(str, 1024, "%s failure %d: %s ; GPU=%d ; hostname=%s ; file=%s ; line=%d ; expr=%s; %s", libName, (int)retCode, CudaErrString(retCode), currentCudaDevice, hostname, - exprString, msg); + file, line, exprString, msg); if constexpr (THRW) { // throw an exception with the error info ORT_THROW(str); @@ -130,19 +130,19 @@ std::conditional_t CudaCall( } } -template Status CudaCall(cudaError retCode, const char* exprString, const char* libName, cudaError successCode, const char* msg); -template void CudaCall(cudaError retCode, const char* exprString, const char* libName, cudaError successCode, const char* msg); -template Status CudaCall(cublasStatus_t retCode, const char* exprString, const char* libName, cublasStatus_t successCode, const char* msg); -template void CudaCall(cublasStatus_t retCode, const char* exprString, const char* libName, cublasStatus_t successCode, const char* msg); -template Status CudaCall(cudnnStatus_t retCode, const char* exprString, const char* libName, cudnnStatus_t successCode, const char* msg); -template void CudaCall(cudnnStatus_t retCode, const char* exprString, const char* libName, cudnnStatus_t successCode, const char* msg); -template Status CudaCall(curandStatus_t retCode, const char* exprString, const char* libName, curandStatus_t successCode, const char* msg); -template void CudaCall(curandStatus_t retCode, const char* exprString, const char* libName, curandStatus_t successCode, const char* msg); -template Status CudaCall(cufftResult retCode, const char* exprString, const char* libName, cufftResult successCode, const char* msg); -template void CudaCall(cufftResult retCode, const char* exprString, const char* libName, cufftResult successCode, const char* msg); +template Status CudaCall(cudaError retCode, const char* exprString, const char* libName, cudaError successCode, const char* msg, const char* file, const int line); +template void CudaCall(cudaError retCode, const char* exprString, const char* libName, cudaError successCode, const char* msg, const char* file, const int line); +template Status CudaCall(cublasStatus_t retCode, const char* exprString, const char* libName, cublasStatus_t successCode, const char* msg, const char* file, const int line); +template void CudaCall(cublasStatus_t retCode, const char* exprString, const char* libName, cublasStatus_t successCode, const char* msg, const char* file, const int line); +template Status CudaCall(cudnnStatus_t retCode, const char* exprString, const char* libName, cudnnStatus_t successCode, const char* msg, const char* file, const int line); +template void CudaCall(cudnnStatus_t retCode, const char* exprString, const char* libName, cudnnStatus_t successCode, const char* msg, const char* file, const int line); +template Status CudaCall(curandStatus_t retCode, const char* exprString, const char* libName, curandStatus_t successCode, const char* msg, const char* file, const int line); +template void CudaCall(curandStatus_t retCode, const char* exprString, const char* libName, curandStatus_t successCode, const char* msg, const char* file, const int line); +template Status CudaCall(cufftResult retCode, const char* exprString, const char* libName, cufftResult successCode, const char* msg, const char* file, const int line); +template void CudaCall(cufftResult retCode, const char* exprString, const char* libName, cufftResult successCode, const char* msg, const char* file, const int line); #ifdef ORT_USE_NCCL -template Status CudaCall(ncclResult_t retCode, const char* exprString, const char* libName, ncclResult_t successCode, const char* msg); -template void CudaCall(ncclResult_t retCode, const char* exprString, const char* libName, ncclResult_t successCode, const char* msg); +template Status CudaCall(ncclResult_t retCode, const char* exprString, const char* libName, ncclResult_t successCode, const char* msg, const char* file, const int line); +template void CudaCall(ncclResult_t retCode, const char* exprString, const char* libName, ncclResult_t successCode, const char* msg, const char* file, const int line); #endif } // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/cuda_provider_factory.cc b/onnxruntime/core/providers/cuda/cuda_provider_factory.cc index d059f93ceb..647fb98455 100644 --- a/onnxruntime/core/providers/cuda/cuda_provider_factory.cc +++ b/onnxruntime/core/providers/cuda/cuda_provider_factory.cc @@ -109,8 +109,8 @@ struct ProviderInfo_CUDA_Impl : ProviderInfo_CUDA { return cuda::Impl_Cast(static_cast(stream), input_data, output_data, count); } - Status CudaCall_false(int retCode, const char* exprString, const char* libName, int successCode, const char* msg) override { return CudaCall(cudaError(retCode), exprString, libName, cudaError(successCode), msg); } - void CudaCall_true(int retCode, const char* exprString, const char* libName, int successCode, const char* msg) override { CudaCall(cudaError(retCode), exprString, libName, cudaError(successCode), msg); } + Status CudaCall_false(int retCode, const char* exprString, const char* libName, int successCode, const char* msg, const char* file, const int line) override { return CudaCall(cudaError(retCode), exprString, libName, cudaError(successCode), msg, file, line); } + void CudaCall_true(int retCode, const char* exprString, const char* libName, int successCode, const char* msg, const char* file, const int line) override { CudaCall(cudaError(retCode), exprString, libName, cudaError(successCode), msg, file, line); } void CopyGpuToCpu(void* dst_ptr, const void* src_ptr, const size_t size, const OrtMemoryInfo& dst_location, const OrtMemoryInfo& src_location) override { ORT_ENFORCE(dst_location.device.Type() == OrtDevice::CPU); diff --git a/onnxruntime/core/providers/cuda/cuda_provider_factory.h b/onnxruntime/core/providers/cuda/cuda_provider_factory.h index 28c9c169e0..47c9768625 100644 --- a/onnxruntime/core/providers/cuda/cuda_provider_factory.h +++ b/onnxruntime/core/providers/cuda/cuda_provider_factory.h @@ -20,7 +20,6 @@ class NvtxRangeCreator; } struct ProviderInfo_CUDA { - virtual OrtStatus* SetCurrentGpuDeviceId(_In_ int device_id) = 0; virtual OrtStatus* GetCurrentGpuDeviceId(_In_ int* device_id) = 0; @@ -33,8 +32,8 @@ struct ProviderInfo_CUDA { virtual void cuda__Impl_Cast(void* stream, const double* input_data, float* output_data, size_t count) = 0; virtual void cuda__Impl_Cast(void* stream, const float* input_data, double* output_data, size_t count) = 0; - virtual Status CudaCall_false(int retCode, const char* exprString, const char* libName, int successCode, const char* msg) = 0; - virtual void CudaCall_true(int retCode, const char* exprString, const char* libName, int successCode, const char* msg) = 0; + virtual Status CudaCall_false(int retCode, const char* exprString, const char* libName, int successCode, const char* msg, const char* file, const int line) = 0; + virtual void CudaCall_true(int retCode, const char* exprString, const char* libName, int successCode, const char* msg, const char* file, const int line) = 0; virtual void CopyGpuToCpu(void* dst_ptr, const void* src_ptr, const size_t size, const OrtMemoryInfo& dst_location, const OrtMemoryInfo& src_location) = 0; virtual void cudaMemcpy_HostToDevice(void* dst, const void* src, size_t count) = 0; @@ -60,7 +59,7 @@ struct ProviderInfo_CUDA { virtual bool TestAll() = 0; #endif -protected: + protected: ~ProviderInfo_CUDA() = default; // Can only be destroyed through a subclass instance }; diff --git a/onnxruntime/core/providers/cuda/shared_inc/cuda_call.h b/onnxruntime/core/providers/cuda/shared_inc/cuda_call.h index 4e76c083e0..51a5631b93 100644 --- a/onnxruntime/core/providers/cuda/shared_inc/cuda_call.h +++ b/onnxruntime/core/providers/cuda/shared_inc/cuda_call.h @@ -13,32 +13,32 @@ namespace onnxruntime { template std::conditional_t CudaCall( - ERRTYPE retCode, const char* exprString, const char* libName, ERRTYPE successCode, const char* msg = ""); + ERRTYPE retCode, const char* exprString, const char* libName, ERRTYPE successCode, const char* msg, const char* file, const int line); -#define CUDA_CALL(expr) (CudaCall((expr), #expr, "CUDA", cudaSuccess)) -#define CUBLAS_CALL(expr) (CudaCall((expr), #expr, "CUBLAS", CUBLAS_STATUS_SUCCESS)) +#define CUDA_CALL(expr) (CudaCall((expr), #expr, "CUDA", cudaSuccess, "", __FILE__, __LINE__)) +#define CUBLAS_CALL(expr) (CudaCall((expr), #expr, "CUBLAS", CUBLAS_STATUS_SUCCESS, "", __FILE__, __LINE__)) -#define CUSPARSE_CALL(expr) (CudaCall((expr), #expr, "CUSPARSE", CUSPARSE_STATUS_SUCCESS)) -#define CURAND_CALL(expr) (CudaCall((expr), #expr, "CURAND", CURAND_STATUS_SUCCESS)) -#define CUDNN_CALL(expr) (CudaCall((expr), #expr, "CUDNN", CUDNN_STATUS_SUCCESS)) -#define CUDNN_CALL2(expr, m) (CudaCall((expr), #expr, "CUDNN", CUDNN_STATUS_SUCCESS, m)) +#define CUSPARSE_CALL(expr) (CudaCall((expr), #expr, "CUSPARSE", CUSPARSE_STATUS_SUCCESS, "", __FILE__, __LINE__)) +#define CURAND_CALL(expr) (CudaCall((expr), #expr, "CURAND", CURAND_STATUS_SUCCESS, "", __FILE__, __LINE__)) +#define CUDNN_CALL(expr) (CudaCall((expr), #expr, "CUDNN", CUDNN_STATUS_SUCCESS, "", __FILE__, __LINE__)) +#define CUDNN_CALL2(expr, m) (CudaCall((expr), #expr, "CUDNN", CUDNN_STATUS_SUCCESS, m, __FILE__, __LINE__)) -#define CUFFT_CALL(expr) (CudaCall((expr), #expr, "CUFFT", CUFFT_SUCCESS)) +#define CUFFT_CALL(expr) (CudaCall((expr), #expr, "CUFFT", CUFFT_SUCCESS, "", __FILE__, __LINE__)) -#define CUDA_CALL_THROW(expr) (CudaCall((expr), #expr, "CUDA", cudaSuccess)) -#define CUBLAS_CALL_THROW(expr) (CudaCall((expr), #expr, "CUBLAS", CUBLAS_STATUS_SUCCESS)) +#define CUDA_CALL_THROW(expr) (CudaCall((expr), #expr, "CUDA", cudaSuccess, "", __FILE__, __LINE__)) +#define CUBLAS_CALL_THROW(expr) (CudaCall((expr), #expr, "CUBLAS", CUBLAS_STATUS_SUCCESS, "", __FILE__, __LINE__)) -#define CUSPARSE_CALL_THROW(expr) (CudaCall((expr), #expr, "CUSPARSE", CUSPARSE_STATUS_SUCCESS)) -#define CURAND_CALL_THROW(expr) (CudaCall((expr), #expr, "CURAND", CURAND_STATUS_SUCCESS)) +#define CUSPARSE_CALL_THROW(expr) (CudaCall((expr), #expr, "CUSPARSE", CUSPARSE_STATUS_SUCCESS, "", __FILE__, __LINE__)) +#define CURAND_CALL_THROW(expr) (CudaCall((expr), #expr, "CURAND", CURAND_STATUS_SUCCESS, "", __FILE__, __LINE__)) // the cudnn configuration call that doesn't need set stream -#define CUDNN_CALL_THROW(expr) (CudaCall((expr), #expr, "CUDNN", CUDNN_STATUS_SUCCESS)) +#define CUDNN_CALL_THROW(expr) (CudaCall((expr), #expr, "CUDNN", CUDNN_STATUS_SUCCESS, "", __FILE__, __LINE__)) -#define CUFFT_CALL_THROW(expr) (CudaCall((expr), #expr, "CUFFT", CUFFT_SUCCESS)) +#define CUFFT_CALL_THROW(expr) (CudaCall((expr), #expr, "CUFFT", CUFFT_SUCCESS, "", __FILE__, __LINE__)) #ifdef ORT_USE_NCCL -#define NCCL_CALL(expr) (CudaCall((expr), #expr, "NCCL", ncclSuccess)) -#define NCCL_CALL_THROW(expr) (CudaCall((expr), #expr, "NCCL", ncclSuccess)) +#define NCCL_CALL(expr) (CudaCall((expr), #expr, "NCCL", ncclSuccess, "", __FILE__, __LINE__)) +#define NCCL_CALL_THROW(expr) (CudaCall((expr), #expr, "NCCL", ncclSuccess, "", __FILE__, __LINE__)) #endif } // namespace onnxruntime diff --git a/onnxruntime/core/providers/migraphx/migraphx_call.cc b/onnxruntime/core/providers/migraphx/migraphx_call.cc index b3d976f361..cd947420b7 100644 --- a/onnxruntime/core/providers/migraphx/migraphx_call.cc +++ b/onnxruntime/core/providers/migraphx/migraphx_call.cc @@ -31,7 +31,7 @@ const char* RocmErrString(hipError_t x) { template std::conditional_t RocmCall( - ERRTYPE retCode, const char* exprString, const char* libName, ERRTYPE successCode, const char* msg) { + ERRTYPE retCode, const char* exprString, const char* libName, ERRTYPE successCode, const char* msg, const char* file, const int line) { if (retCode != successCode) { try { char hostname[HOST_NAME_MAX]; @@ -41,10 +41,10 @@ std::conditional_t RocmCall( (void)hipGetDevice(¤tHipDevice); (void)hipGetLastError(); // clear last HIP error static char str[1024]; - snprintf(str, 1024, "%s failure %d: %s ; GPU=%d ; hostname=%s ; expr=%s; %s", + snprintf(str, 1024, "%s failure %d: %s ; GPU=%d ; hostname=%s ; file=%s ; line=%d ; expr=%s; %s", libName, (int)retCode, RocmErrString(retCode), currentHipDevice, hostname, - exprString, msg); + file, line, exprString, msg); if constexpr (THRW) { // throw an exception with the error info ORT_THROW(str); @@ -66,11 +66,11 @@ std::conditional_t RocmCall( } } -template Status RocmCall(hipError_t retCode, const char* exprString, const char* libName, hipError_t successCode, const char* msg); -template void RocmCall(hipError_t retCode, const char* exprString, const char* libName, hipError_t successCode, const char* msg); -template Status RocmCall(rocblas_status retCode, const char* exprString, const char* libName, rocblas_status successCode, const char* msg); -template void RocmCall(rocblas_status retCode, const char* exprString, const char* libName, rocblas_status successCode, const char* msg); -template Status RocmCall(miopenStatus_t retCode, const char* exprString, const char* libName, miopenStatus_t successCode, const char* msg); -template void RocmCall(miopenStatus_t retCode, const char* exprString, const char* libName, miopenStatus_t successCode, const char* msg); +template Status RocmCall(hipError_t retCode, const char* exprString, const char* libName, hipError_t successCode, const char* msg, const char* file, const int line); +template void RocmCall(hipError_t retCode, const char* exprString, const char* libName, hipError_t successCode, const char* msg, const char* file, const int line); +template Status RocmCall(rocblas_status retCode, const char* exprString, const char* libName, rocblas_status successCode, const char* msg, const char* file, const int line); +template void RocmCall(rocblas_status retCode, const char* exprString, const char* libName, rocblas_status successCode, const char* msg, const char* file, const int line); +template Status RocmCall(miopenStatus_t retCode, const char* exprString, const char* libName, miopenStatus_t successCode, const char* msg, const char* file, const int line); +template void RocmCall(miopenStatus_t retCode, const char* exprString, const char* libName, miopenStatus_t successCode, const char* msg, const char* file, const int line); } // namespace onnxruntime diff --git a/onnxruntime/core/providers/migraphx/migraphx_call.h b/onnxruntime/core/providers/migraphx/migraphx_call.h index bbcfc4c3c9..15d385a636 100644 --- a/onnxruntime/core/providers/migraphx/migraphx_call.h +++ b/onnxruntime/core/providers/migraphx/migraphx_call.h @@ -14,9 +14,9 @@ namespace onnxruntime { template std::conditional_t RocmCall( - ERRTYPE retCode, const char* exprString, const char* libName, ERRTYPE successCode, const char* msg = ""); + ERRTYPE retCode, const char* exprString, const char* libName, ERRTYPE successCode, const char* msg, const char* file, const int line); -#define HIP_CALL(expr) (RocmCall((expr), #expr, "HIP", hipSuccess)) -#define HIP_CALL_THROW(expr) (RocmCall((expr), #expr, "HIP", hipSuccess)) +#define HIP_CALL(expr) (RocmCall((expr), #expr, "HIP", hipSuccess, "", __FILE__, __LINE__)) +#define HIP_CALL_THROW(expr) (RocmCall((expr), #expr, "HIP", hipSuccess, "", __FILE__, __LINE__)) -} +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/rocm/rocm_call.cc b/onnxruntime/core/providers/rocm/rocm_call.cc index c0e572b344..f6dbfbffb1 100644 --- a/onnxruntime/core/providers/rocm/rocm_call.cc +++ b/onnxruntime/core/providers/rocm/rocm_call.cc @@ -90,7 +90,7 @@ const char* RocmErrString(ncclResult_t e) { template std::conditional_t RocmCall( - ERRTYPE retCode, const char* exprString, const char* libName, ERRTYPE successCode, const char* msg) { + ERRTYPE retCode, const char* exprString, const char* libName, ERRTYPE successCode, const char* msg, const char* file, const int line) { if (retCode != successCode) { try { #ifdef _WIN32 @@ -108,10 +108,10 @@ std::conditional_t RocmCall( ORT_IGNORE_RETURN_VALUE(hipGetDevice(¤tHipDevice)); // void to silence nodiscard ORT_IGNORE_RETURN_VALUE(hipGetLastError()); // clear last ROCM error; void to silence nodiscard static char str[1024]; - snprintf(str, 1024, "%s failure %d: %s ; GPU=%d ; hostname=%s ; expr=%s; %s", + snprintf(str, 1024, "%s failure %d: %s ; GPU=%d ; hostname=%s ; file=%s ; line=%d ; expr=%s; %s", libName, (int)retCode, RocmErrString(retCode), currentHipDevice, hostname, - exprString, msg); + file, line, exprString, msg); if constexpr (THRW) { // throw an exception with the error info ORT_THROW(str); @@ -133,19 +133,19 @@ std::conditional_t RocmCall( } } -template Status RocmCall(hipError_t retCode, const char* exprString, const char* libName, hipError_t successCode, const char* msg); -template void RocmCall(hipError_t retCode, const char* exprString, const char* libName, hipError_t successCode, const char* msg); -template Status RocmCall(rocblas_status retCode, const char* exprString, const char* libName, rocblas_status successCode, const char* msg); -template void RocmCall(rocblas_status retCode, const char* exprString, const char* libName, rocblas_status successCode, const char* msg); -template Status RocmCall(miopenStatus_t retCode, const char* exprString, const char* libName, miopenStatus_t successCode, const char* msg); -template void RocmCall(miopenStatus_t retCode, const char* exprString, const char* libName, miopenStatus_t successCode, const char* msg); -template Status RocmCall(hiprandStatus_t retCode, const char* exprString, const char* libName, hiprandStatus_t successCode, const char* msg); -template void RocmCall(hiprandStatus_t retCode, const char* exprString, const char* libName, hiprandStatus_t successCode, const char* msg); -template Status RocmCall(hipfftResult retCode, const char* exprString, const char* libName, hipfftResult successCode, const char* msg); -template void RocmCall(hipfftResult retCode, const char* exprString, const char* libName, hipfftResult successCode, const char* msg); +template Status RocmCall(hipError_t retCode, const char* exprString, const char* libName, hipError_t successCode, const char* msg, const char* file, const int line); +template void RocmCall(hipError_t retCode, const char* exprString, const char* libName, hipError_t successCode, const char* msg, const char* file, const int line); +template Status RocmCall(rocblas_status retCode, const char* exprString, const char* libName, rocblas_status successCode, const char* msg, const char* file, const int line); +template void RocmCall(rocblas_status retCode, const char* exprString, const char* libName, rocblas_status successCode, const char* msg, const char* file, const int line); +template Status RocmCall(miopenStatus_t retCode, const char* exprString, const char* libName, miopenStatus_t successCode, const char* msg, const char* file, const int line); +template void RocmCall(miopenStatus_t retCode, const char* exprString, const char* libName, miopenStatus_t successCode, const char* msg, const char* file, const int line); +template Status RocmCall(hiprandStatus_t retCode, const char* exprString, const char* libName, hiprandStatus_t successCode, const char* msg, const char* file, const int line); +template void RocmCall(hiprandStatus_t retCode, const char* exprString, const char* libName, hiprandStatus_t successCode, const char* msg, const char* file, const int line); +template Status RocmCall(hipfftResult retCode, const char* exprString, const char* libName, hipfftResult successCode, const char* msg, const char* file, const int line); +template void RocmCall(hipfftResult retCode, const char* exprString, const char* libName, hipfftResult successCode, const char* msg, const char* file, const int line); #ifdef ORT_USE_NCCL -template Status RocmCall(ncclResult_t retCode, const char* exprString, const char* libName, ncclResult_t successCode, const char* msg); -template void RocmCall(ncclResult_t retCode, const char* exprString, const char* libName, ncclResult_t successCode, const char* msg); +template Status RocmCall(ncclResult_t retCode, const char* exprString, const char* libName, ncclResult_t successCode, const char* msg, const char* file, const int line); +template void RocmCall(ncclResult_t retCode, const char* exprString, const char* libName, ncclResult_t successCode, const char* msg, const char* file, const int line); #endif } // namespace onnxruntime diff --git a/onnxruntime/core/providers/rocm/rocm_provider_factory.cc b/onnxruntime/core/providers/rocm/rocm_provider_factory.cc index e0a8152e5e..366e3124c7 100644 --- a/onnxruntime/core/providers/rocm/rocm_provider_factory.cc +++ b/onnxruntime/core/providers/rocm/rocm_provider_factory.cc @@ -104,8 +104,8 @@ struct ProviderInfo_ROCM_Impl : ProviderInfo_ROCM { return rocm::Impl_Cast(static_cast(stream), input_data, output_data, count); } - Status RocmCall_false(int retCode, const char* exprString, const char* libName, int successCode, const char* msg) override { return RocmCall(hipError_t(retCode), exprString, libName, hipError_t(successCode), msg); } - void RocmCall_true(int retCode, const char* exprString, const char* libName, int successCode, const char* msg) override { RocmCall(hipError_t(retCode), exprString, libName, hipError_t(successCode), msg); } + Status RocmCall_false(int retCode, const char* exprString, const char* libName, int successCode, const char* msg, const char* file, const int line) override { return RocmCall(hipError_t(retCode), exprString, libName, hipError_t(successCode), msg, file, line); } + void RocmCall_true(int retCode, const char* exprString, const char* libName, int successCode, const char* msg, const char* file, const int line) override { RocmCall(hipError_t(retCode), exprString, libName, hipError_t(successCode), msg, file, line); } void CopyGpuToCpu(void* dst_ptr, const void* src_ptr, const size_t size, const OrtMemoryInfo& dst_location, const OrtMemoryInfo& src_location) override { ORT_ENFORCE(dst_location.device.Type() == OrtDevice::CPU); diff --git a/onnxruntime/core/providers/rocm/rocm_provider_factory.h b/onnxruntime/core/providers/rocm/rocm_provider_factory.h index af2b673908..90306138ec 100644 --- a/onnxruntime/core/providers/rocm/rocm_provider_factory.h +++ b/onnxruntime/core/providers/rocm/rocm_provider_factory.h @@ -29,8 +29,8 @@ struct ProviderInfo_ROCM { virtual void rocm__Impl_Cast(void* stream, const double* input_data, float* output_data, size_t count) = 0; virtual void rocm__Impl_Cast(void* stream, const float* input_data, double* output_data, size_t count) = 0; - virtual Status RocmCall_false(int retCode, const char* exprString, const char* libName, int successCode, const char* msg) = 0; - virtual void RocmCall_true(int retCode, const char* exprString, const char* libName, int successCode, const char* msg) = 0; + virtual Status RocmCall_false(int retCode, const char* exprString, const char* libName, int successCode, const char* msg, const char* file, const int line) = 0; + virtual void RocmCall_true(int retCode, const char* exprString, const char* libName, int successCode, const char* msg, const char* file, const int line) = 0; virtual void CopyGpuToCpu(void* dst_ptr, const void* src_ptr, const size_t size, const OrtMemoryInfo& dst_location, const OrtMemoryInfo& src_location) = 0; virtual void rocmMemcpy_HostToDevice(void* dst, const void* src, size_t count) = 0; diff --git a/onnxruntime/core/providers/rocm/shared_inc/rocm_call.h b/onnxruntime/core/providers/rocm/shared_inc/rocm_call.h index 9226d83b89..10ce83a614 100644 --- a/onnxruntime/core/providers/rocm/shared_inc/rocm_call.h +++ b/onnxruntime/core/providers/rocm/shared_inc/rocm_call.h @@ -13,27 +13,27 @@ namespace onnxruntime { template std::conditional_t RocmCall( - ERRTYPE retCode, const char* exprString, const char* libName, ERRTYPE successCode, const char* msg = ""); + ERRTYPE retCode, const char* exprString, const char* libName, ERRTYPE successCode, const char* msg, const char* file, const int line); -#define HIP_CALL(expr) (RocmCall((expr), #expr, "HIP", hipSuccess)) -#define ROCBLAS_CALL(expr) (RocmCall((expr), #expr, "ROCBLAS", rocblas_status_success)) -#define HIPSPARSE_CALL(expr) (RocmCall((expr), #expr, "HIPSPARSE", HIPSPARSE_STATUS_SUCCESS)) -#define HIPRAND_CALL(expr) (RocmCall((expr), #expr, "HIPRAND", HIPRAND_STATUS_SUCCESS)) -#define MIOPEN_CALL(expr) (RocmCall((expr), #expr, "MIOPEN", miopenStatusSuccess)) -#define MIOPEN_CALL2(expr, m) (RocmCall((expr), #expr, "MIOPEN", miopenStatusSuccess, m)) -#define HIPFFT_CALL(expr) (RocmCall((expr), #expr, "HIPFFT", HIPFFT_SUCCESS)) +#define HIP_CALL(expr) (RocmCall((expr), #expr, "HIP", hipSuccess, "", __FILE__, __LINE__)) +#define ROCBLAS_CALL(expr) (RocmCall((expr), #expr, "ROCBLAS", rocblas_status_success, "", __FILE__, __LINE__)) +#define HIPSPARSE_CALL(expr) (RocmCall((expr), #expr, "HIPSPARSE", HIPSPARSE_STATUS_SUCCESS, "", __FILE__, __LINE__)) +#define HIPRAND_CALL(expr) (RocmCall((expr), #expr, "HIPRAND", HIPRAND_STATUS_SUCCESS, "", __FILE__, __LINE__)) +#define MIOPEN_CALL(expr) (RocmCall((expr), #expr, "MIOPEN", miopenStatusSuccess, "", __FILE__, __LINE__)) +#define MIOPEN_CALL2(expr, m) (RocmCall((expr), #expr, "MIOPEN", miopenStatusSuccess, m, __FILE__, __LINE__)) +#define HIPFFT_CALL(expr) (RocmCall((expr), #expr, "HIPFFT", HIPFFT_SUCCESS, "", __FILE__, __LINE__)) -#define HIP_CALL_THROW(expr) (RocmCall((expr), #expr, "HIP", hipSuccess)) -#define ROCBLAS_CALL_THROW(expr) (RocmCall((expr), #expr, "ROCBLAS", rocblas_status_success)) -#define HIPSPARSE_CALL_THROW(expr) (RocmCall((expr), #expr, "HIPSPARSE", HIPSPARSE_STATUS_SUCCESS)) -#define HIPRAND_CALL_THROW(expr) (RocmCall((expr), #expr, "HIPRAND", HIPRAND_STATUS_SUCCESS)) -#define MIOPEN_CALL_THROW(expr) (RocmCall((expr), #expr, "MIOPEN", miopenStatusSuccess)) -#define MIOPEN_CALL_THROW2(expr, m) (RocmCall((expr), #expr, "MIOPEN", miopenStatusSuccess, m)) -#define HIPFFT_CALL_THROW(expr) (RocmCall((expr), #expr, "HIPFFT", HIPFFT_SUCCESS)) +#define HIP_CALL_THROW(expr) (RocmCall((expr), #expr, "HIP", hipSuccess, "", __FILE__, __LINE__)) +#define ROCBLAS_CALL_THROW(expr) (RocmCall((expr), #expr, "ROCBLAS", rocblas_status_success, "", __FILE__, __LINE__)) +#define HIPSPARSE_CALL_THROW(expr) (RocmCall((expr), #expr, "HIPSPARSE", HIPSPARSE_STATUS_SUCCESS, "", __FILE__, __LINE__)) +#define HIPRAND_CALL_THROW(expr) (RocmCall((expr), #expr, "HIPRAND", HIPRAND_STATUS_SUCCESS, "", __FILE__, __LINE__)) +#define MIOPEN_CALL_THROW(expr) (RocmCall((expr), #expr, "MIOPEN", miopenStatusSuccess, "", __FILE__, __LINE__)) +#define MIOPEN_CALL_THROW2(expr, m) (RocmCall((expr), #expr, "MIOPEN", miopenStatusSuccess, m, __FILE__, __LINE__)) +#define HIPFFT_CALL_THROW(expr) (RocmCall((expr), #expr, "HIPFFT", HIPFFT_SUCCESS, "", __FILE__, __LINE__)) #ifdef ORT_USE_NCCL -#define NCCL_CALL(expr) (RocmCall((expr), #expr, "NCCL", ncclSuccess)) -#define NCCL_CALL_THROW(expr) (RocmCall((expr), #expr, "NCCL", ncclSuccess)) +#define NCCL_CALL(expr) (RocmCall((expr), #expr, "NCCL", ncclSuccess, "", __FILE__, __LINE__)) +#define NCCL_CALL_THROW(expr) (RocmCall((expr), #expr, "NCCL", ncclSuccess, "", __FILE__, __LINE__)) #endif } // namespace onnxruntime diff --git a/onnxruntime/core/providers/shared_library/provider_interfaces.h b/onnxruntime/core/providers/shared_library/provider_interfaces.h index a6b0130af0..2df766360f 100644 --- a/onnxruntime/core/providers/shared_library/provider_interfaces.h +++ b/onnxruntime/core/providers/shared_library/provider_interfaces.h @@ -165,8 +165,8 @@ struct ProviderHost { virtual void cuda__Impl_Cast(void* stream, const double* input_data, float* output_data, size_t count) = 0; virtual void cuda__Impl_Cast(void* stream, const float* input_data, double* output_data, size_t count) = 0; - virtual Status CudaCall_false(int retCode, const char* exprString, const char* libName, int successCode, const char* msg) = 0; - virtual void CudaCall_true(int retCode, const char* exprString, const char* libName, int successCode, const char* msg) = 0; + virtual Status CudaCall_false(int retCode, const char* exprString, const char* libName, int successCode, const char* msg, const char* file, const int line) = 0; + virtual void CudaCall_true(int retCode, const char* exprString, const char* libName, int successCode, const char* msg, const char* file, const int line) = 0; #endif #ifdef USE_ROCM @@ -179,8 +179,8 @@ struct ProviderHost { virtual void rocm__Impl_Cast(void* stream, const double* input_data, float* output_data, size_t count) = 0; virtual void rocm__Impl_Cast(void* stream, const float* input_data, double* output_data, size_t count) = 0; - virtual Status RocmCall_false(int retCode, const char* exprString, const char* libName, int successCode, const char* msg) = 0; - virtual void RocmCall_true(int retCode, const char* exprString, const char* libName, int successCode, const char* msg) = 0; + virtual Status RocmCall_false(int retCode, const char* exprString, const char* libName, int successCode, const char* msg, const char* file, const int line) = 0; + virtual void RocmCall_true(int retCode, const char* exprString, const char* libName, int successCode, const char* msg, const char* file, const int line) = 0; #endif virtual std::unordered_set GetCpuPreferredNodes(const onnxruntime::GraphViewer& graph, diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc index 10c67963b8..6502719eaa 100644 --- a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc +++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc @@ -194,33 +194,33 @@ void Impl_Cast( } // namespace cuda template <> -Status CudaCall(cudaError retCode, const char* exprString, const char* libName, cudaError successCode, const char* msg) { - return g_host->CudaCall_false(retCode, exprString, libName, successCode, msg); +Status CudaCall(cudaError retCode, const char* exprString, const char* libName, cudaError successCode, const char* msg, const char* file, const int line) { + return g_host->CudaCall_false(retCode, exprString, libName, successCode, msg, file, line); } template <> -void CudaCall(cudaError retCode, const char* exprString, const char* libName, cudaError successCode, const char* msg) { - return g_host->CudaCall_true(retCode, exprString, libName, successCode, msg); +void CudaCall(cudaError retCode, const char* exprString, const char* libName, cudaError successCode, const char* msg, const char* file, const int line) { + return g_host->CudaCall_true(retCode, exprString, libName, successCode, msg, file, line); } template <> -Status CudaCall(cublasStatus_t retCode, const char* exprString, const char* libName, cublasStatus_t successCode, const char* msg) { - return g_host->CudaCall_false(retCode, exprString, libName, successCode, msg); +Status CudaCall(cublasStatus_t retCode, const char* exprString, const char* libName, cublasStatus_t successCode, const char* msg, const char* file, const int line) { + return g_host->CudaCall_false(retCode, exprString, libName, successCode, msg, file, line); } template <> -void CudaCall(cublasStatus_t retCode, const char* exprString, const char* libName, cublasStatus_t successCode, const char* msg) { - return g_host->CudaCall_true(retCode, exprString, libName, successCode, msg); +void CudaCall(cublasStatus_t retCode, const char* exprString, const char* libName, cublasStatus_t successCode, const char* msg, const char* file, const int line) { + return g_host->CudaCall_true(retCode, exprString, libName, successCode, msg, file, line); } template <> -Status CudaCall(cudnnStatus_t retCode, const char* exprString, const char* libName, cudnnStatus_t successCode, const char* msg) { - return g_host->CudaCall_false(retCode, exprString, libName, successCode, msg); +Status CudaCall(cudnnStatus_t retCode, const char* exprString, const char* libName, cudnnStatus_t successCode, const char* msg, const char* file, const int line) { + return g_host->CudaCall_false(retCode, exprString, libName, successCode, msg, file, line); } template <> -void CudaCall(cudnnStatus_t retCode, const char* exprString, const char* libName, cudnnStatus_t successCode, const char* msg) { - return g_host->CudaCall_true(retCode, exprString, libName, successCode, msg); +void CudaCall(cudnnStatus_t retCode, const char* exprString, const char* libName, cudnnStatus_t successCode, const char* msg, const char* file, const int line) { + return g_host->CudaCall_true(retCode, exprString, libName, successCode, msg, file, line); } class Memcpy final : public OpKernel { @@ -1496,8 +1496,7 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector(engine_build_stop - engine_build_start).count() << "ms" << std::endl; + LOGS_DEFAULT(INFO) << "TensorRT engine build for " << trt_node_name_with_precision << " took: " << std::chrono::duration_cast(engine_build_stop - engine_build_start).count() << "ms" << std::endl; } if (engine_cache_enable_) { std::unique_ptr serializedModel(trt_engine->serialize()); @@ -1899,8 +1898,7 @@ common::Status TensorrtExecutionProvider::Compile(const std::vectorbuildEngineWithConfig(*trt_state->network->get(), *trt_config)); if (detailed_build_log_) { auto engine_build_stop = std::chrono::steady_clock::now(); - LOGS_DEFAULT(INFO) << "TensorRT engine build for " << trt_state->trt_node_name_with_precision << " took: " << - std::chrono::duration_cast(engine_build_stop - engine_build_start).count() << "ms" << std::endl; + LOGS_DEFAULT(INFO) << "TensorRT engine build for " << trt_state->trt_node_name_with_precision << " took: " << std::chrono::duration_cast(engine_build_stop - engine_build_start).count() << "ms" << std::endl; } } if (trt_state->engine == nullptr) { diff --git a/onnxruntime/core/session/provider_bridge_ort.cc b/onnxruntime/core/session/provider_bridge_ort.cc index 8e70dd24ac..8c266e34d0 100644 --- a/onnxruntime/core/session/provider_bridge_ort.cc +++ b/onnxruntime/core/session/provider_bridge_ort.cc @@ -205,8 +205,8 @@ struct ProviderHostImpl : ProviderHost { void cuda__Impl_Cast(void* stream, const double* input_data, float* output_data, size_t count) override { return GetProviderInfo_CUDA().cuda__Impl_Cast(stream, input_data, output_data, count); } void cuda__Impl_Cast(void* stream, const float* input_data, double* output_data, size_t count) override { return GetProviderInfo_CUDA().cuda__Impl_Cast(stream, input_data, output_data, count); } - Status CudaCall_false(int retCode, const char* exprString, const char* libName, int successCode, const char* msg) override { return GetProviderInfo_CUDA().CudaCall_false(retCode, exprString, libName, successCode, msg); } - void CudaCall_true(int retCode, const char* exprString, const char* libName, int successCode, const char* msg) override { GetProviderInfo_CUDA().CudaCall_true(retCode, exprString, libName, successCode, msg); } + Status CudaCall_false(int retCode, const char* exprString, const char* libName, int successCode, const char* msg, const char* file, const int line) override { return GetProviderInfo_CUDA().CudaCall_false(retCode, exprString, libName, successCode, msg, file, line); } + void CudaCall_true(int retCode, const char* exprString, const char* libName, int successCode, const char* msg, const char* file, const int line) override { GetProviderInfo_CUDA().CudaCall_true(retCode, exprString, libName, successCode, msg, file, line); } #endif #ifdef USE_ROCM @@ -220,8 +220,8 @@ struct ProviderHostImpl : ProviderHost { void rocm__Impl_Cast(void* stream, const double* input_data, float* output_data, size_t count) override { return GetProviderInfo_ROCM().rocm__Impl_Cast(stream, input_data, output_data, count); } void rocm__Impl_Cast(void* stream, const float* input_data, double* output_data, size_t count) override { return GetProviderInfo_ROCM().rocm__Impl_Cast(stream, input_data, output_data, count); } - Status RocmCall_false(int retCode, const char* exprString, const char* libName, int successCode, const char* msg) override { return GetProviderInfo_ROCM().RocmCall_false(retCode, exprString, libName, successCode, msg); } - void RocmCall_true(int retCode, const char* exprString, const char* libName, int successCode, const char* msg) override { GetProviderInfo_ROCM().RocmCall_true(retCode, exprString, libName, successCode, msg); } + Status RocmCall_false(int retCode, const char* exprString, const char* libName, int successCode, const char* msg, const char* file, const int line) override { return GetProviderInfo_ROCM().RocmCall_false(retCode, exprString, libName, successCode, msg, file, line); } + void RocmCall_true(int retCode, const char* exprString, const char* libName, int successCode, const char* msg, const char* file, const int line) override { GetProviderInfo_ROCM().RocmCall_true(retCode, exprString, libName, successCode, msg, file, line); } #endif std::string GetEnvironmentVar(const std::string& var_name) override { return Env::Default().GetEnvironmentVar(var_name); } @@ -1936,7 +1936,7 @@ ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider_Dnnl, auto factory = onnxruntime::DnnlProviderFactoryCreator::Create(dnnl_options); if (!factory) { return OrtApis::CreateStatus(ORT_FAIL, - "SessionOptionsAppendExecutionProvider_Dnnl: Failed to load shared library"); + "SessionOptionsAppendExecutionProvider_Dnnl: Failed to load shared library"); } options->provider_factories.push_back(factory);