From 9345894c823de67f913d907da91dbc07ac322041 Mon Sep 17 00:00:00 2001 From: RandySheriffH <48490400+RandySheriffH@users.noreply.github.com> Date: Mon, 29 Nov 2021 22:44:50 -0800 Subject: [PATCH] Add build option to enable cuda profiling (#9875) --- cmake/CMakeLists.txt | 6 ++++++ cmake/onnxruntime_providers.cmake | 11 ++++++++--- onnxruntime/core/providers/cuda/cuda_profiler.cc | 2 +- onnxruntime/core/providers/cuda/cuda_profiler.h | 2 +- onnxruntime/test/framework/inference_session_test.cc | 2 +- tools/ci_build/build.py | 5 +++++ .../github/azure-pipelines/linux-gpu-ci-pipeline.yml | 1 + .../github/azure-pipelines/win-gpu-ci-pipeline.yml | 2 +- 8 files changed, 24 insertions(+), 7 deletions(-) diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 23e8369250..acbde7f56a 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -175,6 +175,8 @@ option(onnxruntime_PREBUILT_PYTORCH_PATH "Path to pytorch installation dir") # external transformer src path option(onnxruntime_EXTERNAL_TRANSFORMER_SRC_PATH "Path to external transformer src dir") +option(onnxruntime_ENABLE_CUDA_PROFILING "Enable CUDA kernel profiling" OFF) + if (onnxruntime_USE_CUDA) set(onnxruntime_DISABLE_RTTI OFF) endif() @@ -1700,6 +1702,10 @@ if (onnxruntime_ENABLE_TRAINING_OPS) add_compile_definitions(ENABLE_TRAINING_OPS) endif() +if (onnxruntime_ENABLE_CUDA_PROFILING) + add_compile_definitions(ENABLE_CUDA_PROFILING) +endif() + if (onnxruntime_ENABLE_TRAINING) add_compile_definitions(ENABLE_TRAINING) add_compile_definitions(ENABLE_TRAINING_OPS) diff --git a/cmake/onnxruntime_providers.cmake b/cmake/onnxruntime_providers.cmake index adf121a196..a82629bed4 100644 --- a/cmake/onnxruntime_providers.cmake +++ b/cmake/onnxruntime_providers.cmake @@ -353,13 +353,18 @@ if (onnxruntime_USE_CUDA) endif() add_dependencies(onnxruntime_providers_cuda onnxruntime_providers_shared ${onnxruntime_EXTERNAL_DEPENDENCIES} ${onnxruntime_tvm_dependencies}) - target_link_directories(onnxruntime_providers_cuda PRIVATE ${onnxruntime_CUDA_HOME}/extras/CUPTI/lib64) - target_link_libraries(onnxruntime_providers_cuda PRIVATE cublas cudnn curand cufft cupti ${ONNXRUNTIME_PROVIDERS_SHARED}) - target_include_directories(onnxruntime_providers_cuda PRIVATE ${ONNXRUNTIME_ROOT} ${CMAKE_CURRENT_BINARY_DIR} ${onnxruntime_CUDNN_HOME}/include ${eigen_INCLUDE_DIRS} ${TVM_INCLUDES} PUBLIC ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES} ${onnxruntime_CUDA_HOME}/extras/CUPTI/include) + target_link_libraries(onnxruntime_providers_cuda PRIVATE cublas cudnn curand cufft ${ONNXRUNTIME_PROVIDERS_SHARED}) + target_include_directories(onnxruntime_providers_cuda PRIVATE ${ONNXRUNTIME_ROOT} ${CMAKE_CURRENT_BINARY_DIR} ${onnxruntime_CUDNN_HOME}/include ${eigen_INCLUDE_DIRS} ${TVM_INCLUDES} PUBLIC ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) # ${CMAKE_CURRENT_BINARY_DIR} is so that #include "onnxruntime_config.h" inside tensor_shape.h is found set_target_properties(onnxruntime_providers_cuda PROPERTIES LINKER_LANGUAGE CUDA) set_target_properties(onnxruntime_providers_cuda PROPERTIES FOLDER "ONNXRuntime") + if (onnxruntime_ENABLE_CUDA_PROFILING) # configure cupti for cuda profiling + target_include_directories(onnxruntime_providers_cuda PRIVATE ${onnxruntime_CUDA_HOME}/extras/CUPTI/include) + target_link_directories(onnxruntime_providers_cuda PRIVATE ${onnxruntime_CUDA_HOME}/extras/CUPTI/lib64) + target_link_libraries(onnxruntime_providers_cuda PRIVATE cupti) + endif() + if (onnxruntime_ENABLE_NVTX_PROFILE) target_link_libraries(onnxruntime_providers_cuda PRIVATE nvToolsExt) endif() diff --git a/onnxruntime/core/providers/cuda/cuda_profiler.cc b/onnxruntime/core/providers/cuda/cuda_profiler.cc index de9cbcc09f..adf771fb23 100644 --- a/onnxruntime/core/providers/cuda/cuda_profiler.cc +++ b/onnxruntime/core/providers/cuda/cuda_profiler.cc @@ -1,6 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#if !(defined(USE_ROCM) || defined(ENABLE_TRAINING)) +#if defined(USE_CUDA) && defined(ENABLE_CUDA_PROFILING) #include "cuda_profiler.h" #include diff --git a/onnxruntime/core/providers/cuda/cuda_profiler.h b/onnxruntime/core/providers/cuda/cuda_profiler.h index 2ae6715009..bd625a7c6a 100644 --- a/onnxruntime/core/providers/cuda/cuda_profiler.h +++ b/onnxruntime/core/providers/cuda/cuda_profiler.h @@ -2,7 +2,7 @@ // Licensed under the MIT License. #include "core/common/profiler_common.h" -#if !(defined(USE_ROCM) || defined(ENABLE_TRAINING)) +#if defined(USE_CUDA) && defined(ENABLE_CUDA_PROFILING) #include "core/platform/ort_mutex.h" #include diff --git a/onnxruntime/test/framework/inference_session_test.cc b/onnxruntime/test/framework/inference_session_test.cc index 9d5fa3ec74..23e240a134 100644 --- a/onnxruntime/test/framework/inference_session_test.cc +++ b/onnxruntime/test/framework/inference_session_test.cc @@ -661,7 +661,7 @@ TEST(InferenceSessionTests, CheckRunProfilerWithSessionOptions) { } } -#if defined(USE_CUDA) && !defined(ENABLE_TRAINING) && defined(CUDA_VERSION) && CUDA_VERSION >= 11000 +#if defined(USE_CUDA) && defined(ENABLE_CUDA_PROFILING) ASSERT_TRUE(has_kernel_info); #endif } diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 77dc793ed0..35a95c2c90 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -567,6 +567,10 @@ def parse_arguments(): "--test_external_transformer_example", action='store_true', help="run the example external transformer test, mainly used in CI pipeline.") + parser.add_argument( + "--enable_cuda_profiling", action='store_true', help="enable cuda kernel profiling, \ + cupti library must be added to PATH beforehand.") + return parser.parse_args() @@ -817,6 +821,7 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home "-Donnxruntime_ENABLE_EXTERNAL_CUSTOM_OP_SCHEMAS=" + ("ON" if args.enable_external_custom_op_schemas else "OFF"), "-Donnxruntime_NVCC_THREADS=" + str(args.parallel), + "-Donnxruntime_ENABLE_CUDA_PROFILING=" + ("ON" if args.enable_cuda_profiling else "OFF"), ] if args.external_graph_transformer_path: cmake_args.append("-Donnxruntime_EXTERNAL_TRANSFORMER_SRC_PATH=" + args.external_graph_transformer_path) diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml index 0a4c360f5c..59cdd99fc7 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml @@ -38,6 +38,7 @@ jobs: --parallel \ --build_wheel \ --enable_onnx_tests --use_cuda --cuda_version=11.4 --cuda_home=/usr/local/cuda-11.4 --cudnn_home=/usr/local/cuda-11.4 \ + --enable_cuda_profiling \ --enable_pybind --build_java \ --cmake_extra_defines CMAKE_CUDA_HOST_COMPILER=/opt/rh/devtoolset-10/root/usr/bin/cc CMAKE_CUDA_ARCHITECTURES=52 workingDirectory: $(Build.SourcesDirectory) diff --git a/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml index 7f73482a75..a2d9dc3048 100644 --- a/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml @@ -13,7 +13,7 @@ stages: strategy: matrix: cuda: - additionalBuildFlags: --build_java --build_nodejs --use_cuda --cuda_version=11.4 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.4" --cudnn_home="C:\local\cudnn-11.4-windows-x64-v8.2.2.26\cuda" --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=52 --gen_doc validate + additionalBuildFlags: --build_java --build_nodejs --use_cuda --cuda_version=11.4 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.4" --cudnn_home="C:\local\cudnn-11.4-windows-x64-v8.2.2.26\cuda" --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=52 --gen_doc validate --enable_cuda_profiling EnvSetupScript: setup_env_cuda_11.bat ORT_EP_NAME: CUDA dml: