Revert "Remove openmp flag (#1140)" (#1146)

This reverts commit a7137a0f9d.
2026-06-04 23:59:56 +00:00 · 2019-05-31 18:48:14 -07:00 · 2019-05-31 18:48:14 -07:00 · 6c9d815de5
commit 6c9d815de5
parent 71cbd7679c
16 changed files with 62 additions and 33 deletions
--- a/cmake/CMakeLists.txt
+++ b/cmake/CMakeLists.txt
@ -62,6 +62,7 @@ option(onnxruntime_BUILD_BENCHMARKS "Build ONNXRuntime micro-benchmarks" OFF)
 option(onnxruntime_USE_TVM "Build tvm for code-gen" OFF)
 option(onnxruntime_BUILD_FOR_NATIVE_MACHINE "Enable this option for turning on optimization specific to this machine" OFF)
 option(onnxruntime_USE_LLVM "Build tvm with LLVM" OFF)
+option(onnxruntime_USE_OPENMP "Build with OpenMP support" OFF)
 option(onnxruntime_BUILD_SHARED_LIB "Build a shared library" OFF)
 option(onnxruntime_ENABLE_MICROSOFT_INTERNAL "Use this option to enable/disable microsoft internal only code" OFF)
 option(onnxruntime_USE_NUPHAR "Build with Nupha" OFF)
@ -81,6 +82,19 @@ set(protobuf_BUILD_TESTS OFF CACHE BOOL "Build protobuf tests" FORCE)
 set(NSYNC_ENABLE_TESTS OFF CACHE BOOL "Build protobuf tests" FORCE)
 set(ONNX_ML 1)

+if(onnxruntime_USE_OPENMP AND UNIX)
+  #if you enabled both of them, the code can still be built and run, but you may see
+  # 10x performance degradation, because one process should only have one openmp implementation
+  # mkl(or mklml) depends on Intel OpenMP
+  # GCC does not support linking against the Intel OpenMP runtime library
+  # Clang should be ok, but it's not in our consideration right now.
+  if(onnxruntime_USE_MKLML)
+    message(FATAL_ERROR "Please use only one of onnxruntime_USE_MKLML, onnxruntime_USE_OPENMP")
+  endif()
+  if(onnxruntime_USE_NGRAPH)
+    message(FATAL_ERROR "Please use only one of onnxruntime_USE_NGRAPH, onnxruntime_USE_OPENMP")
+  endif()
+endif()
 if(onnxruntime_ENABLE_LTO)
  #TODO: figure out why nsync doesn't work
  if(onnxruntime_USE_NSYNC)
@ -101,6 +115,15 @@ set(REPO_ROOT ${PROJECT_SOURCE_DIR}/..)
 set(ONNXRUNTIME_ROOT ${PROJECT_SOURCE_DIR}/../onnxruntime)
 file (STRINGS "${REPO_ROOT}/VERSION_NUMBER" VERSION_NUMBER)

+if(onnxruntime_USE_OPENMP)
+  find_package(OpenMP)
+  if (OPENMP_FOUND)
+    set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}")
+    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
+    add_definitions(-DUSE_OPENMP)
+  endif()
+endif()
+
 # Guarantee that the Eigen code that you are #including is licensed
 # under the MPL2 and possibly more permissive licenses (like BSD).
 add_definitions(-DEIGEN_MPL2_ONLY)
@ -465,6 +488,9 @@ if (onnxruntime_USE_MKLDNN)
 endif()

 if (onnxruntime_USE_NGRAPH)
+  #if (onnxruntime_USE_OPENMP)
+  #  message(FATAL_ERROR "Please set onnxruntime_USE_OPENMP=OFF for nGraph execution provider.")
+  #endif()
  if (NOT onnxruntime_USE_FULL_PROTOBUF)
    message(FATAL_ERROR "Please set onnxruntime_USE_FULL_PROTOBUF=ON for nGraph execution provider.")
  endif()
--- a/tools/ci_build/build.py
+++ b/tools/ci_build/build.py
@ -126,6 +126,7 @@ Use the individual flags to only run the specified stages.
    parser.add_argument("--use_preinstalled_eigen", action='store_true', help="Use pre-installed eigen.")
    parser.add_argument("--eigen_path", help="Path to pre-installed eigen.")
    parser.add_argument("--use_tvm", action="store_true", help="Build with tvm")
+    parser.add_argument("--use_openmp", action='store_true', help="Build with OpenMP.")
    parser.add_argument("--use_llvm", action="store_true", help="Build tvm with llvm")
    parser.add_argument("--use_eigenthreadpool", action="store_true", help="Build with eigenthreadpool")
    parser.add_argument("--enable_msinternal", action="store_true", help="Enable for Microsoft internal builds only.")
@ -319,6 +320,7 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home
                 "-Donnxruntime_USE_MKLDNN=" + ("ON" if args.use_mkldnn else "OFF"),
                 "-Donnxruntime_USE_MKLML=" + ("ON" if args.use_mklml else "OFF"),
                 "-Donnxruntime_USE_NGRAPH=" + ("ON" if args.use_ngraph else "OFF"),
+                 "-Donnxruntime_USE_OPENMP=" + ("ON" if args.use_openmp and not args.use_mklml and not args.use_ngraph else "OFF"),
                 "-Donnxruntime_USE_TVM=" + ("ON" if args.use_tvm else "OFF"),
                 "-Donnxruntime_USE_LLVM=" + ("ON" if args.use_llvm else "OFF"),
                 "-Donnxruntime_ENABLE_MICROSOFT_INTERNAL=" + ("ON" if args.enable_msinternal else "OFF"),
--- a/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml
+++ b/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml
@ -32,7 +32,7 @@ jobs:
      displayName: 'Run build script'
      inputs:
        scriptPath: 'tools/ci_build/github/linux/run_dockerbuild.sh'
-        args: '-c Release -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -p $(python.version) -x "--build_wheel"'
+        args: '-c Release -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -p $(python.version) -x "--use_openmp --build_wheel"'

    - task: CopyFiles@2
      displayName: 'Copy Python Wheel to:  $(Build.ArtifactStagingDirectory)'
@ -95,7 +95,7 @@ jobs:
      displayName: 'Run build script'
      inputs:
        scriptPath: 'tools/ci_build/github/linux/run_dockerbuild.sh'
-        args: '-c Release -o ubuntu16.04 -d gpu -c cuda9.1-cudnn7.1 -r $(Build.BinariesDirectory) -p $(python.version) -x "--build_wheel"'
+        args: '-c Release -o ubuntu16.04 -d gpu -c cuda9.1-cudnn7.1 -r $(Build.BinariesDirectory) -p $(python.version) -x "--use_openmp --build_wheel"'

    - task: CopyFiles@2
      displayName: 'Copy Python Wheel to:  $(Build.ArtifactStagingDirectory)'
@ -139,7 +139,7 @@ jobs:
      displayName: 'Run build script'
      inputs:
        filename: 'build.bat'
-        arguments: ' --build_dir $(buildDirectory) --config Release --build_wheel'
+        arguments: ' --build_dir $(buildDirectory) --config Release --use_openmp --build_wheel'
        workingFolder: "$(Build.SourcesDirectory)"

    - task: CopyFiles@2
@ -206,7 +206,7 @@ jobs:
      inputs:
        filename: 'build.bat'
        arguments: ' --use_cuda --cuda_home="C:\local\cuda-9.1.85-windows10-x64-0"
-      --cudnn_home="C:\local\cudnn-9.1-windows10-x64-v7.1\cuda" --build_dir $(buildDirectory) --config Release --build_wheel'
+      --cudnn_home="C:\local\cudnn-9.1-windows10-x64-v7.1\cuda" --build_dir $(buildDirectory) --config Release --use_openmp --build_wheel'
        workingFolder: "$(Build.SourcesDirectory)"

    - task: CopyFiles@2
--- a/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml
+++ b/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml
@ -59,7 +59,7 @@ jobs:

    - script: |
        sudo xcode-select --switch /Applications/Xcode_10.app/Contents/Developer
-        python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_shared_lib --config RelWithDebInfo --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum)
+        python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --use_openmp --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_shared_lib --config RelWithDebInfo --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum)
      displayName: 'Build and Test MacOS'
    - template: templates/c-api-artifacts-package-and-publish-steps-posix.yml
      parameters:
@ -90,6 +90,7 @@ jobs:

    - template: templates/windows-build-and-test-steps.yml
      parameters:
+        buildAdditionalParams: ' --use_openmp '
        buildArch: $(buildArch)
        msbuildPlatform: $(buildArch)
        buildConfig: $(buildConfig)
@ -121,7 +122,7 @@ jobs:

    - template: templates/windows-build-and-test-steps.yml
      parameters:
-        buildAdditionalParams: ' --x86 '
+        buildAdditionalParams: ' --use_openmp --x86 '
        buildArch: $(buildArch)
        msbuildPlatform: 'Win32'
        buildConfig: $(buildConfig)
--- a/tools/ci_build/github/azure-pipelines/linux-ort-srv-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/linux-ort-srv-ci-pipeline.yml
@ -30,7 +30,7 @@ jobs:
        pythonInterpreter: '/usr/bin/python3'
        workingDirectory: $(Build.BinariesDirectory)

-    - script: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -x "--config Debug Release --build_server --use_full_protobuf --enable_server_tests"'
+    - script: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -x "--config Debug Release --build_server --use_openmp --use_full_protobuf --enable_server_tests"'
      displayName: 'Run build script'

    - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
--- a/tools/ci_build/github/azure-pipelines/linux-ort-srv-nightly-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/linux-ort-srv-nightly-pipeline.yml
@ -30,10 +30,10 @@ jobs:
        pythonInterpreter: '/usr/bin/python3'
        workingDirectory: $(Build.BinariesDirectory)

-    - script: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -x "--config RelWithDebInfo --build_server --use_full_protobuf --enable_server_model_tests --cmake_extra_defines onnxruntime_SERVER_VERSION=$(cat ./VERSION_NUMBER)-$(Build.BuildNumber) onnxruntime_LATEST_COMMIT_ID=$(Build.SourceVersion)"'
+    - script: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -x "--config RelWithDebInfo --build_server --use_openmp --use_full_protobuf --enable_server_model_tests --cmake_extra_defines onnxruntime_SERVER_VERSION=$(cat ./VERSION_NUMBER)-$(Build.BuildNumber) onnxruntime_LATEST_COMMIT_ID=$(Build.SourceVersion)"'
      displayName: 'Run build script with model tests'

-    - script: 'tools/ci_build/github/linux/upload_ortsrv_binaries.sh -a $(Build.BinariesDirectory) -r $(Build.BinariesDirectory)/RelWithDebInfo -i $(Build.BuildNumber) -c $(Build.SourceVersion) -b "$(blob.binary_upload_url)" -p "--config RelWithDebInfo --build_server --use_full_protobuf --enable_server_model_tests --cmake_extra_defines onnxruntime_SERVER_VERSION=$(cat ./VERSION_NUMBER)-$(Build.BuildNumber) onnxruntime_LATEST_COMMIT_ID=$(Build.SourceVersion)"'
+    - script: 'tools/ci_build/github/linux/upload_ortsrv_binaries.sh -a $(Build.BinariesDirectory) -r $(Build.BinariesDirectory)/RelWithDebInfo -i $(Build.BuildNumber) -c $(Build.SourceVersion) -b "$(blob.binary_upload_url)" -p "--config RelWithDebInfo --build_server --use_openmp --use_full_protobuf --enable_server_model_tests --cmake_extra_defines onnxruntime_SERVER_VERSION=$(cat ./VERSION_NUMBER)-$(Build.BuildNumber) onnxruntime_LATEST_COMMIT_ID=$(Build.SourceVersion)"'
      displayName: 'Upload binary to blob storage'

    - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
--- a/tools/ci_build/github/azure-pipelines/mac-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/mac-ci-pipeline.yml
@ -21,7 +21,7 @@ jobs:
    - script: |
        sudo python3 -m pip install numpy==1.15.0
        sudo xcode-select --switch /Applications/Xcode_10.app/Contents/Developer
-        python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir $(Build.BinariesDirectory) --build_wheel --skip_submodule_sync --parallel --build_shared_lib --enable_onnx_tests --config Debug Release
+        python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --use_openmp --build_dir $(Build.BinariesDirectory) --build_wheel --skip_submodule_sync --parallel --build_shared_lib --enable_onnx_tests --config Debug Release
      displayName: 'Build and Test OnnxRuntime lib for MacOS'

    - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
--- a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml
@ -13,7 +13,7 @@ jobs:
      displayName: 'Download test data and generate cmake config'
      inputs:
        filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
-        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_tvm --enable_pybind --use_mkldnn --use_mklml --build_shared_lib  --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --gen_doc --update'
+        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_tvm --enable_pybind --use_mkldnn --use_mklml --use_openmp --build_shared_lib  --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --gen_doc --update'
        workingDirectory: "$(Build.BinariesDirectory)"

    - task: VSBuild@1
@ -30,7 +30,7 @@ jobs:
      displayName: 'Test Debug'
      inputs:
        filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
-        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_tvm --enable_pybind --use_mkldnn --use_mklml --build_shared_lib  --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --gen_doc --test'
+        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_tvm --enable_pybind --use_mkldnn --use_mklml --use_openmp --build_shared_lib  --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --gen_doc --test'
        workingFolder: '$(Build.BinariesDirectory)'
    - task: VSBuild@1
      displayName: 'Build C# Debug'
@ -65,7 +65,7 @@ jobs:
      displayName: 'Test Release'
      inputs:
        filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
-        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_tvm --enable_pybind --use_mkldnn --use_mklml --build_shared_lib  --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test'
+        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_tvm --enable_pybind --use_mkldnn --use_mklml --use_openmp --build_shared_lib  --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test'
        workingFolder: "$(Build.BinariesDirectory)"

    - task: VSBuild@1
--- a/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml
@ -15,7 +15,7 @@ jobs:
      displayName: 'Download test data and generate cmake config'
      inputs:
        filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
-        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe  --enable_pybind --use_mkldnn --use_mkldnn --build_shared_lib  --enable_onnx_tests --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --update --msvc_toolset=14.11'
+        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe  --enable_pybind --use_openmp --use_mkldnn --use_mkldnn --build_shared_lib  --enable_onnx_tests --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --update --msvc_toolset=14.11'
        workingDirectory: "$(Build.BinariesDirectory)"

    - task: VSBuild@1
@ -32,7 +32,7 @@ jobs:
      displayName: 'Test Debug'
      inputs:
        filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
-        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe  --enable_pybind --use_mkldnn --use_mkldnn --build_shared_lib  --enable_onnx_tests --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test'
+        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe  --enable_pybind --use_openmp --use_mkldnn --use_mkldnn --build_shared_lib  --enable_onnx_tests --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test'
        workingFolder: '$(Build.BinariesDirectory)'
    - task: VSBuild@1
      displayName: 'Build C# Debug'
@ -68,7 +68,7 @@ jobs:
      displayName: 'Test Release'
      inputs:
        filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
-        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe  --enable_pybind --use_mkldnn --build_shared_lib  --enable_onnx_tests --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test'
+        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe  --enable_pybind --use_openmp --use_mkldnn --build_shared_lib  --enable_onnx_tests --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test'
        workingFolder: "$(Build.BinariesDirectory)"

    - task: VSBuild@1
--- a/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml
@ -20,7 +20,7 @@ jobs:
      displayName: 'Download test data and generate cmake config'
      inputs:
        filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
-        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug Release --build_dir $(Build.BinariesDirectory) --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe  --enable_pybind --use_mkldnn --build_shared_lib  --enable_onnx_tests --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --use_tensorrt --tensorrt_home="C:\local\TensorRT-5.0.4.3" --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --update --msvc_toolset=14.11'
+        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug Release --build_dir $(Build.BinariesDirectory) --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe  --enable_pybind --use_openmp --use_mkldnn --build_shared_lib  --enable_onnx_tests --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --use_tensorrt --tensorrt_home="C:\local\TensorRT-5.0.4.3" --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --update --msvc_toolset=14.11'
        workingDirectory: "$(Build.BinariesDirectory)"

    - task: VSBuild@1
@ -37,7 +37,7 @@ jobs:
      displayName: 'Test Debug'
      inputs:
        filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
-        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe  --enable_pybind --use_mkldnn --build_shared_lib  --enable_onnx_tests --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --use_tensorrt --tensorrt_home="C:\local\TensorRT-5.0.4.3" --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test'
+        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe  --enable_pybind --use_openmp --use_mkldnn --build_shared_lib  --enable_onnx_tests --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --use_tensorrt --tensorrt_home="C:\local\TensorRT-5.0.4.3" --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test'
        workingFolder: '$(Build.BinariesDirectory)'
    - task: VSBuild@1
      displayName: 'Build C# Debug'
@ -73,7 +73,7 @@ jobs:
      displayName: 'Test Release'
      inputs:
        filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
-        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe  --enable_pybind --use_mkldnn --build_shared_lib  --enable_onnx_tests --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --use_tensorrt --tensorrt_home="C:\local\TensorRT-5.0.4.3" --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test'
+        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe  --enable_pybind --use_openmp --use_mkldnn --build_shared_lib  --enable_onnx_tests --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --use_tensorrt --tensorrt_home="C:\local\TensorRT-5.0.4.3" --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test'
        workingFolder: "$(Build.BinariesDirectory)"

    - task: VSBuild@1
--- a/tools/ci_build/github/azure-pipelines/win-mklml-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/win-mklml-ci-pipeline.yml
@ -13,7 +13,7 @@ jobs:
      displayName: 'Download test data and generate cmake config'
      inputs:
        filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
-        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_mklml --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --update'
+        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_mklml --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --update'
        workingDirectory: "$(Build.BinariesDirectory)"

    - task: VSBuild@1
@ -30,7 +30,7 @@ jobs:
      displayName: 'Test Debug'
      inputs:
        filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
-        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_mklml --build_shared_lib  --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test'
+        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_mklml --use_openmp --build_shared_lib  --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test'
        workingFolder: '$(Build.BinariesDirectory)'
    - task: VSBuild@1
      displayName: 'Build C# Debug'
@ -65,7 +65,7 @@ jobs:
      displayName: 'Test Release'
      inputs:
        filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
-        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_mklml --build_shared_lib  --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test'
+        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_mklml --use_openmp --build_shared_lib  --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test'
        workingFolder: "$(Build.BinariesDirectory)"

    - task: VSBuild@1
--- a/tools/ci_build/github/azure-pipelines/win-ngraph-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/win-ngraph-ci-pipeline.yml
@ -13,7 +13,7 @@ jobs:
      displayName: 'Download test data and generate cmake config'
      inputs:
        filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
-        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_ngraph --use_full_protobuf --build_shared_lib  --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --gen_doc --update'
+        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_ngraph --use_full_protobuf --build_shared_lib  --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --gen_doc --update'
        workingDirectory: "$(Build.BinariesDirectory)"
    - task: VSBuild@1
      displayName: 'Build Debug'
@ -29,7 +29,7 @@ jobs:
      displayName: 'Test Debug'
      inputs:
        filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
-        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_ngraph --use_full_protobuf --build_shared_lib  --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --gen_doc --test'
+        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_ngraph --use_full_protobuf --build_shared_lib  --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --gen_doc --test'
        workingFolder: '$(Build.BinariesDirectory)'
    - task: VSBuild@1
      displayName: 'Build Release'
@ -45,7 +45,7 @@ jobs:
      displayName: 'Test Release'
      inputs:
        filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
-        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_ngraph --use_full_protobuf --build_shared_lib  --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test'
+        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_ngraph --use_full_protobuf --build_shared_lib  --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test'
        workingFolder: "$(Build.BinariesDirectory)"

    - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
--- a/tools/ci_build/github/azure-pipelines/win-x86-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/win-x86-ci-pipeline.yml
@ -17,7 +17,7 @@ jobs:
      displayName: 'Download test data and generate cmake config'
      inputs:
        filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
-        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --build_shared_lib  --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --update --x86'
+        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib  --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --update --x86'
        workingDirectory: "$(Build.BinariesDirectory)"

    - task: VSBuild@1
@ -35,7 +35,7 @@ jobs:
      displayName: 'Test Debug'
      inputs:
        filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
-        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --build_shared_lib  --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test --x86'
+        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib  --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test --x86'
        workingFolder: '$(Build.BinariesDirectory)'

    - task: VSBuild@1
@ -72,7 +72,7 @@ jobs:
      displayName: 'Test Release'
      inputs:
        filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
-        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --build_shared_lib  --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test --x86'
+        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib  --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test --x86'
        workingFolder: "$(Build.BinariesDirectory)"

    - task: VSBuild@1
--- a/tools/ci_build/github/azure-pipelines/win-x86-nocontribops-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/win-x86-nocontribops-ci-pipeline.yml
@ -19,7 +19,7 @@ jobs:
      displayName: 'Download test data and generate cmake config'
      inputs:
        filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
-        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --build_shared_lib  --enable_onnx_tests --test_data_url $(TestDataUrlNoContribOps) --test_data_checksum $(TestDataChecksum) --update --x86'
+        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib  --enable_onnx_tests --test_data_url $(TestDataUrlNoContribOps) --test_data_checksum $(TestDataChecksum) --update --x86'
        workingDirectory: "$(Build.BinariesDirectory)"

    - task: VSBuild@1
@ -37,7 +37,7 @@ jobs:
      displayName: 'Test Debug'
      inputs:
        filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
-        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --build_shared_lib  --enable_onnx_tests --test_data_url $(TestDataUrlNoContribOps) --test_data_checksum $(TestDataChecksum) --test --x86 --disable_contrib_ops --enable_msvc_static_runtime'
+        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib  --enable_onnx_tests --test_data_url $(TestDataUrlNoContribOps) --test_data_checksum $(TestDataChecksum) --test --x86 --disable_contrib_ops --enable_msvc_static_runtime'
        workingFolder: '$(Build.BinariesDirectory)'

    - task: VSBuild@1
@ -74,7 +74,7 @@ jobs:
      displayName: 'Test Release'
      inputs:
        filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
-        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --build_shared_lib  --enable_onnx_tests --test_data_url $(TestDataUrlNoContribOps) --test_data_checksum $(TestDataChecksum) --test --x86 --disable_contrib_ops --enable_msvc_static_runtime'
+        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib  --enable_onnx_tests --test_data_url $(TestDataUrlNoContribOps) --test_data_checksum $(TestDataChecksum) --test --x86 --disable_contrib_ops --enable_msvc_static_runtime'
        workingFolder: "$(Build.BinariesDirectory)"

    - task: VSBuild@1
--- a/tools/ci_build/github/linux/run_build.sh
+++ b/tools/ci_build/github/linux/run_build.sh
@ -20,7 +20,7 @@ if [ $BUILD_OS = "android" ]; then
    /opt/cmake/bin/cmake -DCMAKE_TOOLCHAIN_FILE=/android-ndk/build/cmake/android.toolchain.cmake -DANDROID_ABI=arm64-v8a -DONNX_CUSTOM_PROTOC_EXECUTABLE=/usr/bin/protoc ../cmake
    /opt/cmake/bin/cmake --build . -- -j$(nproc)
 else
-    COMMON_BUILD_ARGS="--skip_submodule_sync --enable_onnx_tests --parallel --build_shared_lib"
+    COMMON_BUILD_ARGS="--skip_submodule_sync --enable_onnx_tests --parallel --build_shared_lib --use_openmp"
    if [ $BUILD_DEVICE = "gpu" ]; then
        _CUDNN_VERSION=$(echo $CUDNN_VERSION | cut -d. -f1-2)
        python3 $SCRIPT_DIR/../../build.py --build_dir /build \
--- a/tools/ci_build/github/linux/server_run_build.sh
+++ b/tools/ci_build/github/linux/server_run_build.sh
@ -19,7 +19,7 @@ done
        --config Debug Release \
        --skip_submodule_sync --enable_onnx_tests \
        --parallel --build_shared_lib \
-        --use_cuda \
+        --use_cuda --use_openmp \
        --cuda_home /usr/local/cuda \
        --cudnn_home /usr/local/cudnn-$_CUDNN_VERSION/cuda --build_shared_lib $BUILD_EXTR_PAR
    /home/onnxruntimedev/Release/onnx_test_runner -e cuda /data/onnx