onnxruntime/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml

##### start trigger Don't edit it manually, Please do edit set-trigger-rules.py ####
trigger:
  branches:
    include:
    - main
    - rel-*
  paths:
    exclude:
    - docs/**
    - README.md
    - CONTRIBUTING.md
    - BUILD.md
    - 'js/web'
    - 'onnxruntime/core/providers/js'
pr:
  branches:
    include:
    - main
    - rel-*
  paths:
    exclude:
    - docs/**
    - README.md
    - CONTRIBUTING.md
    - BUILD.md
    - 'js/web'
    - 'onnxruntime/core/providers/js'
#### end trigger ####

resources:
  repositories:
  - repository: manylinux
    type: Github
    endpoint: Microsoft
    name: pypa/manylinux
    ref: 5eda9aded5462201e6310105728d33016e637ea7

variables:
  - template: templates/common-variables.yml

jobs:
- job: Linux_Build
  timeoutInMinutes: 120
  variables:
    skipComponentGovernanceDetection: true
    CCACHE_DIR: $(Pipeline.Workspace)/ccache
  workspace:
    clean: all
  pool: onnxruntime-Ubuntu2004-AMD-CPU
  steps:
  - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
    displayName: 'Clean Agent Directories'
    condition: always()

  - checkout: self
    clean: true
    submodules: none

  - template: templates/get-docker-image-steps.yml
    parameters:
      Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cuda11
      Context: tools/ci_build/github/linux/docker
      DockerBuildArgs: "--network=host --build-arg POLICY=manylinux_2_28 --build-arg PLATFORM=x86_64 --build-arg PREPEND_PATH=/usr/local/cuda/bin --build-arg LD_LIBRARY_PATH_ARG=/usr/local/lib64 --build-arg DEVTOOLSET_ROOTPATH=/usr --build-arg BUILD_UID=$( id -u )"
      Repository: onnxruntimecuda11build

  - task: Cache@2
    inputs:
      key: '"ccache" | "$(Build.SourceBranch)" | "$(Build.SourceVersion)"'
      path: $(CCACHE_DIR)
      restoreKeys: |
        "ccache" | "$(Build.SourceBranch)"
        "ccache"        
      cacheHitVar: CACHE_RESTORED
    displayName: Cach Task

  - script: |
      sudo mkdir -p $(Pipeline.Workspace)/ccache      
    condition: ne(variables.CACHE_RESTORED, 'true')
    displayName: Create Cache Dir

  - task: CmdLine@2
    inputs:
      script: |
        mkdir -p $HOME/.onnx
        docker run -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" --rm \
          --volume /data/onnx:/data/onnx:ro \
          --volume $(Build.SourcesDirectory):/onnxruntime_src \
          --volume $(Build.BinariesDirectory):/build \
          --volume /data/models:/build/models:ro \
          --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
          --volume $(Pipeline.Workspace)/ccache:/cache \
          -e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
          -e NIGHTLY_BUILD \
          -e BUILD_BUILDNUMBER \
          -e CCACHE_DIR=/cache \
          onnxruntimecuda11build \
          /bin/bash -c "
            set -ex; \
            env; \
            ccache -s; \
            /opt/python/cp38-cp38/bin/python3 /onnxruntime_src/tools/ci_build/build.py \
              --build_dir /build --cmake_generator Ninja \
              --config Release --update --build \
              --skip_submodule_sync \
              --build_shared_lib \
              --parallel \
              --build_wheel \
              --enable_onnx_tests --use_cuda --cuda_version=${{variables.common_cuda_version}} --cuda_home=/usr/local/cuda-${{variables.common_cuda_version}} --cudnn_home=/usr/local/cuda-${{variables.common_cuda_version}} \
              --enable_cuda_profiling --enable_cuda_nhwc_ops \
              --enable_pybind --build_java \
              --use_cache \
              --cmake_extra_defines  CMAKE_CUDA_ARCHITECTURES=75; \
                ccache -sv; \
                ccache -z"        
      workingDirectory: $(Build.SourcesDirectory)

  - task: CmdLine@2
    inputs:
      script: |
        rm -rf $(Build.BinariesDirectory)/Release/onnxruntime $(Build.BinariesDirectory)/Release/pybind11
        rm -f $(Build.BinariesDirectory)/Release/models
        find $(Build.BinariesDirectory)/Release/_deps -mindepth 1 ! -regex '^$(Build.BinariesDirectory)/Release/_deps/onnx-src\(/.*\)?' -delete
        cd $(Build.BinariesDirectory)/Release
        find -executable -type f > $(Build.BinariesDirectory)/Release/perms.txt        

  - task: PublishPipelineArtifact@0
    displayName: 'Publish Pipeline Artifact'
    inputs:
      artifactName: 'drop-linux'
      targetPath: '$(Build.BinariesDirectory)/Release'

  - template: templates/explicitly-defined-final-tasks.yml

- job: Linux_Test
  timeoutInMinutes: 180
  variables:
    skipComponentGovernanceDetection: true
  workspace:
    clean: all
  pool: Onnxruntime-Linux-GPU-T4
  dependsOn:
  - Linux_Build
  steps:
  - task: DownloadPipelineArtifact@2
    displayName: 'Download Pipeline Artifact'
    inputs:
      buildType: 'current'
      artifactName: 'drop-linux'
      targetPath: '$(Build.BinariesDirectory)/Release'

  - checkout: self
    clean: true
    submodules: none

  - template: templates/get-docker-image-steps.yml
    parameters:
      Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cuda11
      Context: tools/ci_build/github/linux/docker
      DockerBuildArgs: "--network=host --build-arg POLICY=manylinux_2_28 --build-arg PLATFORM=x86_64 --build-arg PREPEND_PATH=/usr/local/cuda/bin --build-arg LD_LIBRARY_PATH_ARG=/usr/local/lib64 --build-arg DEVTOOLSET_ROOTPATH=/usr --build-arg BUILD_UID=$( id -u )"
      Repository: onnxruntimecuda11build

  - task: CmdLine@2
    inputs:
      script: |
        set -e -x
        mkdir -p $HOME/.onnx
        docker run --gpus all --rm \
          --volume  $(Build.SourcesDirectory):/onnxruntime_src \
          --volume $(Build.BinariesDirectory)/Release:/build/Release \
          --volume /data/models:/build/models:ro \
          --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
          --volume /data/onnx:/data/onnx \
          onnxruntimecuda11build \
          /bin/bash -c "
            set -ex; \
            cp /onnxruntime_src/tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt /tmp/requirements.txt; \
            ln -s /opt/python/cp38-cp38/bin/python3 /tmp/python3; \
            /tmp/python3 -m pip install -r /tmp/requirements.txt; \
            /tmp/python3 -m pip install /build/Release/dist/*.whl; \
            cd /build/Release && xargs -a /build/Release/perms.txt chmod a+x; \
            cd /onnxruntime_src/java && /onnxruntime_src/java/gradlew cmakeCheck -DcmakeBuildDir=/build/Release -DUSE_CUDA=1; \
            cd /tmp; \
            /tmp/python3 /onnxruntime_src/tools/ci_build/build.py \
              --build_dir /build --config Release --test --skip_submodule_sync --build_shared_lib --parallel --build_wheel --enable_onnx_tests \
              --use_cuda --cuda_version=${{variables.common_cuda_version}} --cuda_home=/usr/local/cuda --cudnn_home=/usr/local/cuda \
              --enable_pybind --build_java --ctest_path '' "        

  - template: templates/clean-agent-build-directory-step.yml