##### start trigger Don't edit it manually, Please do edit set-trigger-rules.py ####
### please do rerun set-trigger-rules.py ###
trigger:
  branches:
    include:
    - main
    - rel-*
  paths:
    exclude:
    - docs/**
    - README.md
    - CONTRIBUTING.md
    - BUILD.md
    - 'js/web'
    - 'onnxruntime/core/providers/js'
pr:
  branches:
    include:
    - main
    - rel-*
  paths:
    exclude:
    - docs/**
    - README.md
    - CONTRIBUTING.md
    - BUILD.md
    - 'js/web'
    - 'onnxruntime/core/providers/js'
#### end trigger ####
parameters:
  - name: CudaVersion
    displayName: CUDA version
    type: string
    default: '12.2'
    values:
      - 11.8
      - 12.2

  - name: SpecificArtifact
    displayName: Use Specific Artifact
    type: boolean
    default: false

  - name: BuildId
    displayName: Specific Artifact's BuildId
    type: string
    default: '0'

variables:
  - name: docker_base_image
    ${{ if eq(parameters.CudaVersion, '11.8') }}:
      value: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20241111.1
    ${{ if eq(parameters.CudaVersion, '12.2') }}:
      value: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_x64_ubi8_gcc12:20241111.1

  - name: Repository
    ${{ if eq(parameters.CudaVersion, '11.8') }}:
      value: 'onnxruntimecuda11manylinuxbuild'
    ${{ if eq(parameters.CudaVersion, '12.2') }}:
      value: 'onnxruntimecuda12manylinuxbuild'

stages:
- stage: Linux_Build
  jobs:
  - job: Linux_Build
    timeoutInMinutes: 120
    variables:
      skipComponentGovernanceDetection: true
      CCACHE_DIR: $(Pipeline.Workspace)/ccache
    workspace:
      clean: all
    pool: onnxruntime-Ubuntu2204-AMD-CPU

    steps:
    - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
      displayName: 'Clean Agent Directories'
      condition: always()

    - checkout: self
      clean: true
      submodules: none

    - template: templates/get-docker-image-steps.yml
      parameters:
        Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cuda
        Context: tools/ci_build/github/linux/docker
        DockerBuildArgs: "--build-arg BASEIMAGE=$(docker_base_image) --build-arg BUILD_UID=$( id -u )"
        Repository: $(Repository)

    - task: Cache@2
      inputs:
        key: '"ccache" | "${{parameters.CudaVersion}}" |"$(Build.SourceBranch)" | "$(Build.SourceVersion)"'
        path: $(CCACHE_DIR)
        restoreKeys: |
          "ccache" | "${{parameters.CudaVersion}}" | "$(Build.SourceBranch)"
          "ccache"
        cacheHitVar: CACHE_RESTORED
      displayName: Cach Task

    - script: |
        sudo mkdir -p $(Pipeline.Workspace)/ccache
      condition: ne(variables.CACHE_RESTORED, 'true')
      displayName: Create Cache Dir

    - script: |
        set -e -x
        mkdir -p $HOME/.onnx
        docker run --rm \
          --volume /data/onnx:/data/onnx:ro \
          --volume $(Build.SourcesDirectory):/onnxruntime_src \
          --volume $(Build.BinariesDirectory):/build \
          --volume /data/models:/build/models:ro \
          --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
          --volume $(Pipeline.Workspace)/ccache:/cache \
          -e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
          -e NIGHTLY_BUILD \
          -e BUILD_BUILDNUMBER \
          -e CCACHE_DIR=/cache -w /onnxruntime_src \
          $(Repository) tools/ci_build/github/linux/build_cuda_ci.sh
      workingDirectory: $(Build.SourcesDirectory)
      displayName: Build Onnxruntime

    - script: $(Build.SourcesDirectory)/tools/ci_build/github/linux/delete_unused_files_before_upload.sh

    - task: PublishPipelineArtifact@0
      displayName: 'Publish Pipeline Artifact'
      inputs:
        artifactName: 'drop-linux'
        targetPath: '$(Build.BinariesDirectory)/Release'

    - template: templates/explicitly-defined-final-tasks.yml

- stage: Linux_Test
  dependsOn:
    - Linux_Build
  jobs:
  - job: Linux_Test
    timeoutInMinutes: 180
    variables:
      skipComponentGovernanceDetection: true
    workspace:
      clean: all
    pool: onnxruntime-Linux-GPU-T4
    steps:
    - checkout: self
      clean: true
      submodules: none

    - template: templates/flex-downloadPipelineArtifact.yml
      parameters:
        ArtifactName: 'drop-linux'
        StepName: 'Download Pipeline Artifact - Linux Build'
        TargetPath: '$(Build.BinariesDirectory)/Release'
        SpecificArtifact: ${{ parameters.SpecificArtifact }}
        BuildId: ${{ parameters.BuildId }}

    - template: templates/get-docker-image-steps.yml
      parameters:
        Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cuda
        Context: tools/ci_build/github/linux/docker
        DockerBuildArgs: "--build-arg BASEIMAGE=$(docker_base_image) --build-arg BUILD_UID=$( id -u )"
        Repository: $(Repository)

    - task: CmdLine@2
      inputs:
        script: |
          set -e -x
          mkdir -p $HOME/.onnx
          docker run --gpus all --rm \
            --volume  $(Build.SourcesDirectory):/onnxruntime_src \
            --volume $(Build.BinariesDirectory)/Release:/build/Release \
            --volume /data/models:/build/models:ro \
            --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
            --volume /data/onnx:/data/onnx \
            -e NVIDIA_TF32_OVERRIDE=0 \
            $(Repository) \
            /bin/bash -c '
              set -e
              nvidia-smi; \
              /sbin/ldconfig -N -v $(sed "s/:/ /" <<< $LD_LIBRARY_PATH) 2>/dev/null | grep -E "libcudart.so|libcudnn.so|libnvinfer.so"; \
              cat /usr/local/cuda/include/cuda.h | grep -m1 CUDA_VERSION; \
              cat /usr/include/cudnn_version.h | grep CUDNN_MAJOR -m1 -A 2; \
              export PATH=/opt/python/cp312-cp312/bin:$PATH; \
              python3 -m pip install /build/Release/dist/*.whl; \
              python3 -u -c "from onnxruntime.capi._pybind_state import (OrtDevice as C_OrtDevice) ; \
                        ort_device = C_OrtDevice(C_OrtDevice.cuda(), C_OrtDevice.default_memory(), 0); \
                        print(ort_device); print(ort_device.device_type(), C_OrtDevice.cuda()); \
                        assert(ort_device.device_type()==1); assert(C_OrtDevice.cuda()==1);" \
            '
      displayName: 'Check GPU'

    - task: CmdLine@2
      inputs:
        script: |
          set -e -x
          mkdir -p $HOME/.onnx
          docker run --gpus all --shm-size=1g --ipc=host --ulimit memlock=-1 --ulimit stack=67108864 --rm \
            --volume  $(Build.SourcesDirectory):/onnxruntime_src \
            --volume $(Build.BinariesDirectory)/Release:/build/Release \
            --volume /data/models:/build/models:ro \
            --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
            --volume /data/onnx:/data/onnx \
            -e NVIDIA_TF32_OVERRIDE=0 \
            $(Repository) \
            /bin/bash -c '
              set -ex; \
              cp /onnxruntime_src/tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt /tmp/requirements.txt; \
              export PATH=/opt/python/cp312-cp312/bin:$PATH; \
              python3 -m pip install -r /tmp/requirements.txt; \
              python3 -m pip install /build/Release/dist/*.whl; \
              cd /build/Release && xargs -a /build/Release/perms.txt chmod a+x; \
              cd /onnxruntime_src/java && /onnxruntime_src/java/gradlew cmakeCheck -DcmakeBuildDir=/build/Release -DUSE_CUDA=1; \
              cd /tmp; \
              python3 /onnxruntime_src/tools/ci_build/build.py \
                --build_dir /build --config Release --test --skip_submodule_sync --build_shared_lib --parallel --use_binskim_compliant_compile_flags --build_wheel --enable_onnx_tests \
                --enable_transformers_tool_test --use_cuda --cuda_version=${{parameters.CudaVersion}} --cuda_home=/usr/local/cuda --cudnn_home=/usr/local/cuda \
                --enable_pybind --build_java --ctest_path "" ; \
              '
      displayName: 'Run Tests'

    - template: templates/check_test_result.yml
      parameters:
        FileName: '$(Build.BinariesDirectory)/Release/onnxruntime_test_all.Release.results.xml'

    - template: templates/clean-agent-build-directory-step.yml