onnxruntime/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml

##### start trigger Don't edit it manually, Please do edit set-trigger-rules.py ####
trigger:
  branches:
    include:
    - main
    - rel-*
  paths:
    exclude:
    - docs/**
    - README.md
    - CONTRIBUTING.md
    - BUILD.md
    - 'js/web'
    - 'onnxruntime/core/providers/js'
pr:
  branches:
    include:
    - main
    - rel-*
  paths:
    exclude:
    - docs/**
    - README.md
    - CONTRIBUTING.md
    - BUILD.md
    - 'js/web'
    - 'onnxruntime/core/providers/js'
#### end trigger ####

resources:
  repositories:
  - repository: manylinux # The name used to reference this repository in the checkout step
    type: Github
    endpoint: Microsoft
    name: pypa/manylinux
    ref: 5eda9aded5462201e6310105728d33016e637ea7
stages:
- stage: x64
  dependsOn: []
  jobs:
    - job: Linux_Build
      timeoutInMinutes: 180
      workspace:
        clean: all
      variables:
        skipComponentGovernanceDetection: true
        ORT_CACHE_DIR: $(Agent.TempDirectory)/ort_ccache
        TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)]
      pool: onnxruntime-Ubuntu2004-AMD-CPU
      steps:
      - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
        displayName: 'Clean Agent Directories'
        condition: always()

      - checkout: self
        clean: true
        submodules: none

      - task: NodeTool@0
        inputs:
          versionSpec: '16.x'

      - task: UsePythonVersion@0
        inputs:
          versionSpec: '3.8'
          addToPath: true

      - template: templates/get-docker-image-steps.yml
        parameters:
          Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cpu
          Context: tools/ci_build/github/linux/docker/
          DockerBuildArgs: "--build-arg BUILD_UID=$( id -u ) --build-arg BASEIMAGE=registry.access.redhat.com/ubi8/ubi"
          Repository: onnxruntimecpubuild

      - template: templates/linux-build-step-with-cache.yml
        parameters:
          WithCache: true
          Today: $(TODAY)
          AdditionalKey: onnxruntime
          CacheDir: $(ORT_CACHE_DIR)
          ChangeEveryCommit: true
          BuildStep:
            - task: CmdLine@2
              displayName: 'build'
              inputs:
                script: |
                  mkdir -p $HOME/.onnx
                  docker run --rm \
                    --volume /data/onnx:/data/onnx:ro \
                    --volume $(Build.SourcesDirectory):/onnxruntime_src \
                    --volume $(Build.BinariesDirectory):/build \
                    --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
                    --volume $(ORT_CACHE_DIR):/cache \
                    -e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
                    -e NIGHTLY_BUILD \
                    -e BUILD_BUILDNUMBER \
                    -e CCACHE_DIR=/cache \
                    onnxruntimecpubuild \
                    /bin/bash -c "
                      set -ex; \
                      ccache -s; \
                      /opt/python/cp38-cp38/bin/python3 /onnxruntime_src/tools/ci_build/build.py \
                        --build_dir /build --cmake_generator 'Ninja' \
                        --config Debug Release \
                        --skip_submodule_sync \
                        --build_shared_lib \
                        --parallel \
                        --build_wheel \
                        --build_csharp \
                        --enable_onnx_tests \
                        --enable_transformers_tool_test \
                        --use_cache \
                        --update --build --cmake_extra_defines onnxruntime_BUILD_BENCHMARKS=ON; \
                      ccache -sv; \
                      ccache -z"                  
                workingDirectory: $(Build.SourcesDirectory)

      - script: |
          ln -s /data/models $(Build.BinariesDirectory)/models          
        displayName: link model dir

      - bash: |
          mkdir -p $HOME/.onnx
          docker run --rm \
            --volume /data/onnx:/data/onnx:ro \
            --volume $(Build.SourcesDirectory):/onnxruntime_src \
            --volume $(Build.BinariesDirectory):/build \
            --volume /data/models:/build/models:ro \
            --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
            -e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
            -e NIGHTLY_BUILD \
            -e BUILD_BUILDNUMBER \
            onnxruntimecpubuild \
            /bin/bash -c "
              set -ex; \
              pushd /onnxruntime_src/csharp; \
              dotnet restore /onnxruntime_src/csharp/OnnxRuntime.DesktopOnly.CSharp.sln; \
              dotnet build /onnxruntime_src/csharp/OnnxRuntime.DesktopOnly.CSharp.sln; \
              dotnet test /onnxruntime_src/csharp/OnnxRuntime.DesktopOnly.CSharp.sln -f net6.0 --no-build -l \"console;verbosity=normal\"; \
              popd
              "          
        displayName: 'Dotnet build C# sln and Test'

      - bash: |
          mkdir -p $HOME/.onnx
          docker run --rm \
            --volume /data/onnx:/data/onnx:ro \
            --volume $(Build.SourcesDirectory):/onnxruntime_src \
            --volume $(Build.BinariesDirectory):/build \
            --volume /data/models:/build/models:ro \
            --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
            -e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
            -e NIGHTLY_BUILD \
            -e BUILD_BUILDNUMBER \
            onnxruntimecpubuild \
              /bin/bash -c "
                set -ex; \
                /bin/bash /onnxruntime_src/tools/scripts/python_test.sh /onnxruntime_src /build Release && \
                /bin/bash /onnxruntime_src/tools/scripts/symbolic_shape_infer_test.sh /build
              "          
        displayName: 'Run Release tests and symbolic shape infer test'

      - bash: |
          mkdir -p $HOME/.onnx
          docker run --rm \
            --volume /data/onnx:/data/onnx:ro \
            --volume $(Build.SourcesDirectory):/onnxruntime_src \
            --volume $(Build.BinariesDirectory):/build \
            --volume /data/models:/build/models:ro \
            --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
            -e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
            -e NIGHTLY_BUILD \
            -e BUILD_BUILDNUMBER \
            onnxruntimecpubuild \
                /bin/bash /onnxruntime_src/tools/scripts/python_test.sh /onnxruntime_src /build Debug          
        displayName: 'Run Debug tests'

      - task: PublishTestResults@2
        displayName: 'Publish unit test results'
        inputs:
          testResultsFiles: '**/*.results.xml'
          searchFolder: '$(Build.BinariesDirectory)'
          testRunTitle: 'Unit Test Run'
        condition: succeededOrFailed()

- stage: arm64_build
  dependsOn: []
  jobs:
  - template: templates/py-linux.yml
    parameters:
      arch: 'aarch64'
      machine_pool: 'onnxruntime-linux-ARM64-CPU-2019'
      base_image: 'arm64v8/almalinux:8'
      devtoolset_rootpath: /opt/rh/gcc-toolset-12/root
      ld_library_path_arg: /opt/rh/gcc-toolset-12/root/usr/lib64:/opt/rh/gcc-toolset-12/root/usr/lib:/opt/rh/gcc-toolset-12/root/usr/lib64/dyninst:/opt/rh/gcc-toolset-12/root/usr/lib/dyninst:/usr/local/lib64
      prepend_path: '/opt/rh/gcc-toolset-12/root/usr/bin:'
      with_cache: true
      cmake_build_type: Release

- stage: arm64_test
  dependsOn: ['arm64_build']
  jobs:
  - template: templates/py-packaging-linux-test.yml
    parameters:
      arch: 'aarch64'
      machine_pool: 'onnxruntime-linux-ARM64-CPU-2019'
      device: 'CPU'