onnxruntime/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-daily-perf-pipeline.yml

parameters:

- name: BuildORT
  displayName: Build ORT
  type: boolean
  default: true

- name: PostToDashboard
  displayName: Post to Dashboard
  type: boolean
  default: true

- name: RunNvidiaContainer
  displayName: Run in Nvidia Container (or set trtContainer Variable)
  type: boolean
  default: false

- name: TrtVersion
  displayName: TensorRT Version
  type: string
  default: 8.4.1.5
  values:
  - 8.2.1.8
  - 8.0.1.6
  - 8.4.1.5

- name: ConfigureEPs
  displayName: Configure EPs (set epList variable - separate by spaces)
  type: boolean
  default: false

- name: ModelGroups
  type: object
  default: 
    - "onnx-zoo-models"
    - "partner-models"

- name: MemTest
  displayName: Run Memory Test
  type: boolean
  default: false

- name: TrtEPOptions
  displayName: TensorRT EP options
  type: object
  default:
    - trt_max_workspace_size=4294967296
    - trt_engine_cache_enable=True

- name: CUDAEPOptions
  displayName: CUDA EP options
  type: object
  default: []

jobs:
- job: 'Onnxruntime_Linux_GPU_TensorRT_Perf'
  workspace:
    clean: all
  timeoutInMinutes: 3000
  pool: 'Onnxruntime-Linux-GPU-TensorRT-Perf'
  
  variables:
    - name: trtContainerVersion
      ${{ if eq(parameters.RunNvidiaContainer, true)}}:
        ${{ if eq(parameters.TrtVersion, '8.2.1.8') }}:
          value: 21.12
        ${{ elseif eq(parameters.TrtVersion, '8.0.1.6') }}:
          value: 21.07
      
    - name: nvidia_dockerfile
      ${{ if eq(parameters.RunNvidiaContainer, true)}}:
        value: nvcr.io/nvidia/tensorrt:$(trtContainerVersion)-py3
      

    - name: dockerfile_path
      ${{ if eq(parameters.RunNvidiaContainer, true) }}:
        value: dockerfiles
      ${{ else }}:
        value: tools/ci_build/github/linux/docker

    - name: dockerfile
      ${{ if eq(parameters.RunNvidiaContainer, true) }}:
        value: $(dockerfile_path)/Dockerfile.tensorrt
      ${{ else }}:
        ${{ if eq(parameters.TrtVersion, '8.4.1.5') }}:
          value: $(dockerfile_path)/Dockerfile.ubuntu_cuda11_6_tensorrt8_4
        ${{ elseif eq(parameters.TrtVersion, '8.2.1.8') }}:
          value: $(dockerfile_path)/Dockerfile.ubuntu_cuda11_4_tensorrt8_2
        ${{ elseif eq(parameters.TrtVersion, '8.0.1.6') }}:
          value: $(dockerfile_path)/Dockerfile.ubuntu_cuda11_4_tensorrt8_0

    - name: image
      ${{ if eq(parameters.BuildORT, true) }}:
        value: ort-$(branch)
      ${{ else }}:
        value: $(nvidia_dockerfile)
      
    - name: tensorrt_version
      ${{ if eq(parameters.RunNvidiaContainer, true) }}:
        value: $(trtContainerVersion)
      ${{ else }}:
        value: ${{ parameters.TrtVersion }}

    - name: trtEPOptionsArg
      ${{ if not(eq(length(parameters.TrtEPOptions), 0)) }}:
        value: --trt_ep_options ${{ join(',',parameters.TrtEPOptions) }}

    - name: cudaEPOptionsArg
      ${{ if not(eq(length(parameters.CUDAEPOptions), 0)) }}:
        value: --cuda_ep_options ${{ join(',',parameters.CUDAEPOptions) }}

    - name: optional_arguments
      value: -a "-a -g $(optimizeGraph) -b $(bindInputs) $(trtEPOptionsArg) $(cudaEPOptionsArg)"

  steps:
    - ${{ if and( eq(parameters.BuildORT, false), eq(parameters.TrtVersion, '8.2.1.8')) }}:
      - task: DownloadPipelineArtifact@2
        inputs:
          buildType: 'specific'
          project: 'Lotus'
          preferTriggeringPipeline: true 
          pipeline: '841'
          runBranch: 'ref/heads/master'
          buildVersionToDownload: 'latest'
          allowPartiallySucceededBuilds: false
          artifact: 'onnxruntime_gpu'
          targetPath: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/'
      
      - ${{ if eq(parameters.RunNvidiaContainer, true) }}:
        - script: 'docker pull $(nvidia_dockerfile)'
          displayName: 'Pull Nvidia Dockerfile'
      - ${{ else }}:
        - script: 'Can only run pip package in Nvidia Container'
          displayName: 'Please Re-run'

    - ${{ if eq(parameters.BuildORT, true) }}:
      - script: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/build/build_image.sh -o $(Build.SourcesDirectory)/$(dockerfile) -b $(branch) -t $(tensorrt_version) -i $(image) -c 75 '
        displayName: 'Build latest ORT Image'
        workingDirectory: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/build'

    - ${{ each option in parameters.ModelGroups }}:
      - script: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/run_perf_docker.sh -d $(image) -p $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf -v $(modelVolume) -b ${{ parameters.BuildORT }} -o ${{option}} -m $(${{option}}) -e "$(epList)" $(optional_arguments)'
        displayName: '${{option}} perf'
        workingDirectory: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/'
        
    - ${{ if eq(parameters.MemTest, true) }}:
      - script: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/mem_test/run_mem_test_docker.sh -d $(image) -p $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/mem_test/ -w /code/ -l false'
        displayName: 'Run Memory Test'
        workingDirectory: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/mem_test/'

    # Prepare and Publish Artifacts 
    
    - script: 'mkdir $(Build.SourcesDirectory)/Artifact'
      displayName: 'Prepare Artifacts Directory'
      workingDirectory: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/'
      condition: always() 

    - ${{ if not(eq(length(parameters.ModelGroups), 0)) }}:
      - script: 'cp -r $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/result/ $(Build.SourcesDirectory)/Artifact'
        displayName: 'Copy Artifacts'
        workingDirectory: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/'

    - ${{ if eq(parameters.MemTest, true) }}:
      - script: 'cp -r $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/mem_test/build/result $(Build.SourcesDirectory)/Artifact/result_mem_test'
        displayName: 'Copy Artifacts'
        workingDirectory: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/'
        condition: always()

    - task: PublishBuildArtifacts@1
      inputs:
        pathtoPublish: '$(Build.SourcesDirectory)/Artifact'
        artifactName: 'result-$(Build.BuildNumber)'
        
    - ${{ if eq(parameters.PostToDashboard, true) }}:

      - script: 'python3 -m pip install pandas azure-kusto-data[pandas] azure-kusto-ingest[pandas] coloredlogs'
        displayName: 'Install dashboard dependencies'

      - task: AzureCLI@2
        displayName: 'Azure CLI Post to Dashboard'
        inputs:
          azureSubscription: AIInfraBuildOnnxRuntimeOSS
          scriptLocation: inlineScript
          scriptType: bash
          inlineScript: |
            short_hash=$(git rev-parse --short HEAD) &&
            commit_date=$(git log -1 --date=iso-strict --pretty=format:%cd) &&
            python3 $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/post.py -r $(Build.SourcesDirectory)/Artifact/result -c $short_hash -d $commit_date -u "$(reportUrl)?buildId=$(Build.BuildId)" -t ${{ parameters.TrtVersion }} -b $(branch) --kusto_conn $(kustoConn) --database $(database)            
    
    - template: templates/component-governance-component-detection-steps.yml
      parameters :
        condition : 'succeeded'
        
    - template: templates/clean-agent-build-directory-step.yml