parameters:

- name: PostToDashboard
  displayName: Post EP Perf results to Dashboard
  type: boolean
  default: true

- name: TrtVersion
  displayName: TensorRT Version
  type: string
  default: 10.5.cuda_12_5_cudnn_9
  values:
  - 8.6.cuda_11_8_cudnn_8
  - 8.6.cuda_12_3_cudnn_9
  - 10.5.cuda_11_8_cudnn_8
  - 10.5.cuda_12_5_cudnn_9
  - BIN

- name: UseTensorrtOssParser
  displayName: Use TensorRT-OSS Parser (not compatible with BIN)
  type: boolean
  default: false

- name: ModelGroups
  type: object
  default:
    - "onnx-zoo-models"
    - "partner-models"

- name: MemTest
  displayName: Run Memory and Concurrency Test
  type: boolean
  default: true

- name: ConcurrencyTest
  displayName: Specifies the number of concurrency model test to invoke simultaneously
  type: string
  default: 2

- name: TrtEPOptions
  displayName: TensorRT EP options
  type: object
  default:
    - trt_max_workspace_size=4294967296
    - trt_engine_cache_enable=True

- name: CUDAEPOptions
  displayName: CUDA EP options
  type: object
  default: []

jobs:
- job: 'Onnxruntime_Linux_GPU_TensorRT_Perf'
  workspace:
    clean: all
  timeoutInMinutes: 3000
  pool: 'Onnxruntime-Linux-GPU-TensorRT-Perf'

  variables:
    - name: branchName
      ${{ if not(eq(variables['ortBranchOverride'], '')) }}:
        value: $(ortBranchOverride)
      ${{ else }}:
        value: $[ replace(variables['Build.SourceBranch'], 'refs/heads/', '') ]

    - name: trtVersion
      ${{ if eq(parameters.TrtVersion, 'BIN') }}:
        value: $(tarTrtVersion)
      ${{ else }}:
        value: ${{ parameters.TrtVersion }}

    - name: trtEPOptionsArg
      ${{ if not(eq(length(parameters.TrtEPOptions), 0)) }}:
        value: --trt_ep_options ${{ join(',',parameters.TrtEPOptions) }}

    - name: cudaEPOptionsArg
      ${{ if not(eq(length(parameters.CUDAEPOptions), 0)) }}:
        value: --cuda_ep_options ${{ join(',',parameters.CUDAEPOptions) }}

    - name: optional_arguments
      value: -a "-a -z -g $(optimizeGraph) -b $(bindInputs) $(trtEPOptionsArg) $(cudaEPOptionsArg)"

    - name: image
      value: ort-image-$(Build.BuildId)

    - name: parser
      ${{ if eq(parameters.UseTensorrtOssParser, true) }}:
        value: --use_tensorrt_oss_parser $(parameters.UseTensorrtOssParser) }}

  steps:
    - ${{ if and(eq(parameters.TrtVersion, 'BIN'), eq(parameters.UseTensorrtOssParser, false)) }}:
      - script: 'ls -al $(trtBinsDir)'
        displayName: 'Show available TensorRT .tar.gz packages'

      - script: 'cp $(trtBinsDir)/TensorRT-$(trtVersion).Linux.x86_64-gnu.cuda-$(tarCudaVersion).tar.gz $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/build/'
        displayName: 'Copy TensorRT .tar.gz package into Docker build directory'

      - script: 'python3 $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/build/build_image.py -r $(Build.SourcesDirectory) -i $(image) -b $(branchName) -t $(trtVersion) -a 75 --install_bin --tar_cuda_version=$(tarCudaVersion) --trt_bins_dir=.'
        displayName: 'Install TensorRT $(tarTrtVersion) from binaries and build latest ORT Image'
        workingDirectory: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/build'

    # Build ORT with TensorRT built-in parser
    - ${{ if and(ne(parameters.TrtVersion, 'BIN'), eq(parameters.UseTensorrtOssParser, false)) }}:
      - script: 'python3 $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/build/build_image.py -r $(Build.SourcesDirectory) -i $(image) -b $(branchName) -t $(trtVersion) -a 75'
        displayName: 'Build latest ORT Image with TensorRT built-in parser'
        workingDirectory: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/build'

    # Build ORT with certain version of TensorRT OSS parser
    - ${{ if and(ne(parameters.TrtVersion, 'BIN'), eq(parameters.UseTensorrtOssParser, true), ne(variables.onnxTensorrtCommitId, '')) }}:
      - script: 'python3 $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/build/build_image.py -r $(Build.SourcesDirectory) -i $(image) -b $(branchName) -t $(trtVersion) -a 75 -o $(onnxTensorrtCommitId) --use_tensorrt_oss_parser'
        displayName: 'Build latest ORT Image with TensorRT OSS parser and commit id $(onnxTensorrtCommitId)'
        workingDirectory: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/build'

    # Build ORT with default version of TensorRT OSS parser
    - ${{ if and(ne(parameters.TrtVersion, 'BIN'), eq(parameters.UseTensorrtOssParser, true), eq(variables.onnxTensorrtCommitId, '')) }}:
      - script: 'python3 $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/build/build_image.py -r $(Build.SourcesDirectory) -i $(image) -b $(branchName) -t $(trtVersion) -a 75 --use_tensorrt_oss_parser'
        displayName: 'Build latest ORT Image with TensorRT OSS parser'
        workingDirectory: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/build'

    - ${{ if eq(parameters.MemTest, true) }}:
      - script: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/mem_test/run_mem_test_docker.sh -d $(image) -p $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/mem_test/ -w /code/ -l false -c ${{ parameters.ConcurrencyTest }}'
        displayName: 'Run Memory Test and Concurrency Test'
        workingDirectory: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/mem_test/'

    - ${{ each option in parameters.ModelGroups }}:
      - script: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/run_perf_docker.sh -d $(image) -p $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf -v $(modelVolume) -b true -o ${{option}} -m $(${{option}}) -e "$(epList)" $(optional_arguments)'
        displayName: '${{option}} perf'
        workingDirectory: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/'

    # Prepare and Publish Artifacts

    - script: 'mkdir $(Build.SourcesDirectory)/Artifact'
      displayName: 'Prepare Artifacts Directory'
      workingDirectory: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/'
      condition: always()

    - ${{ if not(eq(length(parameters.ModelGroups), 0)) }}:
      - script: 'cp -r $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/result/ $(Build.SourcesDirectory)/Artifact'
        displayName: 'Copy Artifacts'
        workingDirectory: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/'

    - ${{ if eq(parameters.MemTest, true) }}:
      - script: 'cp -r $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/mem_test/build/result $(Build.SourcesDirectory)/Artifact/result_mem_test'
        displayName: 'Copy Artifacts'
        workingDirectory: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/'
        condition: always()

      - script: 'python3 -m pip install pandas azure-kusto-data[pandas] azure-kusto-ingest[pandas] coloredlogs'
        displayName: 'Install dashboard dependencies'

      - script: |
          az --version || {
              echo "Azure CLI not found, installing..."
              curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
          }
        displayName: 'Check and Install Azure CLI'

      - task: AzureCLI@2
        displayName: 'Parse Memory & Concurrency Test Records and Sync'
        inputs:
          azureSubscription: AIInfraBuildOnnxRuntimeOSS
          scriptLocation: inlineScript
          scriptType: bash
          inlineScript: |
            short_hash=$(git rev-parse --short HEAD) &&
            commit_date=$(git log -1 --date=iso-strict --pretty=format:%cd) &&
            python3 $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/parse_mem_concurrency_test.py -r $(Build.SourcesDirectory)/Artifact/result_mem_test -c $short_hash -d $commit_date -u "$(reportUrl)?buildId=$(Build.BuildId)" -t $(trtVersion) -b $(branchName) --kusto_conn $(kustoConn) --database $(database) $(parser)

    - ${{ if eq(parameters.PostToDashboard, true) }}:

      - script: 'python3 -m pip install pandas azure-kusto-data[pandas] azure-kusto-ingest[pandas] coloredlogs'
        displayName: 'Install dashboard dependencies'

      - script: |
          az --version || {
              echo "Azure CLI not found, installing..."
              curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
          }
        displayName: 'Check and Install Azure CLI'

      - task: AzureCLI@2
        displayName: 'Azure CLI Post to Dashboard'
        inputs:
          azureSubscription: AIInfraBuildOnnxRuntimeOSS
          scriptLocation: inlineScript
          scriptType: bash
          inlineScript: |
            short_hash=$(git rev-parse --short HEAD) &&
            commit_date=$(git log -1 --date=iso-strict --pretty=format:%cd) &&
            python3 $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/post.py -r $(Build.SourcesDirectory)/Artifact/result -c $short_hash -d $commit_date -u "$(reportUrl)?buildId=$(Build.BuildId)" -t $(trtVersion) -b $(branchName) --kusto_conn $(kustoConn) --database $(database) $(parser)

    - task: PublishBuildArtifacts@1
      inputs:
        pathtoPublish: '$(Build.SourcesDirectory)/Artifact'
        artifactName: 'result-$(Build.BuildNumber)'

    - template: templates/component-governance-component-detection-steps.yml
      parameters :
        condition : 'succeeded'

    - template: templates/clean-agent-build-directory-step.yml