onnxruntime/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-e2e-test-ci-pipeline.yml

trigger: none

jobs:
- job: Onnxruntime_Linux_GPU_Training_E2E_Test

  timeoutInMinutes: 60

  steps:
  - checkout: self
    clean: true
    submodules: recursive

  # update these if the E2E test data changes
  - script: |
      orttraining/tools/ci_test/download_azure_blob_archive.py \
      --azure_blob_url https://onnxruntimetestdata.blob.core.windows.net/training/onnxruntime_training_data.zip?snapshot=2020-06-15T23:17:35.8314853Z \
      --target_dir $(Build.BinariesDirectory)/training_e2e_test_data \
      --archive_sha256_digest B01C169B6550D1A0A6F1B4E2F34AE2A8714B52DBB70AC04DA85D371F691BDFF9      
    displayName: 'Download onnxruntime_training_data.zip data'

  - template: templates/run-docker-build-steps.yml
    parameters:
      RunDockerBuildArgs: |
        -o ubuntu16.04 -d gpu -r $(Build.BinariesDirectory) \
        -t onnxruntime_e2e_test_image \
        -x " \
          --config RelWithDebInfo \
          --enable_training \
          --update --build \
          "        
      DisplayName: 'Build'

  - script: |
      docker run \
        --gpus all \
        --rm \
        --volume $(Build.SourcesDirectory):/onnxruntime_src \
        --volume $(Build.BinariesDirectory):/build \
        --volume $(Build.BinariesDirectory)/training_e2e_test_data:/training_e2e_test_data:ro \
        onnxruntime_e2e_test_image \
          /onnxruntime_src/orttraining/tools/ci_test/run_batch_size_test.py \
            --binary_dir /build/RelWithDebInfo \
            --model_root /training_e2e_test_data/models      
    displayName: 'Run batch size test'
    condition: succeededOrFailed() # ensure all tests are run

  - script: |
      docker run \
        --gpus all \
        --rm \
        --volume $(Build.SourcesDirectory):/onnxruntime_src \
        --volume $(Build.BinariesDirectory):/build \
        --volume $(Build.BinariesDirectory)/training_e2e_test_data:/training_e2e_test_data:ro \
        onnxruntime_e2e_test_image \
          /onnxruntime_src/orttraining/tools/ci_test/run_convergence_test.py \
            --binary_dir /build/RelWithDebInfo \
            --model_root /training_e2e_test_data/models \
            --training_data_root /training_e2e_test_data/data      
    displayName: 'Run convergence test'
    condition: succeededOrFailed() # ensure all tests are run

  - template: templates/clean-agent-build-directory-step.yml