mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-30 20:18:08 +00:00
90 lines
3.6 KiB
YAML
90 lines
3.6 KiB
YAML
trigger: none
|
|
|
|
jobs:
|
|
- job: Onnxruntime_Linux_GPU_Training_Perf_Test
|
|
|
|
timeoutInMinutes: 120
|
|
|
|
variables:
|
|
- group: 'ortperf' # variable group
|
|
|
|
steps:
|
|
- checkout: self
|
|
clean: true
|
|
submodules: recursive
|
|
|
|
- template: templates/run-docker-build-steps.yml
|
|
parameters:
|
|
RunDockerBuildArgs: >
|
|
-o ubuntu20.04 -d gpu
|
|
-t onnxruntime_perf_test_image
|
|
-x "
|
|
--config RelWithDebInfo
|
|
--enable_training
|
|
--update --build --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=70
|
|
"
|
|
DisplayName: 'Build performance tests'
|
|
|
|
- bash: tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_test_data.sh -p $(orttrainingtestdatascus-storage-key) -s "//orttrainingtestdatascus.file.core.windows.net/bert-data" -d "/bert_data"
|
|
displayName: 'Mount bert-data'
|
|
condition: succeededOrFailed() # ensure all tests are run
|
|
|
|
- bash: tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_test_data.sh -p $(orttrainingtestdatascus-storage-key) -s "//orttrainingtestdatascus.file.core.windows.net/gpt2-data" -d "/gpt2_data"
|
|
displayName: 'Mount gpt2 test data'
|
|
condition: succeededOrFailed() # ensure all tests are run
|
|
|
|
- script: >
|
|
docker run --gpus all --rm --name onnxruntime-gpu-perf
|
|
--volume $(Build.SourcesDirectory):/onnxruntime_src
|
|
--volume $(Build.BinariesDirectory):/build
|
|
--volume /bert_data/bert_models:/build/bert_models:ro
|
|
--volume /bert_data:/build/bert_data:ro
|
|
-e NIGHTLY_BUILD onnxruntime_perf_test_image
|
|
/usr/bin/python3 /onnxruntime_src/orttraining/tools/ci_test/run_bert_perf_test.py
|
|
--binary_dir /build/RelWithDebInfo
|
|
--training_data_root /build/bert_data
|
|
--model_root /build/bert_models
|
|
displayName: 'Run bert performance tests'
|
|
condition: succeededOrFailed()
|
|
timeoutInMinutes: 120
|
|
|
|
- script: >
|
|
docker run --gpus all --rm --name onnxruntime-gpu-perf
|
|
--volume $(Build.SourcesDirectory):/onnxruntime_src
|
|
--volume $(Build.BinariesDirectory):/build
|
|
--volume /gpt2_data/gpt2_models:/build/gpt2_models:ro
|
|
--volume /gpt2_data:/build/gpt2_data:ro
|
|
-e NIGHTLY_BUILD onnxruntime_perf_test_image
|
|
/usr/bin/python3 /onnxruntime_src/orttraining/tools/ci_test/run_gpt2_perf_test.py
|
|
--binary_dir /build/RelWithDebInfo
|
|
--training_data_root /build/gpt2_data
|
|
--model_root /build/gpt2_models
|
|
displayName: 'Run gpt-2 performance tests'
|
|
condition: succeededOrFailed()
|
|
timeoutInMinutes: 120
|
|
|
|
# generate jdbc.properties
|
|
- script: >
|
|
mkdir -p $(Build.SourcesDirectory)/tools/perf_util/src/main/resources &&
|
|
printf "url=jdbc:mysql://onnxruntimedashboard.mysql.database.azure.com/onnxruntime?serverTimezone=UTC&useUnicode=true&characterEncoding=UTF-8\nuser=powerbi@onnxruntimedashboard\npassword_env=ORT_PERF_PASSWORD"
|
|
> $(Build.SourcesDirectory)/tools/perf_util/src/main/resources/jdbc.properties
|
|
displayName: 'Create resource file'
|
|
|
|
- script: >
|
|
mvn package
|
|
displayName: 'Maven build'
|
|
workingDirectory: $(Build.SourcesDirectory)/tools/perf_util
|
|
|
|
# process json files
|
|
- script: >
|
|
java -cp target/send_perf_metrics-0.0.1-SNAPSHOT-jar-with-dependencies.jar com.msft.send_perf_metrics.App "$(Build.SourcesDirectory)/orttraining/tools/ci_test/results"
|
|
env:
|
|
ORT_PERF_PASSWORD: $(ortperf)
|
|
displayName: 'Populate perf metrics'
|
|
workingDirectory: $(Build.SourcesDirectory)/tools/perf_util
|
|
|
|
- template: templates/component-governance-component-detection-steps.yml
|
|
parameters:
|
|
condition: 'succeeded'
|
|
|
|
- template: templates/clean-agent-build-directory-step.yml
|