From dd8ef4409ae6cbb7bf92fe9858bf56844ccbe787 Mon Sep 17 00:00:00 2001
From: liqunfu <liqfu@microsoft.com>
Date: Wed, 17 Feb 2021 17:48:47 -0800
Subject: [PATCH] Liqun/migrate perf test (#6733)

move ort training perf tests to azure devops
---
 ...aining-linux-gpu-perf-test-ci-pipeline.yml | 18 +++++++++--
 ...azure_scale_set_vm_mount_gpt2_test_data.sh | 30 +++++++++++++++++++
 2 files changed, 45 insertions(+), 3 deletions(-)
 create mode 100755 tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_gpt2_test_data.sh

diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-perf-test-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-perf-test-ci-pipeline.yml
index 9cc888018a..f84c966546 100644
--- a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-perf-test-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-perf-test-ci-pipeline.yml
@@ -25,11 +25,19 @@ jobs:
         "
       DisplayName: 'Build performance tests'
 
+  - bash: tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_test_data.sh $(orttrainingtestdata-storage-key)
+    displayName: 'Map bert test data'
+    condition: succeededOrFailed() # ensure all tests are run
+
+  - bash: tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_gpt2_test_data.sh $(orttrainingtestdata-storage-key)
+    displayName: 'Map gpt2 test data'
+    condition: succeededOrFailed() # ensure all tests are run
+
   - script: >
       docker run --gpus all --rm --name onnxruntime-gpu-perf
       --volume $(Build.SourcesDirectory):/onnxruntime_src
       --volume $(Build.BinariesDirectory):/build
-      --volume /bert_ort/bert_models:/build/bert_models:ro
+      --volume /bert_data/bert_models:/build/bert_models:ro
       --volume /bert_data:/build/bert_data:ro
       -e NIGHTLY_BUILD onnxruntime_perf_test_image
       /usr/bin/python3.6 /onnxruntime_src/orttraining/tools/ci_test/run_bert_perf_test.py
@@ -37,19 +45,23 @@ jobs:
       --training_data_root /build/bert_data
       --model_root /build/bert_models
     displayName: 'Run bert performance tests'
+    condition: succeededOrFailed()
+    timeoutInMinutes: 120
 
   - script: >
       docker run --gpus all --rm --name onnxruntime-gpu-perf
       --volume $(Build.SourcesDirectory):/onnxruntime_src
       --volume $(Build.BinariesDirectory):/build
-      --volume /bert_ort/gpt2_models:/build/gpt2_models:ro
-      --volume /bert_data/gpt2_data:/build/gpt2_data:ro
+      --volume /gpt2_data/gpt2_models:/build/gpt2_models:ro
+      --volume /gpt2_data:/build/gpt2_data:ro
       -e NIGHTLY_BUILD onnxruntime_perf_test_image
       /usr/bin/python3.6 /onnxruntime_src/orttraining/tools/ci_test/run_gpt2_perf_test.py
       --binary_dir /build/RelWithDebInfo
       --training_data_root /build/gpt2_data
       --model_root /build/gpt2_models
     displayName: 'Run gpt-2 performance tests'
+    condition: succeededOrFailed()
+    timeoutInMinutes: 120
 
   # generate jdbc.properties
   - script: >
diff --git a/tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_gpt2_test_data.sh b/tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_gpt2_test_data.sh
new file mode 100755
index 0000000000..8604e147b5
--- /dev/null
+++ b/tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_gpt2_test_data.sh
@@ -0,0 +1,30 @@
+#! /usr/bin/env bash
+if [ -d "/gpt2_data" ]; then
+sudo umount /gpt2_data
+fi
+if [ -d "/gpt2_data" ]; then
+sudo rm -rf /gpt2_data
+fi
+
+sudo mkdir /gpt2_data
+
+if [ ! -d "/etc/smbcredentials" ]; then
+sudo mkdir /etc/smbcredentials
+fi
+
+if [ -f "/etc/smbcredentials/orttrainingtestdata.cred" ]; then
+sudo rm /etc/smbcredentials/orttrainingtestdata.cred
+fi
+
+# to create orttrainingtestdata.cred, I have to do: 'sudo bash -c ...'
+sudo bash -c 'echo "username=orttrainingtestdata" >> /etc/smbcredentials/orttrainingtestdata.cred'
+
+# $1 get removed (do defend injection attack?) if I do 'sudo bash -c...'
+# to enable 'sudo echo...' I need to 'sudo chmod 777...' first.
+sudo chmod 777 /etc/smbcredentials/orttrainingtestdata.cred
+sudo echo "password=$1" >> /etc/smbcredentials/orttrainingtestdata.cred
+
+sudo chmod 600 /etc/smbcredentials/orttrainingtestdata.cred
+
+sudo bash -c 'echo "//orttrainingtestdata.file.core.windows.net/gpt2-data /gpt2_data cifs nofail,vers=3.0,credentials=/etc/smbcredentials/orttrainingtestdata.cred,dir_mode=0777,file_mode=0777,serverino" >> /etc/fstab'
+sudo mount -t cifs //orttrainingtestdata.file.core.windows.net/gpt2-data /gpt2_data -o vers=3.0,credentials=/etc/smbcredentials/orttrainingtestdata.cred,dir_mode=0777,file_mode=0777,serverino