From a7a2a16edddc283b53d7737f897b4bbda5e86209 Mon Sep 17 00:00:00 2001 From: baijumeswani Date: Mon, 22 Mar 2021 21:48:32 -0700 Subject: [PATCH] Pass arguments to azure_scale_set_vm_mount_test_data from perf test ci pipeline (#7094) --- ...aining-linux-gpu-perf-test-ci-pipeline.yml | 8 ++--- ...azure_scale_set_vm_mount_gpt2_test_data.sh | 30 ------------------- 2 files changed, 4 insertions(+), 34 deletions(-) delete mode 100755 tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_gpt2_test_data.sh diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-perf-test-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-perf-test-ci-pipeline.yml index 05795f92cf..77bca03e34 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-perf-test-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-perf-test-ci-pipeline.yml @@ -25,12 +25,12 @@ jobs: " DisplayName: 'Build performance tests' - - bash: tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_test_data.sh $(orttrainingtestdata-storage-key) - displayName: 'Map bert test data' + - bash: tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_test_data.sh -p $(orttrainingtestdata-storage-key) -s "//orttrainingtestdata.file.core.windows.net/bert-data" -d "/bert_data" + displayName: 'Mount bert-data' condition: succeededOrFailed() # ensure all tests are run - - bash: tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_gpt2_test_data.sh $(orttrainingtestdata-storage-key) - displayName: 'Map gpt2 test data' + - bash: tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_test_data.sh -p $(orttrainingtestdata-storage-key) -s "//orttrainingtestdata.file.core.windows.net/gpt2-data" -d "/gpt2_data" + displayName: 'Mount gpt2 test data' condition: succeededOrFailed() # ensure all tests are run - script: > diff --git a/tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_gpt2_test_data.sh b/tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_gpt2_test_data.sh deleted file mode 100755 index 8604e147b5..0000000000 --- a/tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_gpt2_test_data.sh +++ /dev/null @@ -1,30 +0,0 @@ -#! /usr/bin/env bash -if [ -d "/gpt2_data" ]; then -sudo umount /gpt2_data -fi -if [ -d "/gpt2_data" ]; then -sudo rm -rf /gpt2_data -fi - -sudo mkdir /gpt2_data - -if [ ! -d "/etc/smbcredentials" ]; then -sudo mkdir /etc/smbcredentials -fi - -if [ -f "/etc/smbcredentials/orttrainingtestdata.cred" ]; then -sudo rm /etc/smbcredentials/orttrainingtestdata.cred -fi - -# to create orttrainingtestdata.cred, I have to do: 'sudo bash -c ...' -sudo bash -c 'echo "username=orttrainingtestdata" >> /etc/smbcredentials/orttrainingtestdata.cred' - -# $1 get removed (do defend injection attack?) if I do 'sudo bash -c...' -# to enable 'sudo echo...' I need to 'sudo chmod 777...' first. -sudo chmod 777 /etc/smbcredentials/orttrainingtestdata.cred -sudo echo "password=$1" >> /etc/smbcredentials/orttrainingtestdata.cred - -sudo chmod 600 /etc/smbcredentials/orttrainingtestdata.cred - -sudo bash -c 'echo "//orttrainingtestdata.file.core.windows.net/gpt2-data /gpt2_data cifs nofail,vers=3.0,credentials=/etc/smbcredentials/orttrainingtestdata.cred,dir_mode=0777,file_mode=0777,serverino" >> /etc/fstab' -sudo mount -t cifs //orttrainingtestdata.file.core.windows.net/gpt2-data /gpt2_data -o vers=3.0,credentials=/etc/smbcredentials/orttrainingtestdata.cred,dir_mode=0777,file_mode=0777,serverino