Update the ONNX Runtime Server CI pipeline setup (#986)

* Update the ORT-SRV ci pipeline setup

* Update pip package installation for server tests

* Install requests package in build setup

* Check if python dependencies exists before install

tools/ci_build/build.py

@@ -572,7 +572,16 @@ def run_onnx_tests(build_dir, configs, onnx_test_data_dir, provider, enable_para
 
 
 def run_server_tests(build_dir, configs):
-    run_subprocess([sys.executable, '-m', 'pip', 'install', '--trusted-host', 'files.pythonhosted.org', 'requests', 'protobuf', 'numpy'])
+    pip_freeze_result = run_subprocess([sys.executable, '-m', 'pip', 'freeze'], capture=True).stdout
+    installed_packages = [r.decode().split('==')[0] for r in pip_freeze_result.split()]
+    if not (('requests' in installed_packages) and ('protobuf' in installed_packages) and ('numpy' in installed_packages)):
+        if hasattr(sys, 'real_prefix'):
+            # In virtualenv
+            run_subprocess([sys.executable, '-m', 'pip', 'install', '--trusted-host', 'files.pythonhosted.org', 'requests', 'protobuf', 'numpy'])
+        else:
+            # Outside virtualenv
+            run_subprocess([sys.executable, '-m', 'pip', 'install', '--user', '--trusted-host', 'files.pythonhosted.org', 'requests', 'protobuf', 'numpy'])
+    
     for config in configs:
         config_build_dir = get_config_build_dir(build_dir, config)
         if is_windows():

tools/ci_build/github/azure-pipelines/linux-ort-srv-ci-pipeline.yml

@@ -1,19 +1,79 @@
 jobs:
 - job: Debug_Build
-  pool: Hosted Ubuntu 1604
+  pool: Linux-CPU
   steps:
     - template: templates/set-test-data-variables-step.yml
-    - script: 'tools/ci_build/github/linux/server_run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -k $(acr.key) -x "--config Debug --build_server --use_openmp --use_full_protobuf --enable_server_tests"'
-      displayName: 'Debug Build'
+
+    - task: CmdLine@2
+      displayName: 'Clean untagged docker images'
+      inputs:
+        script: |
+          docker rm $(docker ps -a | grep Exited | awk '{print $1;}') || true
+          docker images -q --filter "dangling=true" | xargs -n1 -r docker rmi
+        workingDirectory: $(Build.BinariesDirectory)
+      continueOnError: true
+      condition: always()
+
+    - task: CmdLine@2
+      displayName: 'Download azcopy'
+      inputs:
+        script: |
+          curl -so azcopy.tar.gz -L 'https://aka.ms/downloadazcopy-v10-linux'
+          tar -zxvf azcopy.tar.gz --strip 1
+        workingDirectory: $(Build.BinariesDirectory)
+
+    - task: PythonScript@0
+      displayName: 'Download test data'
+      inputs:
+        scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py'
+        arguments: --test_data_url $(TestDataUrl)
+        pythonInterpreter: '/usr/bin/python3'
+        workingDirectory: $(Build.BinariesDirectory)
+
+    - script: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -x "--config Debug --build_server --use_openmp --use_full_protobuf --enable_server_tests"'
+      displayName: 'Run build script'
+
     - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
       displayName: 'Component Detection'
+      condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI'))
+
     - template: templates/clean-agent-build-directory-step.yml
 - job: Release_Build
-  pool: Hosted Ubuntu 1604
+  pool: Linux-CPU
   steps:
     - template: templates/set-test-data-variables-step.yml
-    - script: 'tools/ci_build/github/linux/server_run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -k $(acr.key) -x "--config Release --build_server --use_openmp --use_full_protobuf --enable_server_tests"'
-      displayName: 'Release Build'
+
+    - task: CmdLine@2
+      displayName: 'Clean untagged docker images'
+      inputs:
+        script: |
+          docker rm $(docker ps -a | grep Exited | awk '{print $1;}') || true
+          docker images -q --filter "dangling=true" | xargs -n1 -r docker rmi
+        workingDirectory: $(Build.BinariesDirectory)
+      continueOnError: true
+      condition: always()
+
+    - task: CmdLine@2
+      displayName: 'Download azcopy'
+      inputs:
+        script: |
+          curl -so azcopy.tar.gz -L 'https://aka.ms/downloadazcopy-v10-linux'
+          tar -zxvf azcopy.tar.gz --strip 1
+        workingDirectory: $(Build.BinariesDirectory)
+
+    - task: PythonScript@0
+      displayName: 'Download test data'
+      inputs:
+        scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py'
+        arguments: --test_data_url $(TestDataUrl)
+        pythonInterpreter: '/usr/bin/python3'
+        workingDirectory: $(Build.BinariesDirectory)
+
+    - script: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -x "--config Release --build_server --use_openmp --use_full_protobuf --enable_server_tests"'
+      displayName: 'Run build script'
+
     - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
       displayName: 'Component Detection'
-    - template: templates/clean-agent-build-directory-step.yml
+      condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI'))
+
+    - template: templates/clean-agent-build-directory-step.yml

tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh

@@ -67,6 +67,7 @@ fi
 
 /usr/bin/python${PYTHON_VER} -m pip install --upgrade --force-reinstall pip==19.0.3
 /usr/bin/python${PYTHON_VER} -m pip install --upgrade --force-reinstall numpy==1.15.0
+/usr/bin/python${PYTHON_VER} -m pip install --upgrade --force-reinstall requests==2.21.0
 rm -rf /var/lib/apt/lists/*
 
 mkdir -p /tmp/azcopy