From c2b412f7be037d309eb92ae58deba1ae43321a5c Mon Sep 17 00:00:00 2001 From: Klein Hu Date: Wed, 8 May 2019 11:37:39 -0700 Subject: [PATCH] Update the ONNX Runtime Server CI pipeline setup (#986) * Update the ORT-SRV ci pipeline setup * Update pip package installation for server tests * Install requests package in build setup * Check if python dependencies exists before install --- tools/ci_build/build.py | 11 ++- .../linux-ort-srv-ci-pipeline.yml | 74 +++++++++++++++++-- .../linux/docker/scripts/install_ubuntu.sh | 1 + 3 files changed, 78 insertions(+), 8 deletions(-) diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 1511e049a2..757fc539ef 100755 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -572,7 +572,16 @@ def run_onnx_tests(build_dir, configs, onnx_test_data_dir, provider, enable_para def run_server_tests(build_dir, configs): - run_subprocess([sys.executable, '-m', 'pip', 'install', '--trusted-host', 'files.pythonhosted.org', 'requests', 'protobuf', 'numpy']) + pip_freeze_result = run_subprocess([sys.executable, '-m', 'pip', 'freeze'], capture=True).stdout + installed_packages = [r.decode().split('==')[0] for r in pip_freeze_result.split()] + if not (('requests' in installed_packages) and ('protobuf' in installed_packages) and ('numpy' in installed_packages)): + if hasattr(sys, 'real_prefix'): + # In virtualenv + run_subprocess([sys.executable, '-m', 'pip', 'install', '--trusted-host', 'files.pythonhosted.org', 'requests', 'protobuf', 'numpy']) + else: + # Outside virtualenv + run_subprocess([sys.executable, '-m', 'pip', 'install', '--user', '--trusted-host', 'files.pythonhosted.org', 'requests', 'protobuf', 'numpy']) + for config in configs: config_build_dir = get_config_build_dir(build_dir, config) if is_windows(): diff --git a/tools/ci_build/github/azure-pipelines/linux-ort-srv-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-ort-srv-ci-pipeline.yml index 16d0fe9599..d83ffe9bdd 100644 --- a/tools/ci_build/github/azure-pipelines/linux-ort-srv-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-ort-srv-ci-pipeline.yml @@ -1,19 +1,79 @@ jobs: - job: Debug_Build - pool: Hosted Ubuntu 1604 + pool: Linux-CPU steps: - template: templates/set-test-data-variables-step.yml - - script: 'tools/ci_build/github/linux/server_run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -k $(acr.key) -x "--config Debug --build_server --use_openmp --use_full_protobuf --enable_server_tests"' - displayName: 'Debug Build' + + - task: CmdLine@2 + displayName: 'Clean untagged docker images' + inputs: + script: | + docker rm $(docker ps -a | grep Exited | awk '{print $1;}') || true + docker images -q --filter "dangling=true" | xargs -n1 -r docker rmi + workingDirectory: $(Build.BinariesDirectory) + continueOnError: true + condition: always() + + - task: CmdLine@2 + displayName: 'Download azcopy' + inputs: + script: | + curl -so azcopy.tar.gz -L 'https://aka.ms/downloadazcopy-v10-linux' + tar -zxvf azcopy.tar.gz --strip 1 + workingDirectory: $(Build.BinariesDirectory) + + - task: PythonScript@0 + displayName: 'Download test data' + inputs: + scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py' + arguments: --test_data_url $(TestDataUrl) + pythonInterpreter: '/usr/bin/python3' + workingDirectory: $(Build.BinariesDirectory) + + - script: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -x "--config Debug --build_server --use_openmp --use_full_protobuf --enable_server_tests"' + displayName: 'Run build script' + - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 displayName: 'Component Detection' + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')) + - template: templates/clean-agent-build-directory-step.yml - job: Release_Build - pool: Hosted Ubuntu 1604 + pool: Linux-CPU steps: - template: templates/set-test-data-variables-step.yml - - script: 'tools/ci_build/github/linux/server_run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -k $(acr.key) -x "--config Release --build_server --use_openmp --use_full_protobuf --enable_server_tests"' - displayName: 'Release Build' + + - task: CmdLine@2 + displayName: 'Clean untagged docker images' + inputs: + script: | + docker rm $(docker ps -a | grep Exited | awk '{print $1;}') || true + docker images -q --filter "dangling=true" | xargs -n1 -r docker rmi + workingDirectory: $(Build.BinariesDirectory) + continueOnError: true + condition: always() + + - task: CmdLine@2 + displayName: 'Download azcopy' + inputs: + script: | + curl -so azcopy.tar.gz -L 'https://aka.ms/downloadazcopy-v10-linux' + tar -zxvf azcopy.tar.gz --strip 1 + workingDirectory: $(Build.BinariesDirectory) + + - task: PythonScript@0 + displayName: 'Download test data' + inputs: + scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py' + arguments: --test_data_url $(TestDataUrl) + pythonInterpreter: '/usr/bin/python3' + workingDirectory: $(Build.BinariesDirectory) + + - script: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -x "--config Release --build_server --use_openmp --use_full_protobuf --enable_server_tests"' + displayName: 'Run build script' + - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 displayName: 'Component Detection' - - template: templates/clean-agent-build-directory-step.yml \ No newline at end of file + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')) + + - template: templates/clean-agent-build-directory-step.yml diff --git a/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh b/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh index 00bda4df84..e8e6111418 100755 --- a/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh +++ b/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh @@ -67,6 +67,7 @@ fi /usr/bin/python${PYTHON_VER} -m pip install --upgrade --force-reinstall pip==19.0.3 /usr/bin/python${PYTHON_VER} -m pip install --upgrade --force-reinstall numpy==1.15.0 +/usr/bin/python${PYTHON_VER} -m pip install --upgrade --force-reinstall requests==2.21.0 rm -rf /var/lib/apt/lists/* mkdir -p /tmp/azcopy