diff --git a/tools/ci_build/github/azure-pipelines/linux-openvino-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-openvino-ci-pipeline.yml index 07ba00ff3b..60e8c03ab2 100644 --- a/tools/ci_build/github/azure-pipelines/linux-openvino-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-openvino-ci-pipeline.yml @@ -1,5 +1,6 @@ jobs: - job: Linux_OpenVINO_CI_Dev + timeoutInMinutes: 240 pool: Linux-CPU steps: - template: templates/set-test-data-variables-step.yml diff --git a/tools/ci_build/github/azure-pipelines/linux-openvino-nightly-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-openvino-nightly-pipeline.yml new file mode 100644 index 0000000000..2858c588c1 --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/linux-openvino-nightly-pipeline.yml @@ -0,0 +1,31 @@ +jobs: +- job: Linux_OpenVINO_CI_Dev + timeoutInMinutes: 240 + pool: OpenVINO + steps: + - task: CmdLine@2 + displayName: 'Clean untagged docker images' + inputs: + script: | + docker rm $(docker ps -a | grep Exited | awk '{print $1;}') || true + docker images -q --filter "dangling=true" | xargs -n1 -r docker rmi + workingDirectory: $(Build.BinariesDirectory) + continueOnError: true + condition: always() + + - task: PythonScript@0 + displayName: 'Unzip test data' + inputs: + scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py' + arguments: --build_dir $(Build.BinariesDirectory) --edge_device + pythonInterpreter: '/usr/bin/python3' + workingDirectory: $(Build.BinariesDirectory) + + - script: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d openvino -r $(Build.BinariesDirectory) -x "--use_openvino GPU_FP32 --build_wheel"' + displayName: 'Command Line Script' + + - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection' + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')) + + - template: templates/clean-agent-build-directory-step.yml diff --git a/tools/ci_build/github/azure-pipelines/templates/linux-ci.yml b/tools/ci_build/github/azure-pipelines/templates/linux-ci.yml index 82efba547b..fd0ced21f5 100644 --- a/tools/ci_build/github/azure-pipelines/templates/linux-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/linux-ci.yml @@ -5,7 +5,7 @@ parameters: DoNugetPack: 'false' NuPackScript: '' ArtifactName: 'drop-linux' - TimeoutInMinutes: 60 + TimeoutInMinutes: 120 jobs: - job: ${{ parameters.JobName }} diff --git a/tools/ci_build/github/download_test_data.py b/tools/ci_build/github/download_test_data.py index 821b6054ec..06347a61ea 100755 --- a/tools/ci_build/github/download_test_data.py +++ b/tools/ci_build/github/download_test_data.py @@ -23,6 +23,7 @@ def parse_arguments(): parser.add_argument("--test_data_url", help="Test data URL.") parser.add_argument("--azure_region", help="Azure region") parser.add_argument("--build_dir", required=True, help="Path to the build directory.") + parser.add_argument("--edge_device", action="store_true", help="Edge device with limit disk space.") return parser.parse_args() @@ -61,21 +62,29 @@ def download_and_unzip(build_dir, url, dest_folder): os.unlink(local_file_name) args = parse_arguments() -hostname = get_server_hostname(args.azure_region) -url = args.test_data_url.replace('onnxruntimetestdata', hostname) -print('data url=%s' % url) -download_and_unzip(args.build_dir, url, 'models') -if is_windows(): - url = 'https://onnxruntimetestdata.blob.core.windows.net/models/cmake-3.15.1-win64-x64.zip' - url = url.replace('onnxruntimetestdata', hostname) - download_and_unzip(args.build_dir, url, 'cmake_temp') - dest_dir = os.path.join(args.build_dir,'cmake') - if os.path.exists(dest_dir): - print('deleting %s' % dest_dir) - shutil.rmtree(dest_dir) - shutil.move(os.path.join(args.build_dir,'cmake_temp','cmake-3.15.1-win64-x64'),dest_dir) - url = 'https://onnxruntimetestdata.blob.core.windows.net/models/OpenCppCoverageSetup-x64-0.9.7.0.exe' - url = url.replace('onnxruntimetestdata', hostname) - dest_folder = os.path.join(args.build_dir, 'installer','opencppcoverage') - os.makedirs(dest_folder,exist_ok=True) - subprocess.run([os.path.join(args.build_dir,'azcopy'),'cp', '--log-level','ERROR', url, os.path.join(dest_folder,'installer.exe')],check=True) +models_folder = 'models' +if args.edge_device: + dest_folder = os.path.join(args.build_dir, models_folder) + #For edge device, the model zip file is persist at /mnt/ubuntu/tmp/model.zip + local_file_name = '/mnt/ubuntu/tmp/model.zip' + if os.path.exists(local_file_name): + subprocess.run(['unzip','-qd', dest_folder ,local_file_name], check=True) +else: + hostname = get_server_hostname(args.azure_region) + url = args.test_data_url.replace('onnxruntimetestdata', hostname) + print('data url=%s' % url) + download_and_unzip(args.build_dir, url, models_folder) + if is_windows(): + url = 'https://onnxruntimetestdata.blob.core.windows.net/models/cmake-3.15.1-win64-x64.zip' + url = url.replace('onnxruntimetestdata', hostname) + download_and_unzip(args.build_dir, url, 'cmake_temp') + dest_dir = os.path.join(args.build_dir,'cmake') + if os.path.exists(dest_dir): + print('deleting %s' % dest_dir) + shutil.rmtree(dest_dir) + shutil.move(os.path.join(args.build_dir,'cmake_temp','cmake-3.15.1-win64-x64'),dest_dir) + url = 'https://onnxruntimetestdata.blob.core.windows.net/models/OpenCppCoverageSetup-x64-0.9.7.0.exe' + url = url.replace('onnxruntimetestdata', hostname) + dest_folder = os.path.join(args.build_dir, 'installer','opencppcoverage') + os.makedirs(dest_folder,exist_ok=True) + subprocess.run([os.path.join(args.build_dir,'azcopy'),'cp', '--log-level','ERROR', url, os.path.join(dest_folder,'installer.exe')],check=True) diff --git a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_openvino b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_openvino index b993c360ef..d923826a4b 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_openvino +++ b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_openvino @@ -6,9 +6,12 @@ ARG OPENVINO_VERSION=2019_R1.1 ADD scripts /tmp/scripts ENV PATH="/opt/cmake/bin:${PATH}" -RUN /tmp/scripts/install_ubuntu.sh -p ${PYTHON_VERSION} && \ +RUN /tmp/scripts/install_ubuntu.sh -p ${PYTHON_VERSION} -d EdgeDevice && \ /tmp/scripts/install_deps.sh +RUN apt update && apt install -y libnuma1 ocl-icd-libopencl1 && \ + rm -rf /var/lib/apt/lists/* + RUN /tmp/scripts/install_openvino.sh -o ${OPENVINO_VERSION} && \ rm -rf /tmp/scripts @@ -21,10 +24,18 @@ ENV LD_LIBRARY_PATH $INTEL_CVSDK_DIR/deployment_tools/inference_engine/lib/intel ENV PATH $INTEL_CVSDK_DIR/deployment_tools/model_optimizer:$PATH ENV PYTHONPATH $INTEL_CVSDK_DIR/deployment_tools/model_optimizer:$INTEL_CVSDK_DIR/tools:$PYTHONPATH +ENV IE_PLUGINS_PATH $INTEL_CVSDK_DIR/deployment_tools/inference_engine/lib/intel64 + +RUN wget https://github.com/intel/compute-runtime/releases/download/19.15.12831/intel-gmmlib_19.1.1_amd64.deb && \ + wget https://github.com/intel/compute-runtime/releases/download/19.15.12831/intel-igc-core_1.0.2-1787_amd64.deb && \ + wget https://github.com/intel/compute-runtime/releases/download/19.15.12831/intel-igc-opencl_1.0.2-1787_amd64.deb && \ + wget https://github.com/intel/compute-runtime/releases/download/19.15.12831/intel-opencl_19.15.12831_amd64.deb && \ + wget https://github.com/intel/compute-runtime/releases/download/19.15.12831/intel-ocloc_19.15.12831_amd64.deb && \ + sudo dpkg -i *.deb && rm -rf *.deb ARG BUILD_UID=1000 ARG BUILD_USER=onnxruntimedev WORKDIR /home/$BUILD_USER RUN adduser --gecos 'onnxruntime Build User' --disabled-password $BUILD_USER --uid $BUILD_UID +RUN adduser $BUILD_USER video USER $BUILD_USER - diff --git a/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh b/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh index 31ea5c0d83..71c2132512 100755 --- a/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh +++ b/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh @@ -4,18 +4,21 @@ while getopts p: parameter_Option do case "${parameter_Option}" in p) PYTHON_VER=${OPTARG};; +d) DEVICE_TYPE=${OPTARG};; esac done PYTHON_VER=${PYTHON_VER:=3.5} +# Some Edge devices only have limited disk space, use this option to exclude some package +DEVICE_TYPE=${DEVICE_TYPE:=Normal} DEBIAN_FRONTEND=noninteractive SYS_LONG_BIT=$(getconf LONG_BIT) apt-get update && apt-get install -y software-properties-common add-apt-repository ppa:deadsnakes/ppa -apt-get update && apt-get install -y --no-install-recommends \ - autotools-dev \ + +PACKAGE_LIST="autotools-dev \ automake \ build-essential \ git apt-transport-https apt-utils \ @@ -45,8 +48,11 @@ apt-get update && apt-get install -y --no-install-recommends \ zip \ rsync libunwind8 libpng16-dev libexpat1-dev \ python3-setuptools python3-numpy python3-wheel python python3-pip python3-pytest \ - libprotobuf-dev libprotobuf9v5 protobuf-compiler \ - libedit-dev libxml2-dev python3-packaging + libprotobuf-dev libprotobuf9v5 protobuf-compiler" +if [ $DEVICE_TYPE = "Normal" ]; then + PACKAGE_LIST="$PACKAGE_LIST libedit-dev libxml2-dev python3-packaging" +fi +apt-get update && apt-get install -y --no-install-recommends $PACKAGE_LIST locale-gen en_US.UTF-8 update-locale LANG=en_US.UTF-8 @@ -76,8 +82,12 @@ fi /usr/bin/python${PYTHON_VER} -m pip install --upgrade --force-reinstall numpy==1.15.0 /usr/bin/python${PYTHON_VER} -m pip install --upgrade --force-reinstall requests==2.21.0 -/usr/bin/python${PYTHON_VER} -m pip install --upgrade --force-reinstall sympy==1.1.1 +if [ $DEVICE_TYPE = "Normal" ]; then + /usr/bin/python${PYTHON_VER} -m pip install --upgrade --force-reinstall sympy==1.1.1 +fi rm -rf /var/lib/apt/lists/* +if [ $DEVICE_TYPE = "Normal" ]; then aria2c -q -d /tmp -o llvm.tar.xz http://releases.llvm.org/6.0.1/clang+llvm-6.0.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz tar --strip 1 -Jxf /tmp/llvm.tar.xz -C /usr +fi diff --git a/tools/ci_build/github/linux/run_dockerbuild.sh b/tools/ci_build/github/linux/run_dockerbuild.sh index 7d40231cb1..b004857f13 100755 --- a/tools/ci_build/github/linux/run_dockerbuild.sh +++ b/tools/ci_build/github/linux/run_dockerbuild.sh @@ -57,9 +57,9 @@ else DOCKER_FILE=Dockerfile.ubuntu_tensorrt docker build -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f $DOCKER_FILE . elif [ $BUILD_DEVICE = "openvino" ]; then - IMAGE="ubuntu16.04" + IMAGE="ubuntu16.04-openvino" DOCKER_FILE=Dockerfile.ubuntu_openvino - docker build -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg OS_VERSION=16.04 --build-arg PYTHON_VERSION=${PYTHON_VER} --build-arg OPENVINO_VERSION=${OPENVINO_VERSION} -f Dockerfile.ubuntu_openvino . + docker build -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg OS_VERSION=16.04 --build-arg PYTHON_VERSION=${PYTHON_VER} --build-arg OPENVINO_VERSION=${OPENVINO_VERSION} -f $DOCKER_FILE . else IMAGE="ubuntu16.04" if [ $BUILD_ARCH = "x86" ]; then @@ -85,13 +85,17 @@ else RUNTIME="--runtime=nvidia" fi +DOCKER_RUN_PARAMETER="--name onnxruntime-$BUILD_DEVICE \ + --volume $SOURCE_ROOT:/onnxruntime_src \ + --volume $BUILD_DIR:/build \ + --volume $HOME/.cache/onnxruntime:/home/onnxruntimedev/.cache/onnxruntime \ + --volume $HOME/.onnx:/home/onnxruntimedev/.onnx" +if [ $BUILD_DEVICE = "openvino" ] && [[ $BUILD_EXTR_PAR == *"--use_openvino GPU_FP"* ]]; then + DOCKER_RUN_PARAMETER="$DOCKER_RUN_PARAMETER --device /dev/dri:/dev/dri" +fi + docker rm -f "onnxruntime-$BUILD_DEVICE" || true -docker run $RUNTIME -h $HOSTNAME \ - --name "onnxruntime-$BUILD_DEVICE" \ - --volume "$SOURCE_ROOT:/onnxruntime_src" \ - --volume "$BUILD_DIR:/build" \ - --volume "$HOME/.cache/onnxruntime:/home/onnxruntimedev/.cache/onnxruntime" \ - --volume "$HOME/.onnx:/home/onnxruntimedev/.onnx" \ +docker run $RUNTIME -h $HOSTNAME $DOCKER_RUN_PARAMETER \ -e NIGHTLY_BUILD \ "onnxruntime-$IMAGE" \ /bin/bash /onnxruntime_src/tools/ci_build/github/linux/run_build.sh \