Enable Openvino nightly build on edge device (#1684)

1. Add openvino GPU nightly build pipeline, this test is running on Intel Up square Edge device. The device are host locally not from Azure VM. We persist a smaller model test data on Edge device. 2. Update the build condition for openvino GPU so it works for GPU_FP32, GPU_FP16 3. add option to install_ubuntu.sh to exclude the package used for nuphar, so that we can save some disk space as the Edge device usually have limited disk space.
2026-06-17 01:44:45 +00:00 · 2019-09-11 16:36:12 -07:00 · 2019-09-11 16:36:12 -07:00 · 2b8677b210
commit 2b8677b210
parent fe8915863c
7 changed files with 100 additions and 34 deletions
--- a/tools/ci_build/github/azure-pipelines/linux-openvino-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/linux-openvino-ci-pipeline.yml
@ -1,5 +1,6 @@
 jobs:
 - job: Linux_OpenVINO_CI_Dev
+  timeoutInMinutes: 240
  pool: Linux-CPU
  steps:
    - template: templates/set-test-data-variables-step.yml
--- a/tools/ci_build/github/azure-pipelines/linux-openvino-nightly-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/linux-openvino-nightly-pipeline.yml
@ -0,0 +1,31 @@
+jobs:
+- job: Linux_OpenVINO_CI_Dev
+  timeoutInMinutes: 240
+  pool: OpenVINO
+  steps:
+    - task: CmdLine@2
+      displayName: 'Clean untagged docker images'
+      inputs:
+        script: |
+          docker rm $(docker ps -a | grep Exited | awk '{print $1;}') || true
+          docker images -q --filter "dangling=true" | xargs -n1 -r docker rmi
+        workingDirectory: $(Build.BinariesDirectory)
+      continueOnError: true
+      condition: always()
+
+    - task: PythonScript@0
+      displayName: 'Unzip test data'
+      inputs:
+        scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py'
+        arguments: --build_dir $(Build.BinariesDirectory) --edge_device
+        pythonInterpreter: '/usr/bin/python3'
+        workingDirectory: $(Build.BinariesDirectory)
+        
+    - script: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d openvino -r $(Build.BinariesDirectory) -x "--use_openvino GPU_FP32 --build_wheel"'
+      displayName: 'Command Line Script'
+
+    - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
+      displayName: 'Component Detection'
+      condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI'))
+
+    - template: templates/clean-agent-build-directory-step.yml
--- a/tools/ci_build/github/azure-pipelines/templates/linux-ci.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/linux-ci.yml
@ -5,7 +5,7 @@ parameters:
  DoNugetPack: 'false'
  NuPackScript: ''
  ArtifactName: 'drop-linux'
-  TimeoutInMinutes: 60
+  TimeoutInMinutes: 120

 jobs:
 - job: ${{ parameters.JobName }}
--- a/tools/ci_build/github/download_test_data.py
+++ b/tools/ci_build/github/download_test_data.py
@ -23,6 +23,7 @@ def parse_arguments():
    parser.add_argument("--test_data_url", help="Test data URL.")
    parser.add_argument("--azure_region", help="Azure region")
    parser.add_argument("--build_dir", required=True, help="Path to the build directory.")
+    parser.add_argument("--edge_device", action="store_true", help="Edge device with limit disk space.")
    return parser.parse_args()


@ -61,21 +62,29 @@ def download_and_unzip(build_dir, url, dest_folder):
    os.unlink(local_file_name)

 args = parse_arguments()
-hostname = get_server_hostname(args.azure_region)
-url = args.test_data_url.replace('onnxruntimetestdata', hostname)
-print('data url=%s' % url)
-download_and_unzip(args.build_dir, url, 'models')
-if is_windows():
-    url = 'https://onnxruntimetestdata.blob.core.windows.net/models/cmake-3.15.1-win64-x64.zip'
-    url = url.replace('onnxruntimetestdata', hostname)
-    download_and_unzip(args.build_dir, url, 'cmake_temp')
-    dest_dir = os.path.join(args.build_dir,'cmake')
-    if os.path.exists(dest_dir):
-        print('deleting %s' % dest_dir)
-        shutil.rmtree(dest_dir)
-    shutil.move(os.path.join(args.build_dir,'cmake_temp','cmake-3.15.1-win64-x64'),dest_dir)
-    url = 'https://onnxruntimetestdata.blob.core.windows.net/models/OpenCppCoverageSetup-x64-0.9.7.0.exe'
-    url = url.replace('onnxruntimetestdata', hostname)
-    dest_folder = os.path.join(args.build_dir, 'installer','opencppcoverage')
-    os.makedirs(dest_folder,exist_ok=True)
-    subprocess.run([os.path.join(args.build_dir,'azcopy'),'cp', '--log-level','ERROR', url, os.path.join(dest_folder,'installer.exe')],check=True)
+models_folder = 'models'
+if args.edge_device:
+    dest_folder = os.path.join(args.build_dir, models_folder)
+    #For edge device, the model zip file is persist at /mnt/ubuntu/tmp/model.zip
+    local_file_name = '/mnt/ubuntu/tmp/model.zip'
+    if os.path.exists(local_file_name):
+       subprocess.run(['unzip','-qd', dest_folder ,local_file_name], check=True)
+else:
+    hostname = get_server_hostname(args.azure_region)
+    url = args.test_data_url.replace('onnxruntimetestdata', hostname)
+    print('data url=%s' % url)
+    download_and_unzip(args.build_dir, url, models_folder)
+    if is_windows():
+        url = 'https://onnxruntimetestdata.blob.core.windows.net/models/cmake-3.15.1-win64-x64.zip'
+        url = url.replace('onnxruntimetestdata', hostname)
+        download_and_unzip(args.build_dir, url, 'cmake_temp')
+        dest_dir = os.path.join(args.build_dir,'cmake')
+        if os.path.exists(dest_dir):
+            print('deleting %s' % dest_dir)
+            shutil.rmtree(dest_dir)
+        shutil.move(os.path.join(args.build_dir,'cmake_temp','cmake-3.15.1-win64-x64'),dest_dir)
+        url = 'https://onnxruntimetestdata.blob.core.windows.net/models/OpenCppCoverageSetup-x64-0.9.7.0.exe'
+        url = url.replace('onnxruntimetestdata', hostname)
+        dest_folder = os.path.join(args.build_dir, 'installer','opencppcoverage')
+        os.makedirs(dest_folder,exist_ok=True)
+        subprocess.run([os.path.join(args.build_dir,'azcopy'),'cp', '--log-level','ERROR', url, os.path.join(dest_folder,'installer.exe')],check=True)
--- a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_openvino
+++ b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_openvino
@ -6,9 +6,12 @@ ARG OPENVINO_VERSION=2019_R1.1

 ADD scripts /tmp/scripts
 ENV PATH="/opt/cmake/bin:${PATH}"
-RUN /tmp/scripts/install_ubuntu.sh -p ${PYTHON_VERSION} && \
+RUN /tmp/scripts/install_ubuntu.sh -p ${PYTHON_VERSION} -d EdgeDevice && \
    /tmp/scripts/install_deps.sh

+RUN apt update && apt install -y libnuma1 ocl-icd-libopencl1 && \
+    rm -rf /var/lib/apt/lists/*
+
 RUN /tmp/scripts/install_openvino.sh -o ${OPENVINO_VERSION} && \
    rm -rf /tmp/scripts

@ -21,10 +24,18 @@ ENV LD_LIBRARY_PATH $INTEL_CVSDK_DIR/deployment_tools/inference_engine/lib/intel

 ENV PATH $INTEL_CVSDK_DIR/deployment_tools/model_optimizer:$PATH
 ENV PYTHONPATH $INTEL_CVSDK_DIR/deployment_tools/model_optimizer:$INTEL_CVSDK_DIR/tools:$PYTHONPATH
+ENV IE_PLUGINS_PATH $INTEL_CVSDK_DIR/deployment_tools/inference_engine/lib/intel64
+
+RUN wget https://github.com/intel/compute-runtime/releases/download/19.15.12831/intel-gmmlib_19.1.1_amd64.deb && \
+    wget https://github.com/intel/compute-runtime/releases/download/19.15.12831/intel-igc-core_1.0.2-1787_amd64.deb && \
+    wget https://github.com/intel/compute-runtime/releases/download/19.15.12831/intel-igc-opencl_1.0.2-1787_amd64.deb && \
+    wget https://github.com/intel/compute-runtime/releases/download/19.15.12831/intel-opencl_19.15.12831_amd64.deb && \
+    wget https://github.com/intel/compute-runtime/releases/download/19.15.12831/intel-ocloc_19.15.12831_amd64.deb && \
+    sudo dpkg -i *.deb && rm -rf *.deb

 ARG BUILD_UID=1000
 ARG BUILD_USER=onnxruntimedev
 WORKDIR /home/$BUILD_USER
 RUN adduser --gecos 'onnxruntime Build User' --disabled-password $BUILD_USER --uid $BUILD_UID
+RUN adduser $BUILD_USER video
 USER $BUILD_USER
-
--- a/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh
+++ b/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh
@ -4,18 +4,21 @@ while getopts p: parameter_Option
 do case "${parameter_Option}"
 in
 p) PYTHON_VER=${OPTARG};;
+d) DEVICE_TYPE=${OPTARG};;
 esac
 done

 PYTHON_VER=${PYTHON_VER:=3.5}
+# Some Edge devices only have limited disk space, use this option to exclude some package
+DEVICE_TYPE=${DEVICE_TYPE:=Normal}
 DEBIAN_FRONTEND=noninteractive

 SYS_LONG_BIT=$(getconf LONG_BIT)

 apt-get update && apt-get install -y software-properties-common
 add-apt-repository ppa:deadsnakes/ppa
-apt-get update && apt-get install -y --no-install-recommends \
-        autotools-dev \
+
+PACKAGE_LIST="autotools-dev \
        automake \
        build-essential \
        git apt-transport-https apt-utils \
@ -45,8 +48,11 @@ apt-get update && apt-get install -y --no-install-recommends \
        zip \
        rsync libunwind8 libpng16-dev libexpat1-dev \
        python3-setuptools python3-numpy python3-wheel python python3-pip python3-pytest \
-        libprotobuf-dev libprotobuf9v5 protobuf-compiler \
-        libedit-dev libxml2-dev python3-packaging
+        libprotobuf-dev libprotobuf9v5 protobuf-compiler"
+if [ $DEVICE_TYPE = "Normal" ]; then
+    PACKAGE_LIST="$PACKAGE_LIST libedit-dev libxml2-dev python3-packaging"
+fi
+apt-get update && apt-get install -y --no-install-recommends $PACKAGE_LIST

 locale-gen en_US.UTF-8
 update-locale LANG=en_US.UTF-8
@ -76,8 +82,12 @@ fi

 /usr/bin/python${PYTHON_VER} -m pip install --upgrade --force-reinstall numpy==1.15.0
 /usr/bin/python${PYTHON_VER} -m pip install --upgrade --force-reinstall requests==2.21.0
-/usr/bin/python${PYTHON_VER} -m pip install --upgrade --force-reinstall sympy==1.1.1
+if [ $DEVICE_TYPE = "Normal" ]; then
+    /usr/bin/python${PYTHON_VER} -m pip install --upgrade --force-reinstall sympy==1.1.1
+fi
 rm -rf /var/lib/apt/lists/*

+if [ $DEVICE_TYPE = "Normal" ]; then
 aria2c -q -d /tmp -o llvm.tar.xz http://releases.llvm.org/6.0.1/clang+llvm-6.0.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz
 tar --strip 1 -Jxf /tmp/llvm.tar.xz -C /usr
+fi
--- a/tools/ci_build/github/linux/run_dockerbuild.sh
+++ b/tools/ci_build/github/linux/run_dockerbuild.sh
@ -57,9 +57,9 @@ else
        DOCKER_FILE=Dockerfile.ubuntu_tensorrt
        docker build -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f $DOCKER_FILE .
    elif [ $BUILD_DEVICE = "openvino" ]; then
-        IMAGE="ubuntu16.04"
+        IMAGE="ubuntu16.04-openvino"
        DOCKER_FILE=Dockerfile.ubuntu_openvino
-        docker build -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg OS_VERSION=16.04 --build-arg PYTHON_VERSION=${PYTHON_VER} --build-arg OPENVINO_VERSION=${OPENVINO_VERSION} -f Dockerfile.ubuntu_openvino .
+        docker build -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg OS_VERSION=16.04 --build-arg PYTHON_VERSION=${PYTHON_VER} --build-arg OPENVINO_VERSION=${OPENVINO_VERSION} -f $DOCKER_FILE .
    else
        IMAGE="ubuntu16.04"
        if [ $BUILD_ARCH = "x86" ]; then
@ -85,13 +85,17 @@ else
    RUNTIME="--runtime=nvidia"
 fi

+DOCKER_RUN_PARAMETER="--name onnxruntime-$BUILD_DEVICE \
+                      --volume $SOURCE_ROOT:/onnxruntime_src \
+                      --volume $BUILD_DIR:/build \
+                      --volume $HOME/.cache/onnxruntime:/home/onnxruntimedev/.cache/onnxruntime \
+                      --volume $HOME/.onnx:/home/onnxruntimedev/.onnx"
+if [ $BUILD_DEVICE = "openvino" ] && [[ $BUILD_EXTR_PAR == *"--use_openvino GPU_FP"* ]]; then
+    DOCKER_RUN_PARAMETER="$DOCKER_RUN_PARAMETER --device /dev/dri:/dev/dri"
+fi
+
 docker rm -f "onnxruntime-$BUILD_DEVICE" || true
-docker run $RUNTIME -h $HOSTNAME \
-    --name "onnxruntime-$BUILD_DEVICE" \
-    --volume "$SOURCE_ROOT:/onnxruntime_src" \
-    --volume "$BUILD_DIR:/build" \
-    --volume "$HOME/.cache/onnxruntime:/home/onnxruntimedev/.cache/onnxruntime" \
-    --volume "$HOME/.onnx:/home/onnxruntimedev/.onnx" \
+docker run $RUNTIME -h $HOSTNAME $DOCKER_RUN_PARAMETER \
    -e NIGHTLY_BUILD \
    "onnxruntime-$IMAGE" \
    /bin/bash /onnxruntime_src/tools/ci_build/github/linux/run_build.sh \