From 5e4ebbd9d9b190e5b5bc3bff4cdf0b726921070e Mon Sep 17 00:00:00 2001 From: PeixuanZuo <94887879+PeixuanZuo@users.noreply.github.com> Date: Wed, 28 Sep 2022 10:59:30 +0800 Subject: [PATCH] [ROCm] add MIGraphX ci pipeline (#11569) **Description**: Describe your changes. Add migraphx ci pipeline, test build and unit tests. This PR is based on #11492 Pipeline : https://dev.azure.com/onnxruntime/onnxruntime/_build/results?buildId=765711&view=results --- .../linux-migraphx-ci-pipeline.yml | 80 +++++++++++++++++++ .../pai/migraphx-ci-pipeline-env.Dockerfile | 38 +++++++++ .../github/pai/migraphx-excluded-tests.txt | 2 + .../github/pai/migraphx_test_launcher.sh | 15 ++++ 4 files changed, 135 insertions(+) create mode 100644 tools/ci_build/github/azure-pipelines/linux-migraphx-ci-pipeline.yml create mode 100644 tools/ci_build/github/pai/migraphx-ci-pipeline-env.Dockerfile create mode 100644 tools/ci_build/github/pai/migraphx-excluded-tests.txt create mode 100755 tools/ci_build/github/pai/migraphx_test_launcher.sh diff --git a/tools/ci_build/github/azure-pipelines/linux-migraphx-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-migraphx-ci-pipeline.yml new file mode 100644 index 0000000000..4ef5789f90 --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/linux-migraphx-ci-pipeline.yml @@ -0,0 +1,80 @@ +trigger: none + +name: 'linux_ci_$(Date:yyyyMMdd)_$(Rev:r)' +jobs: +- job: AMDMIGraphX_CI + pool: 'AMD-GPU' + timeoutInMinutes: 60 + + # gid of video and render group on gcr-openpai-35 and -36 + variables: + - name: video + value: 44 + - name: render + value: 109 + - name: RocmVersion + value: 5.2.3 + + steps: + - checkout: self + clean: true + submodules: recursive + + - template: templates/get-docker-image-steps.yml + parameters: + Dockerfile: tools/ci_build/github/pai/migraphx-ci-pipeline-env.Dockerfile + Context: tools/ci_build/github/pai + Repository: onnxruntimetrainingrocm-cibuild-rocm$(RocmVersion) + + - task: CmdLine@2 + inputs: + script: | + docker run --rm \ + -e HIP_VISIBLE_DEVICES \ + --privileged \ + --security-opt seccomp=unconfined \ + --shm-size=1024m \ + --device=/dev/kfd \ + --device=/dev/dri \ + --group-add $(video) \ + --group-add $(render) \ + --volume $(Build.SourcesDirectory):/onnxruntime_src \ + --volume $(Build.BinariesDirectory):/build \ + --workdir /onnxruntime_src \ + onnxruntimetrainingrocm-cibuild-rocm$(RocmVersion) \ + python tools/ci_build/build.py \ + --config RelWithDebInfo \ + --mpi_home /opt/ompi \ + --use_migraphx \ + --rocm_version=5.2.3 \ + --rocm_home /opt/rocm \ + --nccl_home /opt/rocm \ + --update \ + --build_dir /build \ + --build \ + --parallel 8 \ + --build_wheel \ + --skip_submodule_sync \ + --skip_tests + workingDirectory: $(Build.SourcesDirectory) + displayName: 'Build onnxruntime' + + - task: CmdLine@2 + inputs: + script: | + docker run --rm \ + -e HIP_VISIBLE_DEVICES \ + --privileged \ + --security-opt seccomp=unconfined \ + --shm-size=1024m \ + --device=/dev/kfd \ + --device=/dev/dri \ + --group-add $(video) \ + --group-add $(render) \ + --volume $(Build.SourcesDirectory):/onnxruntime_src \ + --volume $(Build.BinariesDirectory):/build \ + --workdir /build/RelWithDebInfo \ + onnxruntimetrainingrocm-cibuild-rocm$(RocmVersion) \ + /onnxruntime_src/tools/ci_build/github/pai/migraphx_test_launcher.sh + workingDirectory: $(Build.SourcesDirectory) + displayName: 'Run onnxruntime unit tests' diff --git a/tools/ci_build/github/pai/migraphx-ci-pipeline-env.Dockerfile b/tools/ci_build/github/pai/migraphx-ci-pipeline-env.Dockerfile new file mode 100644 index 0000000000..74ee83c2f6 --- /dev/null +++ b/tools/ci_build/github/pai/migraphx-ci-pipeline-env.Dockerfile @@ -0,0 +1,38 @@ +FROM rocm/pytorch:rocm5.2.3_ubuntu20.04_py3.7_pytorch_1.12.1 + +ENV DEBIAN_FRONTEND noninteractive +ENV MIGRAPHX_DISABLE_FAST_GELU=1 + +RUN apt-get clean && apt-get update && apt-get install -y locales +RUN locale-gen en_US.UTF-8 +RUN update-locale LANG=en_US.UTF-8 +ENV LC_ALL C.UTF-8 +ENV LANG C.UTF-8 + +WORKDIR /stage + +# from rocm/pytorch's image, work around ucx's dlopen replacement conflicting with shared provider +RUN cd /opt/mpi_install/ucx/build &&\ + make clean &&\ + ../contrib/configure-release --prefix=/opt/ucx --without-rocm &&\ + make -j $(nproc) &&\ + make install + +# CMake +ENV CMAKE_VERSION=3.18.2 +RUN cd /usr/local && \ + wget -q -O - https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz | tar zxf - +ENV PATH=/usr/local/cmake-${CMAKE_VERSION}-Linux-x86_64/bin:${PATH} + +RUN apt-get update &&\ + apt-get install -y half libnuma-dev + +# Install rbuild +RUN pip3 install https://github.com/RadeonOpenCompute/rbuild/archive/master.tar.gz numpy yapf==0.28.0 + +# Install MIGraphX from source +RUN mkdir -p /migraphx +RUN cd /migraphx && git clone --depth=1 --branch migraphx_for_ort https://github.com/ROCmSoftwarePlatform/AMDMIGraphX src +RUN cd /migraphx && rbuild package --cxx /opt/rocm/llvm/bin/clang++ -d /migraphx/deps -B /migraphx/build -S /migraphx/src/ -DPYTHON_EXECUTABLE=/usr/bin/python3 +RUN dpkg -i /migraphx/build/*.deb +RUN rm -rf /migraphx diff --git a/tools/ci_build/github/pai/migraphx-excluded-tests.txt b/tools/ci_build/github/pai/migraphx-excluded-tests.txt new file mode 100644 index 0000000000..5e38f982d7 --- /dev/null +++ b/tools/ci_build/github/pai/migraphx-excluded-tests.txt @@ -0,0 +1,2 @@ +GatherOpTest.Gather_invalid_index_cpu +Scatter.InvalidIndex diff --git a/tools/ci_build/github/pai/migraphx_test_launcher.sh b/tools/ci_build/github/pai/migraphx_test_launcher.sh new file mode 100755 index 0000000000..c2d593b641 --- /dev/null +++ b/tools/ci_build/github/pai/migraphx_test_launcher.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +build_dir=${1:-"."} +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +echo "Warning: The following tests are EXCLUDED on MIGraphX agent:" +gtest_filter="-" +while read line; do + gtest_filter="$gtest_filter:$line" + echo "$line" +done <$script_dir/migraphx-excluded-tests.txt +echo "" + +echo "Running ./onnxruntime_test_all .." +$build_dir/onnxruntime_test_all --gtest_filter=$gtest_filter