[ROCm] add MIGraphX ci pipeline (#11569)

**Description**: Describe your changes.
Add migraphx ci pipeline, test build and unit tests.
This PR is based on #11492 

Pipeline :
https://dev.azure.com/onnxruntime/onnxruntime/_build/results?buildId=765711&view=results
This commit is contained in:
PeixuanZuo 2022-09-28 10:59:30 +08:00 committed by GitHub
parent 19774f9230
commit 5e4ebbd9d9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 135 additions and 0 deletions

View file

@ -0,0 +1,80 @@
trigger: none
name: 'linux_ci_$(Date:yyyyMMdd)_$(Rev:r)'
jobs:
- job: AMDMIGraphX_CI
pool: 'AMD-GPU'
timeoutInMinutes: 60
# gid of video and render group on gcr-openpai-35 and -36
variables:
- name: video
value: 44
- name: render
value: 109
- name: RocmVersion
value: 5.2.3
steps:
- checkout: self
clean: true
submodules: recursive
- template: templates/get-docker-image-steps.yml
parameters:
Dockerfile: tools/ci_build/github/pai/migraphx-ci-pipeline-env.Dockerfile
Context: tools/ci_build/github/pai
Repository: onnxruntimetrainingrocm-cibuild-rocm$(RocmVersion)
- task: CmdLine@2
inputs:
script: |
docker run --rm \
-e HIP_VISIBLE_DEVICES \
--privileged \
--security-opt seccomp=unconfined \
--shm-size=1024m \
--device=/dev/kfd \
--device=/dev/dri \
--group-add $(video) \
--group-add $(render) \
--volume $(Build.SourcesDirectory):/onnxruntime_src \
--volume $(Build.BinariesDirectory):/build \
--workdir /onnxruntime_src \
onnxruntimetrainingrocm-cibuild-rocm$(RocmVersion) \
python tools/ci_build/build.py \
--config RelWithDebInfo \
--mpi_home /opt/ompi \
--use_migraphx \
--rocm_version=5.2.3 \
--rocm_home /opt/rocm \
--nccl_home /opt/rocm \
--update \
--build_dir /build \
--build \
--parallel 8 \
--build_wheel \
--skip_submodule_sync \
--skip_tests
workingDirectory: $(Build.SourcesDirectory)
displayName: 'Build onnxruntime'
- task: CmdLine@2
inputs:
script: |
docker run --rm \
-e HIP_VISIBLE_DEVICES \
--privileged \
--security-opt seccomp=unconfined \
--shm-size=1024m \
--device=/dev/kfd \
--device=/dev/dri \
--group-add $(video) \
--group-add $(render) \
--volume $(Build.SourcesDirectory):/onnxruntime_src \
--volume $(Build.BinariesDirectory):/build \
--workdir /build/RelWithDebInfo \
onnxruntimetrainingrocm-cibuild-rocm$(RocmVersion) \
/onnxruntime_src/tools/ci_build/github/pai/migraphx_test_launcher.sh
workingDirectory: $(Build.SourcesDirectory)
displayName: 'Run onnxruntime unit tests'

View file

@ -0,0 +1,38 @@
FROM rocm/pytorch:rocm5.2.3_ubuntu20.04_py3.7_pytorch_1.12.1
ENV DEBIAN_FRONTEND noninteractive
ENV MIGRAPHX_DISABLE_FAST_GELU=1
RUN apt-get clean && apt-get update && apt-get install -y locales
RUN locale-gen en_US.UTF-8
RUN update-locale LANG=en_US.UTF-8
ENV LC_ALL C.UTF-8
ENV LANG C.UTF-8
WORKDIR /stage
# from rocm/pytorch's image, work around ucx's dlopen replacement conflicting with shared provider
RUN cd /opt/mpi_install/ucx/build &&\
make clean &&\
../contrib/configure-release --prefix=/opt/ucx --without-rocm &&\
make -j $(nproc) &&\
make install
# CMake
ENV CMAKE_VERSION=3.18.2
RUN cd /usr/local && \
wget -q -O - https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz | tar zxf -
ENV PATH=/usr/local/cmake-${CMAKE_VERSION}-Linux-x86_64/bin:${PATH}
RUN apt-get update &&\
apt-get install -y half libnuma-dev
# Install rbuild
RUN pip3 install https://github.com/RadeonOpenCompute/rbuild/archive/master.tar.gz numpy yapf==0.28.0
# Install MIGraphX from source
RUN mkdir -p /migraphx
RUN cd /migraphx && git clone --depth=1 --branch migraphx_for_ort https://github.com/ROCmSoftwarePlatform/AMDMIGraphX src
RUN cd /migraphx && rbuild package --cxx /opt/rocm/llvm/bin/clang++ -d /migraphx/deps -B /migraphx/build -S /migraphx/src/ -DPYTHON_EXECUTABLE=/usr/bin/python3
RUN dpkg -i /migraphx/build/*.deb
RUN rm -rf /migraphx

View file

@ -0,0 +1,2 @@
GatherOpTest.Gather_invalid_index_cpu
Scatter.InvalidIndex

View file

@ -0,0 +1,15 @@
#!/bin/bash
build_dir=${1:-"."}
script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
echo "Warning: The following tests are EXCLUDED on MIGraphX agent:"
gtest_filter="-"
while read line; do
gtest_filter="$gtest_filter:$line"
echo "$line"
done <$script_dir/migraphx-excluded-tests.txt
echo ""
echo "Running ./onnxruntime_test_all .."
$build_dir/onnxruntime_test_all --gtest_filter=$gtest_filter