mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-30 20:18:08 +00:00
[ROCm] add MIGraphX ci pipeline (#11569)
**Description**: Describe your changes. Add migraphx ci pipeline, test build and unit tests. This PR is based on #11492 Pipeline : https://dev.azure.com/onnxruntime/onnxruntime/_build/results?buildId=765711&view=results
This commit is contained in:
parent
19774f9230
commit
5e4ebbd9d9
4 changed files with 135 additions and 0 deletions
|
|
@ -0,0 +1,80 @@
|
|||
trigger: none
|
||||
|
||||
name: 'linux_ci_$(Date:yyyyMMdd)_$(Rev:r)'
|
||||
jobs:
|
||||
- job: AMDMIGraphX_CI
|
||||
pool: 'AMD-GPU'
|
||||
timeoutInMinutes: 60
|
||||
|
||||
# gid of video and render group on gcr-openpai-35 and -36
|
||||
variables:
|
||||
- name: video
|
||||
value: 44
|
||||
- name: render
|
||||
value: 109
|
||||
- name: RocmVersion
|
||||
value: 5.2.3
|
||||
|
||||
steps:
|
||||
- checkout: self
|
||||
clean: true
|
||||
submodules: recursive
|
||||
|
||||
- template: templates/get-docker-image-steps.yml
|
||||
parameters:
|
||||
Dockerfile: tools/ci_build/github/pai/migraphx-ci-pipeline-env.Dockerfile
|
||||
Context: tools/ci_build/github/pai
|
||||
Repository: onnxruntimetrainingrocm-cibuild-rocm$(RocmVersion)
|
||||
|
||||
- task: CmdLine@2
|
||||
inputs:
|
||||
script: |
|
||||
docker run --rm \
|
||||
-e HIP_VISIBLE_DEVICES \
|
||||
--privileged \
|
||||
--security-opt seccomp=unconfined \
|
||||
--shm-size=1024m \
|
||||
--device=/dev/kfd \
|
||||
--device=/dev/dri \
|
||||
--group-add $(video) \
|
||||
--group-add $(render) \
|
||||
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
||||
--volume $(Build.BinariesDirectory):/build \
|
||||
--workdir /onnxruntime_src \
|
||||
onnxruntimetrainingrocm-cibuild-rocm$(RocmVersion) \
|
||||
python tools/ci_build/build.py \
|
||||
--config RelWithDebInfo \
|
||||
--mpi_home /opt/ompi \
|
||||
--use_migraphx \
|
||||
--rocm_version=5.2.3 \
|
||||
--rocm_home /opt/rocm \
|
||||
--nccl_home /opt/rocm \
|
||||
--update \
|
||||
--build_dir /build \
|
||||
--build \
|
||||
--parallel 8 \
|
||||
--build_wheel \
|
||||
--skip_submodule_sync \
|
||||
--skip_tests
|
||||
workingDirectory: $(Build.SourcesDirectory)
|
||||
displayName: 'Build onnxruntime'
|
||||
|
||||
- task: CmdLine@2
|
||||
inputs:
|
||||
script: |
|
||||
docker run --rm \
|
||||
-e HIP_VISIBLE_DEVICES \
|
||||
--privileged \
|
||||
--security-opt seccomp=unconfined \
|
||||
--shm-size=1024m \
|
||||
--device=/dev/kfd \
|
||||
--device=/dev/dri \
|
||||
--group-add $(video) \
|
||||
--group-add $(render) \
|
||||
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
||||
--volume $(Build.BinariesDirectory):/build \
|
||||
--workdir /build/RelWithDebInfo \
|
||||
onnxruntimetrainingrocm-cibuild-rocm$(RocmVersion) \
|
||||
/onnxruntime_src/tools/ci_build/github/pai/migraphx_test_launcher.sh
|
||||
workingDirectory: $(Build.SourcesDirectory)
|
||||
displayName: 'Run onnxruntime unit tests'
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
FROM rocm/pytorch:rocm5.2.3_ubuntu20.04_py3.7_pytorch_1.12.1
|
||||
|
||||
ENV DEBIAN_FRONTEND noninteractive
|
||||
ENV MIGRAPHX_DISABLE_FAST_GELU=1
|
||||
|
||||
RUN apt-get clean && apt-get update && apt-get install -y locales
|
||||
RUN locale-gen en_US.UTF-8
|
||||
RUN update-locale LANG=en_US.UTF-8
|
||||
ENV LC_ALL C.UTF-8
|
||||
ENV LANG C.UTF-8
|
||||
|
||||
WORKDIR /stage
|
||||
|
||||
# from rocm/pytorch's image, work around ucx's dlopen replacement conflicting with shared provider
|
||||
RUN cd /opt/mpi_install/ucx/build &&\
|
||||
make clean &&\
|
||||
../contrib/configure-release --prefix=/opt/ucx --without-rocm &&\
|
||||
make -j $(nproc) &&\
|
||||
make install
|
||||
|
||||
# CMake
|
||||
ENV CMAKE_VERSION=3.18.2
|
||||
RUN cd /usr/local && \
|
||||
wget -q -O - https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz | tar zxf -
|
||||
ENV PATH=/usr/local/cmake-${CMAKE_VERSION}-Linux-x86_64/bin:${PATH}
|
||||
|
||||
RUN apt-get update &&\
|
||||
apt-get install -y half libnuma-dev
|
||||
|
||||
# Install rbuild
|
||||
RUN pip3 install https://github.com/RadeonOpenCompute/rbuild/archive/master.tar.gz numpy yapf==0.28.0
|
||||
|
||||
# Install MIGraphX from source
|
||||
RUN mkdir -p /migraphx
|
||||
RUN cd /migraphx && git clone --depth=1 --branch migraphx_for_ort https://github.com/ROCmSoftwarePlatform/AMDMIGraphX src
|
||||
RUN cd /migraphx && rbuild package --cxx /opt/rocm/llvm/bin/clang++ -d /migraphx/deps -B /migraphx/build -S /migraphx/src/ -DPYTHON_EXECUTABLE=/usr/bin/python3
|
||||
RUN dpkg -i /migraphx/build/*.deb
|
||||
RUN rm -rf /migraphx
|
||||
2
tools/ci_build/github/pai/migraphx-excluded-tests.txt
Normal file
2
tools/ci_build/github/pai/migraphx-excluded-tests.txt
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
GatherOpTest.Gather_invalid_index_cpu
|
||||
Scatter.InvalidIndex
|
||||
15
tools/ci_build/github/pai/migraphx_test_launcher.sh
Executable file
15
tools/ci_build/github/pai/migraphx_test_launcher.sh
Executable file
|
|
@ -0,0 +1,15 @@
|
|||
#!/bin/bash
|
||||
|
||||
build_dir=${1:-"."}
|
||||
script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
|
||||
echo "Warning: The following tests are EXCLUDED on MIGraphX agent:"
|
||||
gtest_filter="-"
|
||||
while read line; do
|
||||
gtest_filter="$gtest_filter:$line"
|
||||
echo "$line"
|
||||
done <$script_dir/migraphx-excluded-tests.txt
|
||||
echo ""
|
||||
|
||||
echo "Running ./onnxruntime_test_all .."
|
||||
$build_dir/onnxruntime_test_all --gtest_filter=$gtest_filter
|
||||
Loading…
Reference in a new issue