saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-05-14 20:48:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions
onnxruntime/tools/ci_build/github/linux/docker/manylinux.patch
Changming Sun 0e8d4c3d21
Enable Address Sanitizer in CI (#19073) 
### Description
1. Add two build jobs for enabling Address Sanitizer in CI. One for
Windows CPU, One for Linux CPU.
2. Set default compiler flags/linker flags in build.py for normal
Windows/Linux/MacOS build. This can help control compiler flags in a
more centralized way.
3. All Windows binaries in our official packages will be built with
"/PROFILE" flag. Symbols of onnxruntime.dll can be found at [Microsoft
public symbol
server](https://learn.microsoft.com/en-us/windows-hardware/drivers/debugger/microsoft-public-symbols).

Limitations:
1. On Linux Address Sanitizer ignores RPATH settings in ELF binaries.
Therefore once Address Sanitizer is enabled, before running tests we
need to manually set LD_LIBRARY_PATH properly otherwise
libonnxruntime.so may not be able to find custom ops and shared EPs.
4. On Linux we also need to set LD_PRELOAD before running some tests(if
the main executable, like python, is not built with address sanitizer.
On Windows we do not need to.
5. On Windows before running python tests we should manually copy
address sanitizer DLL to the onnxruntime/capi directory, because python
3.8 and above has enabled "Safe DLL Search Mode" that wouldn't use the
information provided by PATH env.
6. On Linux Address Sanitizer found a lot of memory leaks from our
python binding code. Therefore right now we cannot enable Address
Sanitizer when building ONNX Runtime with python binding.
7. Address Sanitizer itself uses a lot of memory address space and
delays memory deallocations, which is easy to cause OOM issues in 32-bit
applications. We cannot run all the tests in onnxruntime_test_all in
32-bit mode with Address Sanitizer due to this reason. However, we still
can run individual tests in such a way. We just cannot run all of them
in one process.

### Motivation and Context
To catch memory issues.
2024-01-12 07:24:40 -08:00

173 lines
8 KiB
Diff
Raw Blame History

diff --git a/build-cpython.sh b/build-cpython.sh
index eea89e2..79c74d8 100755
--- a/build-cpython.sh
+++ b/build-cpython.sh
@@ -49,7 +49,7 @@ fi
CFLAGS_NODIST="${MANYLINUX_CFLAGS} ${MANYLINUX_CPPFLAGS} ${CFLAGS_EXTRA}" \
LDFLAGS_NODIST="${MANYLINUX_LDFLAGS}" \
--prefix=${PREFIX} --disable-shared --with-ensurepip=no > /dev/null
-make > /dev/null
+make -j$(nproc) > /dev/null
make install > /dev/null
popd
rm -rf Python-${CPYTHON_VERSION} Python-${CPYTHON_VERSION}.tgz Python-${CPYTHON_VERSION}.tgz.asc
diff --git a/build-git.sh b/build-git.sh
index 9c0b02d..2e2919c 100755
--- a/build-git.sh
+++ b/build-git.sh
@@ -27,7 +27,7 @@ fetch_source ${GIT_ROOT}.tar.gz ${GIT_DOWNLOAD_URL}
check_sha256sum ${GIT_ROOT}.tar.gz ${GIT_HASH}
tar -xzf ${GIT_ROOT}.tar.gz
pushd ${GIT_ROOT}
-make install prefix=/usr/local NO_GETTEXT=1 NO_TCLTK=1 DESTDIR=/manylinux-rootfs CPPFLAGS="${MANYLINUX_CPPFLAGS}" CFLAGS="${MANYLINUX_CFLAGS}" CXXFLAGS="${MANYLINUX_CXXFLAGS}" LDFLAGS="${MANYLINUX_LDFLAGS}"
+make -j$(nproc) install prefix=/usr/local NO_GETTEXT=1 NO_TCLTK=1 DESTDIR=/manylinux-rootfs CPPFLAGS="${MANYLINUX_CPPFLAGS}" CFLAGS="${MANYLINUX_CFLAGS}" CXXFLAGS="${MANYLINUX_CXXFLAGS}" LDFLAGS="${MANYLINUX_LDFLAGS}"
popd
rm -rf ${GIT_ROOT} ${GIT_ROOT}.tar.gz
diff --git a/build-openssl.sh b/build-openssl.sh
index 668deb6..5f3f5d5 100755
--- a/build-openssl.sh
+++ b/build-openssl.sh
@@ -40,7 +40,7 @@ check_sha256sum ${OPENSSL_ROOT}.tar.gz ${OPENSSL_HASH}
tar -xzf ${OPENSSL_ROOT}.tar.gz
pushd ${OPENSSL_ROOT}
./config no-shared --prefix=/usr/local/ssl --openssldir=/usr/local/ssl CPPFLAGS="${MANYLINUX_CPPFLAGS}" CFLAGS="${MANYLINUX_CFLAGS} -fPIC" CXXFLAGS="${MANYLINUX_CXXFLAGS} -fPIC" LDFLAGS="${MANYLINUX_LDFLAGS} -fPIC" > /dev/null
-make > /dev/null
+make -j$(nproc) > /dev/null
make install_sw > /dev/null
popd
rm -rf ${OPENSSL_ROOT} ${OPENSSL_ROOT}.tar.gz
diff --git a/build_utils.sh b/build_utils.sh
index 961e34d..55ae11b 100755
--- a/build_utils.sh
+++ b/build_utils.sh
@@ -52,7 +52,7 @@ function check_sha256sum {
function do_standard_install {
./configure "$@" CPPFLAGS="${MANYLINUX_CPPFLAGS}" CFLAGS="${MANYLINUX_CFLAGS}" "CXXFLAGS=${MANYLINUX_CXXFLAGS}" LDFLAGS="${MANYLINUX_LDFLAGS}" > /dev/null
- make > /dev/null
+ make -j$(nproc) > /dev/null
make install > /dev/null
}
diff --git a/finalize.sh b/finalize.sh
index 621eab9..4cbcf90 100755
--- a/finalize.sh
+++ b/finalize.sh
@@ -86,6 +86,3 @@ clean_pyc /opt/_internal
rm -rf /root/.cache
hardlink -cv /opt/_internal
-
-# update system packages
-LC_ALL=C ${MY_DIR}/update-system-packages.sh
diff --git a/install-build-packages.sh b/install-build-packages.sh
index 408bc33..b45ceba 100755
--- a/install-build-packages.sh
+++ b/install-build-packages.sh
@@ -9,12 +9,11 @@ set -exuo pipefail
# make sure the corresponding library is added to RUNTIME_DEPS if applicable
if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ]; then
- COMPILE_DEPS="bzip2-devel ncurses-devel readline-devel gdbm-devel libpcap-devel xz-devel openssl openssl-devel keyutils-libs-devel krb5-devel libcom_err-devel libidn-devel curl-devel uuid-devel libffi-devel kernel-headers libdb-devel"
+ COMPILE_DEPS="bzip2-devel ncurses-devel gdbm-devel xz-devel openssl openssl-devel keyutils-libs-devel krb5-devel libcom_err-devel curl-devel libffi-devel kernel-headers libdb-devel"
if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then
PACKAGE_MANAGER=yum
else
PACKAGE_MANAGER=dnf
- COMPILE_DEPS="${COMPILE_DEPS} tk-devel"
fi
elif [ "${AUDITWHEEL_POLICY}" == "musllinux_1_1" ]; then
PACKAGE_MANAGER=apk
diff --git a/install-entrypoint.sh b/install-entrypoint.sh
index 9ef1e99..ec52833 100755
--- a/install-entrypoint.sh
+++ b/install-entrypoint.sh
@@ -26,3 +26,8 @@ fi
if [ "${AUDITWHEEL_POLICY}" = "musllinux_1_1" ]; then
apk add --no-cache bash
fi
+
+if command -v yum 2>&1 ; then
+ yum install -y yum-plugin-versionlock
+ yum versionlock cuda* libcudnn* libnccl*
+fi
\ No newline at end of file
diff --git a/install-runtime-packages.sh b/install-runtime-packages.sh
index 137d2e2..a0ed0c8 100755
--- a/install-runtime-packages.sh
+++ b/install-runtime-packages.sh
@@ -33,7 +33,7 @@ source $MY_DIR/build_utils.sh
# MANYLINUX_DEPS: Install development packages (except for libgcc which is provided by gcc install)
if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ]; then
- MANYLINUX_DEPS="glibc-devel libstdc++-devel glib2-devel libX11-devel libXext-devel libXrender-devel mesa-libGL-devel libICE-devel libSM-devel zlib-devel expat-devel"
+ MANYLINUX_DEPS="glibc-devel libstdc++-devel glib2-devel zlib-devel expat-devel"
elif [ "${AUDITWHEEL_POLICY}" == "musllinux_1_1" ]; then
MANYLINUX_DEPS="musl-dev libstdc++ glib-dev libx11-dev libxext-dev libxrender-dev mesa-dev libice-dev libsm-dev zlib-dev expat-dev"
else
@@ -54,7 +54,7 @@ else
exit 1
fi
-BASETOOLS="autoconf automake bison bzip2 diffutils file make patch unzip"
+BASETOOLS="autoconf automake bzip2 diffutils file make patch unzip"
if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then
PACKAGE_MANAGER=yum
BASETOOLS="${BASETOOLS} hardlink hostname which"
@@ -73,9 +73,11 @@ if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then
if [ "${AUDITWHEEL_ARCH}" == "x86_64" ]; then
# Software collection (for devtoolset-10)
yum -y install centos-release-scl-rh
- # EPEL support (for yasm)
- yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
- TOOLCHAIN_DEPS="${TOOLCHAIN_DEPS} yasm"
+ if [[ -d /opt/rocm ]]; then
+ TOOLCHAIN_DEPS="devtoolset-10-binutils devtoolset-10-gcc devtoolset-10-gcc-c++ devtoolset-10-gcc-gfortran"
+ else
+ TOOLCHAIN_DEPS="devtoolset-11-binutils devtoolset-11-gcc devtoolset-11-gcc-c++ devtoolset-11-gcc-gfortran"
+ fi
elif [ "${AUDITWHEEL_ARCH}" == "aarch64" ] || [ "${AUDITWHEEL_ARCH}" == "ppc64le" ] || [ "${AUDITWHEEL_ARCH}" == "s390x" ]; then
# Software collection (for devtoolset-10)
yum -y install centos-release-scl-rh
@@ -86,19 +88,21 @@ if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then
fi
elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ]; then
PACKAGE_MANAGER=dnf
- BASETOOLS="${BASETOOLS} curl glibc-locale-source glibc-langpack-en hardlink hostname libcurl libnsl libxcrypt which"
+ BASETOOLS="${BASETOOLS} yum-utils curl glibc-locale-source glibc-langpack-en hardlink hostname libcurl libxcrypt which"
# See https://unix.stackexchange.com/questions/41784/can-yum-express-a-preference-for-x86-64-over-i386-packages
echo "multilib_policy=best" >> /etc/yum.conf
# Error out if requested packages do not exist
echo "skip_missing_names_on_install=False" >> /etc/yum.conf
# Make sure that locale will not be removed
sed -i '/^override_install_langs=/d' /etc/yum.conf
- dnf -y upgrade
dnf -y install dnf-plugins-core
- dnf config-manager --set-enabled powertools # for yasm
- TOOLCHAIN_DEPS="gcc-toolset-12-binutils gcc-toolset-12-gcc gcc-toolset-12-gcc-c++ gcc-toolset-12-gcc-gfortran"
- if [ "${AUDITWHEEL_ARCH}" == "x86_64" ]; then
- TOOLCHAIN_DEPS="${TOOLCHAIN_DEPS} yasm"
+ if test -f "/etc/yum.repos.d/ubi.repo"; then
+ sed -i 's/enabled\s*=\s*1/enabled = 1\nexclude=dotnet* aspnet* netstandard*/g' /etc/yum.repos.d/ubi.repo
+ fi
+ if [[ -d /usr/local/cuda ]]; then
+ TOOLCHAIN_DEPS="gcc gcc-c++ libasan"
+ else
+ TOOLCHAIN_DEPS="gcc-toolset-12-binutils gcc-toolset-12-gcc gcc-toolset-12-gcc-c++ gcc-toolset-12-gcc-gfortran gcc-toolset-12-libasan-devel"
fi
elif [ "${AUDITWHEEL_POLICY}" == "musllinux_1_1" ]; then
TOOLCHAIN_DEPS="binutils gcc g++ gfortran"
@@ -121,12 +125,6 @@ else
exit 1
fi
-# update system packages, we already updated them but
-# the following script takes care of cleaning-up some things
-# and since it's also needed in the finalize step, everything's
-# centralized in this script to avoid code duplication
-LC_ALL=C ${MY_DIR}/update-system-packages.sh
-
if [ "${BASE_POLICY}" == "manylinux" ]; then
# we'll be removing libcrypt.so.1 later on
# this is needed to ensure the new one will be found
Reference in a new issue View git blame Copy permalink