mirror of
https://github.com/saymrwulf/pytorch.git
synced 2026-05-14 20:57:59 +00:00
Improve ASAN and TSAN handling in cmake (#93147)
Pull Request resolved: https://github.com/pytorch/pytorch/pull/93147 Approved by: https://github.com/malfet
This commit is contained in:
cyy
PyTorch MergeBot
parent
8dbc549517
commit
666efd8d5d
8 changed files with 142 additions and 70 deletions
|
|
@ -14,17 +14,13 @@ clang --version
|
|||
|
||||
python tools/stats/export_test_times.py
|
||||
|
||||
# detect_leaks=0: Python is very leaky, so we need suppress it
|
||||
# symbolize=1: Gives us much better errors when things go wrong
|
||||
export ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=1:symbolize=1:detect_odr_violation=0
|
||||
if [ -n "$(which conda)" ]; then
|
||||
export CMAKE_PREFIX_PATH=/opt/conda
|
||||
fi
|
||||
|
||||
# TODO: Make the ASAN flags a centralized env var and unify with USE_ASAN option
|
||||
CC="clang" CXX="clang++" LDSHARED="clang --shared" \
|
||||
CFLAGS="-fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -fsanitize-address-use-after-scope -shared-libasan" \
|
||||
USE_ASAN=1 USE_CUDA=0 USE_MKLDNN=0 \
|
||||
UBSAN_FLAGS="-fno-sanitize-recover=all" \
|
||||
python setup.py bdist_wheel
|
||||
pip_install_whl "$(echo dist/*.whl)"
|
||||
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@ if [ -n "$(which conda)" ]; then
|
|||
fi
|
||||
|
||||
CC="clang" CXX="clang++" LDSHARED="clang --shared" \
|
||||
CFLAGS="-fsanitize=thread" \
|
||||
USE_TSAN=1 USE_CUDA=0 USE_MKLDNN=0 \
|
||||
python setup.py bdist_wheel
|
||||
pip_install_whl "$(echo dist/*.whl)"
|
||||
|
|
|
|||
|
|
@ -126,7 +126,7 @@ fi
|
|||
# if you're not careful. Check this if you made some changes and the
|
||||
# ASAN test is not working
|
||||
if [[ "$BUILD_ENVIRONMENT" == *asan* ]]; then
|
||||
export ASAN_OPTIONS=detect_leaks=0:symbolize=1:detect_stack_use_after_return=1:strict_init_order=true:detect_odr_violation=0
|
||||
export ASAN_OPTIONS=detect_leaks=0:symbolize=1:detect_stack_use_after_return=1:strict_init_order=true:detect_odr_violation=1:detect_container_overflow=0
|
||||
export UBSAN_OPTIONS=print_stacktrace=1
|
||||
export PYTORCH_TEST_WITH_ASAN=1
|
||||
export PYTORCH_TEST_WITH_UBSAN=1
|
||||
|
|
|
|||
|
|
@ -931,15 +931,6 @@ else()
|
|||
append_cxx_flag_if_supported("/wd4273" CMAKE_CXX_FLAGS)
|
||||
endif()
|
||||
|
||||
if(USE_ASAN)
|
||||
string(APPEND CMAKE_CXX_FLAGS_DEBUG " -fsanitize=address -fsanitize=undefined")
|
||||
string(APPEND CMAKE_LINKER_FLAGS_DEBUG " -fsanitize=address -fsanitize=undefined")
|
||||
endif()
|
||||
|
||||
if(USE_TSAN)
|
||||
string(APPEND CMAKE_CXX_FLAGS_DEBUG " -fsanitize=thread")
|
||||
string(APPEND CMAKE_LINKER_FLAGS_DEBUG " -fsanitize=thread")
|
||||
endif()
|
||||
|
||||
if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64")
|
||||
include(CheckCSourceCompiles)
|
||||
|
|
|
|||
|
|
@ -84,6 +84,29 @@ if(CAFFE2_CMAKE_BUILDING_WITH_MAIN_REPO AND NOT INTERN_BUILD_MOBILE)
|
|||
enable_ubsan()
|
||||
endif()
|
||||
|
||||
if(USE_ASAN OR USE_TSAN)
|
||||
find_package(Sanitizer REQUIRED)
|
||||
if(USE_ASAN)
|
||||
if(TARGET Sanitizer::address)
|
||||
list(APPEND Caffe2_PUBLIC_DEPENDENCY_LIBS Sanitizer::address)
|
||||
else()
|
||||
message(WARNING "Not ASAN found. Suppress this warning with -DUSE_ASAN=OFF.")
|
||||
caffe2_update_option(USE_ASAN OFF)
|
||||
endif()
|
||||
if(TARGET Sanitizer::undefined)
|
||||
list(APPEND Caffe2_PUBLIC_DEPENDENCY_LIBS Sanitizer::undefined)
|
||||
endif()
|
||||
endif()
|
||||
if(USE_TSAN)
|
||||
if(TARGET Sanitizer::thread)
|
||||
list(APPEND Caffe2_PUBLIC_DEPENDENCY_LIBS Sanitizer::thread)
|
||||
else()
|
||||
message(WARNING "Not TSAN found. Suppress this warning with -DUSE_TSAN=OFF.")
|
||||
caffe2_update_option(USE_TSAN OFF)
|
||||
endif()
|
||||
endif()
|
||||
endif()
|
||||
|
||||
# ---[ Threads
|
||||
find_package(Threads REQUIRED)
|
||||
if(TARGET Threads::Threads)
|
||||
|
|
|
|||
|
|
@ -10,37 +10,6 @@ include(CheckCXXSourceCompiles)
|
|||
include(CheckCXXCompilerFlag)
|
||||
include(CMakePushCheckState)
|
||||
|
||||
if(NOT INTERN_BUILD_MOBILE)
|
||||
# ---[ Check that our programs run. This is different from the native CMake
|
||||
# compiler check, which just tests if the program compiles and links. This is
|
||||
# important because with ASAN you might need to help the compiled library find
|
||||
# some dynamic libraries.
|
||||
cmake_push_check_state(RESET)
|
||||
if(CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_OSX_ARCHITECTURES MATCHES "^(x86_64|arm64)$")
|
||||
list(APPEND CMAKE_REQUIRED_FLAGS "-arch ${CMAKE_HOST_SYSTEM_PROCESSOR}")
|
||||
endif()
|
||||
if(CMAKE_CROSSCOMPILING)
|
||||
CHECK_C_SOURCE_COMPILES("
|
||||
int main() { return 0; }
|
||||
" COMPILER_WORKS)
|
||||
else()
|
||||
CHECK_C_SOURCE_RUNS("
|
||||
int main() { return 0; }
|
||||
" COMPILER_WORKS)
|
||||
endif()
|
||||
if(NOT COMPILER_WORKS)
|
||||
# Force cmake to retest next time around
|
||||
unset(COMPILER_WORKS CACHE)
|
||||
message(FATAL_ERROR
|
||||
"Could not run a simple program built with your compiler. "
|
||||
"If you are trying to use -fsanitize=address, make sure "
|
||||
"libasan is properly installed on your system (you can confirm "
|
||||
"if the problem is this by attempting to build and run a "
|
||||
"small program.)")
|
||||
endif()
|
||||
cmake_pop_check_state()
|
||||
endif()
|
||||
|
||||
set(CAFFE2_USE_EXCEPTION_PTR 1)
|
||||
|
||||
# ---[ Check if we want to turn off deprecated warning due to glog.
|
||||
|
|
@ -150,29 +119,6 @@ if(IOS AND (${IOS_ARCH} MATCHES "armv7*"))
|
|||
add_definitions("-Wno-deprecated-declarations")
|
||||
endif()
|
||||
|
||||
# ---[ If we use asan, turn on the flags.
|
||||
# TODO: This only works with new style gcc and clang (not the old -faddress-sanitizer).
|
||||
# Change if necessary on old platforms.
|
||||
if(USE_ASAN)
|
||||
set(CAFFE2_ASAN_COMPILER_FLAGS "-fsanitize=address -fPIE")
|
||||
set(CAFFE2_ASAN_LINKER_FLAGS "-pie")
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${CAFFE2_ASAN_COMPILER_FLAGS}")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CAFFE2_ASAN_COMPILER_FLAGS}")
|
||||
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${CAFFE2_ASAN_LINKER_FLAGS}")
|
||||
set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} ${CAFFE2_ASAN_LINKER_FLAGS}")
|
||||
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${CAFFE2_ASAN_LINKER_FLAGS}")
|
||||
endif()
|
||||
|
||||
if(USE_TSAN)
|
||||
set(CAFFE2_TSAN_COMPILER_FLAGS "-fsanitize=thread -fPIE")
|
||||
set(CAFFE2_TSAN_LINKER_FLAGS "-pie")
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${CAFFE2_TSAN_COMPILER_FLAGS}")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CAFFE2_TSAN_COMPILER_FLAGS}")
|
||||
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${CAFFE2_TSAN_LINKER_FLAGS}")
|
||||
set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} ${CAFFE2_TSAN_LINKER_FLAGS}")
|
||||
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${CAFFE2_TSAN_LINKER_FLAGS}")
|
||||
endif()
|
||||
|
||||
# ---[ Create CAFFE2_BUILD_SHARED_LIBS for macros.h.in usage.
|
||||
set(CAFFE2_BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS})
|
||||
|
||||
|
|
|
|||
116
cmake/Modules/FindSanitizer.cmake
Normal file
116
cmake/Modules/FindSanitizer.cmake
Normal file
|
|
@ -0,0 +1,116 @@
|
|||
# Find sanitizers
|
||||
#
|
||||
# This module sets the following targets:
|
||||
# Sanitizer::address
|
||||
# Sanitizer::thread
|
||||
# Sanitizer::undefined
|
||||
# Sanitizer::leak
|
||||
# Sanitizer::memory
|
||||
include_guard(GLOBAL)
|
||||
|
||||
option(UBSAN_FLAGS "additional UBSAN flags" OFF)
|
||||
|
||||
get_property(languages GLOBAL PROPERTY ENABLED_LANGUAGES)
|
||||
|
||||
set(_source_code
|
||||
[==[
|
||||
#include <stdio.h>
|
||||
int main() {
|
||||
printf("hello world!");
|
||||
return 0;
|
||||
}
|
||||
]==])
|
||||
|
||||
include(CMakePushCheckState)
|
||||
cmake_push_check_state(RESET)
|
||||
foreach(sanitizer_name IN ITEMS address thread undefined leak memory)
|
||||
if(TARGET Sanitizer::${sanitizer_name})
|
||||
continue()
|
||||
endif()
|
||||
|
||||
set(CMAKE_REQUIRED_FLAGS
|
||||
"-fsanitize=${sanitizer_name};-fno-omit-frame-pointer")
|
||||
if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC" OR CMAKE_C_COMPILER_ID STREQUAL
|
||||
"MSVC")
|
||||
if(sanitizer_name STREQUAL "address")
|
||||
set(CMAKE_REQUIRED_FLAGS "/fsanitize=${sanitizer_name}")
|
||||
else()
|
||||
continue()
|
||||
endif()
|
||||
endif()
|
||||
if(sanitizer_name STREQUAL "address")
|
||||
if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR CMAKE_C_COMPILER_ID STREQUAL
|
||||
"Clang")
|
||||
list(APPEND CMAKE_REQUIRED_FLAGS "-shared-libasan")
|
||||
endif()
|
||||
endif()
|
||||
if(sanitizer_name STREQUAL "undefined" AND UBSAN_FLAGS)
|
||||
list(APPEND CMAKE_REQUIRED_FLAGS "${UBSAN_FLAGS}")
|
||||
endif()
|
||||
if(sanitizer_name STREQUAL "memory")
|
||||
list(APPEND CMAKE_REQUIRED_FLAGS "-fsanitize-memory-track-origins=2")
|
||||
endif()
|
||||
|
||||
set(CMAKE_REQUIRED_QUIET ON)
|
||||
set(_run_res 0)
|
||||
include(CheckSourceRuns)
|
||||
foreach(lang IN LISTS languages)
|
||||
if(lang STREQUAL CXX OR lang STREQUAL C)
|
||||
check_source_runs(${lang} "${_source_code}"
|
||||
__${lang}_${sanitizer_name}_res)
|
||||
if(__${lang}_${sanitizer_name}_res)
|
||||
set(_run_res 1)
|
||||
endif()
|
||||
endif()
|
||||
endforeach()
|
||||
if(_run_res)
|
||||
add_library(Sanitizer::${sanitizer_name} INTERFACE IMPORTED GLOBAL)
|
||||
target_compile_options(
|
||||
Sanitizer::${sanitizer_name}
|
||||
INTERFACE
|
||||
$<$<AND:$<COMPILE_LANGUAGE:CXX>,$<BOOL:$__CXX_${sanitizer_name}_res>>:${CMAKE_REQUIRED_FLAGS}>
|
||||
$<$<AND:$<COMPILE_LANGUAGE:C>,$<BOOL:$__C_${sanitizer_name}_res>>:${CMAKE_REQUIRED_FLAGS}>
|
||||
)
|
||||
if(NOT CMAKE_CXX_COMPILER_ID STREQUAL "MSVC" AND NOT CMAKE_C_COMPILER_ID
|
||||
STREQUAL "MSVC")
|
||||
target_link_options(
|
||||
Sanitizer::${sanitizer_name}
|
||||
INTERFACE
|
||||
$<$<AND:$<COMPILE_LANGUAGE:CXX>,$<BOOL:$__CXX_${sanitizer_name}_res>>:${CMAKE_REQUIRED_FLAGS}>
|
||||
$<$<AND:$<COMPILE_LANGUAGE:C>,$<BOOL:$__C_${sanitizer_name}_res>>:${CMAKE_REQUIRED_FLAGS}>
|
||||
)
|
||||
else()
|
||||
target_link_options(
|
||||
Sanitizer::${sanitizer_name}
|
||||
INTERFACE
|
||||
$<$<AND:$<COMPILE_LANGUAGE:CXX>,$<BOOL:$__CXX_${sanitizer_name}_res>>:/INCREMENTAL:NO>
|
||||
$<$<AND:$<COMPILE_LANGUAGE:C>,$<BOOL:$__C_${sanitizer_name}_res>>:/INCREMENTAL:NO>
|
||||
)
|
||||
endif()
|
||||
|
||||
if(sanitizer_name STREQUAL "address")
|
||||
target_compile_definitions(
|
||||
Sanitizer::${sanitizer_name}
|
||||
INTERFACE
|
||||
$<$<AND:$<COMPILE_LANGUAGE:CXX>,$<BOOL:$__CXX_${sanitizer_name}_res>>:_GLIBCXX_SANITIZE_VECTOR>
|
||||
$<$<AND:$<COMPILE_LANGUAGE:CXX>,$<BOOL:$__CXX_${sanitizer_name}_res>>:_GLIBCXX_SANITIZE_STD_ALLOCATOR>
|
||||
)
|
||||
target_link_options(
|
||||
Sanitizer::${sanitizer_name}
|
||||
INTERFACE
|
||||
$<$<AND:$<COMPILE_LANGUAGE:CXX>,$<BOOL:$__CXX_${sanitizer_name}_res>,$<CXX_COMPILER_ID:GNU>>:-lasan>
|
||||
$<$<AND:$<COMPILE_LANGUAGE:C>,$<BOOL:$__C_${sanitizer_name}_res>,$<C_COMPILER_ID:GNU>>:-lasan>
|
||||
)
|
||||
endif()
|
||||
if(sanitizer_name STREQUAL "undefined")
|
||||
target_link_options(
|
||||
Sanitizer::${sanitizer_name}
|
||||
INTERFACE
|
||||
$<$<AND:$<COMPILE_LANGUAGE:CXX>,$<BOOL:$__CXX_${sanitizer_name}_res>,$<CXX_COMPILER_ID:GNU>>:-lubsan>
|
||||
$<$<AND:$<COMPILE_LANGUAGE:C>,$<BOOL:$__C_${sanitizer_name}_res>,$<C_COMPILER_ID:GNU>>:-lubsan>
|
||||
)
|
||||
endif()
|
||||
endif()
|
||||
endforeach()
|
||||
|
||||
cmake_pop_check_state()
|
||||
|
|
@ -203,6 +203,7 @@ class CMake:
|
|||
# CMakeLists.txt.
|
||||
var: var
|
||||
for var in (
|
||||
"UBSAN_FLAGS",
|
||||
"BLAS",
|
||||
"WITH_BLAS",
|
||||
"BUILDING_WITH_TORCH_LIBS",
|
||||
|
|
|
|||
Loading…
Reference in a new issue