From 39059f2539f33c932513c8d37f4a1911005b66b1 Mon Sep 17 00:00:00 2001 From: pengwa Date: Thu, 19 Aug 2021 09:16:32 +0800 Subject: [PATCH] enable torch interop build (#8493) * fix build - python.h not found * disable --build_shared_lib for ortmodule tests * fix * fix the build flag * disable --build_shared_lib for training path (not only for ortmodule) * fix missing test model files * disable test CApiTest.test_custom_op_library when ENABLE_TRAINING_TORCH_INTEROP is ON * enable custom_op_library build * fix build * fix * merge master and fix build failure * build onnx_test_runner when onnxruntime_ENABLE_TRAINING_TORCH_INTEROP is ON * resolve comments * use --enable_training_torch_interop to replace "onnxruntime_ENABLE_TRAINING_TORCH_INTEROP=ON" --- cmake/onnxruntime_session.cmake | 1 + cmake/onnxruntime_training.cmake | 4 +- cmake/onnxruntime_unittests.cmake | 287 +++++++++--------- .../orttraining-linux-gpu-ci-pipeline.yml | 3 +- ...ortmodule-distributed-test-ci-pipeline.yml | 3 +- tools/ci_build/github/linux/run_build.sh | 2 +- .../ci_build/github/linux/run_dockerbuild.sh | 12 +- 7 files changed, 170 insertions(+), 142 deletions(-) diff --git a/cmake/onnxruntime_session.cmake b/cmake/onnxruntime_session.cmake index 8e8eb29753..edbebcc687 100644 --- a/cmake/onnxruntime_session.cmake +++ b/cmake/onnxruntime_session.cmake @@ -23,6 +23,7 @@ onnxruntime_add_include_to_target(onnxruntime_session onnxruntime_common onnxrun if(onnxruntime_ENABLE_INSTRUMENT) target_compile_definitions(onnxruntime_session PUBLIC ONNXRUNTIME_ENABLE_INSTRUMENT) endif() + if(NOT MSVC) set_source_files_properties(${ONNXRUNTIME_ROOT}/core/session/environment.cc PROPERTIES COMPILE_FLAGS "-Wno-parentheses") endif() diff --git a/cmake/onnxruntime_training.cmake b/cmake/onnxruntime_training.cmake index 68cc34cf31..acbc4818ef 100644 --- a/cmake/onnxruntime_training.cmake +++ b/cmake/onnxruntime_training.cmake @@ -71,7 +71,7 @@ if (onnxruntime_BUILD_UNIT_TESTS) add_dependencies(onnxruntime_training_runner ${onnxruntime_EXTERNAL_DEPENDENCIES} onnx onnxruntime_providers) if (onnxruntime_ENABLE_TRAINING_TORCH_INTEROP) - target_link_libraries(onnxruntime_training_runner PRIVATE Python::Module) + target_link_libraries(onnxruntime_training_runner PRIVATE Python::Python) endif() onnxruntime_add_include_to_target(onnxruntime_training_runner onnxruntime_training onnxruntime_framework onnxruntime_common onnx onnx_proto ${PROTOBUF_LIB} onnxruntime_training flatbuffers) @@ -127,7 +127,7 @@ if (onnxruntime_BUILD_UNIT_TESTS) ) if (onnxruntime_ENABLE_TRAINING_TORCH_INTEROP) - list(APPEND ONNXRUNTIME_LIBS Python::Module) + list(APPEND ONNXRUNTIME_LIBS Python::Python) endif() list(APPEND ONNXRUNTIME_LIBS diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index e64e2b0267..696d32277d 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -703,7 +703,7 @@ if (onnxruntime_USE_ROCM) target_include_directories(onnxruntime_test_all PRIVATE ${onnxruntime_ROCM_HOME}/hipfft/include ${onnxruntime_ROCM_HOME}/include ${onnxruntime_ROCM_HOME}/hiprand/include ${onnxruntime_ROCM_HOME}/rocrand/include ${CMAKE_CURRENT_BINARY_DIR}/amdgpu/onnxruntime ${CMAKE_CURRENT_BINARY_DIR}/amdgpu/orttraining) endif() if (onnxruntime_ENABLE_TRAINING_TORCH_INTEROP) - target_link_libraries(onnxruntime_test_all PRIVATE Python::Module) + target_link_libraries(onnxruntime_test_all PRIVATE Python::Python) endif() if (onnxruntime_BUILD_WEBASSEMBLY) set_target_properties(onnxruntime_test_all PROPERTIES LINK_DEPENDS ${TEST_SRC_DIR}/wasm/dump-test-result-in-nodejs.js) @@ -731,30 +731,30 @@ target_include_directories(onnx_test_data_proto PRIVATE ${CMAKE_CURRENT_BINARY_D set_target_properties(onnx_test_data_proto PROPERTIES FOLDER "ONNXRuntimeTest") onnxruntime_protobuf_generate(APPEND_PATH IMPORT_DIRS external/onnx TARGET onnx_test_data_proto) +# +# onnxruntime_ir_graph test data +# +set(TEST_DATA_SRC ${TEST_SRC_DIR}/testdata) +set(TEST_DATA_DES $/testdata) + +set(TEST_SAMPLES_SRC ${REPO_ROOT}/samples) +set(TEST_SAMPLES_DES $/samples) + +# Copy test data from source to destination. +add_custom_command( + TARGET ${test_data_target} PRE_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_directory + ${TEST_DATA_SRC} + ${TEST_DATA_DES}) + +# Copy test samples from source to destination. +add_custom_command( + TARGET ${test_data_target} PRE_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_directory + ${TEST_SAMPLES_SRC} + ${TEST_SAMPLES_DES}) + if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP) - # - # onnxruntime_ir_graph test data - # - set(TEST_DATA_SRC ${TEST_SRC_DIR}/testdata) - set(TEST_DATA_DES $/testdata) - - set(TEST_SAMPLES_SRC ${REPO_ROOT}/samples) - set(TEST_SAMPLES_DES $/samples) - - # Copy test data from source to destination. - add_custom_command( - TARGET ${test_data_target} PRE_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_directory - ${TEST_DATA_SRC} - ${TEST_DATA_DES}) - - # Copy test samples from source to destination. - add_custom_command( - TARGET ${test_data_target} PRE_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_directory - ${TEST_SAMPLES_SRC} - ${TEST_SAMPLES_DES}) - if (onnxruntime_USE_DNNL) list(APPEND onnx_test_libs dnnl) add_custom_command( @@ -779,45 +779,50 @@ if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP) set(onnx_test_runner_common_srcs ${onnx_test_runner_common_srcs}) set(GETOPT_LIB_WIDE win_getopt_wide) endif() +endif() - set(onnx_test_libs - onnxruntime_test_utils - ${ONNXRUNTIME_TEST_LIBS} - onnx_test_data_proto - ${onnxruntime_EXTERNAL_LIBRARIES}) +set(onnx_test_libs + onnxruntime_test_utils + ${ONNXRUNTIME_TEST_LIBS} + onnx_test_data_proto + ${onnxruntime_EXTERNAL_LIBRARIES}) - if (onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS) - list(APPEND onnx_test_libs onnxruntime_language_interop onnxruntime_pyop) +if (onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS) + list(APPEND onnx_test_libs onnxruntime_language_interop onnxruntime_pyop) +endif() + +onnxruntime_add_executable(onnx_test_runner ${onnx_test_runner_src_dir}/main.cc) +if(MSVC) + target_compile_options(onnx_test_runner PRIVATE "$<$:SHELL:--compiler-options /utf-8>" + "$<$>:/utf-8>") +endif() +if(${CMAKE_SYSTEM_NAME} STREQUAL "iOS") + set_target_properties(onnx_test_runner PROPERTIES + XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED "NO" + ) +endif() + +target_link_libraries(onnx_test_runner PRIVATE onnx_test_runner_common ${GETOPT_LIB_WIDE} ${onnx_test_libs}) +target_include_directories(onnx_test_runner PRIVATE ${ONNXRUNTIME_ROOT}) +if (onnxruntime_ENABLE_TRAINING_TORCH_INTEROP) + target_link_libraries(onnx_test_runner PRIVATE Python::Python) +endif() +set_target_properties(onnx_test_runner PROPERTIES FOLDER "ONNXRuntimeTest") + +if (onnxruntime_USE_TVM) + if (WIN32) + target_link_options(onnx_test_runner PRIVATE "/STACK:4000000") endif() +endif() - onnxruntime_add_executable(onnx_test_runner ${onnx_test_runner_src_dir}/main.cc) - if(MSVC) - target_compile_options(onnx_test_runner PRIVATE "$<$:SHELL:--compiler-options /utf-8>" - "$<$>:/utf-8>") - endif() - if(${CMAKE_SYSTEM_NAME} STREQUAL "iOS") - set_target_properties(onnx_test_runner PROPERTIES - XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED "NO" - ) - endif() - - target_link_libraries(onnx_test_runner PRIVATE onnx_test_runner_common ${GETOPT_LIB_WIDE} ${onnx_test_libs}) - target_include_directories(onnx_test_runner PRIVATE ${ONNXRUNTIME_ROOT}) - set_target_properties(onnx_test_runner PROPERTIES FOLDER "ONNXRuntimeTest") - - if (onnxruntime_USE_TVM) - if (WIN32) - target_link_options(onnx_test_runner PRIVATE "/STACK:4000000") - endif() - endif() - - install(TARGETS onnx_test_runner - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - BUNDLE DESTINATION ${CMAKE_INSTALL_LIBDIR} - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) +install(TARGETS onnx_test_runner + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + BUNDLE DESTINATION ${CMAKE_INSTALL_LIBDIR} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) +if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP) if(onnxruntime_BUILD_BENCHMARKS) SET(BENCHMARK_DIR ${TEST_SRC_DIR}/onnx/microbenchmark) onnxruntime_add_executable(onnxruntime_benchmark @@ -869,46 +874,51 @@ if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP) if (CMAKE_SYSTEM_NAME STREQUAL "Android") list(APPEND android_shared_libs log android) endif() +endif() - #eager mode test - if(onnxruntime_ENABLE_EAGER_MODE) - file(GLOB onnxruntime_eager_mode_test_src CONFIGURE_DEPENDS - "${TEST_SRC_DIR}/eager/*.cc" - ) - add_executable(onnxruntime_eager_mode_test ${onnxruntime_eager_mode_test_src}) - target_include_directories(onnxruntime_eager_mode_test PRIVATE ${ONNXRUNTIME_ROOT} - ${onnxruntime_graph_header} - ${onnxruntime_exec_src_dir} - ${CMAKE_CURRENT_BINARY_DIR} - "${TEST_SRC_DIR}/util/include") - set(onnxruntime_eager_mode_libs - onnxruntime_eager - onnxruntime_session - onnxruntime_optimizer - onnxruntime_providers - onnxruntime_util - onnxruntime_framework - flatbuffers - onnxruntime_graph - onnxruntime_common - onnxruntime_mlas - onnx - onnx_proto - ${PROTOBUF_LIB} - GTest::gtest - re2::re2 - onnxruntime_flatbuffers - ${CMAKE_DL_LIBS} - ) - if(onnxruntime_ENABLE_TRAINING) - list(APPEND onnxruntime_eager_mode_libs onnxruntime_training tensorboard) - endif() - IF(NOT WIN32) - list(APPEND onnxruntime_eager_mode_libs nsync_cpp) - endif() - target_link_libraries(onnxruntime_eager_mode_test PRIVATE ${onnxruntime_eager_mode_libs} Threads::Threads ${onnxruntime_EXTERNAL_LIBRARIES}) +#eager mode test +if(onnxruntime_ENABLE_EAGER_MODE) + file(GLOB onnxruntime_eager_mode_test_src CONFIGURE_DEPENDS + "${TEST_SRC_DIR}/eager/*.cc" + ) + add_executable(onnxruntime_eager_mode_test ${onnxruntime_eager_mode_test_src}) + target_include_directories(onnxruntime_eager_mode_test PRIVATE ${ONNXRUNTIME_ROOT} + ${onnxruntime_graph_header} + ${onnxruntime_exec_src_dir} + ${CMAKE_CURRENT_BINARY_DIR} + "${TEST_SRC_DIR}/util/include") + set(onnxruntime_eager_mode_libs + onnxruntime_eager + onnxruntime_session + onnxruntime_optimizer + onnxruntime_providers + onnxruntime_util + onnxruntime_framework + flatbuffers + onnxruntime_graph + onnxruntime_common + onnxruntime_mlas + onnx + onnx_proto + ${PROTOBUF_LIB} + GTest::gtest + re2::re2 + onnxruntime_flatbuffers + ${CMAKE_DL_LIBS} + ) + if(onnxruntime_ENABLE_TRAINING) + list(APPEND onnxruntime_eager_mode_libs onnxruntime_training tensorboard) endif() + IF(NOT WIN32) + list(APPEND onnxruntime_eager_mode_libs nsync_cpp) + endif() + target_link_libraries(onnxruntime_eager_mode_test PRIVATE ${onnxruntime_eager_mode_libs} Threads::Threads ${onnxruntime_EXTERNAL_LIBRARIES}) + if (onnxruntime_ENABLE_TRAINING_TORCH_INTEROP) + target_link_libraries(onnxruntime_eager_mode_test PRIVATE Python::Python) + endif() +endif() +if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP) #perf test runner set(onnxruntime_perf_test_src_dir ${TEST_SRC_DIR}/perftest) set(onnxruntime_perf_test_src_patterns @@ -1121,20 +1131,22 @@ if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP) set_target_properties(onnxruntime_mlas_test PROPERTIES LINK_FLAGS "-s ALLOW_MEMORY_GROWTH=1") endif() endif() +endif() - onnxruntime_add_shared_library_module(custom_op_library ${TEST_SRC_DIR}/testdata/custom_op_library/custom_op_library.cc) - target_include_directories(custom_op_library PRIVATE ${REPO_ROOT}/include) - if(UNIX) - if (APPLE) - set(ONNXRUNTIME_CUSTOM_OP_LIB_LINK_FLAG "-Xlinker -dead_strip") - else() - set(ONNXRUNTIME_CUSTOM_OP_LIB_LINK_FLAG "-Xlinker --version-script=${TEST_SRC_DIR}/testdata/custom_op_library/custom_op_library.lds -Xlinker --no-undefined -Xlinker --gc-sections -z noexecstack") - endif() +onnxruntime_add_shared_library_module(custom_op_library ${TEST_SRC_DIR}/testdata/custom_op_library/custom_op_library.cc) +target_include_directories(custom_op_library PRIVATE ${REPO_ROOT}/include) +if(UNIX) + if (APPLE) + set(ONNXRUNTIME_CUSTOM_OP_LIB_LINK_FLAG "-Xlinker -dead_strip") else() - set(ONNXRUNTIME_CUSTOM_OP_LIB_LINK_FLAG "-DEF:${TEST_SRC_DIR}/testdata/custom_op_library/custom_op_library.def") + set(ONNXRUNTIME_CUSTOM_OP_LIB_LINK_FLAG "-Xlinker --version-script=${TEST_SRC_DIR}/testdata/custom_op_library/custom_op_library.lds -Xlinker --no-undefined -Xlinker --gc-sections -z noexecstack") endif() - set_property(TARGET custom_op_library APPEND_STRING PROPERTY LINK_FLAGS ${ONNXRUNTIME_CUSTOM_OP_LIB_LINK_FLAG}) +else() + set(ONNXRUNTIME_CUSTOM_OP_LIB_LINK_FLAG "-DEF:${TEST_SRC_DIR}/testdata/custom_op_library/custom_op_library.def") +endif() +set_property(TARGET custom_op_library APPEND_STRING PROPERTY LINK_FLAGS ${ONNXRUNTIME_CUSTOM_OP_LIB_LINK_FLAG}) +if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP) if (onnxruntime_BUILD_JAVA AND NOT onnxruntime_ENABLE_STATIC_ANALYSIS) message(STATUS "Running Java tests") # native-test is added to resources so custom_op_lib can be loaded @@ -1165,36 +1177,39 @@ if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP) endif() set_property(TEST onnxruntime4j_test APPEND PROPERTY DEPENDS onnxruntime4j_jni) endif() - - # limit to only test on windows first, due to a runtime path issue on linux - if (NOT onnxruntime_MINIMAL_BUILD AND NOT onnxruntime_EXTENDED_MINIMAL_BUILD - AND NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin|iOS" - AND NOT (CMAKE_SYSTEM_NAME STREQUAL "Android") - AND NOT onnxruntime_BUILD_WEBASSEMBLY - AND NOT onnxruntime_USE_ROCM) - file(GLOB_RECURSE test_execution_provider_srcs - "${REPO_ROOT}/onnxruntime/test/testdata/custom_execution_provider_library/*.h" - "${REPO_ROOT}/onnxruntime/test/testdata/custom_execution_provider_library/*.cc" - "${ONNXRUNTIME_ROOT}/core/providers/shared_library/*.h" - "${ONNXRUNTIME_ROOT}/core/providers/shared_library/*.cc" - ) - - onnxruntime_add_shared_library_module(test_execution_provider ${test_execution_provider_srcs}) - add_dependencies(test_execution_provider onnxruntime_providers_shared onnx) - target_link_libraries(test_execution_provider PRIVATE onnxruntime_providers_shared) - target_include_directories(test_execution_provider PRIVATE $) - target_include_directories(test_execution_provider PRIVATE $) - target_include_directories(test_execution_provider PRIVATE ${ONNXRUNTIME_ROOT} ${CMAKE_CURRENT_BINARY_DIR} ${ORTTRAINING_ROOT}) - if(APPLE) - set_property(TARGET test_execution_provider APPEND_STRING PROPERTY LINK_FLAGS "-Xlinker -exported_symbols_list ${REPO_ROOT}/onnxruntime/test/testdata/custom_execution_provider_library/exported_symbols.lst") - elseif(UNIX) - set_property(TARGET test_execution_provider APPEND_STRING PROPERTY LINK_FLAGS "-Xlinker --version-script=${REPO_ROOT}/onnxruntime/test/testdata/custom_execution_provider_library/version_script.lds -Xlinker --gc-sections -Xlinker -rpath=\\$ORIGIN") - elseif(WIN32) - set_property(TARGET test_execution_provider APPEND_STRING PROPERTY LINK_FLAGS "-DEF:${REPO_ROOT}/onnxruntime/test/testdata/custom_execution_provider_library/symbols.def") - else() - message(FATAL_ERROR "test_execution_provider unknown platform, need to specify shared library exports for it") - endif() - endif() - - include(onnxruntime_fuzz_test.cmake) endif() + +# limit to only test on windows first, due to a runtime path issue on linux +if (NOT onnxruntime_MINIMAL_BUILD AND NOT onnxruntime_EXTENDED_MINIMAL_BUILD + AND NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin|iOS" + AND NOT (CMAKE_SYSTEM_NAME STREQUAL "Android") + AND NOT onnxruntime_BUILD_WEBASSEMBLY + AND NOT onnxruntime_USE_ROCM) + file(GLOB_RECURSE test_execution_provider_srcs + "${REPO_ROOT}/onnxruntime/test/testdata/custom_execution_provider_library/*.h" + "${REPO_ROOT}/onnxruntime/test/testdata/custom_execution_provider_library/*.cc" + "${ONNXRUNTIME_ROOT}/core/providers/shared_library/*.h" + "${ONNXRUNTIME_ROOT}/core/providers/shared_library/*.cc" + ) + + onnxruntime_add_shared_library_module(test_execution_provider ${test_execution_provider_srcs}) + add_dependencies(test_execution_provider onnxruntime_providers_shared onnx) + target_link_libraries(test_execution_provider PRIVATE onnxruntime_providers_shared) + target_include_directories(test_execution_provider PRIVATE $) + target_include_directories(test_execution_provider PRIVATE $) + target_include_directories(test_execution_provider PRIVATE ${ONNXRUNTIME_ROOT} ${CMAKE_CURRENT_BINARY_DIR} ${ORTTRAINING_ROOT}) + if (onnxruntime_ENABLE_TRAINING_TORCH_INTEROP) + target_link_libraries(test_execution_provider PRIVATE Python::Python) + endif() + if(APPLE) + set_property(TARGET test_execution_provider APPEND_STRING PROPERTY LINK_FLAGS "-Xlinker -exported_symbols_list ${REPO_ROOT}/onnxruntime/test/testdata/custom_execution_provider_library/exported_symbols.lst") + elseif(UNIX) + set_property(TARGET test_execution_provider APPEND_STRING PROPERTY LINK_FLAGS "-Xlinker --version-script=${REPO_ROOT}/onnxruntime/test/testdata/custom_execution_provider_library/version_script.lds -Xlinker --gc-sections -Xlinker -rpath=\\$ORIGIN") + elseif(WIN32) + set_property(TARGET test_execution_provider APPEND_STRING PROPERTY LINK_FLAGS "-DEF:${REPO_ROOT}/onnxruntime/test/testdata/custom_execution_provider_library/symbols.def") + else() + message(FATAL_ERROR "test_execution_provider unknown platform, need to specify shared library exports for it") + endif() +endif() + +include(onnxruntime_fuzz_test.cmake) diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ci-pipeline.yml index 9896686732..cf96ea585f 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ci-pipeline.yml @@ -12,11 +12,12 @@ jobs: -e -x " --enable_training + --enable_training_torch_interop --config $(buildConfig) --use_cuda --cuda_version=11.1 --cuda_home=/usr/local/cuda-11.1 --cudnn_home=/usr/local/cuda-11.1 --build_wheel --enable_nvtx_profile - --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=70 onnxruntime_ENABLE_TRAINING_TORCH_INTEROP=ON + --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=70 " DoNugetPack: 'false' RunInjectedPipeline: 'true' diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ortmodule-distributed-test-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ortmodule-distributed-test-ci-pipeline.yml index 75c2f38fd6..fc521c8f4a 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ortmodule-distributed-test-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ortmodule-distributed-test-ci-pipeline.yml @@ -20,8 +20,9 @@ jobs: --config RelWithDebInfo \ --use_cuda --cuda_version=11.1 --cuda_home=/usr/local/cuda-11.1 --cudnn_home=/usr/local/cuda-11.1 \ --enable_training \ + --enable_training_torch_interop \ --update --build \ - --build_wheel --cmake_extra_defines onnxruntime_ENABLE_TRAINING_TORCH_INTEROP=ON \ + --build_wheel \ " \ -m \ -u \ diff --git a/tools/ci_build/github/linux/run_build.sh b/tools/ci_build/github/linux/run_build.sh index 3443682c87..373fa27d1d 100755 --- a/tools/ci_build/github/linux/run_build.sh +++ b/tools/ci_build/github/linux/run_build.sh @@ -44,7 +44,7 @@ elif [ $BUILD_OS = "yocto" ]; then make -j$(nproc) else - COMMON_BUILD_ARGS="--skip_submodule_sync --enable_onnx_tests --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest" + COMMON_BUILD_ARGS="--skip_submodule_sync --enable_onnx_tests --parallel --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest" if [ $BUILD_DEVICE = "gpu" ]; then _CUDNN_VERSION=$(echo $CUDNN_VERSION | cut -d. -f1-2) diff --git a/tools/ci_build/github/linux/run_dockerbuild.sh b/tools/ci_build/github/linux/run_dockerbuild.sh index 86fdbbfb15..f9f871c5bb 100755 --- a/tools/ci_build/github/linux/run_dockerbuild.sh +++ b/tools/ci_build/github/linux/run_dockerbuild.sh @@ -65,6 +65,8 @@ if [[ -n "${IMAGE_CACHE_CONTAINER_REGISTRY_NAME}" ]]; then fi DOCKER_CMD="docker" + +NEED_BUILD_SHARED_LIB=true cd $SCRIPT_DIR/docker if [ $BUILD_OS = "android" ]; then IMAGE="android" @@ -84,8 +86,12 @@ elif [ $BUILD_OS = "yocto" ]; then --docker-build-args="--build-arg TOOL_CHAIN=$TOOL_CHAIN_SCRIPT --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER}" \ --dockerfile $DOCKER_FILE --context . elif [ $BUILD_DEVICE = "gpu" ]; then - #This code path is only for training. Inferecing pipeline uses CentOS + # This code path is only for training. Inferecing pipeline uses CentOS IMAGE="$BUILD_OS-gpu_training" + # Current build script doesn't support building shared lib with Python dependency. To enable building with PythonOp, + # We need to avoid `--no-undefined` when building shared lib (Otherwise, CIs will report `undefined symbols`), but removing that would bring some other concerns. + # Plus the fact training did not need build shared library, we disable the --build_shared_lib for training CIs. + NEED_BUILD_SHARED_LIB=false INSTALL_DEPS_EXTRA_ARGS="${INSTALL_DEPS_EXTRA_ARGS} -t" if [[ $INSTALL_DEPS_DISTRIBUTED_SETUP = true ]]; then INSTALL_DEPS_EXTRA_ARGS="${INSTALL_DEPS_EXTRA_ARGS} -m" @@ -135,6 +141,10 @@ else --dockerfile $DOCKER_FILE --context . fi +if [[ $NEED_BUILD_SHARED_LIB = true ]]; then + BUILD_EXTR_PAR=" --build_shared_lib ${BUILD_EXTR_PAR}" +fi + if [ -v EXTRA_IMAGE_TAG ]; then ${DOCKER_CMD} tag "onnxruntime-$IMAGE" "${EXTRA_IMAGE_TAG}" fi