saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-05-14 20:48:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fix Python Packaging Pipeline && Build Clean Up (#7993)

Browse source 
* remove link to python

* revert orttraining-linux-ci build env change introduced by pr
https://github.com/microsoft/onnxruntime/pull/7993.

* fix builds

* fix builds

* clean up

* fix builds

* Fix unused params

* fix some comments.
This commit is contained in:
pengwa 2021-06-09 17:35:17 +08:00 committed by GitHub
parent d433aa2459
commit cb5f411da3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 76 additions and 109 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
cmake/CMakeLists.txt
View file

@ -143,7 +143,7 @@ option(onnxruntime_ENABLE_NVTX_PROFILE "Enable NVTX profile." OFF)
option(onnxruntime_ENABLE_MEMORY_PROFILE "Enable memory profile." OFF)
option(onnxruntime_ENABLE_TRAINING "Enable training functionality." OFF)
option(onnxruntime_ENABLE_TRAINING_OPS "Include training operators but no training session support." OFF)
option(onnxruntime_ENABLE_TRAINING_TORCH_INTEROP "Enable training kernels interop with torch." ON)
option(onnxruntime_ENABLE_TRAINING_TORCH_INTEROP "Enable training kernels interop with torch." OFF)
option(onnxruntime_ENABLE_TRAINING_E2E_TESTS "Enable training end-to-end tests." OFF)
option(onnxruntime_ENABLE_CPU_FP16_OPS "Build with advanced instruction sets" ON)
option(onnxruntime_USE_NCCL "Build with NCCL support" OFF)
@ -221,9 +221,8 @@ set(NSYNC_ENABLE_TESTS OFF CACHE BOOL "Build protobuf tests" FORCE)
set(ONNX_ML 1)
if(NOT onnxruntime_ENABLE_PYTHON)
set(onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS OFF)
else()
add_compile_definitions(ENABLE_PYTHON)
endif()
if (onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS)
add_compile_definitions(ENABLE_LANGUAGE_INTEROP_OPS)
endif()
@ -231,8 +230,11 @@ endif()
if (NOT (onnxruntime_ENABLE_PYTHON AND onnxruntime_ENABLE_TRAINING))
set(onnxruntime_ENABLE_TRAINING_TORCH_INTEROP OFF)
endif()
if (onnxruntime_ENABLE_TRAINING_TORCH_INTEROP)
add_compile_definitions(ENABLE_TRAINING_TORCH_INTEROP)
# disable tests because executable projects need link to python so which did not always exist for test envs.
set(onnxruntime_BUILD_UNIT_TESTS OFF)
endif()
# General C# properties
@ -636,7 +638,7 @@ endif()
#Need python to generate def file
if(onnxruntime_BUILD_SHARED_LIB OR onnxruntime_ENABLE_PYTHON)
if(onnxruntime_ENABLE_PYTHON)
if(onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS OR onnxruntime_ENABLE_TRAINING_TORCH_INTEROP)
if(onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS)
find_package(Python 3.6 COMPONENTS Interpreter Development NumPy)
else()
find_package(Python 3.6 COMPONENTS Interpreter Development.Module NumPy)

1
cmake/onnxruntime_framework.cmake
View file

@ -42,7 +42,6 @@ if (onnxruntime_ENABLE_TRAINING OR onnxruntime_ENABLE_TRAINING_OPS)
target_include_directories(onnxruntime_framework PRIVATE ${ORTTRAINING_ROOT})
if (onnxruntime_ENABLE_TRAINING_TORCH_INTEROP)
onnxruntime_add_include_to_target(onnxruntime_framework Python::Module)
target_link_libraries(onnxruntime_framework PRIVATE Python::Python)
endif()
if (onnxruntime_USE_NCCL OR onnxruntime_USE_MPI)
target_include_directories(onnxruntime_framework PUBLIC ${MPI_CXX_INCLUDE_DIRS})

16
cmake/onnxruntime_interop_torch.cmake Normal file
View file

@ -0,0 +1,16 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
# Python dependent files for torch interop
file(GLOB onnxruntime_interop_torch_srcs
"${ORTTRAINING_SOURCE_DIR}/core/training_ops/cpu/torch/*.h"
"${ORTTRAINING_SOURCE_DIR}/core/training_ops/cpu/torch/*.cc"
"${ONNXRUNTIME_ROOT}/core/language_interop_ops/torch/*.h"
"${ONNXRUNTIME_ROOT}/core/language_interop_ops/torch/*.cc"
)
onnxruntime_add_static_library(onnxruntime_interop_torch ${onnxruntime_interop_torch_srcs})
add_dependencies(onnxruntime_interop_torch onnx ${onnxruntime_EXTERNAL_DEPENDENCIES})
onnxruntime_add_include_to_target(onnxruntime_interop_torch onnxruntime_common onnx onnx_proto ${PROTOBUF_LIB} flatbuffers Python::Module)
target_include_directories(onnxruntime_interop_torch PRIVATE ${CMAKE_CURRENT_BINARY_DIR} ${ONNXRUNTIME_ROOT} ${ORTTRAINING_ROOT} PUBLIC ${onnxruntime_graph_header} ${DLPACK_INCLUDE_DIR})
target_link_libraries(onnxruntime_interop_torch PRIVATE onnxruntime_python_interface)

37
cmake/onnxruntime_providers.cmake
View file

@ -135,8 +135,6 @@ if (onnxruntime_ENABLE_TRAINING_OPS)
"${ORTTRAINING_SOURCE_DIR}/training_ops/cpu/tensorboard/*.h"
"${ORTTRAINING_SOURCE_DIR}/training_ops/cpu/aten_ops/*.cc"
"${ORTTRAINING_SOURCE_DIR}/training_ops/cpu/aten_ops/*.h"
"${ORTTRAINING_SOURCE_DIR}/training_ops/cpu/torch/*.cc"
"${ORTTRAINING_SOURCE_DIR}/training_ops/cpu/torch/*.h"
)
list(REMOVE_ITEM onnxruntime_providers_src ${onnxruntime_cpu_full_training_only_srcs})
@ -152,37 +150,21 @@ if (onnxruntime_ENABLE_TRAINING)
"${ORTTRAINING_SOURCE_DIR}/core/framework/communication/*"
)
list(REMOVE_ITEM onnxruntime_cpu_training_ops_srcs
"${ORTTRAINING_SOURCE_DIR}/training_ops/cpu/torch/*.cc"
"${ORTTRAINING_SOURCE_DIR}/training_ops/cpu/torch/*.h"
)
source_group(TREE ${ORTTRAINING_ROOT}/ FILES ${onnxruntime_cpu_training_ops_srcs})
list(APPEND onnxruntime_providers_src ${onnxruntime_cpu_training_ops_srcs})
file(GLOB_RECURSE onnxruntime_providers_dlpack_cpp_interfaces_srcs CONFIGURE_DEPENDS
file(GLOB_RECURSE onnxruntime_providers_dlpack_srcs CONFIGURE_DEPENDS
"${ONNXRUNTIME_ROOT}/core/dlpack/dlpack_converter.cc"
"${ONNXRUNTIME_ROOT}/core/dlpack/dlpack_converter.h"
)
set(onnxruntime_providers_dlpack_srcs ${onnxruntime_providers_dlpack_cpp_interfaces_srcs})
if (onnxruntime_ENABLE_PYTHON)
file(GLOB_RECURSE onnxruntime_providers_dlpack_python_interfaces_srcs CONFIGURE_DEPENDS
"${ONNXRUNTIME_ROOT}/core/dlpack/dlpack_python.cc"
"${ONNXRUNTIME_ROOT}/core/dlpack/dlpack_python.h"
"${ONNXRUNTIME_ROOT}/core/dlpack/python_common.cc"
"${ONNXRUNTIME_ROOT}/core/dlpack/python_common.h"
)
list(APPEND onnxruntime_providers_dlpack_srcs ${onnxruntime_providers_dlpack_python_interfaces_srcs})
endif()
set(onnxruntime_providers_dlpack_srcs ${onnxruntime_providers_dlpack_srcs})
source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_dlpack_srcs})
list(APPEND onnxruntime_providers_src ${onnxruntime_providers_dlpack_srcs})
if (onnxruntime_ENABLE_TRAINING_TORCH_INTEROP)
file(GLOB_RECURSE onnxruntime_language_interop_torch_srcs CONFIGURE_DEPENDS
"${ONNXRUNTIME_ROOT}/core/language_interop_ops/torch/*.h"
"${ONNXRUNTIME_ROOT}/core/language_interop_ops/torch/*.cc"
)
source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_language_interop_torch_srcs})
list(APPEND onnxruntime_providers_src ${onnxruntime_language_interop_torch_srcs})
endif()
endif()
onnxruntime_add_static_library(onnxruntime_providers ${onnxruntime_providers_src})
@ -229,8 +211,7 @@ if (onnxruntime_ENABLE_TRAINING)
add_dependencies(onnxruntime_providers tensorboard)
onnxruntime_add_include_to_target(onnxruntime_providers tensorboard)
if (onnxruntime_ENABLE_TRAINING_TORCH_INTEROP)
onnxruntime_add_include_to_target(onnxruntime_providers tensorboard Python::Module)
target_link_libraries(onnxruntime_providers PRIVATE Python::Python)
onnxruntime_add_include_to_target(onnxruntime_providers Python::Module)
endif()
if (onnxruntime_USE_NCCL OR onnxruntime_USE_MPI)
@ -369,7 +350,6 @@ if (onnxruntime_USE_CUDA)
target_link_libraries(onnxruntime_providers_cuda PRIVATE onnxruntime_training)
if (onnxruntime_ENABLE_TRAINING_TORCH_INTEROP)
onnxruntime_add_include_to_target(onnxruntime_providers_cuda Python::Module)
target_link_libraries(onnxruntime_providers_cuda PRIVATE Python::Python)
endif()
endif()
@ -1122,7 +1102,6 @@ if (onnxruntime_USE_ROCM)
target_include_directories(onnxruntime_providers_rocm PRIVATE ${ORTTRAINING_ROOT} ${CMAKE_CURRENT_BINARY_DIR}/amdgpu/orttraining)
if (onnxruntime_ENABLE_TRAINING_TORCH_INTEROP)
onnxruntime_add_include_to_target(onnxruntime_providers_rocm Python::Module)
target_link_libraries(onnxruntime_providers_rocm PRIVATE Python::Python)
endif()
endif()

23
cmake/onnxruntime_python.cmake
View file

@ -49,10 +49,23 @@ endif()
if (onnxruntime_USE_NCCL)
target_include_directories(onnxruntime_pybind11_state PRIVATE ${NCCL_INCLUDE_DIRS})
endif()
if (onnxruntime_ENABLE_TRAINING)
# DLPack is a header-only dependency
set(DLPACK_INCLUDE_DIR ${PROJECT_SOURCE_DIR}/external/dlpack/include)
target_include_directories(onnxruntime_pybind11_state PRIVATE ${ORTTRAINING_ROOT} ${DLPACK_INCLUDE_DIR})
file(GLOB onnxruntime_python_interface_cc_srcs
"${ONNXRUNTIME_ROOT}/core/dlpack/dlpack_python.cc"
"${ONNXRUNTIME_ROOT}/core/dlpack/dlpack_python.h"
"${ONNXRUNTIME_ROOT}/core/dlpack/python_common.h"
)
onnxruntime_add_static_library(onnxruntime_python_interface ${onnxruntime_python_interface_cc_srcs})
add_dependencies(onnxruntime_python_interface onnx ${onnxruntime_EXTERNAL_DEPENDENCIES})
target_include_directories(onnxruntime_python_interface PRIVATE ${DLPACK_INCLUDE_DIR})
onnxruntime_add_include_to_target(onnxruntime_python_interface onnxruntime_common onnx onnx_proto ${PROTOBUF_LIB} flatbuffers Python::Module)
target_link_libraries(onnxruntime_pybind11_state PRIVATE onnxruntime_python_interface)
endif()
if(APPLE)
@ -67,6 +80,10 @@ if (onnxruntime_ENABLE_TRAINING)
target_link_libraries(onnxruntime_pybind11_state PRIVATE onnxruntime_training)
endif()
if (onnxruntime_ENABLE_TRAINING_TORCH_INTEROP)
target_link_libraries(onnxruntime_pybind11_state PRIVATE onnxruntime_interop_torch)
endif()
target_link_libraries(onnxruntime_pybind11_state PRIVATE
onnxruntime_session
${onnxruntime_libs}
@ -96,8 +113,6 @@ if (onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS)
target_link_libraries(onnxruntime_pybind11_state PRIVATE onnxruntime_language_interop onnxruntime_pyop)
endif()
set(onnxruntime_pybind11_state_dependencies
${onnxruntime_EXTERNAL_DEPENDENCIES}
${pybind11_dep}
@ -499,3 +514,7 @@ endif()
if (onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS)
include(onnxruntime_language_interop_ops.cmake)
endif()
if (onnxruntime_ENABLE_TRAINING_TORCH_INTEROP)
include(onnxruntime_interop_torch.cmake)
endif()

4
onnxruntime/core/dlpack/dlpack_python.cc
View file

@ -1,8 +1,6 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#ifdef ENABLE_PYTHON
#include "core/dlpack/dlpack_python.h"
namespace onnxruntime {
@ -37,5 +35,3 @@ OrtValue FromDlpack(PyObject* dlpack_tensor, const bool is_bool_tensor) {
} // namespace dlpack
} // namespace onnxruntime
#endif

4
onnxruntime/core/dlpack/dlpack_python.h
View file

@ -3,8 +3,6 @@
/// Python level interface for DLPack conversion.
#ifdef ENABLE_PYTHON
#pragma once
#include "core/dlpack/dlpack_converter.h"
@ -25,5 +23,3 @@ OrtValue FromDlpack(PyObject* dlpack_tensor, const bool is_bool_tensor);
} // namespace dlpack
} // namespace onnxruntime
#endif

6
onnxruntime/core/dlpack/python_common.h
View file

@ -1,14 +1,12 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#ifdef ENABLE_PYTHON
#pragma once
// Avoid linking to pythonX_d.lib on Windows in debug build
#ifdef _WIN32
#pragma warning(push)
#pragma warning(disable: 4510 4610 4512 4005)
#pragma warning(disable : 4510 4610 4512 4005)
#ifdef _DEBUG
#define ORT_DISABLE_INCLUDE_DEBUG_PYTHON_LIB
#undef _DEBUG
@ -24,5 +22,3 @@
#endif
#pragma warning(pop)
#endif
#endif

21
onnxruntime/python/onnxruntime_pybind_state.cc
View file

@ -606,9 +606,9 @@ static void RegisterExecutionProviders(InferenceSession* sess, const std::vector
}
}
RegisterExecutionProvider(
sess, *onnxruntime::CreateExecutionProviderFactory_VITISAI(target.c_str(), 0,
export_runtime_module.c_str(),
load_runtime_module.c_str()));
sess, *onnxruntime::CreateExecutionProviderFactory_VITISAI(target.c_str(), 0,
export_runtime_module.c_str(),
load_runtime_module.c_str()));
#endif
} else if (type == kAclExecutionProvider) {
#ifdef USE_ACL
@ -817,20 +817,31 @@ void addGlobalMethods(py::module& m, Environment& env) {
void* p_aten_op_executor = reinterpret_cast<void*>(aten_op_executor_address_int);
contrib::aten_ops::ATenOperatorExecutor::Initialize(p_aten_op_executor);
});
#ifdef ENABLE_TRAINING_TORCH_INTEROP
m.def("register_forward_runner", [](py::object obj) -> void {
#ifdef ENABLE_TRAINING_TORCH_INTEROP
auto& pool = onnxruntime::language_interop_ops::torch::OrtTorchFunctionPool::GetInstance();
pool.RegisterForwardRunner(obj.ptr());
#else
ORT_UNUSED_PARAMETER(obj);
#endif
});
m.def("register_backward_runner", [](py::object obj) -> void {
#ifdef ENABLE_TRAINING_TORCH_INTEROP
auto& pool = onnxruntime::language_interop_ops::torch::OrtTorchFunctionPool::GetInstance();
pool.RegisterBackwardRunner(obj.ptr());
#else
ORT_UNUSED_PARAMETER(obj);
#endif
});
m.def("register_torch_autograd_function", [](std::string key, py::object obj) -> void {
#ifdef ENABLE_TRAINING_TORCH_INTEROP
auto& pool = onnxruntime::language_interop_ops::torch::OrtTorchFunctionPool::GetInstance();
pool.RegisterTorchAutogradFunction(key, obj.ptr());
#else
ORT_UNUSED_PARAMETER(key);
ORT_UNUSED_PARAMETER(obj);
#endif
});
#endif
#endif
#ifdef USE_NUPHAR

2
tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml
View file

@ -11,7 +11,7 @@ jobs:
- template: templates/get-docker-image-steps.yml
parameters:
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_training_cpu
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cpu
Context: tools/ci_build/github/linux/docker
DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )"
Repository: onnxruntimecpubuild

36
tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_training_cpu
View file

@ -1,36 +0,0 @@
FROM quay.io/pypa/manylinux2014_x86_64:latest
COPY build_scripts/build_utils.sh /build_scripts/
COPY build_scripts/cpython-pubkeys.txt /build_scripts/cpython-pubkeys.txt
COPY build_scripts/build-openssl.sh /build_scripts/
RUN export OPENSSL_ROOT=openssl-1.1.1k && \
export OPENSSL_HASH=892a0875b9872acd04a9fde79b1f943075d5ea162415de3047c327df33fbaee5 && \
export OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source && \
manylinux-entrypoint /build_scripts/build-openssl.sh
ADD scripts /tmp/scripts
RUN cd /tmp/scripts && /tmp/scripts/manylinux/install_centos.sh
# Remove existing symbolic link and python installation.
RUN rm -rf /opt/python/cp37-cp37m && rm -rf /opt/_internal/cpython-3.7.10
# Build Python.
COPY build_scripts/build-cpython.sh /build_scripts/
RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.7.10 1
ENV SSL_CERT_FILE=/opt/_internal/certs.pem
ENV PATH /opt/python/cp37-cp37m/bin:$PATH
ENV LD_LIBRARY_PATH /opt/_internal/cpython-3.7.10/lib/:$LD_LIBRARY_PATH
# Create symbolic link pointing to new built python installation.
RUN ln -sf /opt/_internal/cpython-3.7.10 /opt/python/cp37-cp37m && /opt/python/cp37-cp37m/bin/python3.7 -m pip install --upgrade pip
RUN /tmp/scripts/manylinux/install_deps.sh && rm -rf /tmp/scripts
ARG BUILD_UID=1001
ARG BUILD_USER=onnxruntimedev
RUN adduser --uid $BUILD_UID $BUILD_USER
WORKDIR /home/$BUILD_USER
USER $BUILD_USER
ENV PATH /usr/local/gradle/bin:$PATH

23
tools/ci_build/github/linux/docker/build_scripts/build-cpython.sh
View file

@ -14,7 +14,6 @@ source $MY_DIR/build_utils.sh
CPYTHON_VERSION=$1
CPYTHON_DOWNLOAD_URL=https://www.python.org/ftp/python
BUILD_PYTHON_SO="${2:-0}"
function pyver_dist_dir {
# Echoes the dist directory name of given pyver, removing alpha/beta prerelease
@ -34,22 +33,12 @@ tar -xzf Python-${CPYTHON_VERSION}.tgz
pushd Python-${CPYTHON_VERSION}
PREFIX="/opt/_internal/cpython-${CPYTHON_VERSION}"
mkdir -p ${PREFIX}/lib
if test ${BUILD_PYTHON_SO} -eq 0
then
# configure with hardening options only for the interpreter & stdlib C extensions
# do not change the default for user built extension (yet?)
./configure \
CFLAGS_NODIST="${MANYLINUX_CFLAGS} ${MANYLINUX_CPPFLAGS}" \
LDFLAGS_NODIST="${MANYLINUX_LDFLAGS}" \
--prefix=${PREFIX} --disable-shared --with-ensurepip=no > /dev/null
else
./configure \
CFLAGS_NODIST="${MANYLINUX_CFLAGS} ${MANYLINUX_CPPFLAGS}" \
LDFLAGS_NODIST="${MANYLINUX_LDFLAGS}" \
--prefix=${PREFIX} --enable-shared --with-ensurepip=yes > /dev/null
fi
# configure with hardening options only for the interpreter & stdlib C extensions
# do not change the default for user built extension (yet?)
./configure \
CFLAGS_NODIST="${MANYLINUX_CFLAGS} ${MANYLINUX_CPPFLAGS}" \
LDFLAGS_NODIST="${MANYLINUX_LDFLAGS}" \
--prefix=${PREFIX} --disable-shared --with-ensurepip=no > /dev/null
make -j$(nproc) > /dev/null
make install > /dev/null
popd

2
tools/ci_build/github/linux/docker/scripts/manylinux/install_centos.sh
View file

@ -9,7 +9,7 @@ if ! rpm -q --quiet epel-release ; then
fi
echo "installing for os major version : $os_major_version"
yum install -y which gdb redhat-lsb-core expat-devel libcurl-devel tar unzip curl zlib-devel make libunwind icu aria2 rsync bzip2 git bzip2-devel wget libffi-devel
yum install -y which gdb redhat-lsb-core expat-devel libcurl-devel tar unzip curl zlib-devel make libunwind icu aria2 rsync bzip2 git bzip2-devel
if [ "$os_major_version" == "7" ]; then
# install dotnet core dependencies