From 9d66e4a5f8e9202c9c0465aca778eff25487c75b Mon Sep 17 00:00:00 2001 From: ajindal1 Date: Wed, 6 Oct 2021 09:50:03 -0700 Subject: [PATCH] adding MKL library resolution and comments --- cmake/onnxruntime_python.cmake | 71 ++++++++++++++++++- .../orttraining/eager/test/ort_eps_test.py | 11 ++- tools/ci_build/build.py | 5 +- 3 files changed, 81 insertions(+), 6 deletions(-) diff --git a/cmake/onnxruntime_python.cmake b/cmake/onnxruntime_python.cmake index c354d424c2..c0fa20d4f9 100644 --- a/cmake/onnxruntime_python.cmake +++ b/cmake/onnxruntime_python.cmake @@ -31,8 +31,11 @@ endif() if (onnxruntime_ENABLE_EAGER_MODE) list(APPEND CMAKE_PREFIX_PATH ${onnxruntime_PREBUILT_PYTORCH_PATH}) find_package(Torch REQUIRED) - find_library(TORCH_PYTHON_LIBRARY torch_python PATHS "${TORCH_INSTALL_PREFIX}/lib") - + # find_library(TORCH_PYTHON_LIBRARY torch_python PATHS "${TORCH_INSTALL_PREFIX}/lib") + find_library(TORCH_PYTHON_LIBRARY torch_python) + # find_package(MKL REQUIRED) + # find_library(MKL_PYTHON_LIBRARY mkl_intel_ilp64) + file(GLOB onnxruntime_eager_extension_srcs CONFIGURE_DEPENDS "${ORTTRAINING_ROOT}/orttraining/eager/*.cpp" ) @@ -47,12 +50,17 @@ if (onnxruntime_ENABLE_EAGER_MODE) endif() onnxruntime_add_shared_library_module(onnxruntime_pybind11_state ${onnxruntime_pybind_srcs}) +get_target_property(OUT1 onnxruntime_pybind11_state LINK_LIBRARIES) +message("Linked libraries 1: " ${OUT1}) if(MSVC) target_compile_options(onnxruntime_pybind11_state PRIVATE "$<$:SHELL:--compiler-options /utf-8>" "$<$>:/utf-8>") endif() if(HAS_CAST_FUNCTION_TYPE) target_compile_options(onnxruntime_pybind11_state PRIVATE "-Wno-cast-function-type") endif() +get_target_property(OUT1 onnxruntime_pybind11_state LINK_LIBRARIES) +message("Linked libraries 2: " ${OUT1}) + # We export symbols using linker and the compiler does not know anything about it # There is a problem with classes that have pybind types as members. @@ -69,6 +77,8 @@ if (MSVC AND NOT CMAKE_SIZEOF_VOID_P EQUAL 8) #TODO: fix the warnings target_compile_options(onnxruntime_pybind11_state PRIVATE "/wd4244") endif() +get_target_property(OUT1 onnxruntime_pybind11_state LINK_LIBRARIES) +message("Linked libraries 3: " ${OUT1}) onnxruntime_add_include_to_target(onnxruntime_pybind11_state Python::Module Python::NumPy) target_include_directories(onnxruntime_pybind11_state PRIVATE ${ONNXRUNTIME_ROOT} ${pybind11_INCLUDE_DIRS}) @@ -82,6 +92,8 @@ endif() if (onnxruntime_USE_NCCL) target_include_directories(onnxruntime_pybind11_state PRIVATE ${NCCL_INCLUDE_DIRS}) endif() +get_target_property(OUT1 onnxruntime_pybind11_state LINK_LIBRARIES) +message("Linked libraries 4: " ${OUT1}) if(APPLE) set(ONNXRUNTIME_SO_LINK_FLAG "-Xlinker -exported_symbols_list ${ONNXRUNTIME_ROOT}/python/exported_symbols.lst") @@ -97,15 +109,45 @@ endif() if (onnxruntime_ENABLE_TRAINING) target_include_directories(onnxruntime_pybind11_state PRIVATE ${ORTTRAINING_ROOT}) + get_target_property(OUT1 onnxruntime_pybind11_state LINK_LIBRARIES) + message("Linked libraries 1: " ${OUT1}) target_include_directories(onnxruntime_pybind11_state PRIVATE ${PROJECT_SOURCE_DIR}/external/dlpack/include) + get_target_property(OUT1 onnxruntime_pybind11_state LINK_LIBRARIES) + message("Linked libraries 2: " ${OUT1}) target_link_libraries(onnxruntime_pybind11_state PRIVATE onnxruntime_training) + get_target_property(OUT1 onnxruntime_pybind11_state LINK_LIBRARIES) + message("Linked libraries 3: " ${OUT1}) endif() if (onnxruntime_ENABLE_EAGER_MODE) # todo: this is because the prebuild pytorch may use a different version of protobuf headers. # force the build to find the protobuf headers ort using. target_include_directories(onnxruntime_pybind11_state PRIVATE "${REPO_ROOT}/cmake/external/protobuf/src") - target_link_libraries(onnxruntime_pybind11_state PRIVATE onnxruntime_eager ${TORCH_LIBRARIES} ${TORCH_PYTHON_LIBRARY}) + get_target_property(OUT1 onnxruntime_pybind11_state LINK_LIBRARIES) + message("Linked libraries 4: " ${OUT1}) + # target_link_libraries(onnxruntime_pybind11_state PUBLIC onnxruntime_eager ${TORCH_PYTHON_LIBRARY}) + # target_link_libraries(onnxruntime_pybind11_state PUBLIC "torch") + # target_link_libraries(onnxruntime_pybind11_state PUBLIC "torch_library") + # target_link_libraries(onnxruntime_pybind11_state PUBLIC "C:/Users/abjindal/Anaconda3/Lib/site-packages/torch/lib/c10.lib") + # target_link_libraries(onnxruntime_pybind11_state PUBLIC "C:/Users/abjindal/Anaconda3/Lib/site-packages/torch/lib/kineto.lib") + # target_link_libraries(onnxruntime_pybind11_state PUBLIC "C:/Program Files/NVIDIA\ Corporation/NvToolsExt/lib/x64/nvToolsExt64_1.lib") + # target_link_libraries(onnxruntime_pybind11_state PUBLIC "C:/Program Files/NVIDIA\ GPU\ Computing\ Toolkit/CUDA/v10.2/lib/x64/cudart_static.lib") + # target_link_libraries(onnxruntime_pybind11_state PUBLIC "C:/Users/abjindal/Anaconda3/Lib/site-packages/torch/lib/caffe2_nvrtc.lib") + # target_link_libraries(onnxruntime_pybind11_state PUBLIC "C:/Users/abjindal/Anaconda3/Lib/site-packages/torch/lib/c10_cuda.lib") + get_target_property(OUT1 "torch" LINK_LIBRARIES) + get_target_property(OUT2 "torch_library" LINK_LIBRARIES) + message("Linked libraries 4.1: " torch_library) + message("Linked libraries MKL Python Lib: " ${MKL_PYTHON_LIBRARY}) + message("Linked libraries TORCH: " ${TORCH_LIBRARIES}) + target_link_libraries(onnxruntime_pybind11_state PUBLIC onnxruntime_eager ${TORCH_LIBRARIES} ${TORCH_PYTHON_LIBRARY}) + target_link_libraries(onnxruntime_pybind11_state PRIVATE ${MKL_PYTHON_LIBRARY}) + get_target_property(OUT2 onnxruntime_pybind11_state LINK_LIBRARIES) + message("Linked libraries 5: " ${OUT2}) + if (NOT (MKL_PYTHON_LIBRARY STREQUAL "")) + message("Linked libraries TORCH: " ${TORCH_LIBRARIES}) + # get_filename_component(MKL_PYTHON_LIBRARY_PATH ${MKL_PYTHON_LIBRARY} DIRECTORY) + # target_link_directories(onnxruntime_pybind11_state PRIVATE ${MKL_PYTHON_LIBRARY_PATH}) + endif() # the ort_aten.g.cpp is generated from tools. currently it has some limitations. # todo: fix this if (NOT MSVC) @@ -115,6 +157,8 @@ if (onnxruntime_ENABLE_EAGER_MODE) set_source_files_properties("${ORTTRAINING_ROOT}/orttraining/eager/ort_tensor.cpp" PROPERTIES COMPILE_FLAGS -Wno-unused-parameter) endif() endif() +get_target_property(OUT1 onnxruntime_pybind11_state LINK_LIBRARIES) +message("Linked libraries 5.1: " ${OUT1}) target_link_libraries(onnxruntime_pybind11_state PRIVATE onnxruntime_session @@ -141,22 +185,34 @@ target_link_libraries(onnxruntime_pybind11_state PRIVATE onnxruntime_flatbuffers ${pybind11_lib} ) +get_target_property(OUT1 onnxruntime_pybind11_state LINK_LIBRARIES) +message("Linked libraries 6: " ${OUT1}) if (onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS) target_link_libraries(onnxruntime_pybind11_state PRIVATE onnxruntime_language_interop onnxruntime_pyop) endif() +get_target_property(OUT1 onnxruntime_pybind11_state LINK_LIBRARIES) +message("Linked libraries 7: " ${OUT1}) set(onnxruntime_pybind11_state_dependencies ${onnxruntime_EXTERNAL_DEPENDENCIES} ${pybind11_dep} ) +get_target_property(OUT1 onnxruntime_pybind11_state LINK_LIBRARIES) +message("Linked libraries 7.1: " ${OUT1}) set_property(TARGET onnxruntime_pybind11_state APPEND_STRING PROPERTY LINK_FLAGS ${ONNXRUNTIME_SO_LINK_FLAG} ${onnxruntime_DELAYLOAD_FLAGS}) +get_target_property(OUT1 onnxruntime_pybind11_state LINK_LIBRARIES) +message("Linked libraries 7.2: " ${OUT1}) add_dependencies(onnxruntime_pybind11_state ${onnxruntime_pybind11_state_dependencies}) +get_target_property(OUT1 onnxruntime_pybind11_state LINK_LIBRARIES) +message("Linked libraries 8: " ${OUT1}) if (MSVC) set_target_properties(onnxruntime_pybind11_state PROPERTIES LINK_FLAGS "${ONNXRUNTIME_SO_LINK_FLAG}") # if MSVC, pybind11 looks for release version of python lib (pybind11/detail/common.h undefs _DEBUG) target_link_libraries(onnxruntime_pybind11_state PRIVATE Python::Module) + get_target_property(OUT1 onnxruntime_pybind11_state LINK_LIBRARIES) + message("Linked libraries 9: " ${OUT1}) elseif (APPLE) set_target_properties(onnxruntime_pybind11_state PROPERTIES LINK_FLAGS "${ONNXRUNTIME_SO_LINK_FLAG} -undefined dynamic_lookup") set_target_properties(onnxruntime_pybind11_state PROPERTIES @@ -183,6 +239,8 @@ if (onnxruntime_ENABLE_EXTERNAL_CUSTOM_OP_SCHEMAS) else() target_link_libraries(onnxruntime_pybind11_state PRIVATE ${onnxruntime_EXTERNAL_LIBRARIES}) endif() +get_target_property(OUT1 onnxruntime_pybind11_state LINK_LIBRARIES) +message("Linked libraries 10: " ${OUT1}) set_target_properties(onnxruntime_pybind11_state PROPERTIES PREFIX "") set_target_properties(onnxruntime_pybind11_state PROPERTIES FOLDER "ONNXRuntime") @@ -196,6 +254,13 @@ if (MSVC) else() set_target_properties(onnxruntime_pybind11_state PROPERTIES SUFFIX ".so") endif() +get_target_property(OUT1 onnxruntime_pybind11_state LINK_LIBRARIES) +message("Linked libraries 11: " ${OUT1}) +get_cmake_property(_variableNames VARIABLES) +list (SORT _variableNames) +foreach (_variableName ${_variableNames}) + message(STATUS "${_variableName}=${${_variableName}}") +endforeach() # Generate version_info.py in Windows build. # Has to be done before onnxruntime_python_srcs is set. diff --git a/orttraining/orttraining/eager/test/ort_eps_test.py b/orttraining/orttraining/eager/test/ort_eps_test.py index f1cc141fef..0effea51bf 100644 --- a/orttraining/orttraining/eager/test/ort_eps_test.py +++ b/orttraining/orttraining/eager/test/ort_eps_test.py @@ -5,15 +5,22 @@ import unittest import torch import onnxruntime_pybind11_state as torch_ort import os +import sys + +def is_windows(): + return sys.platform.startswith("win") class OrtEPTests(unittest.TestCase): def get_test_execution_provider_path(self): - return os.path.join('.', 'libtest_execution_provider.so') + if is_windows(): + return os.path.join('.', 'test_execution_provider.dll') + else: + return os.path.join('.', 'libtest_execution_provider.so') def test_import_custom_eps(self): torch_ort.set_device(0, 'CPUExecutionProvider', {}) - torch_ort._register_provider_lib('TestExecutionProvider', self.get_test_execution_provider_path(), {}) + torch_ort._register_provider_lib('TestExecutionProvider', self.get_test_execution_provider_path(), {'some_config':'val'}) torch_ort.set_device(1, 'TestExecutionProvider', {'device_id':'0', 'some_config':'val'}) ort_device = torch_ort.device(1) diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index b44902f7c1..7d286a184c 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -613,7 +613,10 @@ def run_subprocess(args, cwd=None, capture_stdout=False, dll_path=None, my_env["LD_LIBRARY_PATH"] = dll_path if python_path: if is_windows(): - my_env["PYTHONPATH"] = python_path + os.pathsep + my_env["PYTHONPATH"] + if "PYTHONPATH" in my_env: + my_env["PYTHONPATH"] = python_path + os.pathsep + my_env["PYTHONPATH"] + else: + my_env["PYTHONPATH"] = python_path else: if "PYTHONPATH" in my_env: my_env["PYTHONPATH"] += os.pathsep + python_path