From 3b0aaa9e0e8a26943dfe3e07139c24bfc4fa68d5 Mon Sep 17 00:00:00 2001 From: Valery Chernov Date: Wed, 13 Jul 2022 11:48:42 +0300 Subject: [PATCH] [TVM EP] support build on Windows (#11851) * add description of build ORT+TVM EP on Windows * fix cmake error related to symlink creation on Windows * add llvm config path to build flags for correct build on Windows * update TVM_EP.md for llvm_config build arg * fix warnings skipping during build on Windows * fix using string or wstring for model path to correct build on Windows (MSVC error) * fix error in custom logger for correct build on Windows * implement glob algorithm for Windows * additional build fixes * update TVM with export of VM symbols for dll * description of nasm issue and workaround * update TVM with export of Executable from VM symbols for dll * description of installation of ipp-crypto dependencies on Windows * cmake key for ipp-crypto build * fix wstring for TVMso EP * fix ipp-crypto build * cmake key onnxruntime_TVM_USE_HASH switch off not specific methods, but full hash functionality * fix absolute path to compiled lib * update TVM_EP.md, fix lint warnings * update TVM_EP.md * small fixes after review * switch on handshake functionality for Linux workflow Co-authored-by: Valery Chernov Co-authored-by: KJlaccHoeUM9l --- .github/workflows/linux.yml | 4 +- cgmanifests/cgmanifest.json | 2 +- cmake/CMakeLists.txt | 32 ++++-- cmake/external/tvm.cmake | 10 +- cmake/onnxruntime_providers.cmake | 37 +++++- docs/TVM_EP.md | 106 ++++++++++++++++-- onnxruntime/core/common/path.cc | 6 - onnxruntime/core/common/path.h | 6 + .../core/providers/tvm/custom_logging.cc | 12 +- onnxruntime/core/providers/tvm/tvm_api.cc | 51 +++++++-- .../providers/tvm/tvm_execution_provider.cc | 3 +- .../tvm/tvm_so_execution_provider.cc | 8 +- .../providers/tvm/tvm_so_execution_provider.h | 2 + tools/ci_build/build.py | 12 ++ 14 files changed, 240 insertions(+), 51 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index e5e71eb4e6..92ddf900d9 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -19,10 +19,10 @@ jobs: architecture: 'x64' - name: 'Setup TVM EP requirements' run: | - set -e -x + set -e -x sudo apt-get update sudo apt-get install -y libtinfo-dev zlib1g-dev build-essential libedit-dev libxml2-dev nasm python3 -m pip install -r ${{ github.workspace }}/tools/ci_build/github/linux/tvm/requirements.txt - name: 'Build and Test' run: | - python3 ${{ github.workspace }}/tools/ci_build/build.py --build_dir build --config Release --skip_submodule_sync --parallel --enable_pybind --disable_contrib_ops --disable_ml_ops --skip_onnx_tests --use_tvm --ctest_path "" + python3 ${{ github.workspace }}/tools/ci_build/build.py --build_dir build --config Release --skip_submodule_sync --parallel --enable_pybind --disable_contrib_ops --disable_ml_ops --skip_onnx_tests --use_tvm --use_tvm_hash --ctest_path "" diff --git a/cgmanifests/cgmanifest.json b/cgmanifests/cgmanifest.json index c25bbdb6fc..a6cbe2c7f8 100644 --- a/cgmanifests/cgmanifest.json +++ b/cgmanifests/cgmanifest.json @@ -46,7 +46,7 @@ "component": { "type": "git", "git": { - "commitHash": "bc492acd7677dd7875b14f9ee46beef658955441", + "commitHash": "3425ed846308a456f98404c79f6df1693bed6377", "repositoryUrl": "https://github.com/apache/tvm.git" }, "comments": "needed for TVM EP" diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 4b105b4215..5ff938e694 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -109,6 +109,8 @@ option(onnxruntime_PREFER_SYSTEM_LIB "Experimental: Build with the preinstalled option(onnxruntime_USE_ROCM "Build with AMD GPU support" OFF) option(onnxruntime_USE_TVM "Build with TVM support" OFF) option(onnxruntime_TVM_CUDA_RUNTIME "Build TVM with CUDA support" OFF) +option(onnxruntime_TVM_USE_LLVM "Build TVM with LLVM. Set customized path to llvm-config.exe here if need" OFF) +option(onnxruntime_TVM_USE_HASH "Build ipp-crypto library for support has algorithm. It is defined for TVM only") option(onnxruntime_USE_XNNPACK "Build with XNNPACK support. Provides an alternative math library on ARM, WebAssembly and x86." OFF) # Options related to reducing the binary size produced by the build @@ -1437,11 +1439,9 @@ if (onnxruntime_USE_NUPHAR_TVM) add_definitions(-DUSE_NUPHAR_TVM_WITH_LLVM) endif() - message(STATUS "TVM BEFORE USE_LLVM=${USE_LLVM} USE_OPENMP=${USE_OPENMP} CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} USE_CUDA=${USE_CUDA} USE_GTEST=${USE_GTEST} CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}") message(STATUS "tvm_SOURCE_DIR=${tvm_SOURCE_DIR}") message(STATUS "tvm_BINARY_DIR=${tvm_BINARY_DIR}") add_subdirectory(${tvm_SOURCE_DIR} ${tvm_BINARY_DIR} EXCLUDE_FROM_ALL) - message(STATUS "TVM AFTER USE_LLVM=${USE_LLVM} USE_OPENMP=${USE_OPENMP} CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} USE_CUDA=${USE_CUDA} USE_GTEST=${USE_GTEST} CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}") set_target_properties(tvm PROPERTIES FOLDER ${tvm_SOURCE_DIR}) set_target_properties(tvm_topi PROPERTIES FOLDER ${tvm_SOURCE_DIR}) @@ -1491,18 +1491,24 @@ if (onnxruntime_USE_TVM) if (NOT TARGET tvm) message(STATUS "Include TVM(*).") include(tvm) - message(STATUS "Include ipp-crypto(*).") - include(ipp-crypto) + if (onnxruntime_TVM_USE_HASH) + message(STATUS "Include ipp-crypto(*).") + include(ipp-crypto) + endif() endif() # ipp-crypto - set(ARCH intel64 CACHE STRING "Only defined for TVM") - add_subdirectory(${ipp_crypto_SOURCE_DIR} ${ipp_crypto_BINARY_DIR} EXCLUDE_FROM_ALL) - set_target_properties(ippcp_s PROPERTIES FOLDER ${ipp_crypto_SOURCE_DIR}) - set(IPP_CRYPTO_INCLUDES ${ipp_crypto_INCLUDE_DIRS}) + if (onnxruntime_TVM_USE_HASH) + set(ARCH intel64 CACHE STRING "Only defined for TVM") + add_subdirectory(${ipp_crypto_SOURCE_DIR} ${ipp_crypto_BINARY_DIR} EXCLUDE_FROM_ALL) + set_target_properties(ippcp_s PROPERTIES FOLDER ${ipp_crypto_SOURCE_DIR}) + set(IPP_CRYPTO_INCLUDES ${ipp_crypto_INCLUDE_DIRS}) + endif() # TVM - if (onnxruntime_USE_LLVM) + if (onnxruntime_TVM_USE_LLVM) + set(USE_LLVM "${onnxruntime_TVM_USE_LLVM}" CACHE STRING "Path to LLVM for correct TVM build") + elseif(onnxruntime_USE_LLVM) set(USE_LLVM ON CACHE BOOL "Only defined for TVM") endif() @@ -1533,8 +1539,12 @@ if (onnxruntime_USE_TVM) set(FS_STDLIB stdc++fs) endif() endif() - list(APPEND onnxruntime_EXTERNAL_LIBRARIES tvm ippcp_s ${FS_STDLIB}) - list(APPEND onnxruntime_EXTERNAL_DEPENDENCIES tvm ippcp_s) + list(APPEND onnxruntime_EXTERNAL_LIBRARIES tvm ${FS_STDLIB}) + list(APPEND onnxruntime_EXTERNAL_DEPENDENCIES tvm) + if (onnxruntime_TVM_USE_HASH) + list(APPEND onnxruntime_EXTERNAL_LIBRARIES ippcp_s) + list(APPEND onnxruntime_EXTERNAL_DEPENDENCIES ippcp_s) + endif() endif() # XNNPACK EP diff --git a/cmake/external/tvm.cmake b/cmake/external/tvm.cmake index 8d7602a926..597355daa7 100644 --- a/cmake/external/tvm.cmake +++ b/cmake/external/tvm.cmake @@ -4,13 +4,19 @@ if (onnxruntime_USE_TVM) FetchContent_Declare( tvm GIT_REPOSITORY https://github.com/apache/tvm.git - GIT_TAG bc492acd7677dd7875b14f9ee46beef658955441 + GIT_TAG 3425ed846308a456f98404c79f6df1693bed6377 ) FetchContent_GetProperties(tvm) if(NOT tvm_POPULATED) FetchContent_Populate(tvm) - file(CREATE_LINK ${tvm_BINARY_DIR} ${tvm_SOURCE_DIR}/build SYMBOLIC) + if (WIN32) + execute_process( + COMMAND ${CMAKE_COMMAND} -E create_symlink ${tvm_BINARY_DIR}/${CMAKE_BUILD_TYPE} ${tvm_SOURCE_DIR}/build + ) + else() + file(CREATE_LINK ${tvm_BINARY_DIR} ${tvm_SOURCE_DIR}/build SYMBOLIC) + endif() endif() set(tvm_INCLUDE_DIRS ${tvm_SOURCE_DIR}/include) diff --git a/cmake/onnxruntime_providers.cmake b/cmake/onnxruntime_providers.cmake index 3ef3900207..e9a85b4bb6 100644 --- a/cmake/onnxruntime_providers.cmake +++ b/cmake/onnxruntime_providers.cmake @@ -1457,11 +1457,22 @@ endif() if (onnxruntime_USE_TVM) add_definitions(-DUSE_TVM=1) + if (onnxruntime_TVM_USE_HASH) + add_definitions(-DUSE_TVM_HASH=1) + endif() - file (GLOB_RECURSE onnxruntime_providers_tvm_cc_srcs CONFIGURE_DEPENDS - "${ONNXRUNTIME_ROOT}/core/providers/tvm/*.h" - "${ONNXRUNTIME_ROOT}/core/providers/tvm/*.cc" + if (onnxruntime_TVM_USE_HASH) + file (GLOB_RECURSE onnxruntime_providers_tvm_cc_srcs CONFIGURE_DEPENDS + "${ONNXRUNTIME_ROOT}/core/providers/tvm/*.h" + "${ONNXRUNTIME_ROOT}/core/providers/tvm/*.cc" ) + else() + file (GLOB onnxruntime_providers_tvm_cc_srcs CONFIGURE_DEPENDS + "${ONNXRUNTIME_ROOT}/core/providers/tvm/*.h" + "${ONNXRUNTIME_ROOT}/core/providers/tvm/*.cc" + ) + endif() + source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_tvm_cc_srcs}) onnxruntime_add_static_library(onnxruntime_providers_tvm ${onnxruntime_providers_tvm_cc_srcs}) @@ -1473,22 +1484,36 @@ if (onnxruntime_USE_TVM) ${TVM_INCLUDES} ${IPP_CRYPTO_INCLUDES} ${PYTHON_INLCUDE_DIRS}) - onnxruntime_add_include_to_target(onnxruntime_providers_tvm onnxruntime_common onnx tvm ippcp_s) + onnxruntime_add_include_to_target(onnxruntime_providers_tvm onnxruntime_common onnx tvm) + if (onnxruntime_TVM_USE_HASH) + onnxruntime_add_include_to_target(onnxruntime_providers_tvm ippcp_s) + endif() add_dependencies(onnxruntime_providers_tvm ${onnxruntime_EXTERNAL_DEPENDENCIES}) target_link_libraries(onnxruntime_providers_tvm PRIVATE onnx tvm - ippcp_s onnxruntime_common onnxruntime_framework ) + if (onnxruntime_TVM_USE_HASH) + target_link_libraries(onnxruntime_providers_tvm PRIVATE + ippcp_s + ) + endif() set_target_properties(onnxruntime_providers_tvm PROPERTIES FOLDER "ONNXRuntime") set_target_properties(onnxruntime_providers_tvm PROPERTIES LINKER_LANGUAGE CXX) - target_compile_options(onnxruntime_providers_tvm PRIVATE -Wno-error=type-limits) + if (WIN32 AND MSVC) + # wd4100: identifier' : unreferenced formal parameter + # wd4127: conditional expression is constant + # wd4244: conversion from 'int' to 'char', possible loss of data + target_compile_options(onnxruntime_providers_tvm PRIVATE "/wd4100" "/wd4127" "/wd4244") + else() + target_compile_options(onnxruntime_providers_tvm PRIVATE "-Wno-error=type-limits") + endif() target_compile_definitions(onnxruntime_providers_tvm PUBLIC DMLC_USE_LOGGING_LIBRARY=) install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/tvm DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers) diff --git a/docs/TVM_EP.md b/docs/TVM_EP.md index f659c27c31..6a31616c2c 100644 --- a/docs/TVM_EP.md +++ b/docs/TVM_EP.md @@ -4,6 +4,8 @@ - [Introduction](#introduction) - [Build](#build-onnx-runtime-with-the-tvm-execution-provider) + - [Linux](#linux) + - [Windows](#windows) - [Configuration options](#configuration-options) - [Performance Tuning](#performance-tuning) - [Using precompiled model](#using-precompiled-model) @@ -14,15 +16,17 @@ ## Introduction TVM is an execution provider for ONNX Runtime that is built on top of Apache TVM. It enables ONNX Runtime users to leverage Apache TVM model optimizations. -TVM EP is currently in "Preview". It's been tested to work on a handful of models on Linux, but not on Windows or MacOS. +TVM EP is currently in "Preview". It's been tested to work on a handful of models on Linux or Windows, but not on MacOS. ## Build ONNX Runtime with the TVM Execution Provider +### **Linux** Install the minimal pre-requisites on Ubuntu/Debian like linux operating systems: ```bash apt-get install -y python3 python3-dev python3-pip python3-setuptools gcc libtinfo-dev zlib1g-dev build-essential cmake libedit-dev libxml2-dev llvm-12 -pip3 install numpy decorator attrs +pip3 install numpy decorator attrs nasm ``` +Note: since ONNX Runtime with TVM EP is built with Intel ipp-crypto library there are new requirements. Compiler gcc (and g++) version should be equal to or higher than 8.2. nasm version should be 2.14.02 or higher. Problem with small nasm version can be seen [here](https://github.com/intel/ipp-crypto/issues/9) or [here](https://bugzilla.nasm.us/show_bug.cgi?id=3392205). For ubuntu LTS 18 `apt-get install nasm` is not enough due to it has version 2.13.02, see how to install from sources instruction [here](https://stackoverflow.com/questions/36144930/steps-to-install-nasm-offline-on-ubuntu). Also, the current implementation has `NVidia GPU` support for TVM EP. For now, you can use only `NVidia GPU` with CUDA Toolkit support. To do this, make sure you have installed the NVidia driver and CUDA Toolkit. @@ -81,6 +85,92 @@ export PYTHONPATH=/build//Release:${PYTHONPATH} export PYTHONPATH=/build//Release/_deps/tvm-src/python:${PYTHONPATH} ``` +### **Windows** +Install the minimal prerequisites on Windows: Git, CMake, Visual Studio, Python, LLVM +- Git: Download Git for Windows from [here](https://git-scm.com/download/win) and install it. Please make sure that the git.exe path is included in the environment variable. By default, it should be added. To check git after the installation use `git --version` in command line (cmd). +- CMake: use [the link](https://cmake.org/download/) to download and install CMake. msi-file is recommended for it. To verify CMake installation use `cmake --version` in cmd. +- Visual Studio: Download from [here](https://visualstudio.microsoft.com/ru/downloads/) and install Visual Studio 20** Community & Visual Studio Build Tools respectively. It is recommended not to change the default installation path. Chose "Desktop development with C++" workload and make sure that both options of “MSVC [contemporary version] C++ build tools” and “Windows 10 SDK” are selected. +- Python: Download Python 3.* from [here](https://www.python.org/downloads/windows/) and install it. Please have a check on the option of “Add Python to PATH”, so the installer will include the Python directory into the environment variable directly. To check python after the installation use `python` from cmd. The expected output is similar to the following: +```cmd +Python 3.10.5 (tags/v3.10.5:f377153, Jun 6 2022, 16:14:13) [MSC v.1929 64 bit (AMD64)] on win32 +Type "help", "copyright", "credits" or "license" for more information. +>>> +``` +Use `quit()` to exit from python interface. +- LLVM: the compiler is not necessary for pure ONNX Runtime installation but it is needed for TVM EP by default. +```cmd +git clone --depth 1 --branch release/11.x https://github.com/llvm/llvm-project.git +cmake -S llvm -B build -DLLVM_ENABLE_PROJECTS="clang;libcxx;libcxxabi" -DLLVM_TARGETS_TO_BUILD=X86 -Thost=x64 -DCMAKE_BUILD_TYPE=Release -G "Visual Studio 17 2022" +cmake --build ./build --config Release +``` +- Dependencies of ipp-crypto:
+1. install asm compiler (nasm) on windows by line: +```cmd +winget install nasm -i +``` +          +Add it to PATH (instruction for Windows GUI can be seen [here](https://www.computerhope.com/issues/ch000549.htm#dospath)) or by cmd: +``` +set PATH="%PATH%;C:\Program Files\NASM" +``` +          +Check by `nasm --version` in prompt command line.
+       +2. install openssl on windows by msi-file from [here](https://slproweb.com/products/Win32OpenSSL.html) +Add path to directory (e.g. "C:\Program Files\OpenSSL-Win64\bin") with executable file to PATH (see instructions above).
+          +Check by `openssl version` in prompt command line. +
+
+ +For using NVIDIA GPU (optional) CUDA and cuDNN should be installed. +- CUDA: Install CUDA by the [link](https://developer.nvidia.com/cuda-11.0-download-archive). +- cuDNN: download cuDNN installer from [here](https://developer.nvidia.com/rdp/cudnn-archive). Choose v8.* for corresponding CUDA v11.*, unzip it, and move cuDNN files as following: +1. [unzipped dir]\bin\ → C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.0\bin +2. [unzipped dir]\include\ → C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.0\include +3. [unzipped dir]\lib\ → C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.0\lib + +To verify the CUDA installation use `nvcc --version` in cmd. +
+
+ +Build ONNX Runtime with TVM Execution Provider from source: +- Use command line and clone sources from github: +```cmd +git clone --recursive https://github.com/Microsoft/onnxruntime +cd onnxruntime +``` +- CPU build: +``` +build.bat --config Release --enable_pybind --build_wheel --skip_tests --parallel --use_tvm --skip_onnx_tests --cmake_generator "Visual Studio 17 2022" --llvm_config /build/Release/bin/llvm-config.exe +``` +- GPU build: +``` +build.bat --config Release --enable_pybind --build_wheel --skip_tests --parallel --use_tvm --skip_onnx_tests --cmake_generator "Visual Studio 17 2022" --llvm_config /build/Release/bin/llvm-config.exe --use_cuda --cudnn_home “C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.*” --cuda_home “C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.*” +``` +In both cases (CPU, GPU) there are the following options for cmake generator: "Visual Studio 15 2017", "Visual Studio 16 2019", "Visual Studio 17 2022" and "Ninja" +- Install python wheel package for ONNX Runtime:
+Default path to the package is `/build/Windows/Release/Release/dist`. Note that it is different in comparison with path to the package on Linux. Before installation check names of wheel packages and use corresponding one. It can be looked like the following: +```cmd +python -m pip install .\onnxruntime\build\Windows\Release\Release\dist\onnxruntime_tvm-1.6.0-cp37-cp37m-win_amd64.whl +``` +- Install python wheel package for TVM due to its python API is used inside TVM EP:
+It can be looked like the following: +```cmd +python -m pip install .\onnxruntime\build\Windows\Release\_deps\tvm-src\python\dist\tvm-0.9.dev1728+g3425ed846-cp39-cp39-win_amd64.whl +``` +- Verify result by python script. Note: python should not be launched from directory containing 'onnxruntime' directory for correct result: +```python +import onnxruntime +print(onnxruntime.__version__) +print(onnxruntime.get_device()) +print(onnxruntime.get_available_providers()) +``` +- Uninstall procedure: +```cmd +pip uninstall onnxruntime-tvm +``` + ## Configuration options TVM Executor Provider can be configured with the following provider options: ```python @@ -102,7 +192,7 @@ tvm_session = onnxruntime.InferenceSession(model_path, providers=["TvmExecutionP
- `executor` is executor type used by TVM. There is choice between two types: GraphExecutor and VirtualMachine which are corresponded to "graph" and "vm" tags. VirtualMachine is used by default. -- `so_folder` is path to folder with set of files (.ro-, .so-files and weights) obtained after model tuning. It uses these files for executor compilation instead of onnx-model. But the latter is still needed for ONNX Runtime. +- `so_folder` is path to folder with set of files (.ro-, .so/.dll-files and weights) obtained after model tuning. It uses these files for executor compilation instead of onnx-model. But the latter is still needed for ONNX Runtime. - `check_hash` means that it is necessary to perform a HASH check for the model obtained in the `so_folder` parameter. It is `False` by default. - `hash_file_path` is path to file that contains the pre-computed HASH for the ONNX model which result of tuning locates in the path passed by `so_folder` parameter. If an empty string was passed as this value, then the file will be searched in the folder that was passed in the `so_folder` parameter. @@ -138,18 +228,18 @@ so.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_DISABLE_ALL tvm_session = onnxruntime.InferenceSession(model_path, sess_options=so, providers=["TvmExecutionProvider"], provider_options=po) ``` -### Using precompiled model +### **Using precompiled model** It is also possible to use a precompiled model. -The compiled model can be obtained using the [OctoML platform](https://onnx.octoml.ai) +The compiled model can be obtained using the [OctoML platform](https://onnx.octoml.ai) or compiled directly (see **Support precompiled model** section in [Sample notebook for ResNet50 inference with TVM EP](https://github.com/microsoft/onnxruntime/blob/master/docs/python/inference/notebooks/onnxruntime-tvm-tutorial.ipynb) for more information on model compilation). In order to use the precompiled model, only need to pass two options: -* **executor** - `vm` (`VirtualMachine`) must be used as a value +* **executor** - `vm` (`VirtualMachine`) must be used as a value (this functionality is not supported for `GraphExecutor`); -* **so_folder** - as a value, you must pass the path to the directory where +* **so_folder** - as a value, you must pass the path to the directory where the files of the precompiled model are located. * **check_hash** - (optional) if you want to check hash, you must pass `True` as the value. * **hash_file_path** - (optional) by default, the file containing the hash for the tuned model will be searched in the directory that is passed in the `so_folder` parameter. @@ -162,7 +252,7 @@ You can read more about these options in section [Configuration options](#config - [Sample notebook for ResNet50 inference with TVM EP](https://github.com/microsoft/onnxruntime/blob/master/docs/python/inference/notebooks/onnxruntime-tvm-tutorial.ipynb) ## Known issues -- At this moment, the TVM EP has only been verified on UNIX/Linux systems. +- At this moment, the TVM EP has only been verified on UNIX/Linux and Windows systems. - Some compatibility issues have been found between ONNX and Google protobuf. `AttributeError: module 'google.protobuf.internal.containers' has no attribute 'MutableMapping'`. This usually occurss during `import onnx` in any python scripts for protobuf version >= 3.19.0 and ONNX version <= 1.8.1. To resolve the issue Google protobuf and ONNX can be reinstalled separately or together using: ``` pip3 uninstall onnx -y diff --git a/onnxruntime/core/common/path.cc b/onnxruntime/core/common/path.cc index 296cf9962b..8b74d2d8c9 100644 --- a/onnxruntime/core/common/path.cc +++ b/onnxruntime/core/common/path.cc @@ -13,12 +13,6 @@ namespace { constexpr auto k_dot = ORT_TSTR("."); constexpr auto k_dotdot = ORT_TSTR(".."); -#ifdef _WIN32 -constexpr PathChar k_preferred_path_separator = ORT_TSTR('\\'); -#else // POSIX -constexpr PathChar k_preferred_path_separator = ORT_TSTR('/'); -#endif - constexpr std::array k_valid_path_separators{ ORT_TSTR('/'), ORT_TSTR('\\')}; diff --git a/onnxruntime/core/common/path.h b/onnxruntime/core/common/path.h index 514a636033..732bbabe8a 100644 --- a/onnxruntime/core/common/path.h +++ b/onnxruntime/core/common/path.h @@ -10,6 +10,12 @@ namespace onnxruntime { +#ifdef _WIN32 +constexpr PathChar k_preferred_path_separator = ORT_TSTR('\\'); +#else // POSIX +constexpr PathChar k_preferred_path_separator = ORT_TSTR('/'); +#endif + // Note: We should use the std::filesystem library after upgrading to C++17. /** A filesystem path. */ diff --git a/onnxruntime/core/providers/tvm/custom_logging.cc b/onnxruntime/core/providers/tvm/custom_logging.cc index 3140683825..dd9098292e 100644 --- a/onnxruntime/core/providers/tvm/custom_logging.cc +++ b/onnxruntime/core/providers/tvm/custom_logging.cc @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -22,7 +23,16 @@ std::string GetTimedLogMessage(const std::string& file, int lineno, const std::s std::stringstream sstream; std::string file_name = GetFileName(file); std::time_t t = std::time(nullptr); - sstream << "[" << std::put_time(std::localtime(&t), "%H:%M:%S") << "][TVM] " + sstream << "[" +#ifdef _WIN32 +// TODO(vvchernov): use #include instead of and localtime_s() approach for WIN32 +#pragma warning(disable : 4996) // _CRT_SECURE_NO_WARNINGS +#endif + << std::put_time(std::localtime(&t), "%H:%M:%S") +#ifdef _WIN32 +#pragma warning(default : 4996) +#endif + << "][TVM] " << file_name << ":" << lineno << ": " + message; return sstream.str(); } diff --git a/onnxruntime/core/providers/tvm/tvm_api.cc b/onnxruntime/core/providers/tvm/tvm_api.cc index 4a96bfce25..2e841f38ff 100644 --- a/onnxruntime/core/providers/tvm/tvm_api.cc +++ b/onnxruntime/core/providers/tvm/tvm_api.cc @@ -1,7 +1,11 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +#ifdef _WIN32 +#include +#else #include // glob(), globfree() +#endif #include // memset() #include #include @@ -12,6 +16,7 @@ #include #include "core/common/common.h" +#include "core/common/path.h" #include "tvm_api.h" @@ -59,30 +64,45 @@ TvmModule TVMCompile(const TvmEPOptions& options, return mod; } -std::vector glob(const std::string& pattern) { +std::vector glob(const std::string& dir, const std::string& extension) { std::vector filenames; - +#ifdef _WIN32 + std::string pattern = dir + "/*." + extension; + WIN32_FIND_DATA fd; + HANDLE hFind = ::FindFirstFile(pattern.c_str(), &fd); + if (hFind != INVALID_HANDLE_VALUE) { + do { + if ( !(fd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) ) { + filenames.push_back( + dir + + ToUTF8String(PathString{k_preferred_path_separator}) + + fd.cFileName); + } + } while (::FindNextFile(hFind, &fd)); + ::FindClose(hFind); + } +#else glob_t glob_result; memset(&glob_result, 0, sizeof(glob_result)); + std::string pattern = dir + "/*." + extension; int return_value = glob(pattern.c_str(), GLOB_TILDE, NULL, &glob_result); ORT_ENFORCE(return_value == 0, "No results of glob for pattern: " + pattern); for (size_t i = 0; i < glob_result.gl_pathc; ++i) { filenames.push_back(std::string(glob_result.gl_pathv[i])); } - globfree(&glob_result); - +#endif return filenames; } -std::string filter_lib_paths(const std::vector& lib_paths) { +std::string filter_lib_paths(const std::vector& lib_paths, const std::string& lib_ext) { std::string lib_path; size_t counter = 0; for (const auto& path : lib_paths) { - if (path.find("libtvm_runtime.so") != std::string::npos || - path.find("liboctomized_model.so") != std::string::npos) { + if (path.find("libtvm_runtime." + lib_ext) != std::string::npos || + path.find("liboctomized_model." + lib_ext) != std::string::npos) { ++counter; } else { lib_path = path; @@ -117,10 +137,17 @@ static std::unordered_map str2dev_type = { }; TvmModule TVMSoCompile(const TvmEPOptions& options) { - const std::string dir = options.so_folder; - const std::string lib_path = filter_lib_paths(glob(dir + "/*.so")); - const std::string consts_path = dir + "/consts"; - const auto& ro_paths = glob(dir + "/*.ro"); + const std::string& dir = options.so_folder; +#ifdef _WIN32 + std::string lib_ext = "dll"; +#else + std::string lib_ext = "so"; +#endif + const std::string lib_path = filter_lib_paths(glob(dir, lib_ext), lib_ext); + const std::string consts_path = dir + + ToUTF8String(PathString{k_preferred_path_separator}) + + "consts"; + const auto& ro_paths = glob(dir, "ro"); ORT_ENFORCE(ro_paths.size() == 1, "It should be only one ro file in folder: " + dir); const std::string vm_exec_code_path = ro_paths[0]; @@ -205,7 +232,7 @@ void TVM_VM_SetInputs(TvmModule& mod, TvmPackedFunc set_input = mod.GetFunction("set_input", false); ::tvm::runtime::TVMRetValue rv; - set_input.CallPacked(::tvm::runtime::TVMArgs(tvm_values.data(), tvm_type_codes.data(), num_total_args), &rv); + set_input.CallPacked(::tvm::runtime::TVMArgs(tvm_values.data(), tvm_type_codes.data(), int(num_total_args)), &rv); } void TVMGetOutputs(TvmModule& mod, diff --git a/onnxruntime/core/providers/tvm/tvm_execution_provider.cc b/onnxruntime/core/providers/tvm/tvm_execution_provider.cc index 070030a926..95316a3498 100644 --- a/onnxruntime/core/providers/tvm/tvm_execution_provider.cc +++ b/onnxruntime/core/providers/tvm/tvm_execution_provider.cc @@ -5,6 +5,7 @@ #include #include +#include "core/common/common.h" #include "core/framework/execution_provider.h" #include "core/framework/tensorprotoutils.h" #include "core/framework/kernel_registry.h" @@ -123,7 +124,7 @@ common::Status TvmExecutionProvider::Compile(const std::vector(std::move(onnx_model_str), - fused_node.ModelPath().ToPathString(), + ToUTF8String(fused_node.ModelPath().ToPathString()), int(opset->version())); InputsInfoMap all_input_shapes; auto mod = compileModel(func_name, graph_body_viewer, all_input_shapes); diff --git a/onnxruntime/core/providers/tvm/tvm_so_execution_provider.cc b/onnxruntime/core/providers/tvm/tvm_so_execution_provider.cc index 5c09a3b29a..aa4b4b8d96 100644 --- a/onnxruntime/core/providers/tvm/tvm_so_execution_provider.cc +++ b/onnxruntime/core/providers/tvm/tvm_so_execution_provider.cc @@ -17,7 +17,9 @@ #include "tvm_allocator.h" // NOLINT(build/include_subdir) #include "tvm_utils.h" // NOLINT(build/include_subdir) #include "tvm_api.h" // NOLINT(build/include_subdir) +#ifdef USE_TVM_HASH #include "hash_alg/hasher.h" // NOLINT(build/include_subdir) +#endif using ONNX_NAMESPACE::TensorShapeProto; @@ -107,10 +109,12 @@ common::Status TvmSoExecutionProvider::Compile(const std::vector(); @@ -150,6 +154,7 @@ void TvmSoExecutionProvider::printOptions() { LOGS(*GetLogger(), INFO) << options_; } +#ifdef USE_TVM_HASH bool TvmSoExecutionProvider::checkHash(const std::string& onnx_path) const { auto hasher = Hasher("sha256"); std::string onnx_str = readFromFile(onnx_path); @@ -164,6 +169,7 @@ bool TvmSoExecutionProvider::checkHash(const std::string& onnx_path) const { } return onnx_hash == hash; } +#endif std::shared_ptr TvmSoExecutionProvider::compileModel(const std::string& func_name, const GraphViewer& graph_viewer, diff --git a/onnxruntime/core/providers/tvm/tvm_so_execution_provider.h b/onnxruntime/core/providers/tvm/tvm_so_execution_provider.h index 1ecca73efa..43d433b53b 100644 --- a/onnxruntime/core/providers/tvm/tvm_so_execution_provider.h +++ b/onnxruntime/core/providers/tvm/tvm_so_execution_provider.h @@ -43,7 +43,9 @@ class TvmSoExecutionProvider : public IExecutionProvider { private: void printOptions(); +#ifdef USE_TVM_HASH bool checkHash(const std::string& onnx_path) const; +#endif std::shared_ptr compileModel(const std::string& func_name, const GraphViewer& graph_viewer, InputsInfoMap& inputs_info); // NOLINT diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index f7567fbee8..c1e01d2291 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -476,6 +476,9 @@ def parse_arguments(): parser.add_argument("--use_nuphar", action="store_true", help="Build with nuphar") parser.add_argument("--use_tvm", action="store_true", help="Build with TVM") parser.add_argument("--tvm_cuda_runtime", action="store_true", default=False, help="Build TVM with CUDA support") + parser.add_argument( + "--use_tvm_hash", action="store_true", help="Build ipp-crypto for hash generation. It is used by TVM EP only" + ) parser.add_argument("--use_tensorrt", action="store_true", help="Build with TensorRT") parser.add_argument( "--tensorrt_placeholder_builder", action="store_true", help="Instantiate Placeholder TensorRT Builder" @@ -485,6 +488,12 @@ def parse_arguments(): parser.add_argument("--migraphx_home", help="Path to MIGraphX installation dir") parser.add_argument("--use_full_protobuf", action="store_true", help="Use the full protobuf library") + parser.add_argument( + "--llvm_config", + type=str, + default="", + help="Path to llvm-config.exe for LLVM buit from sources. It is strongly needed for build on Windows", + ) parser.add_argument( "--skip_onnx_tests", action="store_true", @@ -833,6 +842,7 @@ def generate_build_tree( # set vars for TVM "-Donnxruntime_USE_TVM=" + ("ON" if args.use_tvm else "OFF"), "-Donnxruntime_TVM_CUDA_RUNTIME=" + ("ON" if args.use_tvm and args.tvm_cuda_runtime else "OFF"), + "-Donnxruntime_TVM_USE_HASH=" + ("ON" if args.use_tvm_hash else "OFF"), # set vars for migraphx "-Donnxruntime_USE_MIGRAPHX=" + ("ON" if args.use_migraphx else "OFF"), # By default - we currently support only cross compiling for ARM/ARM64 @@ -915,6 +925,8 @@ def generate_build_tree( cmake_args.append("-Donnxruntime_ROCM_VERSION=" + args.rocm_version) if args.use_tensorrt: cmake_args.append("-Donnxruntime_TENSORRT_HOME=" + tensorrt_home) + if args.llvm_config: + cmake_args.append("-Donnxruntime_TVM_USE_LLVM=" + args.llvm_config) # It should be default ON in CI build pipelines, and OFF in packaging pipelines. # And OFF for the people who are not actively developing onnx runtime.