From 8cfe8d33a3da698e27fe1d62ad36c7b3b14cdbe6 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Wed, 9 Jan 2019 10:40:55 -0800 Subject: [PATCH] Add nsync (#292) * Add nsync * nsync2 * nsync3 * fix build * update comments * fix build option --- .gitmodules | 3 ++ cmake/CMakeLists.txt | 6 +++ cmake/external/nsync | 1 + cmake/onnxruntime_common.cmake | 5 +- cmake/onnxruntime_session.cmake | 2 +- onnxruntime/core/platform/ort_mutex.h | 50 +++++++++++++++++++ onnxruntime/core/platform/posix/ort_mutex.cc | 50 +++++++++++++++++++ onnxruntime/core/session/inference_session.cc | 24 ++++----- tools/ci_build/build.py | 1 + 9 files changed, 128 insertions(+), 14 deletions(-) create mode 160000 cmake/external/nsync create mode 100644 onnxruntime/core/platform/ort_mutex.h create mode 100644 onnxruntime/core/platform/posix/ort_mutex.cc diff --git a/.gitmodules b/.gitmodules index 7d34a0dee9..f5277a4561 100644 --- a/.gitmodules +++ b/.gitmodules @@ -19,3 +19,6 @@ [submodule "cmake/external/gemmlowp"] path = cmake/external/gemmlowp url = https://github.com/google/gemmlowp.git +[submodule "cmake/external/nsync"] + path = cmake/external/nsync + url = https://github.com/google/nsync diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index f5cace6d2a..6408edceac 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -43,6 +43,7 @@ option(onnxruntime_GENERATE_TEST_REPORTS "Enable test report generation" OFF) option(onnxruntime_ENABLE_STATIC_ANALYSIS "Enable static analysis" OFF) option(onnxruntime_ENABLE_PYTHON "Enable python buildings" OFF) option(onnxruntime_USE_CUDA "Build with CUDA support" OFF) +option(onnxruntime_USE_NSYNC "Build with NSYNC support. This option only takes effect on Linux" OFF) option(onnxruntime_USE_EIGEN_FOR_BLAS "Use eign for blas" ON) option(onnxruntime_USE_MLAS "Use optimized blas library for GEMM and 2D Convolution" OFF) option(onnxruntime_USE_MKLDNN "Build with MKL-DNN support" OFF) @@ -79,6 +80,7 @@ if(onnxruntime_USE_OPENMP) add_definitions(-DUSE_OPENMP) endif() endif() + #must after OpenMP settings find_package(Threads) @@ -159,6 +161,7 @@ if(onnxruntime_BUILD_BENCHMARKS) endif() endif() +add_subdirectory(${PROJECT_SOURCE_DIR}/external/nsync) # External dependencies list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/external) @@ -501,6 +504,9 @@ include(onnxruntime_mlas.cmake) if(WIN32) set(onnxruntime_EXTERNAL_LIBRARIES_DEBUG ${onnxruntime_EXTERNAL_LIBRARIES} Dbghelp) else() + if(onnxruntime_USE_NSYNC) + list(APPEND onnxruntime_EXTERNAL_LIBRARIES nsync_cpp) + endif() list(APPEND onnxruntime_EXTERNAL_LIBRARIES Threads::Threads) set(onnxruntime_EXTERNAL_LIBRARIES_DEBUG ${onnxruntime_EXTERNAL_LIBRARIES}) endif() diff --git a/cmake/external/nsync b/cmake/external/nsync new file mode 160000 index 0000000000..8f50e4463c --- /dev/null +++ b/cmake/external/nsync @@ -0,0 +1 @@ +Subproject commit 8f50e4463c2c7ba9b3f580c61ca21abc91197b7c diff --git a/cmake/onnxruntime_common.cmake b/cmake/onnxruntime_common.cmake index 477187a6c7..1ab2242d8d 100644 --- a/cmake/onnxruntime_common.cmake +++ b/cmake/onnxruntime_common.cmake @@ -41,7 +41,10 @@ if(NOT WIN32) target_link_libraries(onnxruntime_common dl) endif() onnxruntime_add_include_to_target(onnxruntime_common gsl date) -target_include_directories(onnxruntime_common PRIVATE ${ONNXRUNTIME_ROOT}) +target_include_directories(onnxruntime_common PRIVATE ${ONNXRUNTIME_ROOT} PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/external/nsync/public") +if(onnxruntime_USE_NSYNC) + target_compile_definitions(onnxruntime_common PUBLIC USE_NSYNC) +endif() #threadpool uses eigen add_dependencies(onnxruntime_common eigen) diff --git a/cmake/onnxruntime_session.cmake b/cmake/onnxruntime_session.cmake index 2cfcfebfb0..3efeea02c0 100644 --- a/cmake/onnxruntime_session.cmake +++ b/cmake/onnxruntime_session.cmake @@ -11,7 +11,7 @@ source_group(TREE ${REPO_ROOT} FILES ${onnxruntime_session_srcs}) add_library(onnxruntime_session ${onnxruntime_session_srcs}) install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/session DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core) -onnxruntime_add_include_to_target(onnxruntime_session onnxruntime_framework gsl onnx onnx_proto protobuf::libprotobuf) +onnxruntime_add_include_to_target(onnxruntime_session onnxruntime_common onnxruntime_framework gsl onnx onnx_proto protobuf::libprotobuf) target_include_directories(onnxruntime_session PRIVATE ${ONNXRUNTIME_ROOT}) add_dependencies(onnxruntime_session ${onnxruntime_EXTERNAL_DEPENDENCIES}) set_target_properties(onnxruntime_session PROPERTIES FOLDER "ONNXRuntime") diff --git a/onnxruntime/core/platform/ort_mutex.h b/onnxruntime/core/platform/ort_mutex.h new file mode 100644 index 0000000000..5d6a580cac --- /dev/null +++ b/onnxruntime/core/platform/ort_mutex.h @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#ifdef _WIN32 +#include +namespace onnxruntime { + using OrtMutex = std::mutex; +} +#else +#ifdef USE_NSYNC +#include "nsync.h" +#else +#include "pthread.h" +#endif +namespace onnxruntime { + +class OrtMutex { +#ifdef USE_NSYNC + nsync::nsync_mu data_ = NSYNC_MU_INIT; +#else + pthread_mutex_t data_ = PTHREAD_MUTEX_INITIALIZER; +#endif + + public: + constexpr OrtMutex() = default; +#ifdef USE_NSYNC + ~OrtMutex() = default; +#else + ~OrtMutex(); +#endif + + private: + OrtMutex(const OrtMutex&); // = delete; + OrtMutex& operator=(const OrtMutex&); // = delete; + + public: + void lock(); + bool try_lock() noexcept; + void unlock() noexcept; + +#ifdef USE_NSYNC + using native_handle_type = nsync::nsync_mu*; +#else + using native_handle_type = pthread_mutex_t*; +#endif + native_handle_type native_handle() { return &data_; } +}; +}; // namespace onnxruntime +#endif \ No newline at end of file diff --git a/onnxruntime/core/platform/posix/ort_mutex.cc b/onnxruntime/core/platform/posix/ort_mutex.cc new file mode 100644 index 0000000000..beeb850b7e --- /dev/null +++ b/onnxruntime/core/platform/posix/ort_mutex.cc @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/platform/ort_mutex.h" +#include +#include +#include + +namespace onnxruntime { +#ifndef USE_NSYNC +OrtMutex::~OrtMutex() { + pthread_mutex_destroy(&data_); +} +#endif + +void +OrtMutex::lock() { +#ifndef USE_NSYNC + int ec = pthread_mutex_lock(&data_); + if (ec) + throw std::system_error(ec, std::system_category(), "mutex lock failed"); +#else + nsync::nsync_mu_lock(&data_); +#endif +} + +bool +OrtMutex::try_lock() +noexcept { +#ifndef USE_NSYNC + return pthread_mutex_trylock(&data_) == 0; +#else + return nsync::nsync_mu_trylock(&data_) == 0; +#endif +} + +void +OrtMutex::unlock() +noexcept +{ +#ifdef USE_NSYNC + nsync::nsync_mu_unlock(&data_); +#else +int ec = pthread_mutex_unlock(&data_); +(void) ec; +//Don't throw! +assert(ec== 0); +#endif +} +} \ No newline at end of file diff --git a/onnxruntime/core/session/inference_session.cc b/onnxruntime/core/session/inference_session.cc index 0345c8dbc4..bdea40748c 100644 --- a/onnxruntime/core/session/inference_session.cc +++ b/onnxruntime/core/session/inference_session.cc @@ -4,7 +4,7 @@ #include "core/session/inference_session.h" #include -#include +#include "core/platform/ort_mutex.h" #include #include #include @@ -124,7 +124,7 @@ class InferenceSession::Impl { common::Status Load(const T& model_uri) { auto tp = session_profiler_.StartTime(); try { - std::lock_guard l(session_mutex_); + std::lock_guard l(session_mutex_); if (is_model_loaded_) { // already loaded LOGS(*session_logger_, ERROR) << "This session already contains a loaded model."; return common::Status(common::ONNXRUNTIME, common::MODEL_LOADED, "This session already contains a loaded model."); @@ -155,7 +155,7 @@ class InferenceSession::Impl { auto tp = session_profiler_.StartTime(); try { LOGS(*session_logger_, INFO) << "Loading model using model_proto"; - std::lock_guard l(session_mutex_); + std::lock_guard l(session_mutex_); if (is_model_loaded_) { // already loaded LOGS(*session_logger_, ERROR) << "This session already contains a loaded model."; return common::Status(common::ONNXRUNTIME, common::MODEL_LOADED, "This session already contains a loaded model."); @@ -188,7 +188,7 @@ class InferenceSession::Impl { auto tp = session_profiler_.StartTime(); try { LOGS(*session_logger_, INFO) << "Loading model using model_proto"; - std::lock_guard l(session_mutex_); + std::lock_guard l(session_mutex_); if (is_model_loaded_) { // already loaded LOGS(*session_logger_, ERROR) << "This session already contains a loaded model."; return common::Status(common::ONNXRUNTIME, common::MODEL_LOADED, "This session already contains a loaded model."); @@ -221,7 +221,7 @@ class InferenceSession::Impl { auto tp = session_profiler_.StartTime(); try { LOGS(*session_logger_, INFO) << "Loading model using istream"; - std::lock_guard l(session_mutex_); + std::lock_guard l(session_mutex_); if (is_model_loaded_) { // already loaded LOGS(*session_logger_, ERROR) << "This session already contains a loaded model."; return common::Status(common::ONNXRUNTIME, common::MODEL_LOADED, "This session already contains a loaded model."); @@ -357,7 +357,7 @@ class InferenceSession::Impl { try { LOGS(*session_logger_, INFO) << "Initializing session."; - std::lock_guard l(session_mutex_); + std::lock_guard l(session_mutex_); if (!is_model_loaded_) { LOGS(*session_logger_, ERROR) << "Model was not loaded"; return common::Status(common::ONNXRUNTIME, common::FAIL, "Model was not loaded."); @@ -794,7 +794,7 @@ class InferenceSession::Impl { try { { - std::lock_guard l(session_mutex_); + std::lock_guard l(session_mutex_); if (!is_inited_) { LOGS(*session_logger_, ERROR) << "Session was not initialized"; retval = Status(common::ONNXRUNTIME, common::FAIL, "Session not initialized."); @@ -862,7 +862,7 @@ class InferenceSession::Impl { std::pair GetModelMetadata() const { { - std::lock_guard l(session_mutex_); + std::lock_guard l(session_mutex_); if (!is_model_loaded_) { LOGS(*session_logger_, ERROR) << "Model was not loaded"; return std::make_pair(common::Status(common::ONNXRUNTIME, common::FAIL, "Model was not loaded."), @@ -875,7 +875,7 @@ class InferenceSession::Impl { std::pair GetModelInputs() const { { - std::lock_guard l(session_mutex_); + std::lock_guard l(session_mutex_); if (!is_model_loaded_) { LOGS(*session_logger_, ERROR) << "Model was not loaded"; return std::make_pair(common::Status(common::ONNXRUNTIME, common::FAIL, "Model was not loaded."), @@ -888,7 +888,7 @@ class InferenceSession::Impl { std::pair GetModelOutputs() const { { - std::lock_guard l(session_mutex_); + std::lock_guard l(session_mutex_); if (!is_model_loaded_) { LOGS(*session_logger_, ERROR) << "Model was not loaded"; return std::make_pair(common::Status(common::ONNXRUNTIME, common::FAIL, "Model was not loaded."), @@ -901,7 +901,7 @@ class InferenceSession::Impl { common::Status NewIOBinding(std::unique_ptr* io_binding) { { - std::lock_guard l(session_mutex_); + std::lock_guard l(session_mutex_); if (!is_inited_) { LOGS(*session_logger_, ERROR) << "Session was not initialized"; return common::Status(common::ONNXRUNTIME, common::FAIL, "Session not initialized."); @@ -1135,7 +1135,7 @@ class InferenceSession::Impl { // Number of concurrently running executors std::atomic current_num_runs_; - mutable std::mutex session_mutex_; // to ensure only one thread can invoke Load/Initialize + mutable onnxruntime::OrtMutex session_mutex_; // to ensure only one thread can invoke Load/Initialize bool is_model_loaded_ = false; // GUARDED_BY(session_mutex_) bool is_inited_ = false; // GUARDED_BY(session_mutex_) diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index f9fc3f7149..bcd84ae3a4 100755 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -261,6 +261,7 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home "-Donnxruntime_DEV_MODE=ON", "-DPYTHON_EXECUTABLE=" + sys.executable, "-Donnxruntime_USE_CUDA=" + ("ON" if args.use_cuda else "OFF"), + "-Donnxruntime_USE_NSYNC=" + ("OFF" if is_windows() else "ON"), "-Donnxruntime_CUDNN_HOME=" + (cudnn_home if args.use_cuda else ""), "-Donnxruntime_USE_JEMALLOC=" + ("ON" if args.use_jemalloc else "OFF"), "-Donnxruntime_ENABLE_PYTHON=" + ("ON" if args.enable_pybind else "OFF"),