Add nsync (#292)

* Add nsync

* nsync2

* nsync3

* fix build

* update comments

* fix build option

.gitmodules

@@ -19,3 +19,6 @@
 [submodule "cmake/external/gemmlowp"]
 	path = cmake/external/gemmlowp
 	url = https://github.com/google/gemmlowp.git
+[submodule "cmake/external/nsync"]
+	path = cmake/external/nsync
+	url = https://github.com/google/nsync

cmake/CMakeLists.txt

@@ -43,6 +43,7 @@ option(onnxruntime_GENERATE_TEST_REPORTS "Enable test report generation" OFF)
 option(onnxruntime_ENABLE_STATIC_ANALYSIS "Enable static analysis" OFF)
 option(onnxruntime_ENABLE_PYTHON "Enable python buildings" OFF)
 option(onnxruntime_USE_CUDA "Build with CUDA support" OFF)
+option(onnxruntime_USE_NSYNC "Build with NSYNC support. This option only takes effect on Linux" OFF)
 option(onnxruntime_USE_EIGEN_FOR_BLAS "Use eign for blas" ON)
 option(onnxruntime_USE_MLAS "Use optimized blas library for GEMM and 2D Convolution" OFF)
 option(onnxruntime_USE_MKLDNN "Build with MKL-DNN support" OFF)
@@ -79,6 +80,7 @@ if(onnxruntime_USE_OPENMP)
     add_definitions(-DUSE_OPENMP)
   endif()
 endif()
+
 #must after OpenMP settings
 find_package(Threads)
 
@@ -159,6 +161,7 @@ if(onnxruntime_BUILD_BENCHMARKS)
   endif()
 endif()
 
+add_subdirectory(${PROJECT_SOURCE_DIR}/external/nsync)
 # External dependencies
 list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/external)
 
@@ -501,6 +504,9 @@ include(onnxruntime_mlas.cmake)
 if(WIN32)
   set(onnxruntime_EXTERNAL_LIBRARIES_DEBUG ${onnxruntime_EXTERNAL_LIBRARIES} Dbghelp)
 else()
+  if(onnxruntime_USE_NSYNC)
+    list(APPEND onnxruntime_EXTERNAL_LIBRARIES nsync_cpp)
+  endif()
   list(APPEND onnxruntime_EXTERNAL_LIBRARIES Threads::Threads)
   set(onnxruntime_EXTERNAL_LIBRARIES_DEBUG ${onnxruntime_EXTERNAL_LIBRARIES})
 endif()

cmake/external/nsync

@@ -0,0 +1 @@
+Subproject commit 8f50e4463c2c7ba9b3f580c61ca21abc91197b7c

cmake/onnxruntime_common.cmake

@@ -41,7 +41,10 @@ if(NOT WIN32)
 	target_link_libraries(onnxruntime_common dl)
 endif()
 onnxruntime_add_include_to_target(onnxruntime_common gsl date)
-target_include_directories(onnxruntime_common PRIVATE ${ONNXRUNTIME_ROOT})
+target_include_directories(onnxruntime_common PRIVATE ${ONNXRUNTIME_ROOT} PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/external/nsync/public")
+if(onnxruntime_USE_NSYNC)
+    target_compile_definitions(onnxruntime_common PUBLIC USE_NSYNC)
+endif()
 #threadpool uses eigen
 add_dependencies(onnxruntime_common eigen)
 

cmake/onnxruntime_session.cmake

@@ -11,7 +11,7 @@ source_group(TREE ${REPO_ROOT} FILES ${onnxruntime_session_srcs})
 
 add_library(onnxruntime_session ${onnxruntime_session_srcs})
 install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/session  DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core)
-onnxruntime_add_include_to_target(onnxruntime_session onnxruntime_framework gsl onnx onnx_proto protobuf::libprotobuf)
+onnxruntime_add_include_to_target(onnxruntime_session onnxruntime_common onnxruntime_framework gsl onnx onnx_proto protobuf::libprotobuf)
 target_include_directories(onnxruntime_session PRIVATE ${ONNXRUNTIME_ROOT})
 add_dependencies(onnxruntime_session ${onnxruntime_EXTERNAL_DEPENDENCIES})
 set_target_properties(onnxruntime_session PROPERTIES FOLDER "ONNXRuntime")

onnxruntime/core/platform/ort_mutex.h

@@ -0,0 +1,50 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+#ifdef _WIN32
+#include <mutex>
+namespace onnxruntime {
+  using OrtMutex = std::mutex;
+}
+#else
+#ifdef USE_NSYNC
+#include "nsync.h"
+#else
+#include "pthread.h"
+#endif
+namespace onnxruntime {
+
+class OrtMutex {
+#ifdef USE_NSYNC
+  nsync::nsync_mu data_ = NSYNC_MU_INIT;
+#else
+  pthread_mutex_t data_ = PTHREAD_MUTEX_INITIALIZER;
+#endif
+
+ public:
+  constexpr OrtMutex() = default;
+#ifdef USE_NSYNC
+  ~OrtMutex() = default;
+#else
+  ~OrtMutex();
+#endif
+
+ private:
+  OrtMutex(const OrtMutex&);             // = delete;
+  OrtMutex& operator=(const OrtMutex&);  // = delete;
+
+ public:
+  void lock();
+  bool try_lock() noexcept;
+  void unlock() noexcept;
+
+#ifdef USE_NSYNC
+  using native_handle_type = nsync::nsync_mu*;
+#else
+  using native_handle_type = pthread_mutex_t*;
+#endif
+  native_handle_type native_handle() { return &data_; }
+};
+};  // namespace onnxruntime
+#endif

onnxruntime/core/platform/posix/ort_mutex.cc

@@ -0,0 +1,50 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#include "core/platform/ort_mutex.h"
+#include <assert.h>
+#include <stdexcept>
+#include <system_error>
+
+namespace onnxruntime {
+#ifndef USE_NSYNC
+OrtMutex::~OrtMutex() {
+  pthread_mutex_destroy(&data_);
+}
+#endif
+
+void
+OrtMutex::lock() {
+#ifndef USE_NSYNC
+  int ec = pthread_mutex_lock(&data_);
+  if (ec)
+    throw std::system_error(ec, std::system_category(), "mutex lock failed");
+#else
+  nsync::nsync_mu_lock(&data_);
+#endif
+}
+
+bool
+OrtMutex::try_lock()
+noexcept {
+#ifndef USE_NSYNC
+  return pthread_mutex_trylock(&data_) == 0;
+#else
+  return nsync::nsync_mu_trylock(&data_) == 0;
+#endif
+}
+
+void
+OrtMutex::unlock()
+noexcept
+{
+#ifdef USE_NSYNC
+  nsync::nsync_mu_unlock(&data_);
+#else
+int ec = pthread_mutex_unlock(&data_);
+(void) ec;
+//Don't throw!
+assert(ec== 0);
+#endif
+}
+}

onnxruntime/core/session/inference_session.cc

@@ -4,7 +4,7 @@
 #include "core/session/inference_session.h"
 
 #include <memory>
-#include <mutex>
+#include "core/platform/ort_mutex.h"
 #include <sstream>
 #include <unordered_set>
 #include <list>
@@ -124,7 +124,7 @@ class InferenceSession::Impl {
   common::Status Load(const T& model_uri) {
     auto tp = session_profiler_.StartTime();
     try {
-      std::lock_guard<std::mutex> l(session_mutex_);
+      std::lock_guard<onnxruntime::OrtMutex> l(session_mutex_);
       if (is_model_loaded_) {  // already loaded
         LOGS(*session_logger_, ERROR) << "This session already contains a loaded model.";
         return common::Status(common::ONNXRUNTIME, common::MODEL_LOADED, "This session already contains a loaded model.");
@@ -155,7 +155,7 @@ class InferenceSession::Impl {
     auto tp = session_profiler_.StartTime();
     try {
       LOGS(*session_logger_, INFO) << "Loading model using model_proto";
-      std::lock_guard<std::mutex> l(session_mutex_);
+      std::lock_guard<onnxruntime::OrtMutex> l(session_mutex_);
       if (is_model_loaded_) {  // already loaded
         LOGS(*session_logger_, ERROR) << "This session already contains a loaded model.";
         return common::Status(common::ONNXRUNTIME, common::MODEL_LOADED, "This session already contains a loaded model.");
@@ -188,7 +188,7 @@ class InferenceSession::Impl {
     auto tp = session_profiler_.StartTime();
     try {
       LOGS(*session_logger_, INFO) << "Loading model using model_proto";
-      std::lock_guard<std::mutex> l(session_mutex_);
+      std::lock_guard<onnxruntime::OrtMutex> l(session_mutex_);
       if (is_model_loaded_) {  // already loaded
         LOGS(*session_logger_, ERROR) << "This session already contains a loaded model.";
         return common::Status(common::ONNXRUNTIME, common::MODEL_LOADED, "This session already contains a loaded model.");
@@ -221,7 +221,7 @@ class InferenceSession::Impl {
     auto tp = session_profiler_.StartTime();
     try {
       LOGS(*session_logger_, INFO) << "Loading model using istream";
-      std::lock_guard<std::mutex> l(session_mutex_);
+      std::lock_guard<onnxruntime::OrtMutex> l(session_mutex_);
       if (is_model_loaded_) {  // already loaded
         LOGS(*session_logger_, ERROR) << "This session already contains a loaded model.";
         return common::Status(common::ONNXRUNTIME, common::MODEL_LOADED, "This session already contains a loaded model.");
@@ -357,7 +357,7 @@ class InferenceSession::Impl {
 
     try {
       LOGS(*session_logger_, INFO) << "Initializing session.";
-      std::lock_guard<std::mutex> l(session_mutex_);
+      std::lock_guard<onnxruntime::OrtMutex> l(session_mutex_);
       if (!is_model_loaded_) {
         LOGS(*session_logger_, ERROR) << "Model was not loaded";
         return common::Status(common::ONNXRUNTIME, common::FAIL, "Model was not loaded.");
@@ -794,7 +794,7 @@ class InferenceSession::Impl {
 
     try {
       {
-        std::lock_guard<std::mutex> l(session_mutex_);
+        std::lock_guard<onnxruntime::OrtMutex> l(session_mutex_);
         if (!is_inited_) {
           LOGS(*session_logger_, ERROR) << "Session was not initialized";
           retval = Status(common::ONNXRUNTIME, common::FAIL, "Session not initialized.");
@@ -862,7 +862,7 @@ class InferenceSession::Impl {
 
   std::pair<common::Status, const ModelMetadata*> GetModelMetadata() const {
     {
-      std::lock_guard<std::mutex> l(session_mutex_);
+      std::lock_guard<onnxruntime::OrtMutex> l(session_mutex_);
       if (!is_model_loaded_) {
         LOGS(*session_logger_, ERROR) << "Model was not loaded";
         return std::make_pair(common::Status(common::ONNXRUNTIME, common::FAIL, "Model was not loaded."),
@@ -875,7 +875,7 @@ class InferenceSession::Impl {
 
   std::pair<common::Status, const InputDefList*> GetModelInputs() const {
     {
-      std::lock_guard<std::mutex> l(session_mutex_);
+      std::lock_guard<onnxruntime::OrtMutex> l(session_mutex_);
       if (!is_model_loaded_) {
         LOGS(*session_logger_, ERROR) << "Model was not loaded";
         return std::make_pair(common::Status(common::ONNXRUNTIME, common::FAIL, "Model was not loaded."),
@@ -888,7 +888,7 @@ class InferenceSession::Impl {
 
   std::pair<common::Status, const OutputDefList*> GetModelOutputs() const {
     {
-      std::lock_guard<std::mutex> l(session_mutex_);
+      std::lock_guard<onnxruntime::OrtMutex> l(session_mutex_);
       if (!is_model_loaded_) {
         LOGS(*session_logger_, ERROR) << "Model was not loaded";
         return std::make_pair(common::Status(common::ONNXRUNTIME, common::FAIL, "Model was not loaded."),
@@ -901,7 +901,7 @@ class InferenceSession::Impl {
 
   common::Status NewIOBinding(std::unique_ptr<IOBinding>* io_binding) {
     {
-      std::lock_guard<std::mutex> l(session_mutex_);
+      std::lock_guard<onnxruntime::OrtMutex> l(session_mutex_);
       if (!is_inited_) {
         LOGS(*session_logger_, ERROR) << "Session was not initialized";
         return common::Status(common::ONNXRUNTIME, common::FAIL, "Session not initialized.");
@@ -1135,7 +1135,7 @@ class InferenceSession::Impl {
   // Number of concurrently running executors
   std::atomic<int> current_num_runs_;
 
-  mutable std::mutex session_mutex_;  // to ensure only one thread can invoke Load/Initialize
+  mutable onnxruntime::OrtMutex session_mutex_;  // to ensure only one thread can invoke Load/Initialize
   bool is_model_loaded_ = false;      // GUARDED_BY(session_mutex_)
   bool is_inited_ = false;            // GUARDED_BY(session_mutex_)
 

tools/ci_build/build.py

@@ -261,6 +261,7 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home
                  "-Donnxruntime_DEV_MODE=ON",
                  "-DPYTHON_EXECUTABLE=" + sys.executable,
                  "-Donnxruntime_USE_CUDA=" + ("ON" if args.use_cuda else "OFF"),
+                 "-Donnxruntime_USE_NSYNC=" + ("OFF" if is_windows() else "ON"),
                  "-Donnxruntime_CUDNN_HOME=" + (cudnn_home if args.use_cuda else ""),
                  "-Donnxruntime_USE_JEMALLOC=" + ("ON" if args.use_jemalloc else "OFF"),
                  "-Donnxruntime_ENABLE_PYTHON=" + ("ON" if args.enable_pybind else "OFF"),