From cb69c6575645f1e316c996882e9eb9551908b269 Mon Sep 17 00:00:00 2001 From: Tracy Sharpe <42477615+tracysh@users.noreply.github.com> Date: Sun, 21 Apr 2019 14:04:15 -0700 Subject: [PATCH] Update MLAS to be able to build standalone again (#874) Change MLAS to be able to build standalone without onnxruntime header dependencies. This is enabled when building with MLAS_NO_ONNXRUNTIME_THREADPOOL defined. mlas.h had been changed to include the ThreadPool header, but this header now just has a forward reference for the class. The header was also doing a "using onnxruntime::concurrency"; that has been removed and the external mlas.h users fixed up as needed. As before, if ThreadPool==nullptr, then MLAS uses OpenMP or falls back to a single threaded implementation. The build option to use the Win32 system thread pool has been removed as onnxruntime can't hit that path and I don't use that option for standalone tests anymore. --- cmake/onnxruntime_mlas.cmake | 44 ++++--- .../contrib_ops/cpu/maxpool_with_mask.h | 6 +- onnxruntime/core/mlas/inc/mlas.h | 25 ++-- onnxruntime/core/mlas/lib/convolve.cpp | 49 ++++---- onnxruntime/core/mlas/lib/mlasi.h | 51 ++++---- onnxruntime/core/mlas/lib/platform.cpp | 18 --- onnxruntime/core/mlas/lib/pooling.cpp | 22 ++-- onnxruntime/core/mlas/lib/sgemm.cpp | 45 ++----- onnxruntime/core/mlas/lib/threading.cpp | 110 ++---------------- onnxruntime/core/providers/cpu/nn/conv.cc | 4 +- onnxruntime/core/providers/cpu/nn/pool.cc | 14 +-- onnxruntime/test/mlas/unittest.cpp | 2 +- 12 files changed, 137 insertions(+), 253 deletions(-) diff --git a/cmake/onnxruntime_mlas.cmake b/cmake/onnxruntime_mlas.cmake index 1bb72d14b5..1e19c11b30 100644 --- a/cmake/onnxruntime_mlas.cmake +++ b/cmake/onnxruntime_mlas.cmake @@ -12,21 +12,21 @@ set(mlas_common_srcs ${ONNXRUNTIME_ROOT}/core/mlas/lib/tanh.cpp ) -if (MSVC) +if(MSVC) - if (CMAKE_GENERATOR_PLATFORM STREQUAL "ARM") + if(CMAKE_GENERATOR_PLATFORM STREQUAL "ARM") set(mlas_platform_srcs ${ONNXRUNTIME_ROOT}/core/mlas/lib/arm/sgemmc.cpp ) - elseif (CMAKE_GENERATOR_PLATFORM STREQUAL "ARM64") + elseif(CMAKE_GENERATOR_PLATFORM STREQUAL "ARM64") set(asm_filename ${ONNXRUNTIME_ROOT}/core/mlas/lib/arm64/sgemma.asm) set(pre_filename ${CMAKE_CURRENT_BINARY_DIR}/sgemma.i) set(obj_filename ${CMAKE_CURRENT_BINARY_DIR}/sgemma.obj) - if (CMAKE_BUILD_TYPE STREQUAL "Debug") + if(CMAKE_BUILD_TYPE STREQUAL "Debug") set(ARMASM_FLAGS "-g") else() set(ARMASM_FLAGS "") @@ -42,7 +42,7 @@ if (MSVC) set(mlas_platform_srcs ${obj_filename}) - elseif (CMAKE_GENERATOR_PLATFORM STREQUAL "Win32") + elseif(CMAKE_GENERATOR_PLATFORM STREQUAL "Win32") enable_language(ASM_MASM) @@ -52,7 +52,7 @@ if (MSVC) ${ONNXRUNTIME_ROOT}/core/mlas/lib/i386/sgemma.asm ) - elseif (CMAKE_GENERATOR_PLATFORM STREQUAL "x64") + elseif(CMAKE_GENERATOR_PLATFORM STREQUAL "x64") enable_language(ASM_MASM) @@ -70,17 +70,23 @@ if (MSVC) endif() elseif(CMAKE_SYSTEM_NAME STREQUAL "Android") - if(CMAKE_ANDROID_ARCH_ABI MATCHES "^arm.*") - if (CMAKE_ANDROID_ARCH_ABI STREQUAL "armeabi-v7a") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mfpu=neon") - endif() - set(mlas_platform_srcs - ${ONNXRUNTIME_ROOT}/core/mlas/lib/arm/sgemmc.cpp - ) - else() - message(FATAL_ERROR "Android build is not supported on non-ARM platform now") + if(CMAKE_ANDROID_ARCH_ABI MATCHES "^arm.*") + + if(CMAKE_ANDROID_ARCH_ABI STREQUAL "armeabi-v7a") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mfpu=neon") endif() + + set(mlas_platform_srcs + ${ONNXRUNTIME_ROOT}/core/mlas/lib/arm/sgemmc.cpp + ) + + else() + + message(FATAL_ERROR "Android build is not supported on non-ARM platform now") + + endif() + else() execute_process( @@ -89,7 +95,7 @@ else() ERROR_QUIET ) - if (dumpmachine_output MATCHES "^arm.*") + if(dumpmachine_output MATCHES "^arm.*") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mfpu=neon") @@ -97,7 +103,7 @@ else() ${ONNXRUNTIME_ROOT}/core/mlas/lib/arm/sgemmc.cpp ) - elseif (dumpmachine_output MATCHES "^aarch64.*") + elseif(dumpmachine_output MATCHES "^aarch64.*") enable_language(ASM) @@ -105,7 +111,7 @@ else() ${ONNXRUNTIME_ROOT}/core/mlas/lib/aarch64/sgemma.s ) - elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "^(i.86|x86?)$") + elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(i.86|x86?)$") enable_language(ASM) @@ -124,7 +130,7 @@ else() ${mlas_platform_srcs_avx} ) - elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") enable_language(ASM) diff --git a/onnxruntime/contrib_ops/cpu/maxpool_with_mask.h b/onnxruntime/contrib_ops/cpu/maxpool_with_mask.h index 06c524f91a..16c2f16e28 100644 --- a/onnxruntime/contrib_ops/cpu/maxpool_with_mask.h +++ b/onnxruntime/contrib_ops/cpu/maxpool_with_mask.h @@ -77,7 +77,7 @@ class MaxpoolWithMask : public OpKernel, public PoolBase { y_d[ph] = Yh; } }; - const_cast(thread_pool)->ParallelFor((int32_t)total_channels, work_object); + const_cast(thread_pool)->ParallelFor((int32_t)total_channels, work_object); break; } @@ -116,7 +116,7 @@ class MaxpoolWithMask : public OpKernel, public PoolBase { } } }; - const_cast(thread_pool)->ParallelFor((int32_t)total_channels, work_object); + const_cast(thread_pool)->ParallelFor((int32_t)total_channels, work_object); break; } @@ -162,7 +162,7 @@ class MaxpoolWithMask : public OpKernel, public PoolBase { } } }; - const_cast(thread_pool)->ParallelFor((int32_t)total_channels, work_object); + const_cast(thread_pool)->ParallelFor((int32_t)total_channels, work_object); break; } diff --git a/onnxruntime/core/mlas/inc/mlas.h b/onnxruntime/core/mlas/inc/mlas.h index 3d656fecc7..849c258c6b 100644 --- a/onnxruntime/core/mlas/inc/mlas.h +++ b/onnxruntime/core/mlas/inc/mlas.h @@ -44,10 +44,19 @@ typedef enum { CblasLeft=141, CblasRight=142} CBLAS_SIDE; #endif // -// External threadpool definition +// Forward declare the thread pool implementation class. // -#include "core/platform/threadpool.h" -using namespace onnxruntime::concurrency; +// N.B. Avoid including onnxruntime headers here to keep the dependencies for +// standalone MLAS test executables smaller. +// + +namespace onnxruntime { + namespace concurrency { + class ThreadPool; + }; +}; + +using MLAS_THREADPOOL = onnxruntime::concurrency::ThreadPool; // // Activation routines. @@ -98,7 +107,7 @@ MlasSgemm( float beta, float* C, size_t ldc, - ThreadPool* ExternalThreadPool + MLAS_THREADPOOL* ThreadPool ); // @@ -127,8 +136,8 @@ struct MLAS_CONV_PARAMETERS { size_t InputSize; size_t OutputSize; size_t K; - size_t ThreadCount; MLAS_CONV_ALGORITHM Algorithm; + int32_t ThreadCount; union { struct { CBLAS_TRANSPOSE TransB; @@ -157,7 +166,7 @@ MlasConvPrepare( size_t FilterCount, const MLAS_ACTIVATION* Activation, size_t* WorkingBufferSize, - int32_t ThreadPoolLimit + MLAS_THREADPOOL* ThreadPool ); void @@ -169,7 +178,7 @@ MlasConv( const float* Bias, float* WorkingBuffer, float* Output, - ThreadPool *ExternalThreadPool + MLAS_THREADPOOL* ThreadPool ); // @@ -194,7 +203,7 @@ MlasPool( const int64_t* OutputShape, const float* Input, float* Output, - ThreadPool *ExternalThreadPool + MLAS_THREADPOOL* ThreadPool ); // diff --git a/onnxruntime/core/mlas/lib/convolve.cpp b/onnxruntime/core/mlas/lib/convolve.cpp index 7e039f3ad9..9caf116259 100644 --- a/onnxruntime/core/mlas/lib/convolve.cpp +++ b/onnxruntime/core/mlas/lib/convolve.cpp @@ -745,7 +745,7 @@ MlasConvTryMultithread( const float* Bias, float* WorkingBuffer, float* Output, - ThreadPool* ExternalThreadPool + MLAS_THREADPOOL* ThreadPool ) /*++ @@ -770,6 +770,9 @@ Arguments: Output - Supplies the output tensor. + ThreadPool - Supplies the thread pool object to use, else nullptr if the + base library threading support should be used. + Return Value: Returns true if the operation was completed across multiple threads, else @@ -777,13 +780,6 @@ Return Value: --*/ { - -#if !defined(MLAS_HAS_THREADING_SUPPORT) - if (ExternalThreadPool == nullptr) { - return false; - } -#endif - MLAS_CONV_WORK_BLOCK WorkBlock; const size_t OutputSize = Parameters->OutputSize; @@ -825,7 +821,7 @@ Return Value: Index++; } - MlasExecuteThreaded(MlasConvOperationThreaded, &WorkBlock, Index, ExternalThreadPool); + MlasExecuteThreaded(MlasConvOperationThreaded, &WorkBlock, Index, ThreadPool); return true; } @@ -839,7 +835,7 @@ MlasConv( const float* Bias, float* WorkingBuffer, float* Output, - ThreadPool* ExternalThreadPool + MLAS_THREADPOOL* ThreadPool ) /*++ @@ -863,6 +859,9 @@ Arguments: Output - Supplies the output tensor. + ThreadPool - Supplies the thread pool object to use, else nullptr if the + base library threading support should be used. + Return Value: None. @@ -890,7 +889,7 @@ Return Value: const size_t BatchGroupCount = BatchCount * GroupCount; - int32_t TargetThreadCount = (int32_t)Parameters->ThreadCount; + int32_t TargetThreadCount = MlasGetMaximumThreadCount(ThreadPool); if (size_t(TargetThreadCount) >= BatchGroupCount) { TargetThreadCount = int32_t(BatchGroupCount); @@ -906,15 +905,9 @@ Return Value: WorkBlock.Output = Output; WorkBlock.TargetThreadCount = TargetThreadCount; -#if defined(MLAS_HAS_THREADING_SUPPORT) - MlasExecuteThreaded(MlasConvGemmDirectThreaded, &WorkBlock, TargetThreadCount, ExternalThreadPool); + MlasExecuteThreaded(MlasConvGemmDirectThreaded, &WorkBlock, TargetThreadCount, ThreadPool); + return; -#else - if (ExternalThreadPool != nullptr) { - MlasExecuteThreaded(MlasConvGemmDirectThreaded, &WorkBlock, TargetThreadCount, ExternalThreadPool); - return; - } -#endif } // @@ -942,7 +935,7 @@ Return Value: MlasSgemm(CblasNoTrans, Parameters->u.GemmDirect.TransB, FilterCount, OutputSize, K, 1.0f, filter, K, Input, Parameters->u.GemmDirect.ldb, 0.0f, - Output, OutputSize, ExternalThreadPool); + Output, OutputSize, ThreadPool); // // Apply the activation with optional bias. @@ -968,7 +961,7 @@ Return Value: } MlasSgemm(CblasNoTrans, CblasNoTrans, FilterCount, OutputSize, K, 1.0f, filter, - K, WorkingBuffer, OutputSize, 0.0f, Output, OutputSize, ExternalThreadPool); + K, WorkingBuffer, OutputSize, 0.0f, Output, OutputSize, ThreadPool); // // Apply the activation with optional bias. @@ -988,7 +981,7 @@ Return Value: // if (!MlasConvTryMultithread(Parameters, Input, filter, bias, WorkingBuffer, - Output, ExternalThreadPool)) { + Output, ThreadPool)) { MlasConvOperation(Parameters, Input, filter, bias, WorkingBuffer, Output, 0, OutputSize); } @@ -1029,7 +1022,7 @@ MlasConvPrepare( size_t FilterCount, const MLAS_ACTIVATION* Activation, size_t* WorkingBufferSize, - int32_t ThreadPoolLimit + MLAS_THREADPOOL* ThreadPool ) /*++ @@ -1073,6 +1066,9 @@ Arguments: WorkingBufferSize - Receives the number of elements to allocate for the working buffer for intermediate results. + ThreadPool - Supplies the thread pool object to use, else nullptr if the + base library threading support should be used. + Return Value: None. @@ -1127,17 +1123,12 @@ Return Value: *WorkingBufferSize = 0; - // Take the thread count either limited by the available pool or the platform limit - int32_t MaximumThreadCount = (ThreadPoolLimit > 0) ? ThreadPoolLimit : MlasPlatform.GetMaximumThreadCount(); - if (AllStridesAreOne && AllPaddingIsZero) { // // Detect a pointwise convolution. // - Parameters->ThreadCount = MaximumThreadCount; - if (K == InputChannels) { Parameters->Algorithm = MlasConvAlgorithmGemmDirect; @@ -1206,6 +1197,8 @@ Return Value: TargetThreadCount = MLAS_MAXIMUM_THREAD_COUNT; } + int32_t MaximumThreadCount = MlasGetMaximumThreadCount(ThreadPool); + if (TargetThreadCount >= MaximumThreadCount) { TargetThreadCount = MaximumThreadCount; } diff --git a/onnxruntime/core/mlas/lib/mlasi.h b/onnxruntime/core/mlas/lib/mlasi.h index 940f779c03..a4d7306f1a 100644 --- a/onnxruntime/core/mlas/lib/mlasi.h +++ b/onnxruntime/core/mlas/lib/mlasi.h @@ -76,13 +76,12 @@ Abstract: // Select the threading model. // +#if !defined(MLAS_NO_ONNXRUNTIME_THREADPOOL) +#include "core/platform/threadpool.h" +#endif + #if defined(_OPENMP) #include -#define MLAS_USE_OPENMP -#define MLAS_HAS_THREADING_SUPPORT -#elif defined(_WIN32) -#define MLAS_USE_WIN32_THREADPOOL -#define MLAS_HAS_THREADING_SUPPORT #endif // @@ -218,7 +217,7 @@ extern "C" { // that workload. See EvaluateThreadingPerformance() in the unit test. // -#if defined(MLAS_USE_OPENMP) +#if defined(_OPENMP) #define MLAS_SGEMM_THREAD_COMPLEXITY (64 * 1024) #else #if defined(MLAS_TARGET_AMD64) @@ -270,23 +269,6 @@ struct MLAS_PLATFORM { PMLAS_TANH_KERNEL_ROUTINE TanhKernelRoutine; #endif -#if defined(MLAS_USE_WIN32_THREADPOOL) - int32_t MaximumThreadCount; -#endif - - int32_t - GetMaximumThreadCount( - void - ) - { -#if defined(MLAS_USE_OPENMP) - return (omp_get_num_threads() == 1) ? omp_get_max_threads() : 1; -#elif defined(MLAS_USE_WIN32_THREADPOOL) - return MaximumThreadCount; -#else - return 1; -#endif - } }; extern MLAS_PLATFORM MlasPlatform; @@ -309,9 +291,30 @@ MlasExecuteThreaded( PMLAS_THREADED_ROUTINE ThreadedRoutine, void* Context, int32_t Iterations, - ThreadPool *ExternalThreadPool + MLAS_THREADPOOL* ThreadPool ); +inline +int32_t +MlasGetMaximumThreadCount( + MLAS_THREADPOOL* ThreadPool + ) +{ +#ifdef MLAS_NO_ONNXRUNTIME_THREADPOOL + MLAS_UNREFERENCED_PARAMETER(ThreadPool); +#else + if (ThreadPool != nullptr) { + return ThreadPool->NumThreads(); + } +#endif + +#ifdef _OPENMP + return (omp_get_num_threads() == 1) ? omp_get_max_threads() : 1; +#else + return 1; +#endif +} + // // Define the missing ARM64 NEON intrinsic macros from arm64_neon.h that enable // cross-compiler support. diff --git a/onnxruntime/core/mlas/lib/platform.cpp b/onnxruntime/core/mlas/lib/platform.cpp index 88c3dd4579..ac290751f7 100644 --- a/onnxruntime/core/mlas/lib/platform.cpp +++ b/onnxruntime/core/mlas/lib/platform.cpp @@ -162,22 +162,4 @@ Return Value: #endif -#if defined(MLAS_USE_WIN32_THREADPOOL) - - // - // Retrieve the number of processors in the system. - // - - SYSTEM_INFO SystemInfo; - - GetSystemInfo(&SystemInfo); - - if (SystemInfo.dwNumberOfProcessors <= MLAS_MAXIMUM_THREAD_COUNT) { - this->MaximumThreadCount = int32_t(SystemInfo.dwNumberOfProcessors); - } else { - this->MaximumThreadCount = MLAS_MAXIMUM_THREAD_COUNT; - } - -#endif - } diff --git a/onnxruntime/core/mlas/lib/pooling.cpp b/onnxruntime/core/mlas/lib/pooling.cpp index fd41318887..904d6655a8 100644 --- a/onnxruntime/core/mlas/lib/pooling.cpp +++ b/onnxruntime/core/mlas/lib/pooling.cpp @@ -1183,7 +1183,7 @@ MlasPool( const int64_t* OutputShape, const float* Input, float* Output, - ThreadPool *ExternalThreadPool + MLAS_THREADPOOL* ThreadPool ) /*++ @@ -1212,6 +1212,9 @@ Arguments: Output - Supplies the output tensor. + ThreadPool - Supplies the thread pool object to use, else nullptr if the + base library threading support should be used. + Return Value: None. @@ -1316,26 +1319,29 @@ Return Value: } } +#ifdef MLAS_NO_ONNXRUNTIME_THREADPOOL + MLAS_UNREFERENCED_PARAMETER(ThreadPool); +#else // // Use an external thread pool if one is provided. // TODO: change to use MlasExecuteThreaded - if (!(ExternalThreadPool == nullptr)) { - std::function WorkObject = [&](int64_t c) { PoolKernelRoutine(&WorkBlock, 1, Input + c * InputSize, Output + c * OutputSize); }; - ExternalThreadPool->ParallelFor((int32_t)TotalChannelCount, WorkObject); - - return; + if (!(ThreadPool == nullptr)) { + std::function WorkObject = [&](int64_t c) { PoolKernelRoutine(&WorkBlock, 1, Input + c * InputSize, Output + c * OutputSize); }; + ThreadPool->ParallelFor((int32_t)TotalChannelCount, WorkObject); + return; } +#endif // // Execute the pooling kernel routine. // -#if defined(MLAS_USE_OPENMP) +#if defined(_OPENMP) #pragma omp parallel for for (int64_t c = 0; c < int64_t(TotalChannelCount); c++) { - PoolKernelRoutine(&WorkBlock, 1, Input + c * InputSize, Output + c * OutputSize); + PoolKernelRoutine(&WorkBlock, 1, Input + c * InputSize, Output + c * OutputSize); } #else diff --git a/onnxruntime/core/mlas/lib/sgemm.cpp b/onnxruntime/core/mlas/lib/sgemm.cpp index 39c0d7634a..f436f17e6c 100644 --- a/onnxruntime/core/mlas/lib/sgemm.cpp +++ b/onnxruntime/core/mlas/lib/sgemm.cpp @@ -1081,7 +1081,7 @@ MlasSgemmTryMultithread( float beta, float* C, size_t ldc, - ThreadPool *ExternalThreadPool + MLAS_THREADPOOL* ThreadPool ) /*++ @@ -1119,6 +1119,9 @@ Arguments: ldc - Supplies the first dimension of matrix C. + ThreadPool - Supplies the thread pool object to use, else nullptr if the + base library threading support should be used. + Return Value: Returns true if the operation was completed across multiple threads, else @@ -1126,9 +1129,6 @@ Return Value: --*/ { - -#if defined(MLAS_HAS_THREADING_SUPPORT) - MLAS_SGEMM_WORK_BLOCK WorkBlock; int32_t TargetThreadCount; @@ -1145,7 +1145,7 @@ Return Value: TargetThreadCount = MLAS_MAXIMUM_THREAD_COUNT; } - int32_t MaximumThreadCount = MlasPlatform.GetMaximumThreadCount(); + int32_t MaximumThreadCount = MlasGetMaximumThreadCount(ThreadPool); if (TargetThreadCount >= MaximumThreadCount) { TargetThreadCount = MaximumThreadCount; @@ -1232,35 +1232,9 @@ Return Value: } } - MlasExecuteThreaded(MlasSgemmOperationThreaded, &WorkBlock, Index, ExternalThreadPool); + MlasExecuteThreaded(MlasSgemmOperationThreaded, &WorkBlock, Index, ThreadPool); return true; - -#else - - // - // No threading implementation is available. - // - - MLAS_UNREFERENCED_PARAMETER(TransA); - MLAS_UNREFERENCED_PARAMETER(TransB); - MLAS_UNREFERENCED_PARAMETER(M); - MLAS_UNREFERENCED_PARAMETER(N); - MLAS_UNREFERENCED_PARAMETER(K); - MLAS_UNREFERENCED_PARAMETER(alpha); - MLAS_UNREFERENCED_PARAMETER(A); - MLAS_UNREFERENCED_PARAMETER(lda); - MLAS_UNREFERENCED_PARAMETER(B); - MLAS_UNREFERENCED_PARAMETER(ldb); - MLAS_UNREFERENCED_PARAMETER(beta); - MLAS_UNREFERENCED_PARAMETER(C); - MLAS_UNREFERENCED_PARAMETER(ldc); - MLAS_UNREFERENCED_PARAMETER(ExternalThreadPool); - - return false; - -#endif - } void @@ -1279,7 +1253,7 @@ MlasSgemm( float beta, float* C, size_t ldc, - ThreadPool *ExternalThreadPool + MLAS_THREADPOOL* ThreadPool ) /*++ @@ -1317,6 +1291,9 @@ Arguments: ldc - Supplies the first dimension of matrix C. + ThreadPool - Supplies the thread pool object to use, else nullptr if the + base library threading support should be used. + Return Value: None. @@ -1328,7 +1305,7 @@ Return Value: // single thread based on the GEMM parameters and system configuration. // - if (!MlasSgemmTryMultithread(TransA, TransB, M, N, K, alpha, A, lda, B, ldb, beta, C, ldc, ExternalThreadPool)) { + if (!MlasSgemmTryMultithread(TransA, TransB, M, N, K, alpha, A, lda, B, ldb, beta, C, ldc, ThreadPool)) { MlasSgemmOperation(TransA, TransB, M, N, K, alpha, A, lda, B, ldb, beta, C, ldc); } } diff --git a/onnxruntime/core/mlas/lib/threading.cpp b/onnxruntime/core/mlas/lib/threading.cpp index 251c6a77a6..4b0e9f765f 100644 --- a/onnxruntime/core/mlas/lib/threading.cpp +++ b/onnxruntime/core/mlas/lib/threading.cpp @@ -16,65 +16,12 @@ Abstract: #include "mlasi.h" -#if defined(MLAS_USE_WIN32_THREADPOOL) - -// -// Define the parameters to execute threaded work using the Windows thread pool -// library. -// - -struct MLAS_THREADED_WORK_BLOCK { - volatile LONG Counter; - PMLAS_THREADED_ROUTINE ThreadedRoutine; - void* Context; -}; - -void -CALLBACK -MlasThreadedWorkCallback( - PTP_CALLBACK_INSTANCE Instance, - void* Context, - PTP_WORK WorkObject - ) -/*++ - -Routine Description: - - This routine is invoked from a worker thread to execute one iteration of a - batch of threaded work. - -Arguments: - - Instance - Supplies the callback instance object. - - Context - Supplies the pointer to the parameters for the operation. - - WorkObject - Supplies the threadpool work object. - -Return Value: - - None. - ---*/ -{ - MLAS_UNREFERENCED_PARAMETER(Instance); - MLAS_UNREFERENCED_PARAMETER(WorkObject); - - MLAS_THREADED_WORK_BLOCK* WorkBlock = (MLAS_THREADED_WORK_BLOCK*)Context; - - LONG Index = InterlockedIncrement(&WorkBlock->Counter) - 1; - - WorkBlock->ThreadedRoutine(WorkBlock->Context, Index); -} - -#endif - void MlasExecuteThreaded( MLAS_THREADED_ROUTINE ThreadedRoutine, void* Context, int32_t Iterations, - ThreadPool *ExternalThreadPool + MLAS_THREADPOOL* ThreadPool ) { // @@ -86,63 +33,24 @@ MlasExecuteThreaded( return; } +#ifdef MLAS_NO_ONNXRUNTIME_THREADPOOL + MLAS_UNREFERENCED_PARAMETER(ThreadPool); +#else // - // Use an external thread pool if one is provided. + // Schedule the threaded iterations using the thread pool object. // - if (!(ExternalThreadPool == nullptr)) { - std::function WorkObject = [&](int32_t tid) { ThreadedRoutine(Context, tid); }; - ExternalThreadPool->ParallelFor(Iterations, WorkObject); - - return; - } - -#if defined(MLAS_USE_WIN32_THREADPOOL) - - // - // Schedule the threaded iterations using a work object. - // - - MLAS_THREADED_WORK_BLOCK WorkBlock; - - PTP_WORK WorkObject = CreateThreadpoolWork(MlasThreadedWorkCallback, &WorkBlock, nullptr); - - if (WorkObject != nullptr) { - - WorkBlock.Counter = 0; - WorkBlock.ThreadedRoutine = ThreadedRoutine; - WorkBlock.Context = Context; - - for (int32_t tid = 1; tid < Iterations; tid++) { - SubmitThreadpoolWork(WorkObject); - } - - // - // Execute the remaining iteration on this thread. - // - - ThreadedRoutine(Context, Iterations - 1); - - // - // Wait for the work object callbacks to complete. - // - - WaitForThreadpoolWorkCallbacks(WorkObject, FALSE); - CloseThreadpoolWork(WorkObject); - + if (ThreadPool != nullptr) { + ThreadPool->ParallelFor(Iterations, [&](int32_t tid) { ThreadedRoutine(Context, tid); }); return; } - - // - // Fallback to a serialized implementation. - // - #endif // // Execute the routine for the specified number of iterations. // -#if defined(_OPENMP) + +#ifdef _OPENMP #pragma omp parallel for #endif for (int32_t tid = 0; tid < Iterations; tid++) { diff --git a/onnxruntime/core/providers/cpu/nn/conv.cc b/onnxruntime/core/providers/cpu/nn/conv.cc index 8b009ef7f7..8df8b00563 100644 --- a/onnxruntime/core/providers/cpu/nn/conv.cc +++ b/onnxruntime/core/providers/cpu/nn/conv.cc @@ -87,7 +87,7 @@ Status Conv::Compute(OpKernelContext* context) const { static_cast(M / group_), &Activation, &WorkingBufferSize, - thread_pool->NumThreads()); + const_cast(thread_pool)); auto working_data = WorkingBufferSize > 0 ? alloc->Alloc(sizeof(float) * WorkingBufferSize) : nullptr; BufferUniquePtr working_buffer(working_data, BufferDeleter(alloc)); @@ -98,7 +98,7 @@ Status Conv::Compute(OpKernelContext* context) const { B != nullptr ? B->template Data() : nullptr, static_cast(working_buffer.get()), Ydata, - const_cast(thread_pool)); + const_cast(thread_pool)); } else { const int64_t input_image_size = input_shape.Size(); const int64_t output_image_size = output_shape.Size(); diff --git a/onnxruntime/core/providers/cpu/nn/pool.cc b/onnxruntime/core/providers/cpu/nn/pool.cc index 8ad6b6b6e6..24958c976d 100644 --- a/onnxruntime/core/providers/cpu/nn/pool.cc +++ b/onnxruntime/core/providers/cpu/nn/pool.cc @@ -69,7 +69,7 @@ Status Pool::Compute(OpKernelContext* context) const { y_d[ph] = Yh; } }; - const_cast(thread_pool)->ParallelFor((int32_t)total_channels, work_object); + const_cast(thread_pool)->ParallelFor((int32_t)total_channels, work_object); break; } @@ -108,7 +108,7 @@ Status Pool::Compute(OpKernelContext* context) const { } } }; - const_cast(thread_pool)->ParallelFor((int32_t)total_channels, work_object); + const_cast(thread_pool)->ParallelFor((int32_t)total_channels, work_object); break; } @@ -155,7 +155,7 @@ Status Pool::Compute(OpKernelContext* context) const { } } }; - const_cast(thread_pool)->ParallelFor((int32_t)total_channels, work_object); + const_cast(thread_pool)->ParallelFor((int32_t)total_channels, work_object); break; } @@ -199,7 +199,7 @@ Status PoolBase::Compute(OpKernelContext* context, MLAS_POOLING_KIND kind) const output_dims.data(), X->template Data(), Y->template MutableData(), - const_cast(thread_pool)); + const_cast(thread_pool)); return Status::OK(); } @@ -276,7 +276,7 @@ Status Pool>::Compute(OpKernelContext* context) co if (i_d != nullptr) i_d[ph] = c * x_step + h_index; } }; - const_cast(thread_pool)->ParallelFor((int32_t)total_channels, work_object); + const_cast(thread_pool)->ParallelFor((int32_t)total_channels, work_object); break; } @@ -320,7 +320,7 @@ Status Pool>::Compute(OpKernelContext* context) co } } }; - const_cast(thread_pool)->ParallelFor((int32_t)total_channels, work_object); + const_cast(thread_pool)->ParallelFor((int32_t)total_channels, work_object); break; } @@ -373,7 +373,7 @@ Status Pool>::Compute(OpKernelContext* context) co } } }; - const_cast(thread_pool)->ParallelFor((int32_t)total_channels, work_object); + const_cast(thread_pool)->ParallelFor((int32_t)total_channels, work_object); break; } diff --git a/onnxruntime/test/mlas/unittest.cpp b/onnxruntime/test/mlas/unittest.cpp index 05b46b3766..682a8fea06 100644 --- a/onnxruntime/test/mlas/unittest.cpp +++ b/onnxruntime/test/mlas/unittest.cpp @@ -517,7 +517,7 @@ TrialConv2D( FilterCount, &Activation, &WorkingBufferSize, - 0); + nullptr); size_t OutputHeight = size_t(OutputHeight64); size_t OutputWidth = size_t(OutputWidth64);