diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 91f6786a30..6d5a0576b4 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -62,7 +62,6 @@ option(onnxruntime_USE_EIGEN_FOR_BLAS "Use eign for blas" ON) option(onnxruntime_USE_NNAPI_BUILTIN "Build with builtin NNAPI lib for Android NNAPI support" OFF) option(onnxruntime_USE_RKNPU "Build with RKNPU support" OFF) option(onnxruntime_USE_DNNL "Build with DNNL support" OFF) -option(onnxruntime_USE_MKLML "Build the default cpu provider with MKL-ML binary dependency" OFF) option(onnxruntime_USE_FEATURIZERS "Build ML Featurizers support" OFF) option(onnxruntime_USE_NGRAPH "Build with nGraph support" OFF) option(onnxruntime_USE_OPENBLAS "Use openblas" OFF) @@ -198,12 +197,7 @@ if(onnxruntime_USE_OPENMP) if (OPENMP_FOUND) set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") - include_directories(${OpenMP_CXX_INCLUDE_DIR}) - # MKLML and NGraph depend on their own OpenMP library that may be different with the compiler's. - # Disable the options to build mklml/NGraph and OpenMP together. - if((WIN32 OR APPLE) AND onnxruntime_USE_MKLML) - message(FATAL_ERROR "Please use only one of onnxruntime_USE_MKLML, onnxruntime_USE_OPENMP") - endif() + include_directories(${OpenMP_CXX_INCLUDE_DIR}) if(onnxruntime_USE_NGRAPH) message(FATAL_ERROR "Please use only one of onnxruntime_USE_NGRAPH, onnxruntime_USE_OPENMP") endif() @@ -696,8 +690,7 @@ if (onnxruntime_USE_ARMNN) list(APPEND onnxruntime_EXTERNAL_LIBRARIES armnn pthread arm_compute_core arm_compute arm_compute_graph) endif() -# MKLML -if (onnxruntime_USE_DNNL OR onnxruntime_USE_MKLML) +if (onnxruntime_USE_DNNL) include(dnnl) endif() @@ -713,15 +706,6 @@ if (onnxruntime_USE_TVM) if (onnxruntime_USE_OPENMP) set(USE_OPENMP "gnu") endif() - if (onnxruntime_USE_MKLML) - set(USE_OPENMP "intel") - # make sure MKLML in ORT is used by TVM - if (WIN32) - set(OMP_LIBRARY ${MKLML_LIB_DIR}/${IOMP5MD_IMPORT_LIB}) - else() - set(OMP_LIBRARY ${MKLML_LIB_DIR}/${IOMP5MD_SHARED_LIB}) - endif() - endif() add_subdirectory(${PROJECT_SOURCE_DIR}/external/tvm EXCLUDE_FROM_ALL) set_target_properties(tvm PROPERTIES FOLDER "External/tvm") @@ -729,13 +713,6 @@ if (onnxruntime_USE_TVM) set_target_properties(tvm_runtime PROPERTIES FOLDER "External/tvm") set_target_properties(nnvm_compiler PROPERTIES FOLDER "External/tvm") - if (onnxruntime_USE_MKLML) - add_dependencies(tvm project_mklml) - add_dependencies(tvm_topi project_mklml) - add_dependencies(tvm_runtime project_mklml) - add_dependencies(nnvm_compiler project_mklml) - endif() - set(TVM_INCLUDES ${PROJECT_SOURCE_DIR}/external/tvm/include ${PROJECT_SOURCE_DIR}/external/tvm/3rdparty/dmlc-core/include ${PROJECT_SOURCE_DIR}/external/tvm/3rdparty/dlpack/include @@ -784,22 +761,6 @@ endif() set_target_properties(onnx PROPERTIES FOLDER "External/ONNX") set_target_properties(onnx_proto PROPERTIES FOLDER "External/ONNX") -if (onnxruntime_USE_MKLML) - add_definitions(-DUSE_MKLML=1 -DUSE_MKLML_FOR_BLAS=1) - if (WIN32 OR APPLE) - list(APPEND onnxruntime_EXTERNAL_LIBRARIES mklml) - else() - if(onnxruntime_USE_OPENMP) - list(APPEND onnxruntime_EXTERNAL_LIBRARIES mklml_gnu) - else() - list(APPEND onnxruntime_EXTERNAL_LIBRARIES mklml_intel) - endif() - endif() - list(APPEND onnxruntime_EXTERNAL_DEPENDENCIES project_mklml) - include_directories(${MKLML_INCLUDE_DIR}) - link_directories(${MKLML_LIB_DIR}) -endif() - if (onnxruntime_USE_NGRAPH) if (NOT onnxruntime_USE_FULL_PROTOBUF) message(FATAL_ERROR "Please set onnxruntime_USE_FULL_PROTOBUF=ON for nGraph execution provider.") diff --git a/cmake/external/dnnl.cmake b/cmake/external/dnnl.cmake index 8975537b0d..e9a24fccc7 100644 --- a/cmake/external/dnnl.cmake +++ b/cmake/external/dnnl.cmake @@ -1,64 +1,25 @@ include (ExternalProject) set(DNNL_URL https://github.com/intel/mkl-dnn.git) -# If DNNL_TAG is updated, check if MKLML_VERSION and platform.cmake.patch need to be updated. +# If DNNL_TAG is updated, check if platform.cmake.patch need to be updated. set(DNNL_TAG v1.1.1) -set(MKLML_VERSION 2019.0.5.20190502) if(WIN32) set(MKLML_OS_VERSION_STR "win") set(MKLML_FILE_EXTENSION "zip") set(DNNL_SHARED_LIB dnnl.dll) - set(DNNL_IMPORT_LIB dnnl.lib) - if(onnxruntime_USE_MKLML) - # Windows-only updated MKLML binary which contains fix for thread cleanup hang. - set(MKLML_VERSION 2020.0.20190813) - set(MKLML_SHARED_LIB mklml.dll) - set(MKLML_IMPORT_LIB mklml.lib) - set(IOMP5MD_SHARED_LIB libiomp5md.dll) - set(IOMP5MD_IMPORT_LIB libiomp5md.lib) - endif() + set(DNNL_IMPORT_LIB dnnl.lib) else() set(MKLML_FILE_EXTENSION "tgz") if (APPLE) set(DNNL_SHARED_LIB libdnnl.1.dylib) - set(MKLML_OS_VERSION_STR "mac") - if(onnxruntime_USE_MKLML) - set(MKLML_SHARED_LIB libmklml.dylib) - set(IOMP5MD_SHARED_LIB libiomp5.dylib) - endif() + set(MKLML_OS_VERSION_STR "mac") else() set(DNNL_SHARED_LIB libdnnl.so.1) - set(MKLML_OS_VERSION_STR "lnx") - if(onnxruntime_USE_MKLML) - if(onnxruntime_USE_OPENMP) - set(MKLML_SHARED_LIB libmklml_gnu.so) - else() - set(MKLML_SHARED_LIB libmklml_intel.so) - set(IOMP5MD_SHARED_LIB libiomp5.so) - endif() - endif() + set(MKLML_OS_VERSION_STR "lnx") endif() endif() -if (onnxruntime_USE_MKLML) - set(MKLDNN_VERSION_SHORT v0.20) - set(MKLML_URL https://github.com/intel/mkl-dnn/releases/download/${MKLDNN_VERSION_SHORT}/mklml_${MKLML_OS_VERSION_STR}_${MKLML_VERSION}.${MKLML_FILE_EXTENSION}) - - ExternalProject_Add(project_mklml - PREFIX mklml - URL ${MKLML_URL} - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - UPDATE_COMMAND "" - INSTALL_COMMAND "" ) - - set(MKML_DIR ${CMAKE_CURRENT_BINARY_DIR}/mklml/src/project_mklml) - set(MKLML_INCLUDE_DIR "${MKML_DIR}/include") - set(MKLML_LIB_DIR "${MKML_DIR}/lib") - link_directories(${MKLML_LIB_DIR}) -endif() - if (onnxruntime_USE_DNNL) set(DNNL_SOURCE ${CMAKE_CURRENT_BINARY_DIR}/dnnl/src/dnnl/src) set(DNNL_INSTALL ${CMAKE_CURRENT_BINARY_DIR}/dnnl/install) @@ -88,7 +49,4 @@ if (onnxruntime_USE_DNNL) CMAKE_ARGS -DDNNL_BUILD_TESTS=OFF -DDNNL_BUILD_EXAMPLES=OFF -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX=${DNNL_INSTALL} ) link_directories(${DNNL_LIB_DIR}) - #if (onnxruntime_USE_MKLML) - # add_dependencies(project_dnnl project_mklml) - #endif() endif() diff --git a/cmake/onnxruntime_python.cmake b/cmake/onnxruntime_python.cmake index f238adb4c7..27ca6a0ae0 100644 --- a/cmake/onnxruntime_python.cmake +++ b/cmake/onnxruntime_python.cmake @@ -378,15 +378,6 @@ if (onnxruntime_USE_TVM) ) endif() -if (onnxruntime_USE_MKLML) - add_custom_command( - TARGET onnxruntime_pybind11_state POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy - ${MKLML_LIB_DIR}/${MKLML_SHARED_LIB} ${MKLML_LIB_DIR}/${IOMP5MD_SHARED_LIB} - $/onnxruntime/capi/ - ) -endif() - if (onnxruntime_USE_NUPHAR) file(GLOB onnxruntime_python_nuphar_python_srcs CONFIGURE_DEPENDS "${ONNXRUNTIME_ROOT}/core/providers/nuphar/scripts/*" diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index c7989d1143..5e63298a4a 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -645,14 +645,6 @@ if (onnxruntime_USE_DNNL) COMMAND ${CMAKE_COMMAND} -E copy ${DNNL_DLL_PATH} $ ) endif() -if (onnxruntime_USE_MKLML) - add_custom_command( - TARGET ${test_data_target} POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy - ${MKLML_LIB_DIR}/${MKLML_SHARED_LIB} ${MKLML_LIB_DIR}/${IOMP5MD_SHARED_LIB} - $ - ) -endif() if(WIN32) if (onnxruntime_USE_NGRAPH) add_custom_command( diff --git a/onnxruntime/contrib_ops/cpu/bert/attention.cc b/onnxruntime/contrib_ops/cpu/bert/attention.cc index 0ec32bc376..24a8cb18fc 100644 --- a/onnxruntime/contrib_ops/cpu/bert/attention.cc +++ b/onnxruntime/contrib_ops/cpu/bert/attention.cc @@ -21,9 +21,7 @@ class Attention : public OpKernel, public AttentionCPUBase { explicit Attention(const OpKernelInfo& info); Status Compute(OpKernelContext* context) const override; -#if !defined(USE_MKLML_FOR_BLAS) Status PrePack(const Tensor& tensor, int input_idx, bool& is_packed) override; -#endif private: BufferUniquePtr packed_weights_; @@ -178,7 +176,6 @@ template Attention::Attention(const OpKernelInfo& info) : OpKernel(info), AttentionCPUBase(info) { } -#if !defined(USE_MKLML_FOR_BLAS) template Status Attention::PrePack(const Tensor& weights, int input_idx, bool& is_packed) { @@ -225,8 +222,6 @@ Status Attention::PrePack(const Tensor& weights, int input_idx, bool& is_pack return Status::OK(); } -#endif - template Status Attention::Compute(OpKernelContext* context) const { const Tensor* input = context->Input(0); diff --git a/onnxruntime/contrib_ops/cpu/cdist.cc b/onnxruntime/contrib_ops/cpu/cdist.cc index f6768bc731..9747084a4a 100644 --- a/onnxruntime/contrib_ops/cpu/cdist.cc +++ b/onnxruntime/contrib_ops/cpu/cdist.cc @@ -56,8 +56,8 @@ static void CalculateSqeuclidean(const Tensor& a, const Tensor& b, Tensor& c, co // in Xij and Yjk are very similar, so subtracting can be problematic. // Due to that we calculate -2*sum_k(Xik*Yjk) using GEMM, add sum_k(Xik**2) next, and add sum_k(Yjk**2) last. -// use MLAS on 64-bit (no 32-bit dgemm), or MKL on 32-bit or 64-bit -#if defined(_M_AMD64) || defined(__x86_64__) || defined(USE_MKLML_FOR_BLAS) +// use MLAS on 64-bit (no 32-bit dgemm) +#if defined(_M_AMD64) || defined(__x86_64__) // Use GEMM of A and B^T with -2 as alpha to calculate -2*sum_k(Xik*Yjk) math::Gemm(CBLAS_TRANSPOSE::CblasNoTrans, CBLAS_TRANSPOSE::CblasTrans, m, n, k, diff --git a/onnxruntime/core/providers/cpu/math/matmul.cc b/onnxruntime/core/providers/cpu/math/matmul.cc index 3e58c402f8..508bdee3a6 100644 --- a/onnxruntime/core/providers/cpu/math/matmul.cc +++ b/onnxruntime/core/providers/cpu/math/matmul.cc @@ -125,7 +125,6 @@ Status MatMul::Compute(OpKernelContext* ctx) const { return Status::OK(); } -#if !defined(USE_MKLML_FOR_BLAS) Status MatMul::PrePack(const Tensor& tensor, int input_idx, bool& is_packed) { is_packed = false; @@ -162,7 +161,6 @@ Status MatMul::PrePack(const Tensor& tensor, int input_idx, bool& is_pack } return Status::OK(); } -#endif Status MatMul::Compute(OpKernelContext* ctx) const { concurrency::ThreadPool* thread_pool = ctx->GetOperatorThreadPool(); @@ -190,7 +188,6 @@ Status MatMul::Compute(OpKernelContext* ctx) const { // TODO: replace it with GemmBatch for performance, it's OK for now as GemmBatch unrolls as well size_t max_len = helper.OutputOffsets().size(); for (size_t i = 0; i < max_len; i++) { -#if !defined(USE_MKLML_FOR_BLAS) if (packed_b_) { MlasGemm( trans_a ? CblasTrans : CblasNoTrans, @@ -207,7 +204,6 @@ Status MatMul::Compute(OpKernelContext* ctx) const { thread_pool); continue; } -#endif math::Gemm( trans_a ? CblasTrans : CblasNoTrans, trans_b ? CblasTrans : CblasNoTrans, diff --git a/onnxruntime/core/providers/cpu/math/matmul.h b/onnxruntime/core/providers/cpu/math/matmul.h index 12e31749a1..9f75db3905 100644 --- a/onnxruntime/core/providers/cpu/math/matmul.h +++ b/onnxruntime/core/providers/cpu/math/matmul.h @@ -24,9 +24,7 @@ class MatMul final : public OpKernel { info.GetAttrOrDefault("alpha", &alpha_attr_, 1.0); } -#if !defined(USE_MKLML_FOR_BLAS) Status PrePack(const Tensor& tensor, int input_idx, bool& is_packed) override; -#endif Status Compute(OpKernelContext* context) const override; diff --git a/onnxruntime/core/providers/cpu/rnn/deep_cpu_lstm.cc b/onnxruntime/core/providers/cpu/rnn/deep_cpu_lstm.cc index 5df45aebb4..2a275a0cfd 100644 --- a/onnxruntime/core/providers/cpu/rnn/deep_cpu_lstm.cc +++ b/onnxruntime/core/providers/cpu/rnn/deep_cpu_lstm.cc @@ -327,7 +327,6 @@ Status DeepCpuLstmOp::TryPackWeights(const Tensor& weights, PackedWeights& packe return Status::OK(); } -#if !defined(USE_MKLML_FOR_BLAS) Status DeepCpuLstmOp::PrePack(const Tensor& tensor, int input_idx, bool& is_packed) { is_packed = false; @@ -341,7 +340,6 @@ Status DeepCpuLstmOp::PrePack(const Tensor& tensor, int input_idx, bool& is_pack return Status::OK(); } -#endif Status DeepCpuLstmOp::Compute(OpKernelContext* context) const { const Tensor& X = *context->Input(0); // inputs. [seq_length, batch_size, input_size] diff --git a/onnxruntime/core/providers/cpu/rnn/deep_cpu_lstm.h b/onnxruntime/core/providers/cpu/rnn/deep_cpu_lstm.h index 7c2f978b0c..df48aaddaa 100644 --- a/onnxruntime/core/providers/cpu/rnn/deep_cpu_lstm.h +++ b/onnxruntime/core/providers/cpu/rnn/deep_cpu_lstm.h @@ -50,9 +50,7 @@ class DeepCpuLstmOp final : public OpKernel { activation_func_betas); } -#if !defined(USE_MKLML_FOR_BLAS) Status PrePack(const Tensor& tensor, int input_idx, bool& is_packed) override; -#endif Status Compute(OpKernelContext* context) const override; ~DeepCpuLstmOp() override = default; diff --git a/onnxruntime/core/providers/nuphar/common/nuphar_settings.cc b/onnxruntime/core/providers/nuphar/common/nuphar_settings.cc index 328b5b57ce..672875bddb 100644 --- a/onnxruntime/core/providers/nuphar/common/nuphar_settings.cc +++ b/onnxruntime/core/providers/nuphar/common/nuphar_settings.cc @@ -58,7 +58,7 @@ void SetDefaultOptions(std::map& options) { options.insert(std::make_pair(cache_so_name_opt, cache_so_name_default)); std::string parallel_min_workloads_opt(kNupharParallelMinWorkloads); -#if defined(_OPENMP) || defined(USE_MKLML) +#if defined(_OPENMP) // a rough estimate of workloads based on static dimensions for each thread, when using parallel schedule // user may change it to 0 to turn it off, // or use OMP_NUM_THREADS to control TVM thread pool similar to control MKL diff --git a/onnxruntime/core/providers/nuphar/extern/igemv_mkl.cc b/onnxruntime/core/providers/nuphar/extern/igemv_mkl.cc deleted file mode 100644 index d96c8aca82..0000000000 --- a/onnxruntime/core/providers/nuphar/extern/igemv_mkl.cc +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "igemv_mkl.h" - -namespace onnxruntime { -#ifdef NUPHAR_USE_MKL -void MKLIntGemvS16S16S32R( - int16_t* matrixA, - int16_t* matrixB, - int M, - int N, - int K, - int32_t* output) { - MKL_INT32 co = 0; - cblas_gemm_s16s16s32(CBLAS_LAYOUT::CblasColMajor, CBLAS_TRANSPOSE::CblasTrans, CBLAS_TRANSPOSE::CblasNoTrans, CBLAS_OFFSET::CblasFixOffset, - M, N, K, - 1, matrixA, K, - 0, matrixB, K, 0, 0, output, M, &co); -} -void MKLIntGemvS8U8S32R( - int8_t* matrixA, - uint8_t* matrixB, - int M, - int N, - int K, - int32_t* output) { - MKL_INT32 co = 0; - cblas_gemm_s8u8s32(CBLAS_LAYOUT::CblasColMajor, CBLAS_TRANSPOSE::CblasTrans, CBLAS_TRANSPOSE::CblasNoTrans, CBLAS_OFFSET::CblasFixOffset, - M, N, K, - 1, matrixA, K, - 0, matrixB, K, 0, 0, output, M, &co); -} -#endif - -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/extern/igemv_mkl.h b/onnxruntime/core/providers/nuphar/extern/igemv_mkl.h deleted file mode 100644 index e8a390d994..0000000000 --- a/onnxruntime/core/providers/nuphar/extern/igemv_mkl.h +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once -#include - -#ifdef NUPHAR_USE_MKL -// Need to build with USE_MKLML -#include -#endif // NUPHAR_USE_MKL - -namespace onnxruntime { -#ifdef NUPHAR_USE_MKL -void MKLIntGemvS16S16S32R( - int16_t* matrixA, - int16_t* matrixB, - int M, - int N, - int K, - int32_t* output); - -void MKLIntGemvS8U8S32R( - int8_t* matrixA, - uint8_t* matrixB, - int M, - int N, - int K, - int32_t* output); -#endif -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/mti_x86/quantize/imatmul16_extern.cc b/onnxruntime/core/providers/nuphar/mti_x86/quantize/imatmul16_extern.cc index fcad4a1580..a107022f64 100644 --- a/onnxruntime/core/providers/nuphar/mti_x86/quantize/imatmul16_extern.cc +++ b/onnxruntime/core/providers/nuphar/mti_x86/quantize/imatmul16_extern.cc @@ -5,7 +5,6 @@ #include "core/common/common.h" #include "core/codegen/mti/mti_tvm_utils.h" -#include "core/providers/nuphar/extern/igemv_mkl.h" #include "core/providers/nuphar/extern/igemv_avx2.h" #include diff --git a/onnxruntime/core/providers/nuphar/mti_x86/quantize/imatmul_extern.cc b/onnxruntime/core/providers/nuphar/mti_x86/quantize/imatmul_extern.cc index 4896ab7690..67b638727d 100644 --- a/onnxruntime/core/providers/nuphar/mti_x86/quantize/imatmul_extern.cc +++ b/onnxruntime/core/providers/nuphar/mti_x86/quantize/imatmul_extern.cc @@ -5,7 +5,6 @@ #include "core/common/common.h" #include "core/codegen/mti/mti_tvm_utils.h" -#include "core/providers/nuphar/extern/igemv_mkl.h" #include "core/providers/nuphar/extern/igemv_avx2.h" #include diff --git a/onnxruntime/core/util/math.h b/onnxruntime/core/util/math.h index b5253a5946..16138e680f 100644 --- a/onnxruntime/core/util/math.h +++ b/onnxruntime/core/util/math.h @@ -20,16 +20,11 @@ // still keep it simple, so all platforms would be able to support it fairly // easily. -#ifdef USE_MKLML_FOR_BLAS -// when USE_MKLML is defined, use MKLML cblas for GEMM -#include "mkl_cblas.h" -#define CBLAS_ENUM_DEFINED_H -#else + // We include the cblas header here so that we can obtain the macros from cblas. extern "C" { #include "core/framework/cblas.h" } -#endif #include "core/common/common.h" #include "core/framework/tensor.h" diff --git a/onnxruntime/core/util/math_cpu.cc b/onnxruntime/core/util/math_cpu.cc index 42ccc24b3c..b9455d6340 100644 --- a/onnxruntime/core/util/math_cpu.cc +++ b/onnxruntime/core/util/math_cpu.cc @@ -59,8 +59,6 @@ EIGEN_MATMUL_FUNCTION(uint64_t) // will delegate the Caffe math functions that are BLAS-related to either the // CBLAS call or the Eigen implementation. //////////////////////////////////////////////////////////////////////////////// -// when USE_MKLML is defined, use cblas APIs for MKLML -#if !defined(USE_MKLML_FOR_BLAS) // Caffe2 gemm provides a simpler interface to the gemm functions, with the // limitation that the data has to be contiguous in memory. @@ -212,79 +210,6 @@ template void Gemv(const CBLAS_TRANSPOSE TransA, int M, int SPECIALIZED_AXPY(float) #undef SPECIALIZED_AXPY -#else - -template <> -void Gemm(const CBLAS_TRANSPOSE TransA, const CBLAS_TRANSPOSE TransB, const int64_t M, - const int64_t N, const int64_t K, float alpha, const float* A, const float* B, float beta, - float* C, ThreadPool* /*context*/) { - int lda = gsl::narrow_cast((TransA == CblasNoTrans) ? K : M); - int ldb = gsl::narrow_cast((TransB == CblasNoTrans) ? N : K); - cblas_sgemm(CblasRowMajor, TransA, TransB, - gsl::narrow_cast(M), - gsl::narrow_cast(N), - gsl::narrow_cast(K), - alpha, A, lda, B, ldb, - beta, C, gsl::narrow_cast(N)); -} - -template <> -void Gemm(const CBLAS_TRANSPOSE TransA, const CBLAS_TRANSPOSE TransB, const int64_t M, - const int64_t N, const int64_t K, double alpha, const double* A, const double* B, double beta, - double* C, ThreadPool* /*context*/) { - int lda = gsl::narrow_cast((TransA == CblasNoTrans) ? K : M); - int ldb = gsl::narrow_cast((TransB == CblasNoTrans) ? N : K); - cblas_dgemm(CblasRowMajor, TransA, TransB, - gsl::narrow_cast(M), - gsl::narrow_cast(N), - gsl::narrow_cast(K), - alpha, A, lda, B, ldb, - beta, C, gsl::narrow_cast(N)); -} - -template <> -void MatMul(int M, int N, int K, const float* A, const float* B, float* C, ThreadPool*) { - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, M, N, K, 1, A, K, B, N, 0, C, N); -} - -template <> -void MatMul(int M, int N, int K, const double* A, const double* B, double* C, ThreadPool*) { - cblas_dgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, M, N, K, 1, A, K, B, N, 0, C, N); -} - -template <> -void GemmEx(const CBLAS_TRANSPOSE TransA, const CBLAS_TRANSPOSE TransB, int M, int N, int K, - float alpha, const float* A, int lda, const float* B, int ldb, float beta, float* C, - int ldc, ThreadPool* /*context*/) { - cblas_sgemm(CblasRowMajor, TransA, TransB, M, N, K, alpha, A, lda, B, ldb, - beta, C, ldc); -} - -template <> -void Gemv(const CBLAS_TRANSPOSE TransA, int M, int N, float alpha, const float* A, const float* x, - float beta, float* y, CPUMathUtil* /*context*/) { - cblas_sgemv(CblasRowMajor, TransA, M, N, alpha, A, N, x, 1, beta, y, 1); -} - -template <> -void Gemv(const CBLAS_TRANSPOSE TransA, int M, int N, float alpha, const double* A, const double* x, - float beta, double* y, CPUMathUtil* /*context*/) { - cblas_dgemv(CblasRowMajor, TransA, M, N, alpha, A, N, x, 1, beta, y, 1); -} - -#define CAFFE2_SPECIALIZED_AXPY(T, prefix) \ - template <> \ - void Axpy(int N, const T alpha, const T* x, T* y, CPUMathUtil*) { \ - cblas_##prefix##axpy(N, alpha, x, 1, y, 1); \ - } \ - template <> \ - void Axpy(int N, const T* alpha, const T* x, T* y, CPUMathUtil*) { \ - cblas_##prefix##axpy(N, *alpha, x, 1, y, 1); \ - } -CAFFE2_SPECIALIZED_AXPY(float, s) -#undef CAFFE2_SPECIALIZED_AXPY - -#endif #define DELEGATE_SIMPLE_UNARY_FUNCTION(T, Funcname, expr) \ template <> \ diff --git a/onnxruntime/python/onnxruntime_pybind_state.cc b/onnxruntime/python/onnxruntime_pybind_state.cc index 1609aee0a4..793277d949 100644 --- a/onnxruntime/python/onnxruntime_pybind_state.cc +++ b/onnxruntime/python/onnxruntime_pybind_state.cc @@ -50,12 +50,6 @@ struct OrtStatus { #define BACKEND_DNNL "" #endif -#if USE_MKLML -#define BACKEND_MKLML "-MKL-ML" -#else -#define BACKEND_MKLML "" -#endif - #if USE_NGRAPH #define BACKEND_NGRAPH "-NGRAPH" #include "core/providers/ngraph/ngraph_execution_provider.h" @@ -129,7 +123,7 @@ struct OrtStatus { #define BACKEND_DML "" #endif -#define BACKEND_DEVICE BACKEND_PROC BACKEND_DNNL BACKEND_MKLML BACKEND_NGRAPH BACKEND_OPENVINO BACKEND_NUPHAR BACKEND_OPENBLAS BACKEND_MIGRAPHX BACKEND_ACL BACKEND_ARMNN BACKEND_DML +#define BACKEND_DEVICE BACKEND_PROC BACKEND_DNNL BACKEND_NGRAPH BACKEND_OPENVINO BACKEND_NUPHAR BACKEND_OPENBLAS BACKEND_MIGRAPHX BACKEND_ACL BACKEND_ARMNN BACKEND_DML #include "core/session/onnxruntime_cxx_api.h" #include "core/providers/providers.h" #include "core/providers/cpu/cpu_execution_provider.h" diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index f70045418b..c69e6c5477 100755 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -260,8 +260,6 @@ def parse_arguments(): "--use_openblas", action='store_true', help="Build with OpenBLAS.") parser.add_argument( "--use_dnnl", action='store_true', help="Build with DNNL.") - parser.add_argument( - "--use_mklml", action='store_true', help="Build with MKLML.") parser.add_argument( "--use_featurizers", action='store_true', help="Build with ML Featurizer support.") @@ -619,13 +617,12 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home "OFF" if args.use_openblas else "ON"), "-Donnxruntime_USE_OPENBLAS=" + ("ON" if args.use_openblas else "OFF"), "-Donnxruntime_USE_DNNL=" + ("ON" if args.use_dnnl else "OFF"), - "-Donnxruntime_USE_MKLML=" + ("ON" if args.use_mklml else "OFF"), "-Donnxruntime_USE_NGRAPH=" + ("ON" if args.use_ngraph else "OFF"), "-Donnxruntime_USE_NNAPI_BUILTIN=" + ("ON" if args.use_nnapi else "OFF"), "-Donnxruntime_USE_RKNPU=" + ("ON" if args.use_rknpu else "OFF"), "-Donnxruntime_USE_OPENMP=" + ( "ON" if args.use_openmp and not ( - args.use_nnapi or (args.use_mklml and (is_macOS() or is_windows())) or args.use_ngraph or + args.use_nnapi or args.use_ngraph or args.android or (args.ios and is_macOS()) or args.use_rknpu) else "OFF"), @@ -1308,8 +1305,6 @@ def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs): build_dir, config, "external", "tvm", config)) if args.use_tensorrt: dll_path_list.append(os.path.join(args.tensorrt_home, 'lib')) - if args.use_mklml: - dll_path_list.append(os.path.join(build_dir, config, "mklml", "src", "project_mklml", "lib")) if not is_windows(): # A workaround for making libonnxruntime_providers_shared.so loadable. dll_path_list.append(os.path.join(build_dir, config)) @@ -1509,7 +1504,7 @@ def derive_linux_build_property(): return "/p:IsLinuxBuild=\"true\"" -def build_nuget_package(source_dir, build_dir, configs, use_cuda, use_openvino, use_tensorrt, use_dnnl, use_mklml): +def build_nuget_package(source_dir, build_dir, configs, use_cuda, use_openvino, use_tensorrt, use_dnnl): if not (is_windows() or is_linux()): raise BuildError( 'Currently csharp builds and nuget package creation is only supportted ' @@ -1532,8 +1527,6 @@ def build_nuget_package(source_dir, build_dir, configs, use_cuda, use_openvino, package_name = "/p:OrtPackageId=\"Microsoft.ML.OnnxRuntime.DNNL\"" elif use_cuda: package_name = "/p:OrtPackageId=\"Microsoft.ML.OnnxRuntime.Gpu\"" - elif use_mklml: - package_name = "/p:OrtPackageId=\"Microsoft.ML.OnnxRuntime.MKLML\"" else: pass @@ -1925,8 +1918,7 @@ def main(): args.use_cuda, args.use_openvino, args.use_tensorrt, - args.use_dnnl, - args.use_mklml + args.use_dnnl ) if args.test and args.build_nuget: diff --git a/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml index d39ca75337..44adb70eef 100644 --- a/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml @@ -39,7 +39,7 @@ jobs: --use_openmp \ --enable_onnx_tests \ --enable_symbolic_shape_infer_tests \ - --use_mklml --enable_pybind --build_java --build_nodejs + --enable_pybind --build_java --build_nodejs workingDirectory: $(Build.SourcesDirectory) - task: Docker@2 displayName: logout diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-cuda-11-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-cuda-11-pipeline.yml index eee57e90f8..7c00d25c8b 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-cuda-11-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-cuda-11-pipeline.yml @@ -38,7 +38,7 @@ jobs: --build_wheel \ --use_openmp \ --enable_onnx_tests --use_cuda --cuda_version=11.0 --cuda_home=/usr/local/cuda-11.0 --cudnn_home=/usr/local/cuda-11.0 \ - --use_mklml --enable_pybind --build_java --build_nodejs \ + --enable_pybind --build_java --build_nodejs \ --cmake_extra_defines PYTHON_INCLUDE_DIR=/opt/python/cp37-cp37m/include/python3.7m PYTHON_LIBRARY=/usr/lib64/librt.so CMAKE_CUDA_ARCHITECTURES=52 workingDirectory: $(Build.SourcesDirectory) - task: Docker@2 diff --git a/tools/ci_build/github/azure-pipelines/nuget/cpu-mklml-esrp-pipeline.yml b/tools/ci_build/github/azure-pipelines/nuget/cpu-mklml-esrp-pipeline.yml deleted file mode 100644 index 2257ac9399..0000000000 --- a/tools/ci_build/github/azure-pipelines/nuget/cpu-mklml-esrp-pipeline.yml +++ /dev/null @@ -1,16 +0,0 @@ -schedules: -- cron: "0 10 * * *" - displayName: Daily Build - branches: - include: - - master - always: true - -variables: - PackageName: 'Microsoft.ML.OnnxRuntime.MKLML' - -jobs: -- template: templates/cpu-mklml.yml - parameters: - AgentPool : 'Win-CPU-2019' - DoEsrp: 'true' diff --git a/tools/ci_build/github/azure-pipelines/nuget/cpu-mklml-pipeline.yml b/tools/ci_build/github/azure-pipelines/nuget/cpu-mklml-pipeline.yml deleted file mode 100644 index 482d4dd1fe..0000000000 --- a/tools/ci_build/github/azure-pipelines/nuget/cpu-mklml-pipeline.yml +++ /dev/null @@ -1,8 +0,0 @@ -variables: - PackageName: 'Microsoft.ML.OnnxRuntime.MKLML' - -jobs: -- template: templates/cpu-mklml.yml - parameters: - AgentPool : 'Win-CPU-2019' - DoEsrp: 'false' diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml deleted file mode 100644 index e152ebcb1b..0000000000 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml +++ /dev/null @@ -1,190 +0,0 @@ -parameters: - DoEsrp: 'false' - -jobs: -- template: ../../templates/win-ci-2019.yml - parameters: - AgentPool : 'Win-CPU-2019' - JobName: 'Windows_CI_Dev' - BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_mklml --use_winml --enable_wcos --build_shared_lib --enable_onnx_tests --cmake_generator "Visual Studio 16 2019"' - BuildArch: 'x64' - EnvSetupScript: 'setup_env.bat' - sln_platform: 'x64' - DoDebugBuild: 'false' - DoNugetPack : 'true' - DoCompliance: 'false' - DoEsrp: ${{ parameters.DoEsrp }} - OrtPackageId: 'Microsoft.ML.OnnxRuntime.MKLML' - NuPackScript: | - msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /t:CreatePackage /p:OrtPackageId=Microsoft.ML.OnnxRuntime.MKLML - copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) - copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) - mkdir $(Build.ArtifactStagingDirectory)\testdata - copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\custom_op_library.* $(Build.ArtifactStagingDirectory)\testdata - -- job: 'Linux_CI_Dev' - workspace: - clean: all - timeoutInMinutes: 120 - pool: $(AgentPoolLinux) - steps: - - template: ../../templates/set-version-number-variables-step.yml - - template: ../../templates/linux-set-variables-and-download.yml - - task: Docker@2 - displayName: login - inputs: - containerRegistry: onnxruntimeregistry - command: login - addPipelineData: false - - task: CmdLine@2 - inputs: - script: | - mkdir -p $HOME/.onnx - docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecentoscpubuild:ch9m /bin/bash -c "python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_openmp --enable_onnx_tests --use_mklml && cd /build/Release && make install DESTDIR=/build/linux-x64" - workingDirectory: $(Build.SourcesDirectory) - - task: Docker@2 - displayName: logout - inputs: - containerRegistry: onnxruntimeregistry - command: logout - addPipelineData: false - - script: | - set -e -x - mv $(Build.BinariesDirectory)/linux-x64/usr/local/lib64 $(Build.BinariesDirectory)/linux-x64/linux-x64 - cp $(Build.BinariesDirectory)/Release/mklml/src/project_mklml/lib/libmklml_gnu.so $(Build.BinariesDirectory)/linux-x64/linux-x64 - ldd $(Build.BinariesDirectory)/linux-x64/linux-x64/libonnxruntime.so - cd $(Build.BinariesDirectory)/linux-x64 - zip -r linux-x64.zip linux-x64 - cp $(Build.BinariesDirectory)/linux-x64/linux*.zip $(Build.ArtifactStagingDirectory) - mkdir $(Build.ArtifactStagingDirectory)/testdata - cp $(Build.BinariesDirectory)/Release/libcustom_op_library.so* $(Build.ArtifactStagingDirectory)/testdata - ls -al $(Build.ArtifactStagingDirectory) - displayName: 'Create Artifacts' - - task: PublishPipelineArtifact@0 - displayName: 'Publish Pipeline Artifact' - inputs: - artifactName: 'drop-linux' - targetPath: '$(Build.ArtifactStagingDirectory)' - - template: ../../templates/component-governance-component-detection-steps.yml - parameters : - condition : 'succeeded' - - template: ../../templates/clean-agent-build-directory-step.yml - -- template: ../../templates/mac-ci.yml - parameters: - AgentPool : $(AgentPoolMacOS) - JobName: 'MacOS_CI_Dev' - BuildCommand: 'python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_shared_lib --use_mklml --config RelWithDebInfo' - DoNugetPack : 'true' - NuPackScript: | - set -e -x - mkdir $(Build.BinariesDirectory)/osx-x64 - find $(Build.BinariesDirectory) - cp $(Build.BinariesDirectory)/RelWithDebInfo/libonnxruntime.dylib $(Build.BinariesDirectory)/osx-x64/ - cp $(Build.BinariesDirectory)/RelWithDebInfo/mklml/src/project_mklml/lib/libmklml.dylib $(Build.BinariesDirectory)/osx-x64/ - cp $(Build.BinariesDirectory)/RelWithDebInfo/mklml/src/project_mklml/lib/libiomp5.dylib $(Build.BinariesDirectory)/osx-x64/ - dsymutil $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib -o $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib.dSYM - strip -S -x $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib - find $(Build.BinariesDirectory)/osx-x64 -ls - install_name_tool -change "@rpath/libmklml.dylib" "@loader_path/libmklml.dylib" $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib - install_name_tool -change "@rpath/libiomp5.dylib" "@loader_path/libiomp5.dylib" $(Build.BinariesDirectory)/osx-x64/libmklml.dylib - otool -L $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib - otool -L $(Build.BinariesDirectory)/osx-x64/libmklml.dylib - otool -L $(Build.BinariesDirectory)/osx-x64/libiomp5.dylib - cwd=`pwd` - cd $(Build.BinariesDirectory) - zip -r osx-x64.zip osx-x64 - cp $(Build.BinariesDirectory)/osx-x64.zip $(Build.ArtifactStagingDirectory) - mkdir $(Build.ArtifactStagingDirectory)/testdata - cp $(Build.BinariesDirectory)/RelWithDebInfo/libcustom_op_library.dylib $(Build.ArtifactStagingDirectory)/testdata - cd $cwd - -- job: NuGet_Packaging - workspace: - clean: all - pool: 'Win-CPU-2019' - dependsOn: - - Windows_CI_Dev - - Linux_CI_Dev - - MacOS_CI_Dev - condition: succeeded() - steps: - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - NuGet' - inputs: - artifactName: 'drop-nuget' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - continueOnError: true - - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - Linux' - inputs: - artifactName: 'drop-linux' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - continueOnError: true - - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - MacOS' - inputs: - artifactName: 'drop-osx' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - continueOnError: true - - - script: | - pushd $(Build.BinariesDirectory)\nuget-artifact - dir - powershell -Command "Invoke-WebRequest http://stahlworks.com/dev/unzip.exe -OutFile unzip.exe" - powershell -Command "Invoke-WebRequest http://stahlworks.com/dev/zip.exe -OutFile zip.exe" - set PATH=%CD%;%PATH% - SETLOCAL EnableDelayedExpansion - FOR /R %%i IN (*.nupkg) do ( - set filename=%%~ni - IF NOT "!filename:~25,7!"=="Managed" ( - rename %%~ni.nupkg %%~ni.zip - unzip %%~ni.zip -d %%~ni - del /Q %%~ni.zip - unzip linux-x64.zip -d linux-x64 - mkdir %%~ni\runtimes\linux-x64 - mkdir %%~ni\runtimes\linux-x64\native - move linux-x64\linux-x64\libonnxruntime.so %%~ni\runtimes\linux-x64\native\libonnxruntime.so - move linux-x64\linux-x64\libmklml_gnu.so %%~ni\runtimes\linux-x64\native\libmklml_gnu.so - unzip osx-x64.zip -d osx-x64 - dir osx-x64 /s - mkdir %%~ni\runtimes\osx-x64 - mkdir %%~ni\runtimes\osx-x64\native - move osx-x64\osx-x64\libonnxruntime.dylib %%~ni\runtimes\osx-x64\native\libonnxruntime.dylib - move osx-x64\osx-x64\libonnxruntime.dylib.dSYM %%~ni\runtimes\osx-x64\native\libonnxruntime.dylib.dSYM - move osx-x64\osx-x64\libmklml.dylib %%~ni\runtimes\osx-x64\native\libmklml.dylib - move osx-x64\osx-x64\libiomp5.dylib %%~ni\runtimes\osx-x64\native\libiomp5.dylib - pushd %%~ni - zip -r ..\%%~ni.zip . - popd - move %%~ni.zip %%~ni.nupkg - ) - ) - popd - copy $(Build.BinariesDirectory)\nuget-artifact\*.nupkg $(Build.ArtifactStagingDirectory) - displayName: 'Bundle Native NuGet and other binaries' - - - template: ../../templates/esrp_nuget.yml - parameters: - DisplayName: 'ESRP - sign NuGet package' - FolderPath: '$(Build.ArtifactStagingDirectory)' - DoEsrp: ${{ parameters.DoEsrp }} - - - template: ../../templates/validate-nuget.yml - parameters: - NugetPath: '$(Build.ArtifactStagingDirectory)' - NugetPackage: 'Microsoft.ML.OnnxRuntime.MKLML*nupkg' - PlatformsSupported: 'win-x64,linux-x64,osx-x64' - VerifyNugetSigning: ${{ parameters.DoEsrp }} - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Pipeline NuGet Artifact' - inputs: - artifactName: 'drop-signed-nuget' - targetPath: '$(Build.ArtifactStagingDirectory)' - -- template: test_all_os.yml - parameters: - Skipx86Tests: 'true' diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml index b0a6987f0c..d696cc7b91 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml @@ -38,7 +38,7 @@ jobs: --build_wheel \ --use_openmp \ --enable_onnx_tests \ - --use_mklml --enable_pybind --build_java --build_nodejs --enable_training \ + --enable_pybind --build_java --build_nodejs --enable_training \ --cmake_extra_defines PYTHON_INCLUDE_DIR=/opt/python/cp37-cp37m/include/python3.7m PYTHON_LIBRARY=/usr/lib64/librt.so workingDirectory: $(Build.SourcesDirectory) - task: Docker@2 diff --git a/tools/ci_build/github/azure-pipelines/templates/linux-ci.yml b/tools/ci_build/github/azure-pipelines/templates/linux-ci.yml index 2272785a47..512af00c44 100644 --- a/tools/ci_build/github/azure-pipelines/templates/linux-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/linux-ci.yml @@ -2,7 +2,7 @@ parameters: AgentPool : 'Linux-CPU' JobName : 'Linux_CI_Dev' SubmoduleCheckoutMode: '' - BuildCommand: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -x "--use_mklml --use_tvm --build_wheel"' + BuildCommand: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -x "--use_tvm --build_wheel"' DoNodejsPack: 'false' DoNugetPack: 'false' NuPackScript: '' diff --git a/tools/ci_build/github/azure-pipelines/win-mklml-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-mklml-ci-pipeline.yml deleted file mode 100644 index 154f46ab77..0000000000 --- a/tools/ci_build/github/azure-pipelines/win-mklml-ci-pipeline.yml +++ /dev/null @@ -1,6 +0,0 @@ -jobs: -- template: templates/win-ci.yml - parameters: - JobName: 'Windows_CI_Dev' - BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_mklml --build_shared_lib --build_csharp --enable_onnx_tests' -