diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 6596ed2f45..f56709c525 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -11,6 +11,7 @@ endif() # Support OS X versions 10.12+ # This variable is ignored on non-Apple platforms and needs to be set prior to the first project(...) invocation +# TODO: Make the miniumum deployment target MacOSX version configurable set(CMAKE_OSX_DEPLOYMENT_TARGET "10.12" CACHE STRING "Minimum OS X deployment version for ORT" FORCE) # Project diff --git a/cmake/onnxruntime_mlas.cmake b/cmake/onnxruntime_mlas.cmake index 6cb94816e2..f85e8d9f85 100644 --- a/cmake/onnxruntime_mlas.cmake +++ b/cmake/onnxruntime_mlas.cmake @@ -165,6 +165,12 @@ else() elseif(X86_64) enable_language(ASM) + # Forward the flags for the minimum target platform version from the C + # compiler to the assembler. This works around CMakeASMCompiler.cmake.in + # not including the logic to set this flag for the assembler. + + set(CMAKE_ASM${ASM_DIALECT}_OSX_DEPLOYMENT_TARGET_FLAG "${CMAKE_C_OSX_DEPLOYMENT_TARGET_FLAG}") + # The LLVM assembler does not support the .arch directive to enable instruction # set extensions and also doesn't support AVX-512F instructions without # turning on support via command-line option. Group the sources by the diff --git a/include/onnxruntime/core/graph/node_arg.h b/include/onnxruntime/core/graph/node_arg.h index 6a71ecd697..461e217b0b 100644 --- a/include/onnxruntime/core/graph/node_arg.h +++ b/include/onnxruntime/core/graph/node_arg.h @@ -55,6 +55,10 @@ class NodeArg { @returns TensorShapeProto if shape is set. nullptr if there's no shape specified. */ const ONNX_NAMESPACE::TensorShapeProto* Shape() const; + /** Return an indicator. + @returns true if NodeArg is a normal tensor with a non-empty shape or a scalar with an empty shape. Otherwise, returns false. */ + bool HasTensorOrScalarShape() const; + /** Sets the shape. @remarks Shape can only be set if the TypeProto was provided to the ctor, or #SetType has been called, as the shape information is stored as part of TypeProto. */ diff --git a/onnxruntime/contrib_ops/cpu/quantization/attention_quant.cc b/onnxruntime/contrib_ops/cpu/quantization/attention_quant.cc index 145276c73c..c6f313322e 100644 --- a/onnxruntime/contrib_ops/cpu/quantization/attention_quant.cc +++ b/onnxruntime/contrib_ops/cpu/quantization/attention_quant.cc @@ -110,14 +110,6 @@ Status QAttention::Compute(OpKernelContext* context) const { auto V = K + batch_size * sequence_length * hidden_size; T* QKV[3] = {Q, K, V}; - auto gemm_data_quant = allocator->Alloc(SafeInt(batch_size) * sequence_length * 3 * hidden_size * sizeof(int32_t)); - BufferUniquePtr gemm_buffer_quant(gemm_data_quant, BufferDeleter(allocator)); - - auto Q_quant = reinterpret_cast(gemm_data_quant); - auto K_quant = Q_quant + batch_size * sequence_length * hidden_size; - auto V_quant = K_quant + batch_size * sequence_length * hidden_size; - int32_t* QKV_quant[3] = {Q_quant, K_quant, V_quant}; - { const int loop_len = 3 * batch_size * num_heads_; const auto input_data = input->template Data(); @@ -134,7 +126,7 @@ Status QAttention::Compute(OpKernelContext* context) const { int input_offset = batch_index * sequence_length * hidden_size; int weights_offset = qkv_index * hidden_size + head_index * head_size; - int32_t* qkv_dest = QKV_quant[qkv_index]; + float* qkv_dest = QKV[qkv_index]; int qkv_offset = (batch_index * num_heads_ + head_index) * (sequence_length * head_size); // original transposed iteration @@ -152,19 +144,10 @@ Status QAttention::Compute(OpKernelContext* context) const { weight_zero_point, // weight zero point qkv_dest + qkv_offset, // C head_size, // ldc + &dequant_scale, // output scale + bias_data + weights_offset, // bias nullptr // use single-thread ); - - // dequantize and add bias - // broadcast 3NH -> (3.B.N.S.H) - const T* bias_src = bias_data + weights_offset; - int32_t* gemm_quant_src = QKV_quant[qkv_index] + qkv_offset; - T* data_dest = QKV[qkv_index] + qkv_offset; - for (int seq_index = 0; seq_index < sequence_length; seq_index++) { - for (int head_idx = 0; head_idx < head_size; head_idx++) { - *data_dest++ = *gemm_quant_src++ * dequant_scale + bias_src[head_idx]; - } - } } }); } diff --git a/onnxruntime/contrib_ops/cpu/quantization/dynamic_quantize_matmul.cc b/onnxruntime/contrib_ops/cpu/quantization/dynamic_quantize_matmul.cc index 2e7566e27b..6fbf3eed0a 100644 --- a/onnxruntime/contrib_ops/cpu/quantization/dynamic_quantize_matmul.cc +++ b/onnxruntime/contrib_ops/cpu/quantization/dynamic_quantize_matmul.cc @@ -69,7 +69,7 @@ Status DynamicQuantizeMatMul::Compute(OpKernelContext* ctx) const { } // calculate quantization parameter of a - const float* a_data = a->template Data(); + const auto* a_data = a->template Data(); int64_t num_of_elements = a->Shape().Size(); float a_scale; @@ -86,8 +86,12 @@ Status DynamicQuantizeMatMul::Compute(OpKernelContext* ctx) const { MatMulComputeHelper helper; ORT_RETURN_IF_ERROR(helper.Compute(a->Shape(), b->Shape())); - int32_t* matmul_output = static_cast(allocator->Alloc(SafeInt(helper.OutputShape().Size()) * sizeof(int32_t))); - BufferUniquePtr matmul_output_holder(matmul_output, BufferDeleter(allocator)); + const auto* b_data = b->template Data(); + + Tensor* y = ctx->Output(0, helper.OutputShape()); + auto* y_data = y->template MutableData(); + + const float multiplier = a_scale * b_scale; concurrency::ThreadPool* thread_pool = ctx->GetOperatorThreadPool(); for (size_t i = 0; i < helper.OutputOffsets().size(); i++) { @@ -97,21 +101,16 @@ Status DynamicQuantizeMatMul::Compute(OpKernelContext* ctx) const { a_data_quant + helper.LeftOffsets()[i], static_cast(helper.K()), a_zp, - b->template Data() + helper.RightOffsets()[i], + b_data + helper.RightOffsets()[i], static_cast(helper.N()), b_zp, - matmul_output + helper.OutputOffsets()[i], + y_data + helper.OutputOffsets()[i], static_cast(helper.N()), + &multiplier, + nullptr, thread_pool); } - Tensor* y = ctx->Output(0, helper.OutputShape()); - float* y_data = y->template MutableData(); - - float multiplier = a_scale * b_scale; - for (int64_t i = 0; i < helper.OutputShape().Size(); i++) { - y_data[i] = matmul_output[i] * multiplier; - } return Status::OK(); } diff --git a/onnxruntime/contrib_ops/cpu/quantization/dynamic_quantize_matmul.h b/onnxruntime/contrib_ops/cpu/quantization/dynamic_quantize_matmul.h index 28cd21bc00..ba2fce147b 100644 --- a/onnxruntime/contrib_ops/cpu/quantization/dynamic_quantize_matmul.h +++ b/onnxruntime/contrib_ops/cpu/quantization/dynamic_quantize_matmul.h @@ -12,9 +12,10 @@ namespace contrib { template class DynamicQuantizeMatMul final : public OpKernel { public: - DynamicQuantizeMatMul(const OpKernelInfo& info) : OpKernel(info) {} + DynamicQuantizeMatMul(const OpKernelInfo& info) : OpKernel(info) {} Status Compute(OpKernelContext* context) const override; }; + } // namespace contrib } // namespace onnxruntime diff --git a/onnxruntime/core/framework/session_state.cc b/onnxruntime/core/framework/session_state.cc index 8f32f8b226..09660295c4 100644 --- a/onnxruntime/core/framework/session_state.cc +++ b/onnxruntime/core/framework/session_state.cc @@ -190,16 +190,26 @@ static int64_t CalculateMemoryPatternsKey(const std::vector& feeds, std::unordered_map& out) { +Status ResolveDimParams(const GraphViewer& graph, + const std::map& feeds, + std::unordered_map& out) { for (const auto* input : graph.GetInputs()) { auto* shape = input->Shape(); auto it = feeds.find(input->Name()); - if (it == feeds.end()) - return Status(ONNXRUNTIME, FAIL, "Graph input " + input->Name() + " is not found in the feed list, unable to resolve the value for dynamic shape."); - if (!shape || shape->dim_size() != static_cast(it->second.NumDimensions())) + if (it == feeds.end()) { + return Status(ONNXRUNTIME, FAIL, + "Graph input " + input->Name() + + " is not found in the feed list, unable to resolve the value for dynamic shape."); + } + if (it->second.NumDimensions() == 0 && !shape) { + // This is a scalar, which has nothing to do with symbolic shapes. + continue; + } + if (!shape || shape->dim_size() != static_cast(it->second.NumDimensions())) { return Status(ONNXRUNTIME, FAIL, "Graph input " + input->Name() + - "'s shape is not present or its shape doesn't match feed's shape." - "Unable to resolve the value for dynamic shape"); + "'s shape is not present or its shape doesn't match feed's shape." + "Unable to resolve the value for dynamic shape"); + } for (int k = 0, end = shape->dim_size(); k < end; ++k) { if (shape->dim()[k].has_dim_param()) { out.insert({shape->dim()[k].dim_param(), it->second.GetDims()[k]}); @@ -320,7 +330,7 @@ const MemoryPatternGroup* SessionState::GetMemoryPatternGroup(const std::vector< void SessionState::ResolveMemoryPatternFlag() { if (enable_mem_pattern_) { for (auto* input : graph_viewer_->GetInputs()) { - if (!input->Shape()) { + if (!input->HasTensorOrScalarShape()) { enable_mem_pattern_ = false; break; } diff --git a/onnxruntime/core/graph/graph.cc b/onnxruntime/core/graph/graph.cc index 0e11eb1ba3..8d6fc7457a 100644 --- a/onnxruntime/core/graph/graph.cc +++ b/onnxruntime/core/graph/graph.cc @@ -172,6 +172,26 @@ const TensorShapeProto* NodeArg::Shape() const { } } +bool NodeArg::HasTensorOrScalarShape() const { + const TypeProto* type = TypeAsProto(); + if (!type) return false; + const auto type_case = type->value_case(); + switch (type_case) { + case TypeProto::kTensorType: + case TypeProto::kSparseTensorType: + // Standard tensor has a valid shape field while + // scalar's shape is empty. Thus, we don't need to + // check shape here. + return true; + case TypeProto::kSequenceType: + case TypeProto::kMapType: + case TypeProto::kOpaqueType: + case TypeProto::VALUE_NOT_SET: + default: + return false; + } +} + void NodeArg::SetShape(const TensorShapeProto& shape) { const auto type_case = node_arg_info_.type().value_case(); switch (type_case) { diff --git a/onnxruntime/core/mlas/inc/mlas.h b/onnxruntime/core/mlas/inc/mlas.h index b342b712fe..fd5e62da1a 100644 --- a/onnxruntime/core/mlas/inc/mlas.h +++ b/onnxruntime/core/mlas/inc/mlas.h @@ -165,6 +165,26 @@ MlasGemm( MLAS_THREADPOOL* ThreadPool ); +template +void +MLASCALL +MlasGemm( + size_t M, + size_t N, + size_t K, + const AType* A, + size_t lda, + AType offa, + const BType* B, + size_t ldb, + BType offb, + float* C, + size_t ldc, + const float* Scale, + const float* Bias, + MLAS_THREADPOOL* ThreadPool + ); + // // Convolution routines. // diff --git a/onnxruntime/core/mlas/lib/mlasi.h b/onnxruntime/core/mlas/lib/mlasi.h index 323414eb85..f136debc8e 100644 --- a/onnxruntime/core/mlas/lib/mlasi.h +++ b/onnxruntime/core/mlas/lib/mlasi.h @@ -254,12 +254,7 @@ typedef MLAS_SGEMM_TRANSPOSE_PACKB_BLOCK_ROUTINE* PMLAS_SGEMM_TRANSPOSE_PACKB_BL typedef void (MLASCALL MLAS_GEMM_U8X8_OPERATION)( - const struct MLAS_GEMM_U8X8_WORK_BLOCK* WorkBlock, - size_t M, - size_t N, - const uint8_t* A, - const uint8_t* B, - int32_t* C + const struct MLAS_GEMM_U8X8_WORK_BLOCK* WorkBlock ); typedef MLAS_GEMM_U8X8_OPERATION* PMLAS_GEMM_U8X8_OPERATION; @@ -334,7 +329,7 @@ void size_t OutputCount, size_t OutputCountRightPad, const float* Bias, - unsigned Flags + unsigned KernelFlags ); typedef MLAS_CONV_FLOAT_KERNEL* PMLAS_CONV_FLOAT_KERNEL; @@ -357,7 +352,7 @@ void size_t OutputCount, size_t OutputCountRightPad, const float* Bias, - unsigned Flags + unsigned KernelFlags ); typedef MLAS_CONV_DEPTHWISE_FLOAT_KERNEL* PMLAS_CONV_DEPTHWISE_FLOAT_KERNEL; @@ -376,7 +371,7 @@ void size_t OutputStride, size_t OutputCount, const float* Bias, - unsigned Flags + unsigned KernelFlags ); typedef MLAS_CONV_POINTWISE_FLOAT_KERNEL* PMLAS_CONV_POINTWISE_FLOAT_KERNEL; diff --git a/onnxruntime/core/mlas/lib/qgemm.cpp b/onnxruntime/core/mlas/lib/qgemm.cpp index 6ed2d89a19..8c553728bb 100644 --- a/onnxruntime/core/mlas/lib/qgemm.cpp +++ b/onnxruntime/core/mlas/lib/qgemm.cpp @@ -34,21 +34,19 @@ struct MLAS_GEMM_U8X8_WORK_BLOCK { size_t ldb; int32_t* C; size_t ldc; + const float* Scale; + const float* BiasFloat; uint8_t offa; uint8_t offb; bool BTypeIsSigned; + bool CTypeIsFloat; }; template MLAS_FORCEINLINE void MlasGemmU8X8Operation( - const MLAS_GEMM_U8X8_WORK_BLOCK* WorkBlock, - size_t M, - size_t N, - const uint8_t* A, - const uint8_t* B, - int32_t* C + const MLAS_GEMM_U8X8_WORK_BLOCK* WorkBlock ) /*++ @@ -61,16 +59,6 @@ Arguments: WorkBlock - Supplies the structure containing the GEMM parameters. - M - Supplies the number of rows of matrix A and matrix C. - - N - Supplies the number of columns of matrix B and matrix C. - - A - Supplies the address of matrix A. - - B - Supplies the address of matrix B. - - C - Supplies the address of matrix C. - Return Value: None. @@ -83,6 +71,10 @@ Return Value: MLAS_DECLSPEC_ALIGN(int32_t RowSumVector[KernelType::StrideM], 64); MLAS_DECLSPEC_ALIGN(int32_t ColumnSumVector[KernelType::StrideN], 64); + const uint8_t* A = WorkBlock->A; + const uint8_t* B = WorkBlock->B; + int32_t* C = WorkBlock->C; + const size_t lda = WorkBlock->lda; const size_t ldb = WorkBlock->ldb; const size_t ldc = WorkBlock->ldc; @@ -103,6 +95,8 @@ Return Value: // Step through each slice of matrix B along the K dimension. // + const size_t M = WorkBlock->M; + const size_t N = WorkBlock->N; const size_t K = WorkBlock->K; size_t CountK; @@ -124,7 +118,7 @@ Return Value: // Copy a panel of matrix B to a local packed buffer. // - KernelType::CopyPackB(PanelB, B + n + k * ldb, ldb, CountN, CountK, + KernelType::CopyPackB(PanelB, B + n, ldb, CountN, CountK, ColumnSumVector, -offa, WorkBlock->BTypeIsSigned); // @@ -146,8 +140,8 @@ Return Value: // Copy a panel of matrix A to a local packed buffer. // - KernelType::CopyPackA(PanelA, A + k + m * lda, lda, CountM, - CountK, RowSumVector, -offb); + KernelType::CopyPackA(PanelA, A + m * lda, lda, CountM, CountK, + RowSumVector, -offb); // // Step through the rows of the local packed buffer. @@ -157,13 +151,20 @@ Return Value: int32_t* RowSums = RowSumVector; size_t RowsRemaining = CountM; + bool ZeroMode = (k == 0); + bool PostProcess = (k + CountK == K); + while (RowsRemaining > 0) { size_t RowsHandled; RowsHandled = KernelType::Kernel(pa, PanelB, c, PackedCountK, RowsRemaining, CountN, ldc, RowSums, ColumnSumVector, - DepthValue, k == 0); + DepthValue, ZeroMode); + + if (PostProcess && WorkBlock->CTypeIsFloat) { + KernelType::OutputFloat(WorkBlock, c, n, RowsHandled, CountN); + } c += ldc * RowsHandled; pa += KernelType::PackedK * PackedCountK * RowsHandled; @@ -172,6 +173,9 @@ Return Value: } } } + + A += CountK; + B += CountK * ldb; } } @@ -702,6 +706,112 @@ Return Value: } } +void +MlasGemmU8X8OutputFloatSse( + const MLAS_GEMM_U8X8_WORK_BLOCK* WorkBlock, + int32_t* C, + size_t StartN, + size_t CountM, + size_t CountN + ) +/*++ + +Routine Description: + + This routine is an inner kernel to compute matrix multiplication for a + single row. + +Arguments: + + WorkBlock - Supplies the structure containing the GEMM parameters. + + C - Supplies the address of matrix C. + + StartN - Supplies the starting column offset relative to the base of the + work block. This is used to offset into column vectors accessed via the + work block. + + CountM - Supplies the number of rows of the output matrix to process. + + CountN - Supplies the number of columns of the output matrix to process. + +Return Value: + + None. + +--*/ +{ + const size_t ldc = WorkBlock->ldc; + __m128 ScaleVector = _mm_load_ps1(WorkBlock->Scale); + + // + // Check if the optional bias vector was supplied. + // + + const float* BiasFloat = WorkBlock->BiasFloat; + + if (BiasFloat != nullptr) { + + BiasFloat += StartN; + + while (CountM-- > 0) { + + const float* bias = BiasFloat; + int32_t* c = C; + size_t n = CountN; + + while (n >= 4) { + + __m128 FloatVector = _mm_cvtepi32_ps(_mm_loadu_si128((__m128i*)c)); + FloatVector = _mm_mul_ps(FloatVector, ScaleVector); + FloatVector = _mm_add_ps(FloatVector, _mm_loadu_ps(bias)); + _mm_storeu_ps((float*)c, FloatVector); + + bias += 4; + c += 4; + n -= 4; + } + + for (size_t offset = 0; offset < n; offset++) { + + __m128 FloatVector = _mm_set_ss(float(c[offset])); + FloatVector = _mm_mul_ss(FloatVector, ScaleVector); + FloatVector = _mm_add_ss(FloatVector, _mm_load_ss(&bias[offset])); + _mm_store_ss((float*)&c[offset], FloatVector); + } + + C += ldc; + } + + } else { + + while (CountM-- > 0) { + + int32_t* c = C; + size_t n = CountN; + + while (n >= 4) { + + __m128 FloatVector = _mm_cvtepi32_ps(_mm_loadu_si128((__m128i*)c)); + FloatVector = _mm_mul_ps(FloatVector, ScaleVector); + _mm_storeu_ps((float*)c, FloatVector); + + c += 4; + n -= 4; + } + + for (size_t offset = 0; offset < n; offset++) { + + __m128 FloatVector = _mm_set_ss((float)c[offset]); + FloatVector = _mm_mul_ss(FloatVector, ScaleVector); + _mm_store_ss((float*)&c[offset], FloatVector); + } + + C += ldc; + } + } +} + struct MLAS_GEMM_U8X8_KERNEL_SSE { typedef int16_t PackedAType; @@ -772,6 +882,20 @@ struct MLAS_GEMM_U8X8_KERNEL_SSE return 1; } + + MLAS_FORCEINLINE + static + void + OutputFloat( + const MLAS_GEMM_U8X8_WORK_BLOCK* WorkBlock, + int32_t* C, + size_t StartN, + size_t CountM, + size_t CountN + ) + { + MlasGemmU8X8OutputFloatSse(WorkBlock, C, StartN, CountM, CountN); + } }; constexpr size_t MLAS_GEMM_U8X8_KERNEL_SSE::PackedK; @@ -782,12 +906,7 @@ constexpr size_t MLAS_GEMM_U8X8_KERNEL_SSE::StrideK; void MLASCALL MlasGemmU8X8OperationSse( - const MLAS_GEMM_U8X8_WORK_BLOCK* WorkBlock, - size_t M, - size_t N, - const uint8_t* A, - const uint8_t* B, - int32_t* C + const MLAS_GEMM_U8X8_WORK_BLOCK* WorkBlock ) /*++ @@ -802,23 +921,13 @@ Arguments: WorkBlock - Supplies the structure containing the GEMM parameters. - M - Supplies the number of rows of matrix A and matrix C. - - N - Supplies the number of columns of matrix B and matrix C. - - A - Supplies the address of matrix A. - - B - Supplies the address of matrix B. - - C - Supplies the address of matrix C. - Return Value: None. --*/ { - return MlasGemmU8X8Operation(WorkBlock, M, N, A, B, C); + return MlasGemmU8X8Operation(WorkBlock); } #endif @@ -953,6 +1062,20 @@ struct MLAS_GEMM_U8S8_KERNEL_AVX2 return MlasPlatform.GemmU8S8Kernel(A, B, C, PackedCountK, CountM, CountN, ldc, RowSumVector, ColumnSumVector, DepthValue, ZeroMode); } + + MLAS_FORCEINLINE + static + void + OutputFloat( + const MLAS_GEMM_U8X8_WORK_BLOCK* WorkBlock, + int32_t* C, + size_t StartN, + size_t CountM, + size_t CountN + ) + { + MlasGemmU8X8OutputFloatSse(WorkBlock, C, StartN, CountM, CountN); + } }; constexpr size_t MLAS_GEMM_U8S8_KERNEL_AVX2::PackedK; @@ -963,12 +1086,7 @@ constexpr size_t MLAS_GEMM_U8S8_KERNEL_AVX2::StrideK; void MLASCALL MlasGemmU8S8OperationAvx2( - const MLAS_GEMM_U8X8_WORK_BLOCK* WorkBlock, - size_t M, - size_t N, - const uint8_t* A, - const uint8_t* B, - int32_t* C + const MLAS_GEMM_U8X8_WORK_BLOCK* WorkBlock ) /*++ @@ -983,31 +1101,23 @@ Arguments: WorkBlock - Supplies the structure containing the GEMM parameters. - M - Supplies the number of rows of matrix A and matrix C. - - N - Supplies the number of columns of matrix B and matrix C. - - A - Supplies the address of matrix A. - - B - Supplies the address of matrix B. - - C - Supplies the address of matrix C. - Return Value: None. --*/ { - if (M == 1 && WorkBlock->offa == 0 && WorkBlock->offb == 0) { + if ((WorkBlock->M == 1) && WorkBlock->BTypeIsSigned && !WorkBlock->CTypeIsFloat && + (WorkBlock->offa == 0) && (WorkBlock->offb == 0)) { if (MlasPlatform.GemvU8S8Kernel != nullptr) { - MlasPlatform.GemvU8S8Kernel(A, B, C, WorkBlock->K, N, WorkBlock->ldb); + MlasPlatform.GemvU8S8Kernel(WorkBlock->A, WorkBlock->B, WorkBlock->C, + WorkBlock->K, WorkBlock->N, WorkBlock->ldb); return; } } - return MlasGemmU8X8Operation(WorkBlock, M, N, A, B, C); + return MlasGemmU8X8Operation(WorkBlock); } struct MLAS_GEMM_U8U8_KERNEL_AVX2 @@ -1076,6 +1186,20 @@ struct MLAS_GEMM_U8U8_KERNEL_AVX2 return MlasPlatform.GemmU8U8Kernel(A, B, C, PackedCountK, CountM, CountN, ldc, RowSumVector, ColumnSumVector, DepthValue, ZeroMode); } + + MLAS_FORCEINLINE + static + void + OutputFloat( + const MLAS_GEMM_U8X8_WORK_BLOCK* WorkBlock, + int32_t* C, + size_t StartN, + size_t CountM, + size_t CountN + ) + { + MlasGemmU8X8OutputFloatSse(WorkBlock, C, StartN, CountM, CountN); + } }; constexpr size_t MLAS_GEMM_U8U8_KERNEL_AVX2::PackedK; @@ -1086,12 +1210,7 @@ constexpr size_t MLAS_GEMM_U8U8_KERNEL_AVX2::StrideK; void MLASCALL MlasGemmU8U8OperationAvx2( - const MLAS_GEMM_U8X8_WORK_BLOCK* WorkBlock, - size_t M, - size_t N, - const uint8_t* A, - const uint8_t* B, - int32_t* C + const MLAS_GEMM_U8X8_WORK_BLOCK* WorkBlock ) /*++ @@ -1106,23 +1225,13 @@ Arguments: WorkBlock - Supplies the structure containing the GEMM parameters. - M - Supplies the number of rows of matrix A and matrix C. - - N - Supplies the number of columns of matrix B and matrix C. - - A - Supplies the address of matrix A. - - B - Supplies the address of matrix B. - - C - Supplies the address of matrix C. - Return Value: None. --*/ { - return MlasGemmU8X8Operation(WorkBlock, M, N, A, B, C); + return MlasGemmU8X8Operation(WorkBlock); } #endif @@ -1195,23 +1304,29 @@ Return Value: // Dispatch the partitioned operation. // - const uint8_t* a = WorkBlock->A + m * WorkBlock->lda; - const uint8_t* b = WorkBlock->B + n; - int32_t* c = WorkBlock->C + n + m * WorkBlock->ldc; + MLAS_GEMM_U8X8_WORK_BLOCK LocalWorkBlock; - PMLAS_GEMM_U8X8_OPERATION GemmU8X8Operation; + memcpy(&LocalWorkBlock, WorkBlock, sizeof(MLAS_GEMM_U8X8_WORK_BLOCK)); + + LocalWorkBlock.M = CountM; + LocalWorkBlock.N = CountN; + LocalWorkBlock.A += m * LocalWorkBlock.lda; + LocalWorkBlock.B += n; + LocalWorkBlock.C += m * LocalWorkBlock.ldc + n; + + if (LocalWorkBlock.BiasFloat != nullptr) { + LocalWorkBlock.BiasFloat += n; + } #if defined(MLAS_TARGET_AMD64) if (WorkBlock->BTypeIsSigned) { - GemmU8X8Operation = MlasPlatform.GemmU8S8Operation; + MlasPlatform.GemmU8S8Operation(&LocalWorkBlock); } else { - GemmU8X8Operation = MlasPlatform.GemmU8U8Operation; + MlasPlatform.GemmU8U8Operation(&LocalWorkBlock); } #else - GemmU8X8Operation = MlasGemmU8X8OperationSse; + MlasGemmU8X8OperationSse(&LocalWorkBlock); #endif - - GemmU8X8Operation(WorkBlock, CountM, CountN, a, b, c); } void @@ -1360,6 +1475,8 @@ Return Value: // Capture the GEMM parameters to the work block. // + memset(&WorkBlock, 0, sizeof(MLAS_GEMM_U8X8_WORK_BLOCK)); + WorkBlock.M = M; WorkBlock.N = N; WorkBlock.K = K; @@ -1416,4 +1533,135 @@ MlasGemm( MLAS_THREADPOOL* ThreadPool ); +template +void +MLASCALL +MlasGemm( + size_t M, + size_t N, + size_t K, + const AType* A, + size_t lda, + AType offa, + const BType* B, + size_t ldb, + BType offb, + float* C, + size_t ldc, + const float* Scale, + const float* Bias, + MLAS_THREADPOOL* ThreadPool + ) +/*++ + +Routine Description: + + This module implements the quantized integer matrix/matrix multiply + operation (QGEMM). + +Arguments: + + M - Supplies the number of rows of matrix A and matrix C. + + N - Supplies the number of columns of matrix B and matrix C. + + K - Supplies the number of columns of matrix A and the number of rows of + matrix B. + + A - Supplies the address of matrix A. + + lda - Supplies the first dimension of matrix A. + + offa - Supplies the zero point offset of matrix A. + + B - Supplies the address of matrix B. + + ldb - Supplies the first dimension of matrix B. + + offb - Supplies the zero point offset of matrix B. + + C - Supplies the address of matrix C. + + ldc - Supplies the first dimension of matrix C. + + ThreadPool - Supplies the thread pool object to use, else nullptr if the + base library threading support should be used. + +Return Value: + + None. + +--*/ +{ + MLAS_GEMM_U8X8_WORK_BLOCK WorkBlock; + + // + // Capture the GEMM parameters to the work block. + // + + memset(&WorkBlock, 0, sizeof(MLAS_GEMM_U8X8_WORK_BLOCK)); + + WorkBlock.M = M; + WorkBlock.N = N; + WorkBlock.K = K; + WorkBlock.A = A; + WorkBlock.lda = lda; + WorkBlock.B = (const uint8_t*)B; + WorkBlock.ldb = ldb; + WorkBlock.C = (int32_t*)C; + WorkBlock.ldc = ldc; + WorkBlock.Scale = Scale; + WorkBlock.BiasFloat = Bias; + WorkBlock.offa = offa; + WorkBlock.offb = offb; + WorkBlock.BTypeIsSigned = std::is_signed::value; + WorkBlock.CTypeIsFloat = true; + + // + // Schedule the operation across a set of worker threads. + // + + MlasGemmU8X8Schedule(&WorkBlock, ThreadPool); +} + +template +void +MLASCALL +MlasGemm( + size_t M, + size_t N, + size_t K, + const uint8_t* A, + size_t lda, + uint8_t offa, + const int8_t* B, + size_t ldb, + int8_t offb, + float* C, + size_t ldc, + const float* Scale, + const float* Bias, + MLAS_THREADPOOL* ThreadPool + ); + +template +void +MLASCALL +MlasGemm( + size_t M, + size_t N, + size_t K, + const uint8_t* A, + size_t lda, + uint8_t offa, + const uint8_t* B, + size_t ldb, + uint8_t offb, + float* C, + size_t ldc, + const float* Scale, + const float* Bias, + MLAS_THREADPOOL* ThreadPool + ); + #endif diff --git a/onnxruntime/core/util/qmath.cc b/onnxruntime/core/util/qmath.cc index 166793486d..67f84b32b4 100644 --- a/onnxruntime/core/util/qmath.cc +++ b/onnxruntime/core/util/qmath.cc @@ -69,4 +69,73 @@ void QGemm( #endif } +template +void QGemm( + int M, + int N, + int K, + const LeftScalar* lhs_data, + int lda, + const LeftScalar lhs_offset, + const RightScalar* rhs_data, + int ldb, + const RightScalar rhs_offset, + float* result_data, + int ldc, + const float* result_scale, + const float* bias, + concurrency::ThreadPool* thread_pool) { +#ifdef MLAS_SUPPORTS_GEMM_U8X8 + MlasGemm(M, N, K, lhs_data, lda, lhs_offset, rhs_data, ldb, rhs_offset, result_data, ldc, result_scale, bias, thread_pool); +#else + QGemm(M, N, K, lhs_data, lda, lhs_offset, rhs_data, ldb, rhs_offset, reinterpret_cast(result_data), ldc, thread_pool); + for (int m = 0; m < M; m++) { + if (bias != nullptr) { + for (int n = 0; n < N; n++) { + result_data[n] = static_cast(reinterpret_cast(result_data)[n]) * result_scale[0] + bias[n]; + } + } else { + for (int n = 0; n < N; n++) { + result_data[n] = static_cast(reinterpret_cast(result_data)[n]) * result_scale[0]; + } + } + result_data += ldc; + } +#endif +} + +template +void QGemm( + int M, + int N, + int K, + const uint8_t* lhs_data, + int lda, + const uint8_t lhs_offset, + const int8_t* rhs_data, + int ldb, + const int8_t rhs_offset, + float* result_data, + int ldc, + const float* result_scale, + const float* bias, + concurrency::ThreadPool* thread_pool); + +template +void QGemm( + int M, + int N, + int K, + const uint8_t* lhs_data, + int lda, + const uint8_t lhs_offset, + const uint8_t* rhs_data, + int ldb, + const uint8_t rhs_offset, + float* result_data, + int ldc, + const float* result_scale, + const float* bias, + concurrency::ThreadPool* thread_pool); + } // namespace onnxruntime diff --git a/onnxruntime/core/util/qmath.h b/onnxruntime/core/util/qmath.h index b79775b3f8..9d333023d5 100644 --- a/onnxruntime/core/util/qmath.h +++ b/onnxruntime/core/util/qmath.h @@ -29,6 +29,23 @@ void QGemm( int ldc, concurrency::ThreadPool* thread_pool); +template +void QGemm( + int M, + int N, + int K, + const LeftScalar* lhs_data, + int lda, + const LeftScalar lhs_offset, + const RightScalar* rhs_data, + int ldb, + const RightScalar rhs_offset, + float* result_data, + int ldc, + const float* result_scale, + const float* bias, + concurrency::ThreadPool* thread_pool); + inline float RoundHalfToEven(float input) { std::fesetround(FE_TONEAREST); auto result = std::nearbyintf(input); diff --git a/onnxruntime/python/onnxruntime_pybind_state.cc b/onnxruntime/python/onnxruntime_pybind_state.cc index 5e46ce88ee..95a01fe187 100644 --- a/onnxruntime/python/onnxruntime_pybind_state.cc +++ b/onnxruntime/python/onnxruntime_pybind_state.cc @@ -339,9 +339,6 @@ void RegisterExecutionProviders(InferenceSession* sess, const std::vector(INT_MAX); - arena_extend_strategy = ArenaExtendStrategy::kNextPowerOfTwo; #endif } else if (type == kDnnlExecutionProvider) { #ifdef USE_DNNL @@ -439,7 +436,7 @@ void addGlobalMethods(py::module& m, const Environment& env) { std::vector> factories = { onnxruntime::CreateExecutionProviderFactory_CPU(0), #ifdef USE_CUDA - onnxruntime::CreateExecutionProviderFactory_CUDA(0), + onnxruntime::CreateExecutionProviderFactory_CUDA(cuda_device_id, cuda_mem_limit, arena_extend_strategy), #endif #ifdef USE_DNNL onnxruntime::CreateExecutionProviderFactory_Dnnl(1), @@ -514,7 +511,8 @@ void addOpSchemaSubmodule(py::module& m) { auto schemadef = m.def_submodule("schemadef"); schemadef.doc() = "Schema submodule"; - py::class_ op_schema(schemadef, "OpSchema"); + // Keep this binding local to this module + py::class_ op_schema(schemadef, "OpSchema", py::module_local()); op_schema.def_property_readonly("file", &ONNX_NAMESPACE::OpSchema::file) .def_property_readonly("line", &ONNX_NAMESPACE::OpSchema::line) .def_property_readonly("support_level", &ONNX_NAMESPACE::OpSchema::support_level) @@ -540,7 +538,8 @@ void addOpSchemaSubmodule(py::module& m) { return v == std::numeric_limits::max(); }); - py::class_(op_schema, "Attribute") + // Keep this binding local to this module + py::class_(op_schema, "Attribute", py::module_local()) .def_readonly("name", &ONNX_NAMESPACE::OpSchema::Attribute::name) .def_readonly("description", &ONNX_NAMESPACE::OpSchema::Attribute::description) .def_readonly("type", &ONNX_NAMESPACE::OpSchema::Attribute::type) @@ -553,7 +552,8 @@ void addOpSchemaSubmodule(py::module& m) { }) .def_readonly("required", &ONNX_NAMESPACE::OpSchema::Attribute::required); - py::class_(op_schema, "TypeConstraintParam") + // Keep this binding local to this module + py::class_(op_schema, "TypeConstraintParam", py::module_local()) .def_readonly( "type_param_str", &ONNX_NAMESPACE::OpSchema::TypeConstraintParam::type_param_str) .def_readonly("description", &ONNX_NAMESPACE::OpSchema::TypeConstraintParam::description) @@ -561,12 +561,14 @@ void addOpSchemaSubmodule(py::module& m) { "allowed_type_strs", &ONNX_NAMESPACE::OpSchema::TypeConstraintParam::allowed_type_strs); - py::enum_(op_schema, "FormalParameterOption") + // Keep this binding local to this module + py::enum_(op_schema, "FormalParameterOption", py::module_local()) .value("Single", ONNX_NAMESPACE::OpSchema::Single) .value("Optional", ONNX_NAMESPACE::OpSchema::Optional) .value("Variadic", ONNX_NAMESPACE::OpSchema::Variadic); - py::class_(op_schema, "FormalParameter") + // Keep this binding local to this module + py::class_(op_schema, "FormalParameter", py::module_local()) .def_property_readonly("name", &ONNX_NAMESPACE::OpSchema::FormalParameter::GetName) .def_property_readonly("types", &ONNX_NAMESPACE::OpSchema::FormalParameter::GetTypes) .def_property_readonly("typeStr", &ONNX_NAMESPACE::OpSchema::FormalParameter::GetTypeStr) @@ -576,7 +578,8 @@ void addOpSchemaSubmodule(py::module& m) { .def_property_readonly( "isHomogeneous", &ONNX_NAMESPACE::OpSchema::FormalParameter::GetIsHomogeneous); - py::enum_(op_schema, "AttrType") + // Keep this binding local to this module + py::enum_(op_schema, "AttrType", py::module_local()) .value("FLOAT", ONNX_NAMESPACE::AttributeProto::FLOAT) .value("INT", ONNX_NAMESPACE::AttributeProto::INT) .value("STRING", ONNX_NAMESPACE::AttributeProto::STRING) @@ -588,7 +591,8 @@ void addOpSchemaSubmodule(py::module& m) { .value("TENSORS", ONNX_NAMESPACE::AttributeProto::TENSORS) .value("GRAPHS", ONNX_NAMESPACE::AttributeProto::GRAPHS); - py::enum_(op_schema, "SupportType") + // Keep this binding local to this module + py::enum_(op_schema, "SupportType", py::module_local()) .value("COMMON", ONNX_NAMESPACE::OpSchema::SupportType::COMMON) .value("EXPERIMENTAL", ONNX_NAMESPACE::OpSchema::SupportType::EXPERIMENTAL); } diff --git a/onnxruntime/test/contrib_ops/dynamic_quantize_matmul_test.cc b/onnxruntime/test/contrib_ops/dynamic_quantize_matmul_test.cc index 2b4be7cc8b..839278b8a7 100644 --- a/onnxruntime/test/contrib_ops/dynamic_quantize_matmul_test.cc +++ b/onnxruntime/test/contrib_ops/dynamic_quantize_matmul_test.cc @@ -7,6 +7,7 @@ #include "test/framework/test_utils.h" #include "test/providers/provider_test_utils.h" #include "test/util/include/default_providers.h" +#include "core/util/qmath.h" #include #include @@ -146,6 +147,7 @@ class DynamicQuantizeMatMulOpTester : public OpTester { }; TEST(DynamicQuantizeMatMul, Int8_test) { +#ifdef MLAS_SUPPORTS_GEMM_U8X8 std::vector A_dims{4, 128}; std::vector B_dims{128, 128}; std::vector Y_dims{4, 128}; @@ -155,6 +157,7 @@ TEST(DynamicQuantizeMatMul, Int8_test) { Y_dims, "testdata/dynamic_quantize_matmul_int8.onnx"); test.Run(); +#endif } TEST(DynamicQuantizeMatMul, UInt8_test) { diff --git a/onnxruntime/test/mlas/unittest.cpp b/onnxruntime/test/mlas/unittest.cpp index ab5677b85e..f145e6eac3 100644 --- a/onnxruntime/test/mlas/unittest.cpp +++ b/onnxruntime/test/mlas/unittest.cpp @@ -47,7 +47,7 @@ Abstract: MLAS_THREADPOOL* threadpool = nullptr; -template +template class MatrixGuardBuffer { public: @@ -202,7 +202,7 @@ public: } }; -template +template class MlasFgemmTest : public MlasTestBase { private: @@ -254,6 +254,7 @@ private: // Sensitive to comparing positive/negative zero. if (C[f] != CReference[f]) { printf("mismatch TransA=%d, TransB=%d, M=%zd, N=%zd, K=%zd, alpha=%f, beta=%f %f %f!\n", TransA, TransB, M, N, K, alpha, beta, float(C[f]), float(CReference[f])); + break; } } } @@ -467,8 +468,11 @@ public: #ifdef MLAS_HAS_QGEMM_U8X8 -template -class MlasQgemmU8X8Test : public MlasTestBase +template +class MlasQgemmU8X8Test; + +template +class MlasQgemmU8X8Test : public MlasTestBase { private: void @@ -644,6 +648,124 @@ public: } }; +template +class MlasQgemmU8X8Test : public MlasTestBase +{ +private: + void + Test( + size_t M, + size_t N, + size_t K, + uint8_t offa, + uint8_t offb + ) + { + const uint8_t* A = BufferA.GetBuffer(K * M); + const xint8_t* B = BufferB.GetBuffer(N * K); + float* C = BufferC.GetBuffer(N * M); + float* CReference = BufferCReference.GetBuffer(N * M); + const float* Bias = BufferBias.GetBuffer(N); + + const float AScale = 0.5f; + float* AFloat = BufferAFloat.GetBuffer(K * M); + DequantizeLinear(A, AFloat, K * M, AScale, offa); + + const float BScale = 0.25f; + float* BFloat = BufferBFloat.GetBuffer(N * K); + DequantizeLinear(B, BFloat, N * K, BScale, xint8_t(offb)); + + const float CScale = AScale * BScale; + + Test(M, N, K, A, AFloat, K, offa, B, BFloat, N, xint8_t(offb), C, CReference, N, CScale, nullptr); + Test(M, N, K, A, AFloat, K, offa, B, BFloat, N, xint8_t(offb), C, CReference, N, CScale, Bias); + } + + void + Test( + size_t M, + size_t N, + size_t K, + const uint8_t* A, + const float* AFloat, + size_t lda, + uint8_t offa, + const xint8_t* B, + const float* BFloat, + size_t ldb, + xint8_t offb, + float* C, + float* CReference, + size_t ldc, + float CScale, + const float* Bias + ) + { + MlasGemm(CblasNoTrans, CblasNoTrans, M, N, K, 1.0f, AFloat, lda, BFloat, ldb, 0.0f, CReference, ldc, threadpool); + + if (Bias != nullptr) { + for (size_t m = 0; m < M; m++) { + for (size_t n = 0; n < N; n++) { + CReference[m * ldc + n] += Bias[n]; + } + } + } + + MlasGemm(M, N, K, A, lda, offa, B, ldb, offb, C, ldc, &CScale, Bias, threadpool); + + for (size_t f = 0; f < M * N; f++) { + // Sensitive to comparing positive/negative zero. + if (C[f] != CReference[f]) { + printf("mismatch M=%zd, N=%zd, K=%zd, offa=%d, offb=%d! %f %f\n", M, N, K, (int)offa, (int)offb, C[f], CReference[f]); + } + } + } + + template + void + DequantizeLinear( + const qint8_t* Input, + float* Output, + size_t N, + float scale, + qint8_t offset + ) + { + for (size_t n = 0; n < N; n++) { + Output[n] = float((int32_t(Input[n]) - offset)) * scale; + } + } + + MatrixGuardBuffer BufferA; + MatrixGuardBuffer BufferB; + MatrixGuardBuffer BufferAFloat; + MatrixGuardBuffer BufferBFloat; + MatrixGuardBuffer BufferC; + MatrixGuardBuffer BufferCReference; + MatrixGuardBuffer BufferBias; + +public: + void + ExecuteShort( + void + ) override + { + for (size_t b = 1; b < 16; b++) { + Test(b, b, b, 34, 46); + } + for (size_t b = 16; b <= 256; b <<= 1) { + Test(b, b, b, 15, 191); + } + for (size_t b = 256; b < 320; b += 32) { + Test(b, b, b, 223, 73); + } + for (size_t b = 1; b < 96; b++) { + Test(1, b, 32, 0, 0); + } + Test(1024, 1024, 256, 13, 15); + } +}; + #endif class MlasConv2DTest : public MlasTestBase @@ -2254,9 +2376,14 @@ RunThreadedTests( #endif #ifdef MLAS_HAS_QGEMM_U8X8 - printf("QGEMM tests.\n"); - onnxruntime::make_unique>()->ExecuteShort(); - onnxruntime::make_unique>()->ExecuteShort(); + printf("QGEMM U8S8=int32_t tests.\n"); + onnxruntime::make_unique>()->ExecuteShort(); + printf("QGEMM U8U8=int32_t tests.\n"); + onnxruntime::make_unique>()->ExecuteShort(); + printf("QGEMM U8S8=float tests.\n"); + onnxruntime::make_unique>()->ExecuteShort(); + printf("QGEMM U8U8=float tests.\n"); + onnxruntime::make_unique>()->ExecuteShort(); #endif printf("Conv2D tests.\n"); diff --git a/orttraining/orttraining/core/graph/pipeline_transformer.cc b/orttraining/orttraining/core/graph/pipeline_transformer.cc index 6fde902670..2f6762be49 100644 --- a/orttraining/orttraining/core/graph/pipeline_transformer.cc +++ b/orttraining/orttraining/core/graph/pipeline_transformer.cc @@ -41,9 +41,12 @@ void AddNewNodeArg(Graph& graph, new_node_args.push_back(&new_node_arg); } -// gradient graph can contain some dangling leaf nodes. Add them all to WaitEvent -// backward node's input. -void FindLeafNodes(Graph& graph, std::vector& input_args) { +// Gradient graph can contain some dangling leaf nodes. This function collects +// their first output using the returned vector. +std::vector FindBackwardLeafNodes(Graph& graph) { + // leaf_node_args[i] is the i-th leaf node's first output in the backward + // pass. + std::vector leaf_node_args; for (auto& node : graph.Nodes()) { if (!IsBackward(node)) { // only check backward node @@ -59,11 +62,52 @@ void FindLeafNodes(Graph& graph, std::vector& input_args) { } } if (!find_consumer_nodes && outputs.size() > 0) { - input_args.push_back(outputs[0]); + leaf_node_args.push_back(outputs[0]); } } + + return leaf_node_args; }; +// This function converts tensor NodeArg to a boolean scalar so that last +// backward RecordEvent doesn't block the early release of large gradient +// tensors. If we connect gradient tensors directly to that RecordEvent, +// we need a memory block (as large as a whole model) to store gradient +// for each trainable tensor until the end of backward pass. +// +// The newly created boolean scalar may be appended to signal_args. If +// signal_args is empty, the source of signal_args[i] would be tensor_args[i]. +void ConvertTensorToBoolSignal( + Graph& graph, + const std::vector& tensor_args, + std::vector& signal_args) { + + for (auto tensor_arg: tensor_args) { + // Declare the scalar signal this "tensor_arg" will be converted into. + auto signal_arg = &CreateTypedNodeArg( + graph, + ONNX_NAMESPACE::TensorProto_DataType_BOOL, + "signal_" + tensor_arg->Name() + ); + + // Add the new scalar to user-specified vector. + signal_args.push_back(signal_arg); + + // Add tensor-to-scalar conversion node. + const auto name = graph.GenerateNodeName("tensor_to_scalar_signal"); + std::vector input_args{tensor_arg}; + std::vector output_args{signal_arg}; + graph.AddNode( + name, + "Group", + "", + input_args, + output_args, + nullptr, + kMSDomain); + } +} + // Return mirror variables for node_arg with a different name. NodeArg& CreateNodeArg(Graph& graph, const NodeArg* base_arg) { const auto& new_name = graph.GenerateNodeArgName(base_arg->Name()); @@ -133,7 +177,19 @@ Node* AddBackwardRecord(Graph& graph, std::begin(backward_send->MutableOutputDefs()), std::end(backward_send->MutableOutputDefs())); } - FindLeafNodes(graph, input_args); + + // Find all leaf nodes' frist inputs. They are used togehter as control edges + // to determine if backward pass is finished. + auto backward_leaf_node_args = FindBackwardLeafNodes(graph); + + // For each leaf tensor in the backward pass, we use "Group" operator to + // convert it to a boolean scalar so that the original leaf's memory can be + // released earlier. + + // TODO: use full list instead of the first element after changining + // topological sort to depth-first from inputs. + std::vector sub_backward_leaf_node_args{backward_leaf_node_args[0]}; + ConvertTensorToBoolSignal(graph, backward_leaf_node_args, input_args); // Optimizer will be added after applying pipeline transformer. To support partial graph evaluation, // the added Record backward op will have its first passthrough input as output. @@ -1040,8 +1096,17 @@ common::Status GenerateSubgraph(Graph& graph, const Node* start_node) { } } - // update the grah with only visited inputs and outputs - graph.SetInputs({visited_inputs.begin(), visited_inputs.end()}); + // If the following line is uncommented, middle and last pipeline stages may + // have unresolved symbolic shapes. The reason is that some symbolic shapes + // are defined for the original inputs, if original inputs are removed, we + // loss the hit to resolve symbolic shapes. For example, if an original + // input's shape is [batch, sequence, 1024], that input should be provided as + // a feed to all pipeline stages. Otherwise, we don't know the actual values + // of "batch" and "sequence". + // + // graph.SetInputs({visited_inputs.begin(), visited_inputs.end()}); + + // update the grah with only visited outputs graph.SetOutputs({visited_outputs.begin(), visited_outputs.end()}); graph.SetGraphResolveNeeded(); graph.SetGraphProtoSyncNeeded(); diff --git a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml index fce059db1b..ec50ce12d8 100644 --- a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml @@ -69,7 +69,7 @@ jobs: displayName: 'Generate cmake config' inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_wheel --use_featurizers --use_dnnl --use_winml --use_openmp --build_shared_lib --enable_onnx_tests --enable_wcos --build_java --build_nodejs' + arguments: '--gen_doc --config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_wheel --use_featurizers --use_dnnl --use_winml --use_openmp --build_shared_lib --enable_onnx_tests --enable_wcos --build_java --build_nodejs' workingDirectory: '$(Build.BinariesDirectory)' - task: VSBuild@1