From f49a4b63298df2c5f890dfea210cfb3c702c2516 Mon Sep 17 00:00:00 2001 From: Zhang Lei Date: Tue, 25 May 2021 17:08:42 -0700 Subject: [PATCH 01/47] Decrease lock contention in qlstm by memory allocation. (#7815) * Decrease lock contention in qlstm caused by memory allocation. --- .../core/providers/cpu/rnn/rnn_helpers.cc | 16 +++++----- .../core/providers/cpu/rnn/rnn_helpers.h | 6 ++-- .../providers/cpu/rnn/uni_directional_lstm.cc | 30 ++++++++++++++++--- .../providers/cpu/rnn/uni_directional_lstm.h | 11 +++++++ 4 files changed, 48 insertions(+), 15 deletions(-) diff --git a/onnxruntime/core/providers/cpu/rnn/rnn_helpers.cc b/onnxruntime/core/providers/cpu/rnn/rnn_helpers.cc index a16fc0992e..0311a20e26 100644 --- a/onnxruntime/core/providers/cpu/rnn/rnn_helpers.cc +++ b/onnxruntime/core/providers/cpu/rnn/rnn_helpers.cc @@ -214,7 +214,8 @@ void ComputeGemm(const int M, float* C, float* C_end, const int ldc, - AllocatorPtr /*allocator*/, + uint8_t* /* quantized_A_buffer */, + int32_t* /* quantize_agg_C_buffer */, concurrency::ThreadPool* thread_pool) { // validate all the inputs // need to use the lda/ldb/ldc strides which should be >= the columns for the span @@ -249,7 +250,8 @@ void ComputeGemm(const int M, float* C, float* C_end, const int ldc, - AllocatorPtr allocator, + uint8_t* quantized_A_buffer, + int32_t* quantize_agg_C_buffer, concurrency::ThreadPool* thread_pool) { // validate all the inputs // need to use the lda/ldb/ldc strides which should be >= the columns for the span @@ -262,10 +264,8 @@ void ComputeGemm(const int M, uint8_t a_zero_point; GetQuantizationParameter(A, M * K, a_scale, a_zero_point, thread_pool); - uint8_t* a_data_quant = static_cast(allocator->Alloc(SafeInt(M * K) * sizeof(uint8_t))); - BufferUniquePtr a_buffer_quant_holder(a_data_quant, BufferDeleter(allocator)); // quantize the data - ParQuantizeLinear(A, a_data_quant, M * K, a_scale, a_zero_point, thread_pool); + ParQuantizeLinear(A, quantized_A_buffer, M * K, a_scale, a_zero_point, thread_pool); bool b_is_signed = weights.quant_para_->is_signed; uint8_t b_zero_point = weights.quant_para_->zero_point ? *static_cast(weights.quant_para_->zero_point) : 0; @@ -277,11 +277,9 @@ void ComputeGemm(const int M, size_t ld_C_buffer = ldc; int32_t* C_buffer = reinterpret_cast(C); - BufferUniquePtr tmp_res_buffer_holder; if (beta == 1.0f) { - C_buffer = static_cast(allocator->Alloc(SafeInt(M * N) * sizeof(int32_t))); + C_buffer = quantize_agg_C_buffer; ld_C_buffer = static_cast(N); - tmp_res_buffer_holder = BufferUniquePtr(C_buffer, BufferDeleter(allocator)); } MLAS_QGEMM_SCALE_BIAS_OUTPUT_PROCESSOR output_processor( @@ -296,7 +294,7 @@ void ComputeGemm(const int M, gemm_shape.BIsSigned = b_is_signed; MLAS_GEMM_U8X8_DATA_PARAMS gemm_params; - gemm_params.A = a_data_quant; + gemm_params.A = quantized_A_buffer; gemm_params.lda = static_cast(K); gemm_params.ZeroPointA = a_zero_point; gemm_params.B = weights.buffer_; diff --git a/onnxruntime/core/providers/cpu/rnn/rnn_helpers.h b/onnxruntime/core/providers/cpu/rnn/rnn_helpers.h index 2b534c7295..e23d516350 100644 --- a/onnxruntime/core/providers/cpu/rnn/rnn_helpers.h +++ b/onnxruntime/core/providers/cpu/rnn/rnn_helpers.h @@ -227,7 +227,8 @@ void ComputeGemm(const int M, float* C, float* C_end, const int ldc, - AllocatorPtr /*allocator*/, + uint8_t* /* quantized_A_buffer */, + int32_t* /* quantize_agg_C_buffer */, concurrency::ThreadPool* thread_pool); void ComputeGemm(const int M, @@ -241,7 +242,8 @@ void ComputeGemm(const int M, float* C, float* C_end, const int ldc, - AllocatorPtr allocator, + uint8_t* quantized_A_buffer, + int32_t* quantize_agg_C_buffer, concurrency::ThreadPool* thread_pool); // helper to convert a span to a raw pointer diff --git a/onnxruntime/core/providers/cpu/rnn/uni_directional_lstm.cc b/onnxruntime/core/providers/cpu/rnn/uni_directional_lstm.cc index bbf4beddb1..39a2d86296 100644 --- a/onnxruntime/core/providers/cpu/rnn/uni_directional_lstm.cc +++ b/onnxruntime/core/providers/cpu/rnn/uni_directional_lstm.cc @@ -198,6 +198,20 @@ void UniDirectionalLstm::LoadBias(const gsl::span& WbRb_values) { */ } +template +template +void UniDirectionalLstm::AllocateQuantizeBuffers(int max_sequence_length) { + // Can not specialize on WeightT without specify T explicitly, so use sizeof + if (sizeof(WeightT) == 1) { + const int hidden_size_x4 = 4 * hidden_size_; + const int total_rows = max_sequence_length * batch_size_; + + int input_or_a_size = std::max(total_rows * input_size_, batch_size_ * hidden_size_); + quantized_input_or_a_ = Allocate(allocator_, input_or_a_size, quantized_input_or_a_ptr_, false); + quantized_C_buffer_ = Allocate(allocator_, batch_size_ * hidden_size_x4, quantized_C_buffer_ptr_, false); + } +} + template template void UniDirectionalLstm::Compute(const gsl::span& inputs_arg, @@ -247,8 +261,8 @@ void UniDirectionalLstm::Compute(const gsl::span& inputs_arg, // Calculate the max and min length const auto min_max_pair = std::minmax_element(sequence_lengths.cbegin(), sequence_lengths.cend()); - int32_t max_sequence_length = *min_max_pair.second; - int32_t min_sequence_length = std::min(seq_length_, *min_max_pair.first); + int max_sequence_length = *min_max_pair.second; + int min_sequence_length = std::min(seq_length_, *min_max_pair.first); ///**************************LSTM Calculations****************************/ float alpha = 1.0f; @@ -257,10 +271,15 @@ void UniDirectionalLstm::Compute(const gsl::span& inputs_arg, const int hidden_size_x4 = 4 * hidden_size_; const int total_rows = max_sequence_length * batch_size_; + AllocateQuantizeBuffers(max_sequence_length); + // apply the weights to all the inputs and save to output_IOFC ComputeGemm(total_rows, hidden_size_x4, input_size_, alpha, inputs.cbegin(), inputs.cend(), input_weights, - beta, output_iofc_.begin(), output_iofc_.end(), hidden_size_x4, allocator_, thread_pool_); + beta, output_iofc_.begin(), output_iofc_.end(), hidden_size_x4, + quantized_input_or_a_.begin(), + nullptr, + thread_pool_); DumpMatrix("Xt*(W[iofc]^T)", output_iofc_.data(), total_rows, hidden_size_x4); @@ -311,7 +330,10 @@ void UniDirectionalLstm::Compute(const gsl::span& inputs_arg, previous_state, previous_state_end, // Ht-1 recurrent_weights, // R[iofc] beta, step_out_IOFC, output_iofc_.end(), // input contains Xt*(W[iofc]^T) - hidden_size_x4, allocator_, ttp); + hidden_size_x4, + quantized_input_or_a_.begin() + (seq_start * hidden_size_), + quantized_C_buffer_.begin() + (seq_start * hidden_size_x4), + ttp); DumpMatrix("Xt*(W[iofc]^T) + Ht-t*R[iofc]" + row_str, &*step_out_IOFC, num_seq_to_compute_adjusted, hidden_size_x4); diff --git a/onnxruntime/core/providers/cpu/rnn/uni_directional_lstm.h b/onnxruntime/core/providers/cpu/rnn/uni_directional_lstm.h index 9c18b79c72..9eb71112d2 100644 --- a/onnxruntime/core/providers/cpu/rnn/uni_directional_lstm.h +++ b/onnxruntime/core/providers/cpu/rnn/uni_directional_lstm.h @@ -109,6 +109,17 @@ class UniDirectionalLstm { ActivationInfo activation_h_; concurrency::ThreadPool* thread_pool_; + + // Quantized operation related allocation members + template + void AllocateQuantizeBuffers(int max_sequence_length); + + // Buffer shared for quantized input whole, and quantized a each sequence step + IAllocatorUniquePtr quantized_input_or_a_ptr_; + gsl::span quantized_input_or_a_; + + IAllocatorUniquePtr quantized_C_buffer_ptr_; + gsl::span quantized_C_buffer_; }; } // namespace lstm From 3d12e957a7b99bbe41c2344f0dc6d859530e1ba1 Mon Sep 17 00:00:00 2001 From: Jesse Benson Date: Sun, 23 May 2021 12:23:08 -0700 Subject: [PATCH 02/47] Workaround for miopenReduceTensor() behavior difference in ROCm 4.2 --- .../core/providers/rocm/miopen_common.cc | 24 ++++++++ .../core/providers/rocm/miopen_common.h | 12 ++++ .../providers/rocm/reduction/reduction_ops.cc | 21 +++++-- .../cpu/reduction/reduction_ops_test.cc | 57 +++++++++++++++++++ 4 files changed, 110 insertions(+), 4 deletions(-) diff --git a/onnxruntime/core/providers/rocm/miopen_common.cc b/onnxruntime/core/providers/rocm/miopen_common.cc index 6c18b202a7..c473894cfb 100644 --- a/onnxruntime/core/providers/rocm/miopen_common.cc +++ b/onnxruntime/core/providers/rocm/miopen_common.cc @@ -88,5 +88,29 @@ const float Consts::Zero = 0; const float Consts::One = 1; +// As of ROCm 4.2, miopenReduceTensor() requires alpha/beta to be the same data +// type as the input type. This differs from cudnnReduceTensor() and other +// MIOpen/cuDNN APIs where alpha/beta are float when input type is half (float16). +// +// NOTE: this workaround can be removed in ROCm 4.3: +// https://github.com/ROCmSoftwarePlatform/MIOpen/pull/914 +template <> +const half ReduceConsts::One = 1.f; + +template <> +const float ReduceConsts::One = 1; + +template <> +const double ReduceConsts::One = 1; + +template <> +const half ReduceConsts::Zero = 0.f; + +template <> +const float ReduceConsts::Zero = 0; + +template <> +const double ReduceConsts::Zero = 0; + } // namespace rocm } // namespace onnxruntime diff --git a/onnxruntime/core/providers/rocm/miopen_common.h b/onnxruntime/core/providers/rocm/miopen_common.h index 73d865dcfd..32491b0cb8 100644 --- a/onnxruntime/core/providers/rocm/miopen_common.h +++ b/onnxruntime/core/providers/rocm/miopen_common.h @@ -44,5 +44,17 @@ struct Consts { static const float One; }; +// As of ROCm 4.2, miopenReduceTensor() requires alpha/beta to be the same data +// type as the input type. This differs from cudnnReduceTensor() and other +// MIOpen/cuDNN APIs where alpha/beta are float when input type is half (float16). +// +// NOTE: this workaround can be removed in ROCm 4.3: +// https://github.com/ROCmSoftwarePlatform/MIOpen/pull/914 +template +struct ReduceConsts { + static const ElemType Zero; + static const ElemType One; +}; + } // namespace rocm } // namespace onnxruntime diff --git a/onnxruntime/core/providers/rocm/reduction/reduction_ops.cc b/onnxruntime/core/providers/rocm/reduction/reduction_ops.cc index 886a8ab2c6..907e9404d8 100644 --- a/onnxruntime/core/providers/rocm/reduction/reduction_ops.cc +++ b/onnxruntime/core/providers/rocm/reduction/reduction_ops.cc @@ -185,8 +185,15 @@ Status ReduceKernel::ReduceKernelShared( ORT_RETURN_IF_ERROR(reduce_desc.Set(miopen_reduce_op, MiopenTensor::GetDataType(), ReduceTensorIndices)); else ORT_RETURN_IF_ERROR(reduce_desc.Set(miopen_reduce_op, miopen_type_X, ReduceTensorIndices)); - const auto one = Consts::One; - const auto zero = Consts::Zero; + + // As of ROCm 4.2, miopenReduceTensor() requires alpha/beta to be the same data + // type as the input type. This differs from cudnnReduceTensor() and other + // MIOpen/cuDNN APIs where alpha/beta are float when input type is half (float16). + // + // NOTE: this workaround can be removed in ROCm 4.3: + // https://github.com/ROCmSoftwarePlatform/MIOpen/pull/914 + const auto one = ReduceConsts::One; + const auto zero = ReduceConsts::Zero; MiopenTensor input_tensor; MiopenTensor output_tensor; ORT_RETURN_IF_ERROR(input_tensor.Set(input_dims_miopen, miopen_type_X)); @@ -483,8 +490,14 @@ Status ReduceComputeCore(ROCMExecutionProvider& rocm_ep, const Tensor& input, Pr ORT_RETURN_IF_ERROR(reduce_desc.Set(miopen_reduce_op, miopen_type_X, ReduceTensorIndices)); } - const auto one = Consts::One; - const auto zero = Consts::Zero; + // As of ROCm 4.2, miopenReduceTensor() requires alpha/beta to be the same data + // type as the input type. This differs from cudnnReduceTensor() and other + // MIOpen/cuDNN APIs where alpha/beta are float when input type is half (float16). + // + // NOTE: this workaround can be removed in ROCm 4.3: + // https://github.com/ROCmSoftwarePlatform/MIOpen/pull/914 + const auto one = ReduceConsts::One; + const auto zero = ReduceConsts::Zero; MiopenTensor input_tensor; MiopenTensor output_tensor; ORT_RETURN_IF_ERROR(input_tensor.Set(input_dims_miopen, miopen_type_X)); diff --git a/onnxruntime/test/providers/cpu/reduction/reduction_ops_test.cc b/onnxruntime/test/providers/cpu/reduction/reduction_ops_test.cc index 799412b42a..01b0671261 100644 --- a/onnxruntime/test/providers/cpu/reduction/reduction_ops_test.cc +++ b/onnxruntime/test/providers/cpu/reduction/reduction_ops_test.cc @@ -557,6 +557,25 @@ TEST(ReductionOpTest, ReduceLogSumExp_double) { test.Run(); } +#if defined(USE_CUDA) || defined(USE_ROCM) +TEST(ReductionOpTest, ReduceLogSumExp_half) { + OpTester test("ReduceLogSumExp"); + test.AddAttribute("axes", std::vector{0, 2}); + test.AddAttribute("keepdims", (int64_t)1); + test.AddInput("data", {3, 2, 2}, + FloatsToMLFloat16s({1.0f, 2.0f, + 3.0f, 4.0f, + + 5.0f, 6.0f, + 7.0f, 8.0f, + + 9.0f, 10.0f, + 11.0f, 12.0f})); + test.AddOutput("reduced", {1, 2, 1}, FloatsToMLFloat16s({10.33174133f, 12.33174133f})); + test.Run(); +} +#endif // defined(USE_CUDA) || defined(USE_ROCM) + TEST(ReductionOpTest, ReduceLogSumExp_int32) { OpTester test("ReduceLogSumExp"); test.AddAttribute("axes", std::vector{0, 2}); @@ -700,6 +719,25 @@ TEST(ReductionOpTest, ReduceMax_double) { test.Run(); } +#if defined(USE_CUDA) || defined(USE_ROCM) +TEST(ReductionOpTest, ReduceMax_half) { + OpTester test("ReduceMax"); + test.AddAttribute("axes", std::vector{1, 2}); + test.AddAttribute("keepdims", (int64_t)1); + test.AddInput("data", {3, 2, 2}, + FloatsToMLFloat16s({1.0f, 2.0f, + 3.0f, 4.0f, + + 5.0f, 6.0f, + 7.0f, 8.0f, + + 9.0f, 10.0f, + 11.0f, 12.0f})); + test.AddOutput("reduced", {3, 1, 1}, FloatsToMLFloat16s({4.0f, 8.0f, 12.0f})); + test.Run(); +} +#endif // defined(USE_CUDA) || defined(USE_ROCM) + TEST(ReductionOpTest, ReduceMax_int32) { OpTester test("ReduceMax"); test.AddAttribute("axes", std::vector{1, 2}); @@ -1167,6 +1205,25 @@ TEST(ReductionOpTest, ReduceMin_double) { test.Run(); } +#if defined(USE_CUDA) || defined(USE_ROCM) +TEST(ReductionOpTest, ReduceMin_half) { + OpTester test("ReduceMin"); + test.AddAttribute("axes", std::vector{0, 2}); + test.AddAttribute("keepdims", (int64_t)1); + test.AddInput("data", {3, 2, 2}, + FloatsToMLFloat16s({1.0f, 2.0f, + 3.0f, 4.0f, + + 5.0f, 6.0f, + 7.0f, 8.0f, + + 9.0f, 10.0f, + 11.0f, 12.0f})); + test.AddOutput("reduced", {1, 2, 1}, FloatsToMLFloat16s({1.0f, 3.0f})); + test.Run(); +} +#endif // defined(USE_CUDA) || defined(USE_ROCM) + TEST(ReductionOpTest, ReduceMin_int32) { OpTester test("ReduceMin"); test.AddAttribute("axes", std::vector{0, 2}); From 29c68888af36941fd0653232a0848704ca7d0f5a Mon Sep 17 00:00:00 2001 From: Jesse Benson Date: Mon, 24 May 2021 12:33:54 -0700 Subject: [PATCH 03/47] Update BERT convergence baseline. --- .../bert_base.convergence.baseline.mi100.csv | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/orttraining/tools/ci_test/results/bert_base.convergence.baseline.mi100.csv b/orttraining/tools/ci_test/results/bert_base.convergence.baseline.mi100.csv index 0bdb749881..8d027b983d 100644 --- a/orttraining/tools/ci_test/results/bert_base.convergence.baseline.mi100.csv +++ b/orttraining/tools/ci_test/results/bert_base.convergence.baseline.mi100.csv @@ -1,11 +1,11 @@ step,total_loss,mlm_loss,nsp_loss -0,11.217,10.5178,0.699256 -5,9.67644,7.52047,2.15598 -10,8.31964,7.54136,0.778281 -15,8.22823,7.54625,0.681978 -20,8.17299,7.49675,0.676236 -25,8.2415,7.5356,0.705902 -30,8.0874,7.39312,0.694279 -35,7.99095,7.25612,0.734829 -40,7.92988,7.25608,0.673804 -45,7.94762,7.27291,0.674713 +0,11.2171,10.5178,0.699279 +5,9.6935,7.51946,2.17404 +10,8.72874,7.60452,1.12422 +15,8.25456,7.54113,0.713431 +20,8.17125,7.47469,0.696562 +25,8.21603,7.52277,0.693259 +30,8.08864,7.39777,0.69087 +35,7.9672,7.25153,0.715668 +40,7.94141,7.25788,0.683527 +45,7.94186,7.27316,0.668707 From 6ca1ee77332ab89ec60064ea4542fa68fed71b3c Mon Sep 17 00:00:00 2001 From: Pranav Sharma Date: Tue, 25 May 2021 17:36:15 -0700 Subject: [PATCH 04/47] Fix rpath issue with pybind. (#7829) * Fix rpath issue with pybind * Address PR comment --- cmake/onnxruntime.cmake | 3 +++ cmake/onnxruntime_python.cmake | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/cmake/onnxruntime.cmake b/cmake/onnxruntime.cmake index 256e07023a..b542490cee 100644 --- a/cmake/onnxruntime.cmake +++ b/cmake/onnxruntime.cmake @@ -75,6 +75,9 @@ elseif(onnxruntime_BUILD_APPLE_FRAMEWORK) ) else() onnxruntime_add_shared_library(onnxruntime ${CMAKE_CURRENT_BINARY_DIR}/generated_source.c) + if (onnxruntime_USE_CUDA) + set_property(TARGET onnxruntime APPEND_STRING PROPERTY LINK_FLAGS " -Xlinker -rpath=\\$ORIGIN") + endif() endif() add_dependencies(onnxruntime onnxruntime_generate_def ${onnxruntime_EXTERNAL_DEPENDENCIES}) diff --git a/cmake/onnxruntime_python.cmake b/cmake/onnxruntime_python.cmake index 419fdca22c..16103dca5d 100644 --- a/cmake/onnxruntime_python.cmake +++ b/cmake/onnxruntime_python.cmake @@ -143,7 +143,7 @@ elseif (APPLE) INSTALL_RPATH_USE_LINK_PATH FALSE) else() target_link_libraries(onnxruntime_pybind11_state PRIVATE ${onnxruntime_pybind11_state_libs} ${onnxruntime_EXTERNAL_LIBRARIES}) - set_property(TARGET onnxruntime_pybind11_state APPEND_STRING PROPERTY LINK_FLAGS " -Xlinker -rpath=\$ORIGIN") + set_property(TARGET onnxruntime_pybind11_state APPEND_STRING PROPERTY LINK_FLAGS " -Xlinker -rpath=\\$ORIGIN") endif() set_target_properties(onnxruntime_pybind11_state PROPERTIES PREFIX "") From ea1a4f8fb5e00839e225853e68661b43d95dbd5f Mon Sep 17 00:00:00 2001 From: Tixxx Date: Tue, 25 May 2021 17:43:43 -0700 Subject: [PATCH 05/47] [JS]support running super resolution model using ortweb (#7677) * migrated changes to support running super resolution model using ortweb * reverted benchmarking tool related changes which will be in a separate pr * added kernel tests to op and node tests * minor change to the order of variables * added one more unit test for packed matmul --- .../backends/webgl/glsl-coordinate-lib.ts | 101 ++++++- .../onnxjs/backends/webgl/op-resolve-rules.ts | 2 + .../onnxjs/backends/webgl/ops/conv-pack.ts | 12 +- .../backends/webgl/ops/depth-to-space.ts | 81 ++++++ .../onnxjs/backends/webgl/ops/matmul-pack.ts | 70 ++--- .../onnxjs/backends/webgl/webgl-context.ts | 4 +- js/web/lib/onnxjs/ops/depth-to-space.ts | 53 ++++ js/web/test/data/ops/depth-to-space.jsonc | 133 +++++++++ js/web/test/test-suite-whitelist.jsonc | 5 + .../backends/webgl/test-depth-to-space.ts | 177 ++++++++++++ .../backends/webgl/test-matmul-packed.ts | 262 ++++++++++++++++++ .../backends/webgl/test-pack-unpack.ts | 32 +++ .../unittests/backends/webgl/test-utils.ts | 10 + js/web/test/unittests/index.ts | 2 + 14 files changed, 903 insertions(+), 41 deletions(-) create mode 100644 js/web/lib/onnxjs/backends/webgl/ops/depth-to-space.ts create mode 100644 js/web/lib/onnxjs/ops/depth-to-space.ts create mode 100644 js/web/test/data/ops/depth-to-space.jsonc create mode 100644 js/web/test/unittests/backends/webgl/test-depth-to-space.ts create mode 100644 js/web/test/unittests/backends/webgl/test-matmul-packed.ts diff --git a/js/web/lib/onnxjs/backends/webgl/glsl-coordinate-lib.ts b/js/web/lib/onnxjs/backends/webgl/glsl-coordinate-lib.ts index cbe0bf5724..a43dfd682e 100644 --- a/js/web/lib/onnxjs/backends/webgl/glsl-coordinate-lib.ts +++ b/js/web/lib/onnxjs/backends/webgl/glsl-coordinate-lib.ts @@ -154,6 +154,10 @@ export class CoordsGlslLib extends GlslLib { result[funcName] = this.getOutputUnpacked5DCoords( outShape as [number, number, number, number, number], outTexShape as [number, number]); break; + case 6: + result[funcName] = this.getOutputUnpacked6DCoords( + outShape as [number, number, number, number, number, number], outTexShape as [number, number]); + break; default: throw new Error(`Unsupported output dimensionality: ${outShape.length}`); } @@ -472,9 +476,9 @@ export class CoordsGlslLib extends GlslLib { source = ` ivec5 getOutputCoords() { - ivec2 resTexRC = ivec2(TexCoords.yx * + ivec2 resTexRC = ivec2(TexCoords.xy * vec2(${texShape[0]}, ${texShape[1]})); - int index = resTexRC.y * ${texShape[1]} + resTexRC.x; + int index = resTexRC.y * ${texShape[0]} + resTexRC.x; ${coordsFromIndexSnippet} return ivec5(r, c, d, d2, d3); } @@ -482,6 +486,49 @@ export class CoordsGlslLib extends GlslLib { return new GlslLibRoutine(source); } + /** + * Unpacked 6D output coordinates. + */ + protected getOutputUnpacked6DCoords(shape: [number, number, number, number, number, number], texShape: [ + number, number + ]): GlslLibRoutine { + let source = ''; + const rank = shape.length; + + let strides = null; + if (rank < 2) { + strides = []; + } + + strides = new Array(rank - 1); + strides[rank - 2] = shape[rank - 1]; + for (let i = rank - 3; i >= 0; --i) { + strides[i] = strides[i + 1] * shape[i + 1]; + } + const coordsToCompute = ['r', 'c', 'd', 'd2', 'd3', 'd4']; + const coordsFromIndexSnippet = + strides + .map((stride, i) => { + const line1 = `int ${coordsToCompute[i]} = index / ${stride}`; + const line2 = i === strides.length - 1 ? + `int ${coordsToCompute[i + 1]} = index - ${coordsToCompute[i]} * ${stride}` : + `index -= ${coordsToCompute[i]} * ${stride}`; + return `${line1}; ${line2};`; + }) + .join(''); + + source = ` + ivec6 getOutputCoords() { + ivec2 resTexRC = ivec2(TexCoords.xy * + vec2(${texShape[0]}, ${texShape[1]})); + int index = resTexRC.y * ${texShape[0]} + resTexRC.x; + ${coordsFromIndexSnippet} + return ivec6(r, c, d, d2, d3, d4); + } + `; + return new GlslLibRoutine(source); + } + /** * Generates code for common UV coords computation utility functions. */ @@ -730,6 +777,8 @@ export class CoordsGlslLib extends GlslLib { return this.getUnpackedSampler4D(funcName, name, inputLayout); case 5: return this.getUnpackedSampler5D(funcName, name, inputLayout); + case 6: + return this.getUnpackedSampler6D(funcName, name, inputLayout); default: // TODO support more dimensionalities throw new Error(`Unsupported dimension ${shape.length}-D`); @@ -1092,8 +1141,7 @@ export class CoordsGlslLib extends GlslLib { return sampleTexture(${name}, uv); } `; - return new GlslLibRoutine( - source, ['coordinates.uvFromFlat', 'coordinates.sampleTexture', 'coordinates.coordsToOffset']); + return new GlslLibRoutine(source, ['coordinates.uvFromFlat', 'coordinates.sampleTexture']); } /** @@ -1133,6 +1181,49 @@ export class CoordsGlslLib extends GlslLib { return sampleTexture(${name}, uv); } `; + return new GlslLibRoutine(source, ['coordinates.sampleTexture', 'coordinates.uvFromFlat']); + } + + /** + * Unpacked 6D snippet. + */ + protected getUnpackedSampler6D(funcName: string, name: string, inputLayout: TextureLayout): GlslLibRoutine { + const shape = inputLayout.unpackedShape; + const stride4 = shape[5]; + const stride3 = shape[4] * stride4; + const stride2 = shape[3] * stride3; + const stride1 = shape[2] * stride2; + const stride0 = shape[1] * stride1; + + const {newShape, keptDims} = squeezeShape(shape as number[]); + if (newShape.length < shape.length) { + const newInputShape = squeezeInputShape(shape, newShape); + const params = ['row', 'col', 'depth', 'depth2', 'depth3', 'depth4']; + // Deep copy of input texture layout. + const newInputLayout: TextureLayout = JSON.parse(JSON.stringify(inputLayout)); + newInputLayout.unpackedShape = newInputShape; + + const source = ` + ${this.getUnpackedSamplerFromInput(funcName, name, newInputLayout).routineBody} + float ${funcName}(int row, int col, int depth, + int depth2, int depth3, int depth4) { + return ${funcName}(${getSqueezedParams(params, keptDims)}); + } + `; + return new GlslLibRoutine(source, ['coordinates.sampleTexture', 'coordinates.uvFromFlat']); + } + + const texNumR = inputLayout.width; + const texNumC = inputLayout.height; + const source = ` + float ${funcName}(int row, int col, int depth, + int depth2, int depth3, int depth4) { + int index = row * ${stride0} + col * ${stride1} + depth * ${stride2} + + depth2 * ${stride3} + depth3 * ${stride4} + depth4; + vec2 uv = uvFromFlat(${texNumR}, ${texNumC}, index); + return sampleTexture(${name}, uv); + } + `; return new GlslLibRoutine( source, ['coordinates.uvFromFlat', 'coordinates.sampleTexture', 'coordinates.coordsToOffset']); } @@ -1181,7 +1272,7 @@ export class CoordsGlslLib extends GlslLib { const result: {[name: string]: GlslLibRoutine} = {}; this.context.programInfo.samplers.forEach((name, i) => { const layout = programInfo.inputLayouts[i]; - const shape = layout.shape; + const shape = layout.unpackedShape.length > 0 ? layout.unpackedShape : layout.shape; const rank = shape.length; let funcName = `_${name}`; result[funcName] = new GlslLibRoutine( diff --git a/js/web/lib/onnxjs/backends/webgl/op-resolve-rules.ts b/js/web/lib/onnxjs/backends/webgl/op-resolve-rules.ts index 7e7bc5abf6..6dd11c91b9 100644 --- a/js/web/lib/onnxjs/backends/webgl/op-resolve-rules.ts +++ b/js/web/lib/onnxjs/backends/webgl/op-resolve-rules.ts @@ -9,6 +9,7 @@ import * as binaryOps from './ops/binary-op'; import {WebGLClip} from './ops/clip'; import {WebGLConcat} from './ops/concat'; import {WebGLConv} from './ops/conv'; +import {WebGLDepthToSpace} from './ops/depth-to-space'; import {WebGLDropout} from './ops/dropout'; import {WebGLElu} from './ops/elu'; import {WebGLFlatten} from './ops/flatten'; @@ -50,6 +51,7 @@ export const WEBGL_OP_RESOLVE_RULES: readonly OpSet.ResolveRule[] = [ ['Cos', '', '7+', () => new unaryOps.WebGLUnaryOp(FLOAT_TYPES, unaryOps.glslCos())], ['Div', '', '7+', () => new binaryOps.WebGLBinaryOp(NUMBER_TYPES, binaryOps.glslDiv())], ['Dropout', '', '7+', () => new WebGLDropout()], + ['DepthToSpace', '', '1+', () => new WebGLDepthToSpace()], ['Equal', '', '7+', () => new binaryOps.WebGLBinaryOp(NUMBER_TYPES, binaryOps.glslEqual(), undefined, 'bool')], ['Elu', '', '6+', () => new WebGLElu()], ['Exp', '', '6+', () => new unaryOps.WebGLUnaryOp(FLOAT_TYPES, unaryOps.glslExp())], diff --git a/js/web/lib/onnxjs/backends/webgl/ops/conv-pack.ts b/js/web/lib/onnxjs/backends/webgl/ops/conv-pack.ts index 08ec52430a..68c962b5e5 100644 --- a/js/web/lib/onnxjs/backends/webgl/ops/conv-pack.ts +++ b/js/web/lib/onnxjs/backends/webgl/ops/conv-pack.ts @@ -16,6 +16,8 @@ import {WebGLReshapePacked} from './reshape-packed'; export class WebGLConvPacked extends Conv { protected artifacts: Artifact[]; protected programInfo: ProgramInfo[]; + protected outputShape: number[]; + private kernelReshape = new WebGLReshapePacked(); private im2col: WebGLIm2ColPacked; private matmul = new WebGLMatMulPacked(); @@ -38,9 +40,11 @@ export class WebGLConvPacked extends Conv { `autpPad:${this.autoPad}, dilations:${this.dilations}, group:${this.group}, kernelShape:${ this.kernelShape}, pads:${this.pads}, strides:${this.strides}`); - const outputShape = WebGLConv.calcOutputShape(xshape, kshape, this.dilations, this.pads, this.strides); + if (!this.outputShape) { + this.outputShape = WebGLConv.calcOutputShape(xshape, kshape, this.dilations, this.pads, this.strides); + } if (this.im2col === undefined) { - this.im2col = new WebGLIm2ColPacked(outputShape, kshape, this.dilations, this.pads, this.strides); + this.im2col = new WebGLIm2ColPacked(this.outputShape, kshape, this.dilations, this.pads, this.strides); } if (this.activation) { const attributes = new Attribute(undefined); @@ -90,8 +94,8 @@ export class WebGLConvPacked extends Conv { // reshape output const outputShapeTensor = new Tensor( - [outputShape.length], 'int32', undefined, undefined, - new Int32Array([outputShape[0], outputShape[1], outputShape[2], outputShape[3]])); + [this.outputShape.length], 'int32', undefined, undefined, + new Int32Array([this.outputShape[0], this.outputShape[1], this.outputShape[2], this.outputShape[3]])); assert(this.artifacts.length > 2, () => 'expect at least 3 artifacts created'); if (this.artifacts.length === 3) { diff --git a/js/web/lib/onnxjs/backends/webgl/ops/depth-to-space.ts b/js/web/lib/onnxjs/backends/webgl/ops/depth-to-space.ts new file mode 100644 index 0000000000..75b461e256 --- /dev/null +++ b/js/web/lib/onnxjs/backends/webgl/ops/depth-to-space.ts @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import {Attribute} from '../../../attribute'; +import {DepthToSpace} from '../../../ops/depth-to-space'; +import {Tensor} from '../../../tensor'; +import {WebGLInferenceHandler} from '../inference-handler'; +import {Artifact, ProgramInfo, RunData} from '../types'; + +import {reshape} from './reshape'; +import {WebGLTranspose} from './transpose'; + +export class WebGLDepthToSpace extends DepthToSpace { + protected transposeProgramInfo: ProgramInfo; + + protected transposeArtifact: Artifact; + + run(inferenceHandler: WebGLInferenceHandler, inputs: Tensor[]): Tensor[] { + const programManager = inferenceHandler.session.programManager; + const transposePerm = this.mode === 'DCR' ? [0, 3, 4, 1, 5, 2] : [0, 1, 4, 2, 5, 3]; + const firstReshapeShape = this.mode === 'DCR' ? + [ + inputs[0].dims[0], this.blocksize, this.blocksize, inputs[0].dims[1] / this.blocksizeSqr, inputs[0].dims[2], + inputs[0].dims[3] + ] : + [ + inputs[0].dims[0], inputs[0].dims[1] / this.blocksizeSqr, this.blocksize, this.blocksize, inputs[0].dims[2], + inputs[0].dims[3] + ]; + + const transpose = new WebGLTranspose(); + const attributes = new Attribute(undefined); + attributes.set('perm', 'ints', transposePerm); + transpose.initialize(attributes); + + // First reshape + + const firstReshapedTensor = reshape(inferenceHandler, inputs[0], firstReshapeShape); + + // transpose + if (!this.transposeProgramInfo) { + this.transposeProgramInfo = transpose.createProgramInfo(inferenceHandler, [firstReshapedTensor]); + this.transposeArtifact = programManager.build(this.transposeProgramInfo); + } + const runDataTranspose = + transpose.createRunData(inferenceHandler, this.transposeProgramInfo, [firstReshapedTensor]); + inferenceHandler.checkAndUpdateTextureForm(this.transposeArtifact, runDataTranspose); + programManager.run(this.transposeArtifact, runDataTranspose); + const transposeOutput = runDataTranspose.outputTextureData.tensor; + + // Second reshape + const result = reshape(inferenceHandler, transposeOutput, [ + inputs[0].dims[0], inputs[0].dims[1] / this.blocksizeSqr, inputs[0].dims[2] * this.blocksize, + inputs[0].dims[3] * this.blocksize + ]); + return [result]; + } + + protected getOutShape(input: Tensor): number[] { + const batchSize = input.dims[0]; + const inputDepth = input.dims[1]; + const inputHeight = input.dims[2]; + const inputWidth = input.dims[3]; + if (inputDepth % (this.blocksizeSqr) !== 0) { + throw new Error('Input depth must be divisible by squared blocksize.'); + } + const outputDepth = inputDepth / this.blocksizeSqr; + const outputHeight = inputHeight * this.blocksize; + const outputWidth = inputWidth * this.blocksize; + return [batchSize, outputDepth, outputHeight, outputWidth]; + } + + createRunData(handler: WebGLInferenceHandler, programInfo: ProgramInfo, inputs: Tensor[]): RunData { + const inputTDs = inputs.map((t, i) => handler.getOrCreateTextureData(t, programInfo.inputLayouts[i])); + return { + inputTextureDatas: inputTDs, + outputTextureData: handler.createTextureDataFromLayout(programInfo.outputLayout, inputTDs[0].tensor.type), + uniformData: {} + }; + } +} \ No newline at end of file diff --git a/js/web/lib/onnxjs/backends/webgl/ops/matmul-pack.ts b/js/web/lib/onnxjs/backends/webgl/ops/matmul-pack.ts index 70535184db..59e052e079 100644 --- a/js/web/lib/onnxjs/backends/webgl/ops/matmul-pack.ts +++ b/js/web/lib/onnxjs/backends/webgl/ops/matmul-pack.ts @@ -4,8 +4,11 @@ import {MatMul} from '../../../ops/matmul'; import {Tensor} from '../../../tensor'; import {BroadcastUtil} from '../../../util'; +import {getGlsl} from '../glsl-source'; import {WebGLInferenceHandler} from '../inference-handler'; import {ProgramInfo, RunData, WebGLOperator} from '../types'; +import {getCoordsDataType} from '../utils'; + import {getActicationSnippet} from './fuse-utils'; export class WebGLMatMulPacked extends MatMul implements WebGLOperator { @@ -14,7 +17,7 @@ export class WebGLMatMulPacked extends MatMul implements WebGLOperator { } createProgramInfo(handler: WebGLInferenceHandler, inputs: Tensor[]): ProgramInfo { const hasBias = inputs.length > 2; - const processBias = hasBias ? 'value += vec4(getBias(a[0]*2).xx, getBias(a[0]*2).yy);' : ''; + const processBias = hasBias ? 'result += getBiasAtOutCoords();' : ''; const aShape = inputs[0].dims; const bShape = inputs[1].dims; const outputShape = BroadcastUtil.calcShape(aShape, bShape, true); @@ -22,31 +25,35 @@ export class WebGLMatMulPacked extends MatMul implements WebGLOperator { if (!outputShape) { throw new Error('Can\'t use matmul on the given tensors'); } - const rank = outputShape.length; + const sharedDim = aShape[aShape.length - 1]; + const sharedDimIndex = Math.ceil(sharedDim / 2); const aRank = aShape.length; const bRank = bShape.length; - const sharedDim = aShape[aShape.length - 1]; + + const glsl = getGlsl(handler.session.backend.glContext.version); + const coordsDataType = getCoordsDataType(outputShape.length); + const outRank = outputShape.length; + const allGlChannels = ['x', 'y', 'z', 'w', 'u', 'v']; const {activationFunction, applyActivation} = getActicationSnippet(this.activation); - // TODO:fix broadcasting const shaderSource = ` ${activationFunction} - vec4 process(int indices[${rank}]) { - int a[${aRank}]; - int b[${bRank}]; - bcastMatmulIndices_A(indices, a); - bcastMatmulIndices_B(indices, b); + void main() { + ${coordsDataType} rc = getOutputCoords(); + int lastDim = rc.${allGlChannels[outRank - 1]}; + rc.${allGlChannels[outRank - 1]} = rc.${allGlChannels[outRank - 2]}; + rc.${allGlChannels[outRank - 2]} = lastDim; - vec4 value; - for (int k=0; k<((${sharedDim}+1)/2); ++k) { - a[${aRank - 1}] = k; - b[${bRank - 2}] = k; - value += ${getA(aRank)}.rrbb * ${getB(bRank)}.rgrg; - value += ${getA(aRank)}.ggaa * ${getB(bRank)}.baba; - } - ${processBias} - ${applyActivation} - return value; + vec4 result = vec4(0); + for (int i = 0; i < ${sharedDimIndex}; i++) { + vec4 a = getA(${getA(allGlChannels, aRank)}); + vec4 b = getB(${getB(allGlChannels, bRank)}); + result += (a.rrbb * b.rgrg); + result += (a.ggaa * b.baba); + } + ${processBias} + ${applyActivation} + ${glsl.output} = result; }`; return { name: 'WebGLMatMulPacked', @@ -55,6 +62,7 @@ export class WebGLMatMulPacked extends MatMul implements WebGLOperator { handler.createTextureLayoutFromShape(outputShape, 4, outputShape, {isPacked: true, reverseWH: true}), samplers: hasBias ? ['A', 'B', 'Bias'] : ['A', 'B'], shaderSource, + hasMain: true, expectPackedInputs: true, expectPackedOutputs: true, }; @@ -70,22 +78,22 @@ export class WebGLMatMulPacked extends MatMul implements WebGLOperator { } } -function getA(outputRank: number): string { - let res = 'getA('; - for (let i = 0; i < outputRank - 2; i++) { - res += `a[${i}], `; +function getA(allGlChannels: string[], rank: number): string { + let res = ''; + for (let i = 0; i < rank - 2; i++) { + res += `rc.${allGlChannels[i]}, `; } - res += `a[${outputRank - 2}]*2, ` + - 'k*2)'; + res += `rc.${allGlChannels[rank - 2]}, ` + + 'i<<1'; return res; } -function getB(outputRank: number): string { - let res = 'getB('; - for (let i = 0; i < outputRank - 2; i++) { - res += `b[${i}], `; +function getB(allGlChannels: string[], rank: number): string { + let res = ''; + for (let i = 0; i < rank - 2; i++) { + res += `rc.${allGlChannels[i]}, `; } - res += 'k*2, ' + - `b[${outputRank - 1}]*2)`; + res += 'i<<1, ' + + `rc.${allGlChannels[rank - 1]}`; return res; } diff --git a/js/web/lib/onnxjs/backends/webgl/webgl-context.ts b/js/web/lib/onnxjs/backends/webgl/webgl-context.ts index c859fddc5f..98dd6912aa 100644 --- a/js/web/lib/onnxjs/backends/webgl/webgl-context.ts +++ b/js/web/lib/onnxjs/backends/webgl/webgl-context.ts @@ -176,7 +176,9 @@ export class WebGLContext { gl.shaderSource(shader, shaderSource); gl.compileShader(shader); if (gl.getShaderParameter(shader, gl.COMPILE_STATUS) === false) { - throw new Error(`Failed to compile shader: ${gl.getShaderInfoLog(shader)}`); + throw new Error(`Failed to compile shader: ${gl.getShaderInfoLog(shader)} +Shader source: +${shaderSource}`); } return shader; } diff --git a/js/web/lib/onnxjs/ops/depth-to-space.ts b/js/web/lib/onnxjs/ops/depth-to-space.ts new file mode 100644 index 0000000000..ac3e5f3250 --- /dev/null +++ b/js/web/lib/onnxjs/ops/depth-to-space.ts @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import {Attribute} from '../attribute'; +import {InferenceHandler} from '../backend'; +import {Operator} from '../operators'; +import {Tensor} from '../tensor'; + +export abstract class DepthToSpace implements Operator { + constructor() {} + + abstract run(inferenceHandler: InferenceHandler, inputs: Tensor[]): Tensor[]|Promise; + + initialize(attributes: Attribute): void { + // processing node attributes + this.blocksize = attributes.getInt('blocksize'); + if (this.blocksize < 1) { + throw new Error(`blocksize must be >= 1, but got : ${this.blocksize} for DepthToSpace`); + } + this.blocksizeSqr = this.blocksize * this.blocksize; + this.mode = attributes.getString('mode', 'DCR'); + if (DepthToSpace.supportedModes.indexOf(this.mode) < 0) { + throw new Error(`unrecognized mode: ${this.mode} for DepthToSpace`); + } + } + + checkInputs(inputs: Tensor[]): boolean { + if (!inputs || inputs.length !== 1) { + return false; + } + + return this.checkInputTypes(inputs); + } + + protected checkInputTypes(inputs: Tensor[]): boolean { + const inputType = inputs[0].type; + const inputDimensionality = inputs[0].dims.length; + + // Input has to be a 4-D tensor + // TODO: Support string depth-to-space. + if (inputType === 'string' || inputDimensionality !== 4) { + return false; + } + + return true; + } + + protected mode: string; + protected blocksize: number; + protected blocksizeSqr: number; + + private static readonly supportedModes = ['DCR', 'CRD']; +} \ No newline at end of file diff --git a/js/web/test/data/ops/depth-to-space.jsonc b/js/web/test/data/ops/depth-to-space.jsonc new file mode 100644 index 0000000000..22425b5472 --- /dev/null +++ b/js/web/test/data/ops/depth-to-space.jsonc @@ -0,0 +1,133 @@ +[ + { + "name": "Depth-to-space of a 4-D tensor with default mode", + "operator": "DepthToSpace", + "attributes": [{ "name": "blocksize", "data": 2, "type": "int" }], + "cases": [ + { + "name": "4D reshape [1, 8, 1, 1]", + "inputs": [ + { + "data": [0, 9, 18, 27, 36, 45, 54, 63], + "dims": [1, 8, 1, 1], + "type": "float32" + } + ], + "outputs": [ + { + "data": [0, 18, 36, 54, 9, 27, 45, 63], + "dims": [1, 2, 2, 2], + "type": "float32" + } + ] + }, + { + "name": "4D reshape [2, 8, 1, 2]", + "inputs": [ + { + "data": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, + 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], + "dims": [2, 8, 1, 2], + "type": "float32" + } + ], + "outputs": [ + { + "data": [0, 4, 1, 5, 8, 12, 9, 13, 2, 6, 3, 7, 10, 14, 11, 15, 16, 20, + 17, 21, 24, 28, 25, 29, 18, 22, 19, 23, 26, 30, 27, 31], + "dims": [2, 2, 2, 4], + "type": "float32" + } + ] + } + ] + }, + { + "name": "Depth-to-space of a 4-D tensor with DCR mode", + "operator": "DepthToSpace", + "attributes": [{ "name": "mode", "data": "DCR", "type": "string" }, + { "name": "blocksize", "data": 2, "type": "int" }], + "cases": [ + { + "name": "4D reshape [1, 8, 1, 1]", + "inputs": [ + { + "data": [0, 9, 18, 27, 36, 45, 54, 63], + "dims": [1, 8, 1, 1], + "type": "float32" + } + ], + "outputs": [ + { + "data": [0, 18, 36, 54, 9, 27, 45, 63], + "dims": [1, 2, 2, 2], + "type": "float32" + } + ] + }, + { + "name": "4D reshape [2, 8, 1, 2]", + "inputs": [ + { + "data": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, + 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], + "dims": [2, 8, 1, 2], + "type": "float32" + } + ], + "outputs": [ + { + "data": [0, 4, 1, 5, 8, 12, 9, 13, 2, 6, 3, 7, 10, 14, 11, 15, 16, 20, + 17, 21, 24, 28, 25, 29, 18, 22, 19, 23, 26, 30, 27, 31], + "dims": [2, 2, 2, 4], + "type": "float32" + } + ] + } + ] + }, + { + "name": "Depth-to-space of a 4-D tensor with CRD mode", + "operator": "DepthToSpace", + "attributes": [{ "name": "mode", "data": "CRD", "type": "string" }, + { "name": "blocksize", "data": 2, "type": "int" }], + "cases": [ + { + "name": "4D reshape [1, 8, 1, 1]", + "inputs": [ + { + "data": [0, 9, 18, 27, 36, 45, 54, 63], + "dims": [1, 8, 1, 1], + "type": "float32" + } + ], + "outputs": [ + { + "data": [0, 9, 18, 27, 36, 45, 54, 63], + "dims": [1, 2, 2, 2], + "type": "float32" + } + ] + }, + { + "name": "4D reshape [2, 8, 1, 2]", + "inputs": [ + { + "data": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, + 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], + "dims": [2, 8, 1, 2], + "type": "float32" + } + ], + "outputs": [ + { + "data": [0, 2, 1, 3, 4, 6, 5, 7, 8, 10, 9, 11, 12, 14, 13, 15, 16, 18, + 17, 19, 20, 22, 21, 23, 24, 26, 25, 27, 28, 30, 29, 31], + "dims": [2, 2, 2, 4], + "type": "float32" + } + ] + } + ] + } +] diff --git a/js/web/test/test-suite-whitelist.jsonc b/js/web/test/test-suite-whitelist.jsonc index 8539d6d150..0af998a9bf 100644 --- a/js/web/test/test-suite-whitelist.jsonc +++ b/js/web/test/test-suite-whitelist.jsonc @@ -63,6 +63,10 @@ "test_div", "test_dropout_default", "test_dropout_random", + "test_depthtospace_crd_mode", + "test_depthtospace_crd_mode_example", + "test_depthtospace_dcr_mode", + "test_depthtospace_example", "test_elu_example", "test_elu", "test_elu_default", @@ -233,6 +237,7 @@ "conv.jsonc", "cos.jsonc", "div.jsonc", + "depth-to-space.jsonc", "equal.jsonc", "exp.jsonc", "floor.jsonc", diff --git a/js/web/test/unittests/backends/webgl/test-depth-to-space.ts b/js/web/test/unittests/backends/webgl/test-depth-to-space.ts new file mode 100644 index 0000000000..8fd5f366df --- /dev/null +++ b/js/web/test/unittests/backends/webgl/test-depth-to-space.ts @@ -0,0 +1,177 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import {expect} from 'chai'; + +import {Attribute} from '../../../../lib/onnxjs/attribute'; +import {Backend, InferenceHandler, resolveBackend, SessionHandler} from '../../../../lib/onnxjs/backend'; +import {WebGLBackend} from '../../../../lib/onnxjs/backends/backend-webgl'; +import {WebGLInferenceHandler} from '../../../../lib/onnxjs/backends/webgl/inference-handler'; +import {WebGLDepthToSpace} from '../../../../lib/onnxjs/backends/webgl/ops/depth-to-space'; +import {Profiler} from '../../../../lib/onnxjs/instrument'; +import {Tensor} from '../../../../lib/onnxjs/tensor'; + +import {createAscendingArray} from './test-utils'; + +interface TestData { + elementCount: number; + blocksize: number; + inputShape: number[]; + outputShape: number[]; + inputTextureShape: number[]; + outputTextureShape: number[]; + expectedOutput: Float32Array; + // If empty, the test will use auto-generated data. + rawInput?: Float32Array; + mode?: string; +} +function getTestData(): TestData[] { + return [ + { + elementCount: 8, + blocksize: 2, + inputShape: [1, 8, 1, 1], + outputShape: [1, 2, 2, 2], + inputTextureShape: [8, 1], + outputTextureShape: [4, 2], + rawInput: new Float32Array([0., 9., 18., 27., 36., 45., 54., 63.]), + expectedOutput: new Float32Array([0., 18., 36., 54., 9., 27., 45., 63.]), + mode: 'DCR', + }, + { + elementCount: 16, + blocksize: 2, + inputShape: [1, 8, 1, 2], + outputShape: [1, 2, 4, 2], + inputTextureShape: [1, 16], + outputTextureShape: [8, 2], + rawInput: new Float32Array([0., 1., 9., 10, 18., 19, 27., 28., 36., 37., 45., 46., 54., 55., 63., 64.]), + expectedOutput: new Float32Array([0, 18, 1, 19, 36, 54, 37, 55, 9, 27, 10, 28, 45, 63, 46, 64]), + mode: 'DCR', + }, + + { + elementCount: 48, + blocksize: 2, + inputShape: [1, 8, 2, 3], + outputShape: [1, 2, 4, 6], + inputTextureShape: [16, 3], + outputTextureShape: [8, 6], + rawInput: new Float32Array([ + 0., 1., 2., 3., 4., 5., 9., 10., 11., 12., 13., 14., 18., 19., 20., 21., + 22., 23., 27., 28., 29., 30., 31., 32., 36., 37., 38., 39., 40., 41., 45., 46., + 47., 48., 49., 50., 54., 55., 56., 57., 58., 59., 63., 64., 65., 66., 67., 68. + ]), + expectedOutput: new Float32Array([ + 0., 18., 1., 19., 2., 20., 36., 54., 37., 55., 38., 56., 3., 21., 4., 22., + 5., 23., 39., 57., 40., 58., 41., 59., 9., 27., 10., 28., 11., 29., 45., 63., + 46., 64., 47., 65., 12., 30., 13., 31., 14., 32., 48., 66., 49., 67., 50., 68. + ]), + mode: 'DCR', + }, + { + elementCount: 8, + blocksize: 2, + inputShape: [1, 8, 1, 1], + outputShape: [1, 2, 2, 2], + inputTextureShape: [8, 1], + outputTextureShape: [4, 2], + rawInput: new Float32Array([0., 9., 18., 27., 36., 45., 54., 63.]), + expectedOutput: new Float32Array([0, 9, 18, 27, 36, 45, 54, 63]), + mode: 'CRD', + }, + { + elementCount: 16, + blocksize: 2, + inputShape: [1, 8, 1, 2], + outputShape: [1, 2, 4, 2], + inputTextureShape: [1, 16], + outputTextureShape: [8, 2], + rawInput: new Float32Array([0., 1., 9., 10, 18., 19, 27., 28., 36., 37., 45., 46., 54., 55., 63., 64.]), + expectedOutput: new Float32Array([0, 9, 1, 10, 18, 27, 19, 28, 36, 45, 37, 46, 54, 63, 55, 64]), + mode: 'CRD', + }, + + { + elementCount: 48, + blocksize: 2, + inputShape: [1, 8, 2, 3], + outputShape: [1, 2, 4, 6], + inputTextureShape: [16, 3], + outputTextureShape: [8, 6], + rawInput: new Float32Array([ + 0., 1., 2., 3., 4., 5., 9., 10., 11., 12., 13., 14., 18., 19., 20., 21., + 22., 23., 27., 28., 29., 30., 31., 32., 36., 37., 38., 39., 40., 41., 45., 46., + 47., 48., 49., 50., 54., 55., 56., 57., 58., 59., 63., 64., 65., 66., 67., 68. + ]), + expectedOutput: new Float32Array([ + 0., 9., 1., 10., 2., 11., 18., 27., 19., 28., 20., 29., 3., 12., 4., 13., + 5., 14., 21., 30., 22., 31., 23., 32., 36., 45., 37., 46., 38., 47., 54., 63., + 55., 64., 56., 65., 39., 48., 40., 49., 41., 50., 57., 66., 58., 67., 59., 68. + ]), + mode: 'CRD', + }, + ]; +} + +let backend: Backend|undefined; +let sessionhandler: SessionHandler|undefined; +let inferenceHandler: InferenceHandler|undefined; + +describe('#UnitTest# - unpacked WebGLDepthToSpace - Tensor WebGLDepthToSpace', () => { + before('Initialize Context', async () => { + const profiler = Profiler.create(); + backend = await resolveBackend('webgl'); + sessionhandler = backend.createSessionHandler({profiler}); + inferenceHandler = sessionhandler.createInferenceHandler(); + }); + + // Set it back to false, apparently this state is sticky throughout all the tests running in same browser session.. + after('Resetting Context', () => { + (backend as WebGLBackend).pack = false; + }); + + const testDataSet = getTestData(); + for (let k = 0; k < testDataSet.length; ++k) { + const testData = testDataSet[k]; + describe(`Test concat ${JSON.stringify(testData)}`, () => {}); + it('Test depth to space ', () => { + const webglInferenceHandler = inferenceHandler as WebGLInferenceHandler; + + // TODO support WebGl 1.0 + if (webglInferenceHandler.session.textureManager.glContext.version === 1) { + console.log('Running depth to space with webgl1 is not supported. Skipping.'); + return; + } + + const op = new WebGLDepthToSpace(); + const attributes = new Attribute(undefined); + const blocksize = testData.blocksize; + attributes.set('blocksize', 'int', blocksize); + attributes.set('mode', 'string', testData.mode as string); + + op.initialize(attributes); + const elementCount = testData.elementCount; + const inputTensorShape = testData.inputShape; + + // create input data and tensor. + const inputData = testData.rawInput ? testData.rawInput : createAscendingArray(elementCount); + const inputTensorA = new Tensor(inputTensorShape, 'float32', undefined, undefined, inputData); + + // manually creat packed texture from inputTensor, and insert in cache + webglInferenceHandler.session.textureManager.glContext.checkError(); + + webglInferenceHandler.session.textureManager.glContext.checkError(); + + const result = op.run(webglInferenceHandler, [inputTensorA]); + + webglInferenceHandler.session.textureManager.glContext.checkError(); + // verify result. + const expectedOutput = testData.expectedOutput; + expect(result[0].data).to.not.equal(null); + + expect(result[0].data).to.have.lengthOf(elementCount); + expect(result[0].data).to.deep.equal(expectedOutput); + }); + } +}); diff --git a/js/web/test/unittests/backends/webgl/test-matmul-packed.ts b/js/web/test/unittests/backends/webgl/test-matmul-packed.ts new file mode 100644 index 0000000000..31b1e99bd8 --- /dev/null +++ b/js/web/test/unittests/backends/webgl/test-matmul-packed.ts @@ -0,0 +1,262 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import {expect} from 'chai'; + +import {Backend, InferenceHandler, resolveBackend, SessionHandler} from '../../../../lib/onnxjs/backend'; +import {WebGLBackend} from '../../../../lib/onnxjs/backends/backend-webgl'; +import {WebGLInferenceHandler} from '../../../../lib/onnxjs/backends/webgl/inference-handler'; +import {WebGLMatMulPacked} from '../../../../lib/onnxjs/backends/webgl/ops/matmul-pack'; +import {Profiler} from '../../../../lib/onnxjs/instrument'; +import {Tensor} from '../../../../lib/onnxjs/tensor'; +import {ShapeUtil} from '../../../../lib/onnxjs/util'; + +import {createAscendingArray, createTextureFromArray} from './test-utils'; + +interface TestData { + elementCountA: number; + elementCountB: number; + inputShapeA: number[]; + inputShapeB: number[]; + outputShape: number[]; + inputTextureShapeA: number[]; + inputTextureShapeB: number[]; + outputTextureShape: number[]; + expectedOutput: Float32Array; + // The value of bias matrix that will be broadcasted to the corresponding shape in matmul. + // i.e. If biasValue = 1, then bias matrix is [1], when being added to 2x2 matmul result, it will be bcasted to + // [1, 1] + // [1, 1] + biasValue?: number; + // If empty, the test will use auto-generated data. + rawInputA?: Float32Array; + // If empty, the test will use auto-generated data. + rawInputB?: Float32Array; +} +function getTestData(): TestData[] { + return [ + // test 2D tensor + { + elementCountA: 4, + elementCountB: 4, + inputShapeA: [2, 2], + inputShapeB: [2, 2], + outputShape: [2, 2], + inputTextureShapeA: [1, 1], + inputTextureShapeB: [1, 1], + outputTextureShape: [1, 1], + expectedOutput: new Float32Array([7, 10, 15, 22]), + }, + { + elementCountA: 4, + elementCountB: 4, + inputShapeA: [2, 2], + inputShapeB: [2, 2], + outputShape: [2, 2], + inputTextureShapeA: [1, 1], + inputTextureShapeB: [1, 1], + outputTextureShape: [1, 1], + biasValue: 1, + expectedOutput: new Float32Array([8, 11, 16, 23]), + }, + { + elementCountA: 6, + elementCountB: 6, + inputShapeA: [2, 3], + inputShapeB: [3, 2], + outputShape: [2, 2], + inputTextureShapeA: [2, 1], + inputTextureShapeB: [1, 2], + outputTextureShape: [1, 1], + expectedOutput: new Float32Array([22, 28, 49, 64]), + rawInputA: new Float32Array([1, 2, 4, 5, 3, 0, 6, 0]), + rawInputB: new Float32Array([1, 2, 3, 4, 5, 6, 0, 0]), + }, + { + elementCountA: 6, + elementCountB: 6, + inputShapeA: [2, 3], + inputShapeB: [3, 2], + outputShape: [2, 2], + inputTextureShapeA: [2, 1], + inputTextureShapeB: [1, 2], + outputTextureShape: [1, 1], + expectedOutput: new Float32Array([23, 29, 50, 65]), + biasValue: 1, + rawInputA: new Float32Array([1, 2, 4, 5, 3, 0, 6, 0]), + rawInputB: new Float32Array([1, 2, 3, 4, 5, 6, 0, 0]), + }, + { + elementCountA: 16, + elementCountB: 16, + inputShapeA: [4, 4], + inputShapeB: [4, 4], + outputShape: [4, 4], + inputTextureShapeA: [2, 2], + inputTextureShapeB: [2, 2], + outputTextureShape: [2, 2], + rawInputA: new Float32Array([1, 2, 5, 6, 3, 4, 7, 8, 9, 10, 13, 14, 11, 12, 15, 16]), + rawInputB: new Float32Array([1, 2, 5, 6, 3, 4, 7, 8, 9, 10, 13, 14, 11, 12, 15, 16]), + biasValue: 2, + expectedOutput: new Float32Array([92, 102, 112, 122, 204, 230, 256, 282, 316, 358, 400, 442, 428, 486, 544, 602]), + }, + { + elementCountA: 12, + elementCountB: 12, + inputShapeA: [2, 2, 3], + inputShapeB: [2, 3, 2], + outputShape: [2, 2, 2], + inputTextureShapeA: [2, 2], + inputTextureShapeB: [1, 4], + outputTextureShape: [2, 1], + expectedOutput: new Float32Array([23, 29, 50, 65, 23, 29, 50, 65]), + biasValue: 1, + rawInputA: new Float32Array([1, 2, 4, 5, 3, 0, 6, 0, 1, 2, 4, 5, 3, 0, 6, 0]), + rawInputB: new Float32Array([1, 2, 3, 4, 5, 6, 0, 0, 1, 2, 3, 4, 5, 6, 0, 0]), + }, + // test bcast + { + elementCountA: 12, + elementCountB: 6, + inputShapeA: [2, 2, 3], + inputShapeB: [3, 2], + outputShape: [2, 2, 2], + inputTextureShapeA: [2, 2], + inputTextureShapeB: [1, 2], + outputTextureShape: [2, 1], + expectedOutput: new Float32Array([23, 29, 50, 65, 23, 29, 50, 65]), + biasValue: 1, + rawInputA: new Float32Array([1, 2, 4, 5, 3, 0, 6, 0, 1, 2, 4, 5, 3, 0, 6, 0]), + rawInputB: new Float32Array([1, 2, 3, 4, 5, 6, 0, 0]), + }, + ]; +} + +let backend: Backend|undefined; +let sessionhandler: SessionHandler|undefined; +let inferenceHandler: InferenceHandler|undefined; + +describe('#UnitTest# - packed matmul - Tensor matmul', () => { + before('Initialize Context', async () => { + const profiler = Profiler.create(); + backend = await resolveBackend('webgl'); + // Explicitly set to true to trigger packed version + (backend as WebGLBackend).pack = true; + sessionhandler = backend.createSessionHandler({profiler}); + inferenceHandler = sessionhandler.createInferenceHandler(); + }); + + // Set it back to false, apparently this state is sticky throughout all the tests running in same browser session.. + after('Resetting Context', () => { + (backend as WebGLBackend).pack = false; + }); + + const testDataSet = getTestData(); + for (let k = 0; k < testDataSet.length; ++k) { + const testData = testDataSet[k]; + describe(`Test matmul ${JSON.stringify(testData)}`, () => {}); + it('Test packed matmul kernel ', () => { + const webglInferenceHandler = inferenceHandler as WebGLInferenceHandler; + + // TODO support WebGl 1.0 + if (webglInferenceHandler.session.textureManager.glContext.version === 1) { + console.log('Running packed matmul with webgl1 is not supported. Skipping.'); + return; + } + + const op = new WebGLMatMulPacked(); + + const elementCountA = testData.elementCountA; + const elementCountB = testData.elementCountB; + + const inputTensorShapeA = testData.inputShapeA; + const inputTextureShapeA = testData.inputTextureShapeA; + + const inputTensorShapeB = testData.inputShapeB; + const inputTextureShapeB = testData.inputTextureShapeB; + + // create input data and tensor. The input data will be used to verify if the output tensor contains the + // same value but possibly different order depending on our packing algorithm. + const inputDataA = createAscendingArray(elementCountA); + const inputDataB = createAscendingArray(elementCountB); + const inputTensorA = new Tensor(inputTensorShapeA, 'float32', undefined, undefined, inputDataA); + const inputTensorB = new Tensor(inputTensorShapeB, 'float32', undefined, undefined, inputDataB); + + // manually creat packed texture from inputTensor, and insert in cache + const gl = webglInferenceHandler.session.textureManager.glContext.gl; + + webglInferenceHandler.session.textureManager.glContext.checkError(); + const webglTextureA = createTextureFromArray( + webglInferenceHandler.session.textureManager.glContext, testData.rawInputA ? testData.rawInputA : inputDataA, + gl.RGBA, inputTextureShapeA[0], inputTextureShapeA[1]); + const webglTextureB = createTextureFromArray( + webglInferenceHandler.session.textureManager.glContext, testData.rawInputB ? testData.rawInputB : inputDataB, + gl.RGBA, inputTextureShapeB[0], inputTextureShapeB[1]); + + webglInferenceHandler.session.textureManager.glContext.checkError(); + const packedShapeA = inputTextureShapeA; + const textureDataA = { + width: inputTextureShapeA[0], + height: inputTextureShapeA[1], + channels: 4 as const, + isPacked: true, + shape: packedShapeA, + strides: ShapeUtil.computeStrides(packedShapeA), + unpackedShape: inputTensorShapeA, + tensor: inputTensorA, + texture: webglTextureA! + }; + + const packedShapeB = inputTextureShapeB; + const textureDataB = { + width: inputTextureShapeB[0], + height: inputTextureShapeB[1], + channels: 4 as const, + isPacked: true, + shape: packedShapeB, + strides: ShapeUtil.computeStrides(packedShapeB), + unpackedShape: inputTensorShapeB, + tensor: inputTensorB, + texture: webglTextureB! + }; + + webglInferenceHandler.setTextureData(inputTensorA.dataId, textureDataA, true); + webglInferenceHandler.setTextureData(inputTensorB.dataId, textureDataB, true); + + const inputList = testData.biasValue ? + [ + inputTensorA, inputTensorB, + new Tensor([1], 'float32', undefined, undefined, new Float32Array([testData.biasValue])) + ] : + [inputTensorA, inputTensorB]; + + // compile shader code + const programInfo = op.createProgramInfo(inferenceHandler! as WebGLInferenceHandler, inputList); + + const artifact = webglInferenceHandler.session.programManager.build(programInfo); + webglInferenceHandler.session.programManager.setArtifact(op, artifact); + + // run kernal and get output + const runData = op.createRunData(webglInferenceHandler, artifact.programInfo, inputList); + webglInferenceHandler.session.programManager.run(artifact, runData); + const result = runData.outputTextureData.tensor.data; + + webglInferenceHandler.session.textureManager.glContext.checkError(); + // verify result. + const expectedOutput = testData.expectedOutput; + expect(result).to.not.equal(null); + let batchMultiplier = 1; + if (testData.inputShapeA.length > 2) { + batchMultiplier = testData.inputShapeA[0]; + } + if (testData.inputShapeB.length > 2) { + batchMultiplier = Math.max(batchMultiplier, testData.inputShapeB[0]); + } + + expect(result).to.have.lengthOf( + batchMultiplier * testData.inputShapeA[testData.inputShapeA.length - 2] * + testData.inputShapeB[testData.inputShapeB.length - 1]); + expect(result).to.deep.equal(expectedOutput); + }); + } +}); diff --git a/js/web/test/unittests/backends/webgl/test-pack-unpack.ts b/js/web/test/unittests/backends/webgl/test-pack-unpack.ts index 19dd606850..fda4e8b276 100644 --- a/js/web/test/unittests/backends/webgl/test-pack-unpack.ts +++ b/js/web/test/unittests/backends/webgl/test-pack-unpack.ts @@ -73,6 +73,21 @@ function getTestData(isPacked = true): TestData[] { inputTextureShape: [], outputTextureShape: [24, 40] }, + // test 6D tensor + { + elementCount: 32, + inputShape: [1, 1, 2, 2, 2, 4], + outputShape: [], + inputTextureShape: [], + outputTextureShape: [4, 2] + }, + { + elementCount: 3840, + inputShape: [1, 1, 2, 24, 2, 40], + outputShape: [], + inputTextureShape: [], + outputTextureShape: [48, 20] + }, ]; } else { return [ @@ -156,6 +171,23 @@ function getTestData(isPacked = true): TestData[] { outputTextureShape: [16, 4], useGeneratedOutput: true, }, + // test 6d tensor + { + elementCount: 32, + inputShape: [1, 1, 2, 2, 2, 4], + outputShape: [1, 1, 2, 2, 2, 4], + inputTextureShape: [2, 4], + outputTextureShape: [8, 4], + useGeneratedOutput: true, + }, + { + elementCount: 64, + inputShape: [1, 2, 1, 2, 4, 4], + outputShape: [1, 2, 1, 2, 4, 4], + inputTextureShape: [2, 8], + outputTextureShape: [16, 4], + useGeneratedOutput: true, + }, ]; } } diff --git a/js/web/test/unittests/backends/webgl/test-utils.ts b/js/web/test/unittests/backends/webgl/test-utils.ts index b3e2f3e39f..acb3f0002c 100644 --- a/js/web/test/unittests/backends/webgl/test-utils.ts +++ b/js/web/test/unittests/backends/webgl/test-utils.ts @@ -7,6 +7,16 @@ export function createAscendingArray(size: number): Float32Array { return new Float32Array(Array.from({length: size}, (v, i) => (i + 1))); } +// Returns an array by injecting 3 zeros after every element in the input array to be used for creating unpacked +// texture. +export function generateArrayForUnpackedTexture(input: Float32Array): Float32Array { + const output = new Float32Array(input.length * 4); + for (let i = 0; i < (input.length * 4); i += 4) { + output[i] = input[i / 4]; + } + return output; +} + // create a webgl texture and fill it with the array content export function createTextureFromArray( glContext: WebGLContext, dataArray: Float32Array, type: GLenum, width: number, height: number): WebGLTexture { diff --git a/js/web/test/unittests/index.ts b/js/web/test/unittests/index.ts index 8e46c394ad..92b86af64e 100644 --- a/js/web/test/unittests/index.ts +++ b/js/web/test/unittests/index.ts @@ -7,6 +7,8 @@ if (typeof window !== 'undefined') { require('./backends/webgl/test-pack-unpack'); require('./backends/webgl/test-concat-packed'); require('./backends/webgl/test-reshape-packed'); + require('./backends/webgl/test-depth-to-space'); + require('./backends/webgl/test-matmul-packed'); } require('./opset'); From f78af4fc8cf3bc5b6d8a486af78473840f12612f Mon Sep 17 00:00:00 2001 From: Ryan Hill <38674843+RyanUnderhill@users.noreply.github.com> Date: Tue, 25 May 2021 19:03:24 -0700 Subject: [PATCH 06/47] Use RTLD_GLOBAL for onnxrutime_providers_shared on unix (#7831) * Use RTLD_GLOBAL for onnxrutime_providers_shared on unix --- cmake/onnxruntime_providers.cmake | 72 ++++++++++--------- onnxruntime/core/framework/ex_lib_loader.cc | 2 +- .../core/framework/provider_bridge_ort.cc | 4 +- onnxruntime/core/platform/env.h | 4 +- onnxruntime/core/platform/posix/env.cc | 4 +- onnxruntime/core/platform/windows/env.cc | 2 +- .../nuphar/common/nuphar_tvm_utils.cc | 2 +- onnxruntime/core/session/onnxruntime_c_api.cc | 2 +- .../python/onnxruntime_pybind_state.cc | 4 +- 9 files changed, 51 insertions(+), 45 deletions(-) diff --git a/cmake/onnxruntime_providers.cmake b/cmake/onnxruntime_providers.cmake index a74633ae7d..6ba1be3d99 100644 --- a/cmake/onnxruntime_providers.cmake +++ b/cmake/onnxruntime_providers.cmake @@ -221,6 +221,40 @@ install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/cp set_target_properties(onnxruntime_providers PROPERTIES LINKER_LANGUAGE CXX) set_target_properties(onnxruntime_providers PROPERTIES FOLDER "ONNXRuntime") +if (NOT onnxruntime_MINIMAL_BUILD AND NOT onnxruntime_EXTENDED_MINIMAL_BUILD + AND NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin|iOS" + AND NOT (CMAKE_SYSTEM_NAME STREQUAL "Android") + AND NOT onnxruntime_BUILD_WEBASSEMBLY) + file(GLOB onnxruntime_providers_shared_cc_srcs CONFIGURE_DEPENDS + "${ONNXRUNTIME_ROOT}/core/providers/shared/*.h" + "${ONNXRUNTIME_ROOT}/core/providers/shared/*.cc" + ) + + source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_shared_cc_srcs}) + onnxruntime_add_shared_library(onnxruntime_providers_shared ${onnxruntime_providers_shared_cc_srcs}) + set_target_properties(onnxruntime_providers_shared PROPERTIES FOLDER "ONNXRuntime") + set_target_properties(onnxruntime_providers_shared PROPERTIES LINKER_LANGUAGE CXX) + + # On Apple/Unix we don't directly link with this library as we load it with RTLD_GLOBAL, so this is only set to the actual library on WIN32 + set(ONNXRUNTIME_PROVIDERS_SHARED) + + if(APPLE) + set_property(TARGET onnxruntime_providers_shared APPEND_STRING PROPERTY LINK_FLAGS "-Xlinker -exported_symbols_list ${ONNXRUNTIME_ROOT}/core/providers/shared/exported_symbols.lst") + elseif(UNIX) + set_property(TARGET onnxruntime_providers_shared APPEND_STRING PROPERTY LINK_FLAGS "-Xlinker --version-script=${ONNXRUNTIME_ROOT}/core/providers/shared/version_script.lds -Xlinker --gc-sections") + elseif(WIN32) + set_property(TARGET onnxruntime_providers_shared APPEND_STRING PROPERTY LINK_FLAGS "-DEF:${ONNXRUNTIME_ROOT}/core/providers/shared/symbols.def") + set(ONNXRUNTIME_PROVIDERS_SHARED onnxruntime_providers_shared) + else() + message(FATAL_ERROR "onnxruntime_providers_shared unknown platform, need to specify shared library exports for it") + endif() + + install(TARGETS onnxruntime_providers_shared + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) +endif() + if (onnxruntime_USE_CUDA) file(GLOB_RECURSE onnxruntime_providers_cuda_cc_srcs CONFIGURE_DEPENDS "${ONNXRUNTIME_ROOT}/core/providers/cuda/*.h" @@ -309,7 +343,7 @@ if (onnxruntime_USE_CUDA) endif() add_dependencies(onnxruntime_providers_cuda onnxruntime_providers_shared ${onnxruntime_EXTERNAL_DEPENDENCIES} ${onnxruntime_tvm_dependencies}) - target_link_libraries(onnxruntime_providers_cuda PRIVATE cudart cublas cudnn curand cufft onnxruntime_providers_shared) + target_link_libraries(onnxruntime_providers_cuda PRIVATE cudart cublas cudnn curand cufft ${ONNXRUNTIME_PROVIDERS_SHARED}) target_include_directories(onnxruntime_providers_cuda PRIVATE ${ONNXRUNTIME_ROOT} ${CMAKE_CURRENT_BINARY_DIR} ${onnxruntime_CUDNN_HOME}/include ${eigen_INCLUDE_DIRS} ${TVM_INCLUDES} PUBLIC ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) # ${CMAKE_CURRENT_BINARY_DIR} is so that #include "onnxruntime_config.h" inside tensor_shape.h is found install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/cuda DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers) @@ -381,36 +415,6 @@ if (onnxruntime_USE_CUDA) endif() -if (NOT onnxruntime_MINIMAL_BUILD AND NOT onnxruntime_EXTENDED_MINIMAL_BUILD - AND NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin|iOS" - AND NOT (CMAKE_SYSTEM_NAME STREQUAL "Android") - AND NOT onnxruntime_BUILD_WEBASSEMBLY) - file(GLOB onnxruntime_providers_shared_cc_srcs CONFIGURE_DEPENDS - "${ONNXRUNTIME_ROOT}/core/providers/shared/*.h" - "${ONNXRUNTIME_ROOT}/core/providers/shared/*.cc" - ) - - source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_shared_cc_srcs}) - onnxruntime_add_shared_library(onnxruntime_providers_shared ${onnxruntime_providers_shared_cc_srcs}) - set_target_properties(onnxruntime_providers_shared PROPERTIES FOLDER "ONNXRuntime") - set_target_properties(onnxruntime_providers_shared PROPERTIES LINKER_LANGUAGE CXX) - - if(APPLE) - set_property(TARGET onnxruntime_providers_shared APPEND_STRING PROPERTY LINK_FLAGS "-Xlinker -exported_symbols_list ${ONNXRUNTIME_ROOT}/core/providers/shared/exported_symbols.lst") - elseif(UNIX) - set_property(TARGET onnxruntime_providers_shared APPEND_STRING PROPERTY LINK_FLAGS "-Xlinker --version-script=${ONNXRUNTIME_ROOT}/core/providers/shared/version_script.lds -Xlinker --gc-sections") - elseif(WIN32) - set_property(TARGET onnxruntime_providers_shared APPEND_STRING PROPERTY LINK_FLAGS "-DEF:${ONNXRUNTIME_ROOT}/core/providers/shared/symbols.def") - else() - message(FATAL_ERROR "onnxruntime_providers_shared unknown platform, need to specify shared library exports for it") - endif() - - install(TARGETS onnxruntime_providers_shared - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) -endif() - if (onnxruntime_USE_DNNL) file(GLOB_RECURSE onnxruntime_providers_dnnl_cc_srcs CONFIGURE_DEPENDS "${ONNXRUNTIME_ROOT}/core/providers/dnnl/*.h" @@ -426,7 +430,7 @@ if (onnxruntime_USE_DNNL) add_dependencies(onnxruntime_providers_dnnl onnxruntime_providers_shared project_dnnl ${onnxruntime_EXTERNAL_DEPENDENCIES}) target_include_directories(onnxruntime_providers_dnnl PRIVATE ${ONNXRUNTIME_ROOT} ${CMAKE_CURRENT_BINARY_DIR} ${eigen_INCLUDE_DIRS} ${DNNL_INCLUDE_DIR} ${DNNL_OCL_INCLUDE_DIR}) # ${CMAKE_CURRENT_BINARY_DIR} is so that #include "onnxruntime_config.h" inside tensor_shape.h is found - target_link_libraries(onnxruntime_providers_dnnl PRIVATE dnnl onnxruntime_providers_shared) + target_link_libraries(onnxruntime_providers_dnnl PRIVATE dnnl ${ONNXRUNTIME_PROVIDERS_SHARED}) install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/dnnl DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers) set_target_properties(onnxruntime_providers_dnnl PROPERTIES FOLDER "ONNXRuntime") set_target_properties(onnxruntime_providers_dnnl PROPERTIES LINKER_LANGUAGE CXX) @@ -510,7 +514,7 @@ if (onnxruntime_USE_TENSORRT) onnxruntime_add_shared_library_module(onnxruntime_providers_tensorrt ${onnxruntime_providers_tensorrt_cc_srcs}) onnxruntime_add_include_to_target(onnxruntime_providers_tensorrt onnxruntime_common onnx flatbuffers) add_dependencies(onnxruntime_providers_tensorrt onnxruntime_providers_shared ${onnxruntime_EXTERNAL_DEPENDENCIES}) - target_link_libraries(onnxruntime_providers_tensorrt PRIVATE ${onnxparser_link_libs} ${trt_link_libs} cudart onnxruntime_providers_shared protobuf::libprotobuf flatbuffers) + target_link_libraries(onnxruntime_providers_tensorrt PRIVATE ${onnxparser_link_libs} ${trt_link_libs} cudart ${ONNXRUNTIME_PROVIDERS_SHARED} protobuf::libprotobuf flatbuffers) target_include_directories(onnxruntime_providers_tensorrt PRIVATE ${ONNXRUNTIME_ROOT} ${CMAKE_CURRENT_BINARY_DIR} ${onnxruntime_CUDNN_HOME}/include ${eigen_INCLUDE_DIRS} PUBLIC ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) # ${CMAKE_CURRENT_BINARY_DIR} is so that #include "onnxruntime_config.h" inside tensor_shape.h is found install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/tensorrt DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers) @@ -637,7 +641,7 @@ if (onnxruntime_USE_OPENVINO) set_target_properties(onnxruntime_providers_openvino PROPERTIES FOLDER "ONNXRuntime") add_dependencies(onnxruntime_providers_openvino onnxruntime_providers_shared ${onnxruntime_EXTERNAL_DEPENDENCIES}) target_include_directories(onnxruntime_providers_openvino SYSTEM PUBLIC ${ONNXRUNTIME_ROOT} ${CMAKE_CURRENT_BINARY_DIR} ${eigen_INCLUDE_DIRS} ${OPENVINO_INCLUDE_DIR_LIST} ${PYTHON_INCLUDE_DIRS}) - target_link_libraries(onnxruntime_providers_openvino onnxruntime_providers_shared ${OPENVINO_LIB_LIST}) + target_link_libraries(onnxruntime_providers_openvino ${ONNXRUNTIME_PROVIDERS_SHARED} ${OPENVINO_LIB_LIST}) if(MSVC) target_compile_options(onnxruntime_providers_openvino PUBLIC /wd4099 /wd4275 /wd4100 /wd4005 /wd4244 /wd4267) diff --git a/onnxruntime/core/framework/ex_lib_loader.cc b/onnxruntime/core/framework/ex_lib_loader.cc index 391106d426..7070cbf9f1 100644 --- a/onnxruntime/core/framework/ex_lib_loader.cc +++ b/onnxruntime/core/framework/ex_lib_loader.cc @@ -38,7 +38,7 @@ common::Status ExLibLoader::LoadExternalLib(const std::string& dso_file_path, } void* lib_handle = nullptr; - ORT_RETURN_IF_ERROR(Env::Default().LoadDynamicLibrary(dso_file_path, &lib_handle)); + ORT_RETURN_IF_ERROR(Env::Default().LoadDynamicLibrary(dso_file_path, false, &lib_handle)); dso_name_data_map_[dso_file_path] = lib_handle; *handle = lib_handle; return Status::OK(); diff --git a/onnxruntime/core/framework/provider_bridge_ort.cc b/onnxruntime/core/framework/provider_bridge_ort.cc index 554b3eea37..155b8c8b6e 100644 --- a/onnxruntime/core/framework/provider_bridge_ort.cc +++ b/onnxruntime/core/framework/provider_bridge_ort.cc @@ -896,7 +896,7 @@ struct ProviderSharedLibrary { return true; std::string full_path = Env::Default().GetRuntimePath() + std::string(LIBRARY_PREFIX "onnxruntime_providers_shared" LIBRARY_EXTENSION); - auto error = Env::Default().LoadDynamicLibrary(full_path, &handle_); + auto error = Env::Default().LoadDynamicLibrary(full_path, true /*shared_globals on unix*/, &handle_); if (!error.IsOK()) { LOGS_DEFAULT(ERROR) << error.ErrorMessage(); return false; @@ -947,7 +947,7 @@ struct ProviderLibrary { return nullptr; std::string full_path = Env::Default().GetRuntimePath() + std::string(filename_); - auto error = Env::Default().LoadDynamicLibrary(full_path, &handle_); + auto error = Env::Default().LoadDynamicLibrary(full_path, false, &handle_); if (!error.IsOK()) { LOGS_DEFAULT(ERROR) << error.ErrorMessage(); return nullptr; diff --git a/onnxruntime/core/platform/env.h b/onnxruntime/core/platform/env.h index 8f412dd84a..3beec3a807 100644 --- a/onnxruntime/core/platform/env.h +++ b/onnxruntime/core/platform/env.h @@ -193,11 +193,13 @@ class Env { // loading a library. The rules for determining the exact location of the // library are platform-specific and are not documented here. // + // global_symbols only has an effect on unix, where a value of true means to load with RTLD_GLOBAL vs RTLD_LOCAL + // // On success, returns a handle to the library in "*handle" and returns // OK from the function. // Otherwise returns nullptr in "*handle" and an error status from the // function. - virtual common::Status LoadDynamicLibrary(const std::string& library_filename, void** handle) const = 0; + virtual common::Status LoadDynamicLibrary(const std::string& library_filename, bool global_symbols, void** handle) const = 0; virtual common::Status UnloadDynamicLibrary(void* handle) const = 0; diff --git a/onnxruntime/core/platform/posix/env.cc b/onnxruntime/core/platform/posix/env.cc index e48a364c35..3a05f424a2 100644 --- a/onnxruntime/core/platform/posix/env.cc +++ b/onnxruntime/core/platform/posix/env.cc @@ -411,9 +411,9 @@ class PosixEnv : public Env { return Status::OK(); } - common::Status LoadDynamicLibrary(const std::string& library_filename, void** handle) const override { + common::Status LoadDynamicLibrary(const std::string& library_filename, bool global_symbols, void** handle) const override { dlerror(); // clear any old error_str - *handle = dlopen(library_filename.c_str(), RTLD_NOW | RTLD_LOCAL); + *handle = dlopen(library_filename.c_str(), RTLD_NOW | (global_symbols ? RTLD_GLOBAL : RTLD_LOCAL)); char* error_str = dlerror(); if (!*handle) { return common::Status(common::ONNXRUNTIME, common::FAIL, diff --git a/onnxruntime/core/platform/windows/env.cc b/onnxruntime/core/platform/windows/env.cc index f33be15444..2fc7e9ae12 100644 --- a/onnxruntime/core/platform/windows/env.cc +++ b/onnxruntime/core/platform/windows/env.cc @@ -494,7 +494,7 @@ class WindowsEnv : public Env { return path.substr(0, slash_index + 1); } - virtual Status LoadDynamicLibrary(const std::string& library_filename, void** handle) const override { + virtual Status LoadDynamicLibrary(const std::string& library_filename, bool /*global_symbols*/, void** handle) const override { #if WINAPI_FAMILY == WINAPI_FAMILY_PC_APP *handle = ::LoadPackagedLibrary(ToWideString(library_filename).c_str(), 0); #else diff --git a/onnxruntime/core/providers/nuphar/common/nuphar_tvm_utils.cc b/onnxruntime/core/providers/nuphar/common/nuphar_tvm_utils.cc index 7e27842f1e..78e6a260fe 100644 --- a/onnxruntime/core/providers/nuphar/common/nuphar_tvm_utils.cc +++ b/onnxruntime/core/providers/nuphar/common/nuphar_tvm_utils.cc @@ -69,7 +69,7 @@ static bool GetCacheSoFilePath(std::string& so_path) { static void* GetFuncFromLibrary(const std::string& so_path, const std::string& func_name, bool throw_if_not_found = true) { void* so_handle; - ORT_ENFORCE(Env::Default().LoadDynamicLibrary(so_path, &so_handle).IsOK()); + ORT_ENFORCE(Env::Default().LoadDynamicLibrary(so_path, false, &so_handle).IsOK()); void* func = nullptr; Status s = Env::Default().GetSymbolFromLibrary(so_handle, func_name, &func); if (throw_if_not_found && !s.IsOK()) diff --git a/onnxruntime/core/session/onnxruntime_c_api.cc b/onnxruntime/core/session/onnxruntime_c_api.cc index 5c2b1fe9ac..e73bdbe18a 100644 --- a/onnxruntime/core/session/onnxruntime_c_api.cc +++ b/onnxruntime/core/session/onnxruntime_c_api.cc @@ -389,7 +389,7 @@ ORT_API_STATUS_IMPL(OrtApis::AddCustomOpDomain, _Inout_ OrtSessionOptions* optio ORT_API_STATUS_IMPL(OrtApis::RegisterCustomOpsLibrary, _Inout_ OrtSessionOptions* options, _In_ const char* library_path, void** library_handle) { API_IMPL_BEGIN - Env::Default().LoadDynamicLibrary(library_path, library_handle); + Env::Default().LoadDynamicLibrary(library_path, false, library_handle); if (!*library_handle) return OrtApis::CreateStatus(ORT_FAIL, "RegisterCustomOpsLibrary: Failed to load library"); diff --git a/onnxruntime/python/onnxruntime_pybind_state.cc b/onnxruntime/python/onnxruntime_pybind_state.cc index 74bf596d48..42d48584ea 100644 --- a/onnxruntime/python/onnxruntime_pybind_state.cc +++ b/onnxruntime/python/onnxruntime_pybind_state.cc @@ -252,7 +252,7 @@ static Env& platform_env = Env::Default(); CustomOpLibrary::CustomOpLibrary(const char* library_path, OrtSessionOptions& ort_so) { { - OrtPybindThrowIfError(platform_env.LoadDynamicLibrary(library_path, &library_handle_)); + OrtPybindThrowIfError(platform_env.LoadDynamicLibrary(library_path, false, &library_handle_)); OrtStatus*(ORT_API_CALL * RegisterCustomOps)(OrtSessionOptions * options, const OrtApiBase* api); @@ -455,7 +455,7 @@ static std::unique_ptr LoadExecutionProvider( const std::string& ep_shared_lib_path, const ProviderOptions& provider_options = {}) { void* handle; - auto error = Env::Default().LoadDynamicLibrary(ep_shared_lib_path, &handle); + auto error = Env::Default().LoadDynamicLibrary(ep_shared_lib_path, false, &handle); if (!error.IsOK()) { throw std::runtime_error(error.ErrorMessage()); } From 1c6b6f696eae4a1841ece99cea98380de6b1c3f7 Mon Sep 17 00:00:00 2001 From: George Wu Date: Tue, 25 May 2021 19:38:59 -0700 Subject: [PATCH 07/47] fixes for cuda centos/manylinux (#7830) * fixes for cuda centos/manylinux * remove providers_shared.so dep processing. --- setup.py | 26 +++++++++++++++---- .../github/linux/java_copy_strip_binary.sh | 6 +++-- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 074d8ed6f8..7f22fb9de9 100644 --- a/setup.py +++ b/setup.py @@ -134,19 +134,33 @@ try: logger.info('copying %s -> %s', source, dest) copyfile(source, dest) result = subprocess.run(['patchelf', '--print-needed', dest], check=True, stdout=subprocess.PIPE, universal_newlines=True) - cuda_dependencies = ['libcublas.so', 'libcudnn.so', 'libcudart.so', 'libcurand.so', 'libcufft.so', 'libnvToolsExt.so'] - cuda_dependencies.extend(['librccl.so', 'libamdhip64.so', 'librocblas.so', 'libMIOpen.so', 'libhsa-runtime64.so', 'libhsakmt.so']) + dependencies = ['librccl.so', 'libamdhip64.so', 'librocblas.so', 'libMIOpen.so', 'libhsa-runtime64.so', 'libhsakmt.so'] to_preload = [] args = ['patchelf', '--debug'] for line in result.stdout.split('\n'): - for dependency in cuda_dependencies: + for dependency in dependencies: if dependency in line: to_preload.append(line) args.extend(['--remove-needed', line]) args.append(dest) - if len(to_preload) > 0: + if len(args) > 3: subprocess.run(args, check=True, stdout=subprocess.PIPE) - self._rewrite_ld_preload(to_preload) + + dest = 'onnxruntime/capi/libonnxruntime_providers_cuda.so' + if path.isfile(dest): + result = subprocess.run(['patchelf', '--print-needed', dest], check=True, stdout=subprocess.PIPE, universal_newlines=True) + cuda_dependencies = ['libcublas.so', 'libcublasLt.so', 'libcudnn.so', 'libcudart.so', 'libcurand.so', 'libcufft.so', 'libnvToolsExt.so'] + args = ['patchelf', '--debug'] + for line in result.stdout.split('\n'): + for dependency in cuda_dependencies: + if dependency in line: + if not dependency in to_preload: + to_preload.append(line) + args.extend(['--remove-needed', line]) + args.append(dest) + if len(args) > 3: + subprocess.run(args, check=True, stdout=subprocess.PIPE) + self._rewrite_ld_preload(to_preload) _bdist_wheel.run(self) if is_manylinux: file = glob(path.join(self.dist_dir, '*linux*.whl'))[0] @@ -165,6 +179,7 @@ except ImportError as error: # Additional binaries if platform.system() == 'Linux': libs = ['onnxruntime_pybind11_state.so', 'libdnnl.so.2', 'libmklml_intel.so', 'libmklml_gnu.so', 'libiomp5.so', 'mimalloc.so'] + dl_libs = ['libonnxruntime_providers_shared.so', 'libonnxruntime_providers_cuda.so'] # DNNL, TensorRT & OpenVINO EPs are built as shared libs libs.extend(['libonnxruntime_providers_shared.so']) libs.extend(['libonnxruntime_providers_dnnl.so']) @@ -201,6 +216,7 @@ else: if is_manylinux: data = ['capi/libonnxruntime_pywrapper.so'] if nightly_build else [] + data += [path.join('capi', x) for x in dl_libs if path.isfile(path.join('onnxruntime', 'capi', x))] ext_modules = [ Extension( 'onnxruntime.capi.onnxruntime_pybind11_state', diff --git a/tools/ci_build/github/linux/java_copy_strip_binary.sh b/tools/ci_build/github/linux/java_copy_strip_binary.sh index 11f097d14e..a3278dbf6f 100755 --- a/tools/ci_build/github/linux/java_copy_strip_binary.sh +++ b/tools/ci_build/github/linux/java_copy_strip_binary.sh @@ -46,8 +46,10 @@ then # Add custom lib cp $BINARY_DIR/$BUILD_CONFIG/libcustom_op_library.so $BINARY_DIR/$ARTIFACT_NAME # Add cuda provider if it exists - cp $BINARY_DIR/$BUILD_CONFIG/$LIB_NAME $BINARY_DIR/$ARTIFACT_NAME/$NATIVE_FOLDER/libonnxruntime_providers_shared.so - cp $BINARY_DIR/$BUILD_CONFIG/$LIB_NAME $BINARY_DIR/$ARTIFACT_NAME/$NATIVE_FOLDER/libonnxruntime_providers_cuda.so + if [[ -f "$BINARY_DIR/$BUILD_CONFIG/libonnxruntime_providers_cuda.so" ]]; then + cp $BINARY_DIR/$BUILD_CONFIG/libonnxruntime_providers_shared.so $BINARY_DIR/$ARTIFACT_NAME/$NATIVE_FOLDER/libonnxruntime_providers_shared.so + cp $BINARY_DIR/$BUILD_CONFIG/libonnxruntime_providers_cuda.so $BINARY_DIR/$ARTIFACT_NAME/$NATIVE_FOLDER/libonnxruntime_providers_cuda.so + fi fi find $BINARY_DIR/$ARTIFACT_NAME -ls From 4fe59c8b29613fcd9b8fb6125a05358164f81bb5 Mon Sep 17 00:00:00 2001 From: harshithapv <54084812+harshithapv@users.noreply.github.com> Date: Tue, 25 May 2021 22:22:13 -0700 Subject: [PATCH 08/47] delete model_copy to save memory allocated in forward call (#7832) * delete model copy * add flag * address comments * address flag comment Co-authored-by: root --- .../orttraining/python/training/ortmodule/_io.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/orttraining/orttraining/python/training/ortmodule/_io.py b/orttraining/orttraining/python/training/ortmodule/_io.py index b7cb3e79a8..8aaa9e908c 100644 --- a/orttraining/orttraining/python/training/ortmodule/_io.py +++ b/orttraining/orttraining/python/training/ortmodule/_io.py @@ -8,7 +8,7 @@ import copy import inspect import torch import warnings - +import gc class _OutputIdentityOp(torch.autograd.Function): '''Internal class used to prepend Identity ops in model's outputs @@ -460,12 +460,14 @@ def parse_outputs_for_onnx_export_and_extract_schema(module, inputs, kwargs): module.eval() output_names = None output_dynamic_axes = None + is_deepcopy = False with torch.no_grad(): # Deepcopy inputs, since input values may change after model run. sample_inputs_copy, sample_kwargs_copy = deepcopy_model_input(*inputs, **kwargs) try: # Deepcopy model, in case model is stateful and changes after model run. model_copy = copy.deepcopy(module) + is_deepcopy = True except Exception: model_copy = module warnings.warn("This model cannot be deep copied (or pickled), " @@ -478,6 +480,9 @@ def parse_outputs_for_onnx_export_and_extract_schema(module, inputs, kwargs): output_names, output_dynamic_axes = _parse_outputs_and_extract_names_and_dynamic_axes(sample_outputs) if is_train_mode: module.train() - + output_schema = _extract_schema(sample_outputs) + if is_deepcopy: + del model_copy + gc.collect() # Return output names, output dynamic axes and output schema - return output_names, output_dynamic_axes, _extract_schema(sample_outputs) + return output_names, output_dynamic_axes, output_schema From 57782b3463735402c582721bd1883363ada2fb4a Mon Sep 17 00:00:00 2001 From: Scott McKay Date: Wed, 26 May 2021 15:57:40 +1000 Subject: [PATCH 09/47] Add supported operators/types documentation for the ORT Mobile package (#7807) * Add ability to generate documentation for the ORT Mobile package using the build configuration as input. --- docs/ORTMobilePackageOperatorTypeSupport.md | 127 ++++++++++++++++++ tools/python/gen_ort_mobile_pkg_doc.py | 81 +++++++++++ .../operator_type_usage_processors.py | 3 + 3 files changed, 211 insertions(+) create mode 100644 docs/ORTMobilePackageOperatorTypeSupport.md create mode 100644 tools/python/gen_ort_mobile_pkg_doc.py diff --git a/docs/ORTMobilePackageOperatorTypeSupport.md b/docs/ORTMobilePackageOperatorTypeSupport.md new file mode 100644 index 0000000000..7e08e06890 --- /dev/null +++ b/docs/ORTMobilePackageOperatorTypeSupport.md @@ -0,0 +1,127 @@ +# ONNX Runtime Mobile Pre-Built Package Operator and Type Support + +## Supported operators and types + +The supported operators and types are based on what is required to support float32 and quantized versions of popular models. The full list of input models used to determine this list is available [here](https://github.com/microsoft/onnxruntime/blob/master/tools/ci_build/github/android/mobile_package.required_operators.readme.txt) + +## Supported data input types + + - float + - int8_t + - uint8_t + +NOTE: Operators used to manipulate dimensions and indices will support int32 and int64. + +## Supported Operators + +|Operator|Opsets| +|--------|------| +|**ai.onnx**|| +|ai.onnx:Abs|12, 13| +|ai.onnx:Add|12, 13| +|ai.onnx:And|12, 13| +|ai.onnx:ArgMax|12, 13| +|ai.onnx:ArgMin|12, 13| +|ai.onnx:AveragePool|12, 13| +|ai.onnx:Cast|12, 13| +|ai.onnx:Ceil|12, 13| +|ai.onnx:Clip|12, 13| +|ai.onnx:Concat|12, 13| +|ai.onnx:ConstantOfShape|12, 13| +|ai.onnx:Conv|12, 13| +|ai.onnx:ConvTranspose|12, 13| +|ai.onnx:Cos|12, 13| +|ai.onnx:CumSum|12, 13| +|ai.onnx:DepthToSpace|12, 13| +|ai.onnx:DequantizeLinear|12, 13| +|ai.onnx:Div|12, 13| +|ai.onnx:DynamicQuantizeLinear|12, 13| +|ai.onnx:Elu|12, 13| +|ai.onnx:Equal|12, 13| +|ai.onnx:Exp|12, 13| +|ai.onnx:Expand|12, 13| +|ai.onnx:Flatten|12, 13| +|ai.onnx:Floor|12, 13| +|ai.onnx:Gather|12, 13| +|ai.onnx:GatherND|12, 13| +|ai.onnx:Gemm|12, 13| +|ai.onnx:GlobalAveragePool|12, 13| +|ai.onnx:Greater|12, 13| +|ai.onnx:GreaterOrEqual|12, 13| +|ai.onnx:Identity|12, 13| +|ai.onnx:If|12, 13| +|ai.onnx:LRN|12, 13| +|ai.onnx:LeakyRelu|12, 13| +|ai.onnx:Less|12, 13| +|ai.onnx:LessOrEqual|12, 13| +|ai.onnx:Log|12, 13| +|ai.onnx:LogSoftmax|12, 13| +|ai.onnx:Loop|12, 13| +|ai.onnx:MatMul|12, 13| +|ai.onnx:MatMulInteger|12, 13| +|ai.onnx:Max|12, 13| +|ai.onnx:MaxPool|12, 13| +|ai.onnx:Mean|12, 13| +|ai.onnx:Min|12, 13| +|ai.onnx:Mul|12, 13| +|ai.onnx:Neg|12, 13| +|ai.onnx:NonMaxSuppression|12, 13| +|ai.onnx:NonZero|12, 13| +|ai.onnx:Not|12, 13| +|ai.onnx:Or|12, 13| +|ai.onnx:PRelu|12, 13| +|ai.onnx:Pad|12, 13| +|ai.onnx:Pow|12, 13| +|ai.onnx:QLinearConv|12, 13| +|ai.onnx:QLinearMatMul|12, 13| +|ai.onnx:QuantizeLinear|12, 13| +|ai.onnx:Range|12, 13| +|ai.onnx:Reciprocal|12, 13| +|ai.onnx:ReduceMax|12, 13| +|ai.onnx:ReduceMean|12, 13| +|ai.onnx:ReduceMin|12, 13| +|ai.onnx:ReduceProd|12, 13| +|ai.onnx:ReduceSum|12, 13| +|ai.onnx:Relu|12, 13| +|ai.onnx:Reshape|12, 13| +|ai.onnx:Resize|12, 13| +|ai.onnx:ReverseSequence|12, 13| +|ai.onnx:Round|12, 13| +|ai.onnx:Scan|12, 13| +|ai.onnx:ScatterND|12, 13| +|ai.onnx:Shape|12, 13| +|ai.onnx:Sigmoid|12, 13| +|ai.onnx:Sin|12, 13| +|ai.onnx:Size|12, 13| +|ai.onnx:Slice|12, 13| +|ai.onnx:Softmax|12, 13| +|ai.onnx:SpaceToDepth|12, 13| +|ai.onnx:Split|12, 13| +|ai.onnx:Sqrt|12, 13| +|ai.onnx:Squeeze|12, 13| +|ai.onnx:Sub|12, 13| +|ai.onnx:Sum|12, 13| +|ai.onnx:Tanh|12, 13| +|ai.onnx:ThresholdedRelu|12, 13| +|ai.onnx:Tile|12, 13| +|ai.onnx:TopK|12, 13| +|ai.onnx:Transpose|12, 13| +|ai.onnx:Unique|12, 13| +|ai.onnx:Unsqueeze|12, 13| +|ai.onnx:Where|12, 13| +||| +|**com.microsoft**|| +|com.microsoft:DynamicQuantizeMatMul|1| +|com.microsoft:FusedConv|1| +|com.microsoft:FusedGemm|1| +|com.microsoft:FusedMatMul|1| +|com.microsoft:MatMulIntegerToFloat|1| +|com.microsoft:NhwcMaxPool|1| +|com.microsoft:QLinearAdd|1| +|com.microsoft:QLinearAveragePool|1| +|com.microsoft:QLinearConv|1| +|com.microsoft:QLinearGlobalAveragePool|1| +|com.microsoft:QLinearLeakyRelu|1| +|com.microsoft:QLinearMul|1| +|com.microsoft:QLinearSigmoid|1| +||| diff --git a/tools/python/gen_ort_mobile_pkg_doc.py b/tools/python/gen_ort_mobile_pkg_doc.py new file mode 100644 index 0000000000..b8de140556 --- /dev/null +++ b/tools/python/gen_ort_mobile_pkg_doc.py @@ -0,0 +1,81 @@ +import argparse +import os +import pathlib +from util import reduced_build_config_parser +from util.ort_format_model.operator_type_usage_processors import GloballyAllowedTypesOpTypeImplFilter + + +def generate_docs(output_file, required_ops, op_type_impl_filter): + with open(output_file, 'w') as out: + out.write('# ONNX Runtime Mobile Pre-Built Package Operator and Type Support\n\n') + + # Description + out.write('## Supported operators and types\n\n') + out.write('The supported operators and types are based on what is required to support float32 and quantized ' + 'versions of popular models. The full list of input models used to determine this list is available ' + '[here](https://github.com/microsoft/onnxruntime/blob/master/tools/ci_build/github/android/mobile_package.required_operators.readme.txt)') # noqa + out.write('\n\n') + + # Globally supported types + out.write('## Supported data input types\n\n') + assert(op_type_impl_filter.__class__ is GloballyAllowedTypesOpTypeImplFilter) + global_types = op_type_impl_filter.global_type_list() + for type in sorted(global_types): + out.write(' - {}\n'.format(type)) + out.write('\n') + out.write('NOTE: Operators used to manipulate dimensions and indices will support int32 and int64.\n\n') + + domain_op_opsets = [] + for domain in sorted(required_ops.keys()): + op_opsets = {} + domain_op_opsets.append((domain, op_opsets)) + for opset in sorted(required_ops[domain].keys()): + str_opset = str(opset) + for op in required_ops[domain][opset]: + op_with_domain = '{}:{}'.format(domain, op) + if op_with_domain not in op_opsets: + op_opsets[op_with_domain] = [] + + op_opsets[op_with_domain].append(str_opset) + + out.write('## Supported Operators\n\n') + out.write('|Operator|Opsets|\n') + out.write('|--------|------|\n') + for domain, op_opsets in domain_op_opsets: + out.write('|**{}**||\n'.format(domain)) + for op in sorted(op_opsets.keys()): + out.write('|{}|{}|\n'.format(op, ', '.join(op_opsets[op]))) + out.write('|||\n') + + +def main(): + script_dir = os.path.dirname(os.path.realpath(__file__)) + + parser = argparse.ArgumentParser( + description='ONNX Runtime Mobile Pre-Built Package Operator and Type Support Documentation Generator', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + + default_config_path = \ + pathlib.Path(os.path.join(script_dir, '../ci_build/github/android/mobile_package.required_operators.config') + ).resolve() + + default_output_path = \ + pathlib.Path(os.path.join(script_dir, '../../docs/ORTMobilePackageOperatorTypeSupport.md')).resolve() + + parser.add_argument('--config_path', help='Path to build configuration used to generate package.', required=False, + type=pathlib.Path, default=default_config_path) + + parser.add_argument('--output_path', help='output markdown file path', required=False, + type=pathlib.Path, default=default_output_path) + + args = parser.parse_args() + config_file = args.config_path.resolve(strict=True) # must exist so strict=True + output_path = args.output_path.resolve() + + enable_type_reduction = True + required_ops, op_type_impl_filter = reduced_build_config_parser.parse_config(config_file, enable_type_reduction) + generate_docs(output_path, required_ops, op_type_impl_filter) + + +if __name__ == '__main__': + main() diff --git a/tools/python/util/ort_format_model/operator_type_usage_processors.py b/tools/python/util/ort_format_model/operator_type_usage_processors.py index b6bfbbf54e..0d44c826d3 100644 --- a/tools/python/util/ort_format_model/operator_type_usage_processors.py +++ b/tools/python/util/ort_format_model/operator_type_usage_processors.py @@ -568,3 +568,6 @@ class GloballyAllowedTypesOpTypeImplFilter(OpTypeImplFilterInterface): def get_cpp_entries(self): return ["ORT_SPECIFY_OP_KERNEL_GLOBAL_ALLOWED_TYPES({});".format( ", ".join(sorted(self._globally_allowed_types)))] + + def global_type_list(self): + return self._globally_allowed_types From d1f0251e3913934e73cee64398e915032618cb01 Mon Sep 17 00:00:00 2001 From: Dmitri Smirnov Date: Wed, 26 May 2021 09:47:41 -0700 Subject: [PATCH 10/47] Python bindings fix ups in preparation to Sparse Tensor introduction (#7817) * Fix up constness in pybindings Fix up return argument treatments. Specifically, for all functions that return pointers or references to the members of other pybind registered classes, we want not to copy them, but internally bump up a reference to the hosting class so they do not disappear before the reference to the returned members is re-claimed. This policy is applied by default to def_property and def_readwrite but not to def_readonly and other def methods. See https://pybind11-jagerman.readthedocs.io/en/stable/advanced.html#return-value-policies https://pybind11.readthedocs.io/en/stable/advanced/functions.html#return-value-policies Move OrtValue binding to a separate file Move IOBinding into separate file. --- onnxruntime/core/session/IOBinding.cc | 2 + onnxruntime/core/session/IOBinding.h | 1 + onnxruntime/core/session/inference_session.cc | 8 + onnxruntime/core/session/inference_session.h | 2 + .../python/onnxruntime_pybind_exceptions.cc | 71 ++ .../python/onnxruntime_pybind_exceptions.h | 47 +- .../python/onnxruntime_pybind_iobinding.cc | 178 +++++ .../python/onnxruntime_pybind_mlvalue.cc | 105 ++- .../python/onnxruntime_pybind_mlvalue.h | 41 ++ .../python/onnxruntime_pybind_ortvalue.cc | 216 ++++++ .../python/onnxruntime_pybind_state.cc | 693 ++---------------- .../python/onnxruntime_pybind_state_common.cc | 49 +- .../python/onnxruntime_pybind_state_common.h | 204 ++++++ 13 files changed, 931 insertions(+), 686 deletions(-) create mode 100644 onnxruntime/python/onnxruntime_pybind_exceptions.cc create mode 100644 onnxruntime/python/onnxruntime_pybind_iobinding.cc create mode 100644 onnxruntime/python/onnxruntime_pybind_ortvalue.cc diff --git a/onnxruntime/core/session/IOBinding.cc b/onnxruntime/core/session/IOBinding.cc index c77b37507d..830caf03c3 100644 --- a/onnxruntime/core/session/IOBinding.cc +++ b/onnxruntime/core/session/IOBinding.cc @@ -109,6 +109,8 @@ void IOBinding::ClearOutputs() { const std::vector& IOBinding::GetOutputNames() const { return output_names_; } +const std::vector& IOBinding::GetOutputs() const { return outputs_; } + std::vector& IOBinding::GetOutputs() { return outputs_; } const std::vector& IOBinding::GetOutputsDeviceInfo() const { diff --git a/onnxruntime/core/session/IOBinding.h b/onnxruntime/core/session/IOBinding.h index 62953f0178..4989308f74 100644 --- a/onnxruntime/core/session/IOBinding.h +++ b/onnxruntime/core/session/IOBinding.h @@ -75,6 +75,7 @@ class IOBinding { * This simply collects the outputs obtained after calling Run() inside the @param outputs. */ const std::vector& GetOutputNames() const; + const std::vector& GetOutputs() const; std::vector& GetOutputs(); const std::vector& GetInputNames() const; diff --git a/onnxruntime/core/session/inference_session.cc b/onnxruntime/core/session/inference_session.cc index 09bb97781d..8ed0212d7c 100644 --- a/onnxruntime/core/session/inference_session.cc +++ b/onnxruntime/core/session/inference_session.cc @@ -2120,10 +2120,18 @@ SessionIOBinding::SessionIOBinding(InferenceSession* session) : sess_(session) { ORT_ENFORCE(session->NewIOBinding(&binding_).IsOK()); } +const InferenceSession* SessionIOBinding::GetInferenceSession() const { + return sess_; +} + InferenceSession* SessionIOBinding::GetInferenceSession() { return sess_; } +const IOBinding* SessionIOBinding::Get() const { + return binding_.get(); +} + IOBinding* SessionIOBinding::Get() { return binding_.get(); } diff --git a/onnxruntime/core/session/inference_session.h b/onnxruntime/core/session/inference_session.h index f49900928b..6033b72731 100644 --- a/onnxruntime/core/session/inference_session.h +++ b/onnxruntime/core/session/inference_session.h @@ -723,7 +723,9 @@ struct SessionIOBinding { public: SessionIOBinding(InferenceSession* session); + const IOBinding* Get() const; IOBinding* Get(); + const InferenceSession* GetInferenceSession() const; InferenceSession* GetInferenceSession(); private: diff --git a/onnxruntime/python/onnxruntime_pybind_exceptions.cc b/onnxruntime/python/onnxruntime_pybind_exceptions.cc new file mode 100644 index 0000000000..21897465bb --- /dev/null +++ b/onnxruntime/python/onnxruntime_pybind_exceptions.cc @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "onnxruntime_pybind_exceptions.h" + +namespace onnxruntime { +namespace python { +namespace py = pybind11; + +void ThrowIfPyErrOccured() { + if (PyErr_Occurred()) { + PyObject *ptype, *pvalue, *ptraceback; + PyErr_Fetch(&ptype, &pvalue, &ptraceback); + + PyObject* pStr = PyObject_Str(ptype); + std::string sType = py::reinterpret_borrow(pStr); + Py_XDECREF(pStr); + pStr = PyObject_Str(pvalue); + sType += ": "; + sType += py::reinterpret_borrow(pStr); + Py_XDECREF(pStr); + throw Fail(sType); + } +} + +void RegisterExceptions(pybind11::module& m) { + pybind11::register_exception(m, "Fail"); + pybind11::register_exception(m, "InvalidArgument"); + pybind11::register_exception(m, "NoSuchFile"); + pybind11::register_exception(m, "NoModel"); + pybind11::register_exception(m, "EngineError"); + pybind11::register_exception(m, "RuntimeException"); + pybind11::register_exception(m, "InvalidProtobuf"); + pybind11::register_exception(m, "ModelLoaded"); + pybind11::register_exception(m, "NotImplemented"); + pybind11::register_exception(m, "InvalidGraph"); + pybind11::register_exception(m, "EPFail"); +} + +void OrtPybindThrowIfError(onnxruntime::common::Status status) { + std::string msg = status.ToString(); + if (!status.IsOK()) { + switch (status.Code()) { + case onnxruntime::common::StatusCode::FAIL: + throw Fail(std::move(msg)); + case onnxruntime::common::StatusCode::INVALID_ARGUMENT: + throw InvalidArgument(std::move(msg)); + case onnxruntime::common::StatusCode::NO_SUCHFILE: + throw NoSuchFile(std::move(msg)); + case onnxruntime::common::StatusCode::NO_MODEL: + throw NoModel(std::move(msg)); + case onnxruntime::common::StatusCode::ENGINE_ERROR: + throw EngineError(std::move(msg)); + case onnxruntime::common::StatusCode::RUNTIME_EXCEPTION: + throw RuntimeException(std::move(msg)); + case onnxruntime::common::StatusCode::INVALID_PROTOBUF: + throw InvalidProtobuf(std::move(msg)); + case onnxruntime::common::StatusCode::NOT_IMPLEMENTED: + throw NotImplemented(std::move(msg)); + case onnxruntime::common::StatusCode::INVALID_GRAPH: + throw InvalidGraph(std::move(msg)); + case onnxruntime::common::StatusCode::EP_FAIL: + throw EPFail(std::move(msg)); + default: + throw std::runtime_error(std::move(msg)); + } + } +} + +} +} \ No newline at end of file diff --git a/onnxruntime/python/onnxruntime_pybind_exceptions.h b/onnxruntime/python/onnxruntime_pybind_exceptions.h index 8723e5daaa..3b42a76ed8 100644 --- a/onnxruntime/python/onnxruntime_pybind_exceptions.h +++ b/onnxruntime/python/onnxruntime_pybind_exceptions.h @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +#pragma once + #include #include #include "core/common/status.h" @@ -43,48 +45,9 @@ struct EPFail : std::runtime_error { explicit EPFail(const std::string& what) : std::runtime_error(what) {} }; -inline void RegisterExceptions(pybind11::module& m) { - pybind11::register_exception(m, "Fail"); - pybind11::register_exception(m, "InvalidArgument"); - pybind11::register_exception(m, "NoSuchFile"); - pybind11::register_exception(m, "NoModel"); - pybind11::register_exception(m, "EngineError"); - pybind11::register_exception(m, "RuntimeException"); - pybind11::register_exception(m, "InvalidProtobuf"); - pybind11::register_exception(m, "ModelLoaded"); - pybind11::register_exception(m, "NotImplemented"); - pybind11::register_exception(m, "InvalidGraph"); - pybind11::register_exception(m, "EPFail"); -} +void RegisterExceptions(pybind11::module& m); + +void OrtPybindThrowIfError(onnxruntime::common::Status status); -inline void OrtPybindThrowIfError(onnxruntime::common::Status status) { - std::string msg = status.ToString(); - if (!status.IsOK()) { - switch (status.Code()) { - case onnxruntime::common::StatusCode::FAIL: - throw Fail(std::move(msg)); - case onnxruntime::common::StatusCode::INVALID_ARGUMENT: - throw InvalidArgument(std::move(msg)); - case onnxruntime::common::StatusCode::NO_SUCHFILE: - throw NoSuchFile(std::move(msg)); - case onnxruntime::common::StatusCode::NO_MODEL: - throw NoModel(std::move(msg)); - case onnxruntime::common::StatusCode::ENGINE_ERROR: - throw EngineError(std::move(msg)); - case onnxruntime::common::StatusCode::RUNTIME_EXCEPTION: - throw RuntimeException(std::move(msg)); - case onnxruntime::common::StatusCode::INVALID_PROTOBUF: - throw InvalidProtobuf(std::move(msg)); - case onnxruntime::common::StatusCode::NOT_IMPLEMENTED: - throw NotImplemented(std::move(msg)); - case onnxruntime::common::StatusCode::INVALID_GRAPH: - throw InvalidGraph(std::move(msg)); - case onnxruntime::common::StatusCode::EP_FAIL: - throw EPFail(std::move(msg)); - default: - throw std::runtime_error(std::move(msg)); - } - } -} } // namespace python } // namespace onnxruntime diff --git a/onnxruntime/python/onnxruntime_pybind_iobinding.cc b/onnxruntime/python/onnxruntime_pybind_iobinding.cc new file mode 100644 index 0000000000..fb7965db15 --- /dev/null +++ b/onnxruntime/python/onnxruntime_pybind_iobinding.cc @@ -0,0 +1,178 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "python/onnxruntime_pybind_exceptions.h" +#include "python/onnxruntime_pybind_mlvalue.h" +#include "python/onnxruntime_pybind_state_common.h" + +#define NO_IMPORT_ARRAY +#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION +#define PY_ARRAY_UNIQUE_SYMBOL onnxruntime_python_ARRAY_API +#include + +#include "core/framework/ml_value.h" +#include "core/framework/tensor.h" +#include "core/framework/tensorprotoutils.h" +#include "core/framework/TensorSeq.h" +#include "core/session/IOBinding.h" + +namespace onnxruntime { +namespace python { + +namespace py = pybind11; + +void addIoBindingMethods(pybind11::module& m) { + py::class_ session_io_binding(m, "SessionIOBinding"); + session_io_binding + .def(py::init([](PyInferenceSession* sess) { + auto sess_io_binding = std::make_unique(sess->GetSessionHandle()); + return sess_io_binding; + })) + // May create Tensor/Sequence based OrtValues. Use bind_ortvalue_input for universal binding. + .def("bind_input", [](SessionIOBinding* io_binding, const std::string& name, py::object& arr_on_cpu) -> void { + InferenceSession* sess = io_binding->GetInferenceSession(); + auto px = sess->GetModelInputs(); + if (!px.first.IsOK() || !px.second) { + throw std::runtime_error("Either failed to get model inputs from the session object or the input def list was null"); + } + + // For now, limit binding support to only non-string Tensors + // TODO: Support non-tensors + const auto& def_list = *px.second; + onnx::TypeProto type_proto; + if (!CheckIfTensor(def_list, name, type_proto)) { + throw std::runtime_error("Only binding Tensors is currently supported"); + } + + ORT_ENFORCE(utils::HasTensorType(type_proto) && utils::HasElemType(type_proto.tensor_type())); + if (type_proto.tensor_type().elem_type() == onnx::TensorProto::STRING) { + throw std::runtime_error("Only binding non-string Tensors is currently supported"); + } + + OrtValue ml_value; + // Set the parameter `accept_only_numpy_array` to `true` (we only support binding Tensors) + CreateGenericMLValue(px.second, GetAllocator(), name, arr_on_cpu, &ml_value, true); + + auto status = io_binding->Get()->BindInput(name, ml_value); + if (!status.IsOK()) { + throw std::runtime_error("Error when bind input: " + status.ErrorMessage()); + } + }) + // This binds input as a Tensor that wraps memory pointer along with the OrtMemoryInfo + .def("bind_input", [](SessionIOBinding* io_binding, const std::string& name, const OrtDevice& device, py::object& element_type, std::vector& shape, int64_t data_ptr) -> void { + ORT_ENFORCE(data_ptr != 0, "Pointer to data memory is not valid"); + + PyArray_Descr* dtype; + if (!PyArray_DescrConverter(element_type.ptr(), &dtype)) { + throw std::runtime_error("Not a valid numpy type"); + } + int type_num = dtype->type_num; + Py_DECREF(dtype); + + OrtMemoryInfo info(GetDeviceName(device), OrtDeviceAllocator, device, device.Id()); + std::unique_ptr p_tensor = + std::make_unique(NumpyTypeToOnnxRuntimeType(type_num), shape, reinterpret_cast(data_ptr), info); + + OrtValue ml_value; + ml_value.Init(p_tensor.release(), + DataTypeImpl::GetType(), + DataTypeImpl::GetType()->GetDeleteFunc()); + + auto status = io_binding->Get()->BindInput(name, ml_value); + if (!status.IsOK()) { + throw std::runtime_error("Error when binding input: " + status.ErrorMessage()); + } + }) + // This binds input as an OrtValue which may contain various types and point to the user pre-allocated + // buffers + .def("bind_ortvalue_input", [](SessionIOBinding* io_binding, const std::string& name, const OrtValue& ml_value) -> void { + auto status = io_binding->Get()->BindInput(name, ml_value); + if (!status.IsOK()) { + throw std::runtime_error("Error when binding input: " + status.ErrorMessage()); + } + }) + // This binds output to a pre-allocated memory as a Tensor + .def("bind_output", [](SessionIOBinding* io_binding, const std::string& name, const OrtDevice& device, py::object& element_type, std::vector& shape, int64_t data_ptr) -> void { + ORT_ENFORCE(data_ptr != 0, "Pointer to data memory is not valid"); + + InferenceSession* sess = io_binding->GetInferenceSession(); + auto px = sess->GetModelOutputs(); + if (!px.first.IsOK() || !px.second) { + throw std::runtime_error("Either failed to get model inputs from the session object or the input def list was null"); + } + + // For now, limit binding support to only non-string Tensors + const auto& def_list = *px.second; + onnx::TypeProto type_proto; + if (!CheckIfTensor(def_list, name, type_proto)) { + throw std::runtime_error("Only binding Tensors is currently supported"); + } + + ORT_ENFORCE(utils::HasTensorType(type_proto) && utils::HasElemType(type_proto.tensor_type())); + if (type_proto.tensor_type().elem_type() == onnx::TensorProto::STRING) { + throw std::runtime_error("Only binding non-string Tensors is currently supported"); + } + + PyArray_Descr* dtype; + if (!PyArray_DescrConverter(element_type.ptr(), &dtype)) { + throw std::runtime_error("Not a valid numpy type"); + } + int type_num = dtype->type_num; + Py_DECREF(dtype); + + OrtMemoryInfo info(GetDeviceName(device), OrtDeviceAllocator, device, device.Id()); + + std::unique_ptr p_tensor = std::make_unique(NumpyTypeToOnnxRuntimeType(type_num), shape, reinterpret_cast(data_ptr), info); + + OrtValue ml_value; + ml_value.Init(p_tensor.release(), + DataTypeImpl::GetType(), + DataTypeImpl::GetType()->GetDeleteFunc()); + + auto status = io_binding->Get()->BindOutput(name, ml_value); + if (!status.IsOK()) { + throw std::runtime_error("Error when binding output: " + status.ErrorMessage()); + } + }) + // This binds output to a device. Meaning that the output OrtValue must be allocated on a specific device. + .def("bind_output", [](SessionIOBinding* io_binding, const std::string& name, const OrtDevice& device) -> void { + auto status = io_binding->Get()->BindOutput(name, device); + if (!status.IsOK()) { + throw std::runtime_error("Error when binding output: " + status.ErrorMessage()); + } + }) + // Binds output to a pre-constructed OrtValue which may contain various elements (e.g. Tensor/SparseTensor/TensorSequece) + .def("bind_ortvalue_output", [](SessionIOBinding* io_binding, const std::string& name, const OrtValue& ml_value) -> void { + auto status = io_binding->Get()->BindOutput(name, ml_value); + if (!status.IsOK()) { + throw std::runtime_error("Error when binding output: " + status.ErrorMessage()); + } + }) + .def("clear_binding_inputs", [](SessionIOBinding* io_binding) -> void { + io_binding->Get()->ClearInputs(); + }) + .def("clear_binding_outputs", [](SessionIOBinding* io_binding) -> void { + io_binding->Get()->ClearOutputs(); + }) + .def( + "get_outputs", [](const SessionIOBinding* io_binding) -> const std::vector& { + return io_binding->Get()->GetOutputs(); + }, + py::return_value_policy::reference_internal) + .def("copy_outputs_to_cpu", [](const SessionIOBinding* io_binding) -> std::vector { + const std::vector& outputs = io_binding->Get()->GetOutputs(); + std::vector rfetch; + rfetch.reserve(outputs.size()); + for (const auto& ort_value : outputs) { + if (ort_value.IsTensor()) { + AddTensorAsPyObj(ort_value, rfetch, &io_binding->GetInferenceSession()->GetDataTransferManager(), nullptr); + } else { + AddNonTensorAsPyObj(ort_value, rfetch, &io_binding->GetInferenceSession()->GetDataTransferManager(), nullptr); + } + } + return rfetch; + }); +} + +} // namespace python +} // namespace onnxruntime \ No newline at end of file diff --git a/onnxruntime/python/onnxruntime_pybind_mlvalue.cc b/onnxruntime/python/onnxruntime_pybind_mlvalue.cc index 8128fbc5ba..7b28fed2d1 100644 --- a/onnxruntime/python/onnxruntime_pybind_mlvalue.cc +++ b/onnxruntime/python/onnxruntime_pybind_mlvalue.cc @@ -2,6 +2,7 @@ // Licensed under the MIT License. #include "onnxruntime_pybind_mlvalue.h" +#include "python/onnxruntime_pybind_state_common.h" #define NO_IMPORT_ARRAY #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION @@ -16,7 +17,13 @@ #include "core/framework/data_types.h" #include "core/framework/onnxruntime_typeinfo.h" -using namespace std; +#include "core/framework/data_transfer_utils.h" +#include "core/framework/data_types_internal.h" +#include "core/providers/get_execution_providers.h" +#include "core/framework/kernel_registry.h" +#include "core/framework/provider_bridge_ort.h" +#include "core/framework/provider_options_utils.h" + namespace onnxruntime { namespace python { @@ -58,6 +65,102 @@ void CpuToCpuMemCpy(void* dst, const void* src, size_t num_bytes) { memcpy(dst, src, num_bytes); } +#ifdef USE_CUDA +void CpuToCudaMemCpy(void* dst, const void* src, size_t num_bytes) { + GetProviderInfo_CUDA()->cudaMemcpy_HostToDevice(dst, src, num_bytes); +} + +void CudaToCpuMemCpy(void* dst, const void* src, size_t num_bytes) { + GetProviderInfo_CUDA()->cudaMemcpy_DeviceToHost(dst, src, num_bytes); +} + +const std::unordered_map* GetCudaToHostMemCpyFunction() { + static std::unordered_map map{ + {OrtDevice::GPU, CudaToCpuMemCpy}}; + + return ↦ +} + +bool IsCudaDeviceIdValid(const onnxruntime::logging::Logger& logger, int id) { + int num_devices = GetProviderInfo_CUDA()->cudaGetDeviceCount(); + + if (0 == num_devices) { + LOGS(logger, WARNING) << "your system does not have a CUDA capable device."; + return false; + } + + if (id < 0 || id >= num_devices) { + LOGS(logger, WARNING) << "cuda_device=" << id << " is invalid, must choose device ID between 0 and " << num_devices - 1; + return false; + } + + return true; +} + +AllocatorPtr GetCudaAllocator(OrtDevice::DeviceId id) { + // Current approach is not thread-safe, but there are some bigger infra pieces to put together in order to make + // multi-threaded CUDA allocation work we need to maintain a per-thread CUDA allocator + + static auto* id_to_allocator_map = new std::unordered_map(); + + if (id_to_allocator_map->find(id) == id_to_allocator_map->end()) { + // TODO: Expose knobs so that users can set fields associated with OrtArenaCfg so that we can pass it to the following method + id_to_allocator_map->insert({id, GetProviderInfo_CUDA()->CreateCudaAllocator(id, gpu_mem_limit, arena_extend_strategy, external_allocator_info, nullptr)}); + } + + return (*id_to_allocator_map)[id]; +} + +#endif + +#ifdef USE_ROCM + +bool IsRocmDeviceIdValid(const onnxruntime::logging::Logger& logger, int id) { + int num_devices = 0; + HIP_CALL_THROW(hipGetDeviceCount(&num_devices)); + + if (0 == num_devices) { + LOGS(logger, WARNING) << "your system does not have a ROCM capable device."; + return false; + } + + if (id < 0 || id >= num_devices) { + LOGS(logger, WARNING) << "rocm_device=" << id << " is invalid, must choose device ID between 0 and " << num_devices - 1; + return false; + } + + return true; +} + +AllocatorPtr GetRocmAllocator(OrtDevice::DeviceId id) { + // Current approach is not thread-safe, but there are some bigger infra pieces to put together in order to make + // multi-threaded ROCM allocation work we need to maintain a per-thread ROCM allocator + static std::unordered_map id_to_allocator_map; + + if (id_to_allocator_map.find(id) == id_to_allocator_map.end()) { + id_to_allocator_map.insert({id, ROCMExecutionProvider::CreateRocmAllocator(id, gpu_mem_limit, arena_extend_strategy, external_allocator_info)}); + } + + return id_to_allocator_map[id]; +} + +void CpuToRocmMemCpy(void* dst, const void* src, size_t num_bytes) { + HIP_CALL_THROW(hipMemcpy(dst, src, num_bytes, hipMemcpyHostToDevice)); +} + +void RocmToCpuMemCpy(void* dst, const void* src, size_t num_bytes) { + HIP_CALL_THROW(hipMemcpy(dst, src, num_bytes, hipMemcpyDeviceToHost)); +} + +const std::unordered_map* GetRocmToHostMemCpyFunction() { + static std::unordered_map map{ + {OrtDevice::GPU, RocmToCpuMemCpy}}; + + return ↦ +} + +#endif + int OnnxRuntimeTensorToNumpyType(const DataTypeImpl* tensor_type) { static std::map type_map{ {DataTypeImpl::GetType(), NPY_BOOL}, diff --git a/onnxruntime/python/onnxruntime_pybind_mlvalue.h b/onnxruntime/python/onnxruntime_pybind_mlvalue.h index 72d5ac568e..2e34c32a82 100644 --- a/onnxruntime/python/onnxruntime_pybind_mlvalue.h +++ b/onnxruntime/python/onnxruntime_pybind_mlvalue.h @@ -16,6 +16,7 @@ #include "core/framework/ml_value.h" #include "core/session/inference_session.h" + namespace onnxruntime { namespace python { @@ -33,7 +34,47 @@ int OnnxRuntimeTensorToNumpyType(const DataTypeImpl* tensor_type); MLDataType NumpyTypeToOnnxRuntimeType(int numpy_type); using MemCpyFunc = void (*)(void*, const void*, size_t); + void CpuToCpuMemCpy(void*, const void*, size_t); + +void AddTensorAsPyObj(const OrtValue& val, std::vector& pyobjs, + const DataTransferManager* data_transfer_manager, + const std::unordered_map* mem_cpy_to_host_functions); + +void AddNonTensorAsPyObj(const OrtValue& val, std::vector& pyobjs, + const DataTransferManager* data_transfer_manager, + const std::unordered_map* mem_cpy_to_host_functions); + + +#ifdef USE_CUDA + +void CpuToCudaMemCpy(void* dst, const void* src, size_t num_bytes); + +void CudaToCpuMemCpy(void* dst, const void* src, size_t num_bytes); + +const std::unordered_map* GetCudaToHostMemCpyFunction(); + +bool IsCudaDeviceIdValid(const onnxruntime::logging::Logger& logger, int id); + +AllocatorPtr GetCudaAllocator(OrtDevice::DeviceId id); + +#endif + +#ifdef USE_ROCM + +bool IsRocmDeviceIdValid(const onnxruntime::logging::Logger& logger, int id); + +AllocatorPtr GetRocmAllocator(OrtDevice::DeviceId id); + +void CpuToRocmMemCpy(void* dst, const void* src, size_t num_bytes); + +void RocmToCpuMemCpy(void* dst, const void* src, size_t num_bytes); + +const std::unordered_map* GetRocmToHostMemCpyFunction(); + +#endif + + void CreateGenericMLValue(const onnxruntime::InputDefList* input_def_list, const AllocatorPtr& alloc, const std::string& name_input, py::object& value, OrtValue* p_mlvalue, bool accept_only_numpy_array = false, bool use_numpy_data_memory = true, MemCpyFunc mem_cpy_to_device = CpuToCpuMemCpy); diff --git a/onnxruntime/python/onnxruntime_pybind_ortvalue.cc b/onnxruntime/python/onnxruntime_pybind_ortvalue.cc new file mode 100644 index 0000000000..43b178e409 --- /dev/null +++ b/onnxruntime/python/onnxruntime_pybind_ortvalue.cc @@ -0,0 +1,216 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "python/onnxruntime_pybind_exceptions.h" +#include "python/onnxruntime_pybind_mlvalue.h" +#include "python/onnxruntime_pybind_state_common.h" + +#define NO_IMPORT_ARRAY +#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION +#define PY_ARRAY_UNIQUE_SYMBOL onnxruntime_python_ARRAY_API +#include + +#include "core/framework/ml_value.h" +#include "core/framework/tensor.h" +#include "core/framework/sparse_tensor.h" +#include "core/framework/TensorSeq.h" + +namespace onnxruntime { +namespace python { + +namespace py = pybind11; + +void addOrtValueMethods(pybind11::module& m) { + py::class_ ortvalue_binding(m, "OrtValue"); + ortvalue_binding + // Factory method to create an OrtValue (Tensor) from the given Numpy object + // The Tensor allocates and manages its own memory (on the specified device) and copies data from the Numpy data buffer + .def_static("ortvalue_from_numpy", [](py::object& array_on_cpu, const OrtDevice& device) { + if (!IsNumericNumpyArray(array_on_cpu)) { + throw std::runtime_error("Creation of OrtValues is currently only supported from non-string numpy arrays"); + } + + auto ml_value = std::make_unique(); + + // The tensor's memory is allocated on the CPU + if (strcmp(GetDeviceName(device), CPU) == 0) { + // InputDeflist is null because OrtValue creation is not tied to a specific model + // Likewise, there is no need to specify the name (as the name was previously used to lookup the def list) + + CreateGenericMLValue(nullptr, GetAllocator(), "", array_on_cpu, ml_value.get(), true); + } else if (strcmp(GetDeviceName(device), CUDA) == 0) { + // The tensor's memory is allocated on CUDA + +#ifdef USE_CUDA + if (!IsCudaDeviceIdValid(logging::LoggingManager::DefaultLogger(), device.Id())) { + throw std::runtime_error("The provided device id doesn't match any available GPUs on the machine."); + } + + // InputDeflist is null because OrtValue creation is not tied to a specific model + // Likewise, there is no need to specify the name (as the name was previously used to lookup the def list) + // TODO: Add check to ensure that string arrays are not passed - we currently don't support string tensors in CUDA + CreateGenericMLValue(nullptr, GetCudaAllocator(device.Id()), "", array_on_cpu, ml_value.get(), true, false, CpuToCudaMemCpy); +#elif USE_ROCM + if (!IsRocmDeviceIdValid(logging::LoggingManager::DefaultLogger(), device.Id())) { + throw std::runtime_error("The provided device id doesn't match any available GPUs on the machine."); + } + + // InputDeflist is null because OrtValue creation is not tied to a specific model + // Likewise, there is no need to specify the name (as the name was previously used to lookup the def list) + // TODO: Add check to ensure that string arrays are not passed - we currently don't support string tensors in CUDA + CreateGenericMLValue(nullptr, GetRocmAllocator(device.Id()), "", array_on_cpu, ml_value.get(), true, false, CpuToRocmMemCpy); + +#else + throw std::runtime_error( + "Can't allocate memory on the CUDA device using this package of OnnxRuntime. " + "Please use the CUDA package of OnnxRuntime to use this feature."); +#endif + } else { + throw std::runtime_error("Unsupported device: Cannot place the OrtValue on this device"); + } + + return ml_value; + }) + + // Factory method to create an OrtValue (Tensor) from the given shape and element type with memory on the specified device + // The memory is left uninitialized + .def_static("ortvalue_from_shape_and_type", [](const std::vector& shape, py::object& element_type, const OrtDevice& device) { + PyArray_Descr* dtype; + if (!PyArray_DescrConverter(element_type.ptr(), &dtype)) { + throw std::runtime_error("Not a valid numpy type"); + } + + int type_num = dtype->type_num; + Py_DECREF(dtype); + + if (!IsNumericNumpyType(type_num)) { + throw std::runtime_error("Creation of OrtValues is currently only supported from non-string numpy arrays"); + } + + auto ml_value = std::make_unique(); + + std::unique_ptr tensor; + // The tensor's memory is allocated on the CPU + if (strcmp(GetDeviceName(device), CPU) == 0) { + tensor = std::make_unique(NumpyTypeToOnnxRuntimeType(type_num), shape, GetAllocator()); + } else if (strcmp(GetDeviceName(device), CUDA) == 0) { + // The tensor's memory is allocated on CUDA +#ifdef USE_CUDA + if (!IsCudaDeviceIdValid(logging::LoggingManager::DefaultLogger(), device.Id())) { + throw std::runtime_error("The provided device id doesn't match any available GPUs on the machine."); + } + + tensor = std::make_unique(NumpyTypeToOnnxRuntimeType(type_num), shape, GetCudaAllocator(device.Id())); +#else + throw std::runtime_error( + "Can't allocate memory on the CUDA device using this package of OnnxRuntime. " + "Please use the CUDA package of OnnxRuntime to use this feature."); +#endif + } else { + throw std::runtime_error("Unsupported device: Cannot place the OrtValue on this device"); + } + + auto ml_tensor = DataTypeImpl::GetType(); + ml_value->Init(tensor.release(), + ml_tensor, + ml_tensor->GetDeleteFunc()); + + return ml_value; + }) + // Get a pointer to Tensor data + .def("data_ptr", [](OrtValue* ml_value) -> int64_t { + // TODO: Assumes that the OrtValue is a Tensor, make this generic to handle non-Tensors + ORT_ENFORCE(ml_value->IsTensor(), "Only OrtValues that are Tensors are currently supported"); + + auto* tensor = ml_value->GetMutable(); + + if (tensor->Shape().Size() == 0) { + return 0; + } + + // Should cover x86 and x64 platforms + return reinterpret_cast(tensor->MutableDataRaw()); + }) + .def("device_name", [](const OrtValue* ort_value) -> std::string { + if (ort_value->IsTensor()) { + return std::string(GetDeviceName(ort_value->Get().Location().device)); + } else { + ORT_THROW("Only OrtValues that are Tensors are currently supported"); + } + }) + .def("shape", [](const OrtValue* ort_value) -> py::list { + // OrtValue can only be a Tensor/SparseTensor, make this generic to handle non-Tensors + ORT_ENFORCE(ort_value->IsTensor() || ort_value->IsSparseTensor(), + "Only OrtValues that are Tensors/SpareTensors are currently supported"); + + py::list shape_arr; + const auto& dims = (ort_value->IsTensor()) + ? ort_value->Get().Shape().GetDims() + : ort_value->Get().Shape().GetDims(); + + for (auto dim : dims) { + // For sequence tensors - we would append a list of dims to the outermost list + // For now only tensors are supported in OrtValue + shape_arr.append(dim); + } + + return shape_arr; + }) + .def("data_type", [](const OrtValue* ort_value) -> std::string { + const ONNX_NAMESPACE::TypeProto* type_proto; + // Handle gutless types first to get the actual type + if (ort_value->IsTensor()) { + auto elem_type = ort_value->Get().GetElementType(); + type_proto = DataTypeImpl::TensorTypeFromONNXEnum(elem_type)->GetTypeProto(); + } else if (ort_value->IsSparseTensor()) { + auto elem_type = ort_value->Get().Values().GetElementType(); + type_proto = DataTypeImpl::SparseTensorTypeFromONNXEnum(elem_type)->GetTypeProto(); + } else if (ort_value->IsTensorSequence()) { + auto elem_type = ort_value->Get().DataType()->AsPrimitiveDataType()->GetDataType(); + type_proto = DataTypeImpl::SequenceTensorTypeFromONNXEnum(elem_type)->GetTypeProto(); + } else { + // Plane sequences and maps probably have their specific type + type_proto = ort_value->Type()->GetTypeProto(); + } + + ORT_ENFORCE(type_proto != nullptr, "Unknown type of OrtValue: ", ort_value->Type()); + + return *ONNX_NAMESPACE::Utils::DataTypeUtils::ToType(*type_proto); + }) + .def("is_tensor", [](const OrtValue* ort_value) -> bool { + return ort_value->IsTensor(); + }) + .def("is_sparse_tensor", [](const OrtValue* ort_value) -> bool { + return ort_value->IsSparseTensor(); + }) + .def("is_tensor_sequence", [](const OrtValue* ort_value) -> bool { + return ort_value->IsTensorSequence(); + }) + // Converts Tensor into a numpy array + .def("numpy", [](const OrtValue* ml_value) -> py::object { + ORT_ENFORCE(ml_value->IsTensor(), "Only OrtValues that are Tensors are convertible to Numpy objects"); + + py::object obj; + +#ifdef USE_CUDA + GetPyObjFromTensor(ml_value->Get(), obj, nullptr, GetCudaToHostMemCpyFunction()); +#elif USE_ROCM + GetPyObjFromTensor(ml_value->Get(), obj, nullptr, GetRocmToHostMemCpyFunction()); +#else + GetPyObjFromTensor(ml_value->Get(), obj, nullptr, nullptr); +#endif + return obj; + }) +#ifdef ENABLE_TRAINING + .def("to_dlpack", [](OrtValue* ort_value) -> py::object { + return ToDlpack(*ort_value); + }) + .def_static("from_dlpack", [](py::object data, bool is_bool_tensor = false) { + return FromDlpack(data, is_bool_tensor); + }) +#endif + ; +} + +} // namespace python +} // namespace onnxruntime \ No newline at end of file diff --git a/onnxruntime/python/onnxruntime_pybind_state.cc b/onnxruntime/python/onnxruntime_pybind_state.cc index 42d48584ea..e352093593 100644 --- a/onnxruntime/python/onnxruntime_pybind_state.cc +++ b/onnxruntime/python/onnxruntime_pybind_state.cc @@ -20,6 +20,7 @@ #include "core/framework/provider_bridge_ort.h" #include "core/framework/provider_options_utils.h" #include "core/framework/random_seed.h" +#include "core/framework/sparse_tensor.h" #include "core/framework/tensorprotoutils.h" #include "core/framework/TensorSeq.h" #include "core/graph/graph_viewer.h" @@ -28,175 +29,9 @@ #include "core/session/abi_session_options_impl.h" #ifdef ENABLE_TRAINING -#include "core/dlpack/dlpack_converter.h" #include "orttraining/training_ops/cpu/aten_ops/aten_op_executor.h" #endif -// execution provider factory creator headers -#include "core/providers/cpu/cpu_provider_factory_creator.h" -#ifdef USE_ROCM -#include "core/providers/rocm/rocm_provider_factory_creator.h" -#endif - -#include "core/providers/dnnl/dnnl_provider_factory.h" -#include "core/providers/shared_library/provider_host_api.h" - -struct OrtStatus { - OrtErrorCode code; - char msg[1]; // a null-terminated string -}; - -#if defined(USE_CUDA) || defined(USE_ROCM) -#define BACKEND_PROC "GPU" -#else -#define BACKEND_PROC "CPU" -#endif - -#if _OPENMP -#define BACKEND_OPENMP "-OPENMP" -#else -#define BACKEND_OPENMP "" -#endif - -#if USE_DNNL -#define BACKEND_DNNL "-DNNL" -#else -#define BACKEND_DNNL "" -#endif - -#if USE_MIGRAPHX -#define BACKEND_MIGRAPHX "-MIGRAPHX" -#else -#define BACKEND_MIGRAPHX "" -#endif - -#ifdef USE_OPENVINO -#if OPENVINO_CONFIG_CPU_FP32 -#define BACKEND_OPENVINO "-OPENVINO_CPU_FP32" - -#elif OPENVINO_CONFIG_GPU_FP32 -#define BACKEND_OPENVINO "-OPENVINO_GPU_FP32" - -#elif OPENVINO_CONFIG_GPU_FP16 -#define BACKEND_OPENVINO "-OPENVINO_GPU_FP16" - -#elif OPENVINO_CONFIG_MYRIAD -#define BACKEND_OPENVINO "-OPENVINO_MYRIAD" - -#elif OPENVINO_CONFIG_VAD_M -#define BACKEND_OPENVINO "-OPENVINO_VAD_M" - -#elif OPENVINO_CONFIG_VAD_F -#define BACKEND_OPENVINO "-OPENVINO_VAD_F" - -#elif OPENVINO_CONFIG_MULTI -#define BACKEND_OPENVINO "-OPENVINO_MULTI" - -#elif OPENVINO_CONFIG_HETERO -#define BACKEND_OPENVINO "-OPENVINO_HETERO" -#endif -#else -#define BACKEND_OPENVINO "" -#endif - -#ifdef USE_NUPHAR -#define BACKEND_NUPHAR "-NUPHAR" -#else -#define BACKEND_NUPHAR "" -#endif - -#if USE_VITISAI -#define BACKEND_VITISAI "-VITISAI" -#include "core/providers/vitisai/vitisai_execution_provider.h" -#else -#define BACKEND_VITISAI "" -#endif - -#if USE_OPENBLAS -#define BACKEND_OPENBLAS "-OPENBLAS" -#else -#define BACKEND_OPENBLAS "" -#endif - -#if USE_ACL -#define BACKEND_ACL "-ACL" -#else -#define BACKEND_ACL "" -#endif - -#if USE_ARMNN -#define BACKEND_ARMNN "-ARMNN" -#else -#define BACKEND_ARMNN "" -#endif - -#if USE_DML -#define BACKEND_DML "-DML" -#else -#define BACKEND_DML "" -#endif - -#define BACKEND_DEVICE BACKEND_PROC BACKEND_DNNL BACKEND_OPENVINO BACKEND_NUPHAR BACKEND_OPENBLAS BACKEND_MIGRAPHX BACKEND_ACL BACKEND_ARMNN BACKEND_DML -#include "core/session/onnxruntime_cxx_api.h" -#include "core/providers/providers.h" -#include "core/providers/cpu/cpu_execution_provider.h" - -#if defined(USE_CUDA) || defined(USE_ROCM) -#ifdef USE_CUDA -#include "core/providers/cuda/cuda_execution_provider_info.h" -// TODO remove deprecated global config -OrtCudnnConvAlgoSearch cudnn_conv_algo_search = OrtCudnnConvAlgoSearch::EXHAUSTIVE; -// TODO remove deprecated global config -bool do_copy_in_default_stream = true; -onnxruntime::CUDAExecutionProviderExternalAllocatorInfo external_allocator_info{}; -#endif - -#ifdef USE_ROCM -#include "core/providers/rocm/rocm_execution_provider.h" -#include "core/providers/rocm/rocm_allocator.h" -onnxruntime::ROCMExecutionProviderExternalAllocatorInfo external_allocator_info{}; -#endif - -// TODO remove deprecated global config -OrtDevice::DeviceId cuda_device_id = 0; -// TODO remove deprecated global config -size_t gpu_mem_limit = std::numeric_limits::max(); -// TODO remove deprecated global config -onnxruntime::ArenaExtendStrategy arena_extend_strategy = onnxruntime::ArenaExtendStrategy::kNextPowerOfTwo; -#endif - -#ifdef USE_CUDA -#include "core/providers/cuda/cuda_provider_factory.h" -#endif -#ifdef USE_TENSORRT -#include "core/providers/tensorrt/tensorrt_provider_factory.h" -#endif -#ifdef USE_MIGRAPHX -#include "core/providers/migraphx/migraphx_provider_factory.h" -#endif -#ifdef USE_OPENVINO -#include "core/providers/openvino/openvino_provider_factory.h" -// TODO remove deprecated global config -std::string openvino_device_type; -#endif -#ifdef USE_NUPHAR -#include "core/providers/nuphar/nuphar_provider_factory.h" -// TODO remove deprecated global config -std::string nuphar_settings; -#endif -#ifdef USE_VITISAI -#include "core/providers/vitisai/vitisai_provider_factory.h" -#endif -#ifdef USE_ACL -#include "core/providers/acl/acl_provider_factory.h" -#endif -#ifdef USE_ARMNN -#include "core/providers/armnn/armnn_provider_factory.h" -#endif -#ifdef USE_DML -#include "core/providers/dml/dml_provider_factory.h" -#endif - // Explicitly provide a definition for the static const var 'GPU' in the OrtDevice struct, // GCC 4.x doesn't seem to define this and it breaks the pipelines based on CentOS as it uses // GCC 4.x. @@ -211,12 +46,6 @@ std::shared_ptr CreateExecutionProviderFactory_MIGrap std::shared_ptr CreateExecutionProviderFactory_Cuda(const OrtCUDAProviderOptions* params); std::shared_ptr CreateExecutionProviderFactory_Dnnl(int use_arena); std::shared_ptr CreateExecutionProviderFactory_OpenVINO(const OrtOpenVINOProviderOptions* params); -#ifdef USE_CUDA -ProviderInfo_CUDA* GetProviderInfo_CUDA(); -#endif -#ifdef USE_OPENVINO -ProviderInfo_OpenVINO* GetProviderInfo_OpenVINO(); -#endif std::shared_ptr CreateExecutionProviderFactory_Nuphar(bool, const char*); std::shared_ptr CreateExecutionProviderFactory_VITISAI(const char* backend_type, int device_id); std::shared_ptr CreateExecutionProviderFactory_ACL(int use_arena); @@ -366,7 +195,7 @@ void GetPyObjFromTensor(const Tensor& rtensor, py::object& obj, } } -static const char* GetDeviceName(const OrtDevice& device) { +const char* GetDeviceName(const OrtDevice& device) { switch (device.Type()) { case OrtDevice::CPU: return CPU; @@ -393,9 +222,9 @@ void AddNonTensor(const OrtValue& val, std::vector& pyobj pyobjs.push_back(py_list); } -static void AddNonTensorAsPyObj(const OrtValue& val, std::vector& pyobjs, - const DataTransferManager* data_transfer_manager, - const std::unordered_map* mem_cpy_to_host_functions) { +void AddNonTensorAsPyObj(const OrtValue& val, std::vector& pyobjs, + const DataTransferManager* data_transfer_manager, + const std::unordered_map* mem_cpy_to_host_functions) { // Should be in sync with core/framework/datatypes.h auto val_type = val.Type(); if (val_type->IsTensorSequenceType()) { @@ -437,9 +266,9 @@ static void AddNonTensorAsPyObj(const OrtValue& val, std::vector& py } } -static void AddTensorAsPyObj(const OrtValue& val, std::vector& pyobjs, - const DataTransferManager* data_transfer_manager, - const std::unordered_map* mem_cpy_to_host_functions) { +void AddTensorAsPyObj(const OrtValue& val, std::vector& pyobjs, + const DataTransferManager* data_transfer_manager, + const std::unordered_map* mem_cpy_to_host_functions) { const Tensor& rtensor = val.Get(); py::object obj; GetPyObjFromTensor(rtensor, obj, data_transfer_manager, mem_cpy_to_host_functions); @@ -468,103 +297,6 @@ static std::unique_ptr LoadExecutionProvider( return ep_factory->CreateProvider(); } -#ifdef USE_CUDA - -static bool IsCudaDeviceIdValid(const onnxruntime::logging::Logger& logger, int id) { - int num_devices = GetProviderInfo_CUDA()->cudaGetDeviceCount(); - - if (0 == num_devices) { - LOGS(logger, WARNING) << "your system does not have a CUDA capable device."; - return false; - } - - if (id < 0 || id >= num_devices) { - LOGS(logger, WARNING) << "cuda_device=" << id << " is invalid, must choose device ID between 0 and " << num_devices - 1; - return false; - } - - return true; -} - -static AllocatorPtr GetCudaAllocator(OrtDevice::DeviceId id) { - // Current approach is not thread-safe, but there are some bigger infra pieces to put together in order to make - // multi-threaded CUDA allocation work we need to maintain a per-thread CUDA allocator - - static auto* id_to_allocator_map = new std::unordered_map(); - - if (id_to_allocator_map->find(id) == id_to_allocator_map->end()) { - // TODO: Expose knobs so that users can set fields associated with OrtArenaCfg so that we can pass it to the following method - id_to_allocator_map->insert({id, GetProviderInfo_CUDA()->CreateCudaAllocator(id, gpu_mem_limit, arena_extend_strategy, external_allocator_info, nullptr)}); - } - - return (*id_to_allocator_map)[id]; -} - -static void CpuToCudaMemCpy(void* dst, const void* src, size_t num_bytes) { - GetProviderInfo_CUDA()->cudaMemcpy_HostToDevice(dst, src, num_bytes); -} - -static void CudaToCpuMemCpy(void* dst, const void* src, size_t num_bytes) { - GetProviderInfo_CUDA()->cudaMemcpy_DeviceToHost(dst, src, num_bytes); -} - -static const std::unordered_map* GetCudaToHostMemCpyFunction() { - static std::unordered_map map{ - {OrtDevice::GPU, CudaToCpuMemCpy}}; - - return ↦ -} - -#endif - -#ifdef USE_ROCM - -static bool IsRocmDeviceIdValid(const onnxruntime::logging::Logger& logger, int id) { - int num_devices = 0; - HIP_CALL_THROW(hipGetDeviceCount(&num_devices)); - - if (0 == num_devices) { - LOGS(logger, WARNING) << "your system does not have a ROCM capable device."; - return false; - } - - if (id < 0 || id >= num_devices) { - LOGS(logger, WARNING) << "rocm_device=" << id << " is invalid, must choose device ID between 0 and " << num_devices - 1; - return false; - } - - return true; -} - -static AllocatorPtr GetRocmAllocator(OrtDevice::DeviceId id) { - // Current approach is not thread-safe, but there are some bigger infra pieces to put together in order to make - // multi-threaded ROCM allocation work we need to maintain a per-thread ROCM allocator - static std::unordered_map id_to_allocator_map; - - if (id_to_allocator_map.find(id) == id_to_allocator_map.end()) { - id_to_allocator_map.insert({id, ROCMExecutionProvider::CreateRocmAllocator(id, gpu_mem_limit, arena_extend_strategy, external_allocator_info)}); - } - - return id_to_allocator_map[id]; -} - -static void CpuToRocmMemCpy(void* dst, const void* src, size_t num_bytes) { - HIP_CALL_THROW(hipMemcpy(dst, src, num_bytes, hipMemcpyHostToDevice)); -} - -static void RocmToCpuMemCpy(void* dst, const void* src, size_t num_bytes) { - HIP_CALL_THROW(hipMemcpy(dst, src, num_bytes, hipMemcpyDeviceToHost)); -} - -static const std::unordered_map* GetRocmToHostMemCpyFunction() { - static std::unordered_map map{ - {OrtDevice::GPU, RocmToCpuMemCpy}}; - - return ↦ -} - -#endif - /* * Register execution provider with options. */ @@ -962,9 +694,9 @@ void InitializeSession(InferenceSession* sess, const std::vector& p OrtPybindThrowIfError(sess->Initialize()); } -static bool CheckIfTensor(const std::vector& def_list, - const std::string& name, - /*out*/ onnx::TypeProto& type_proto) { +bool CheckIfTensor(const std::vector& def_list, + const std::string& name, + /*out*/ onnx::TypeProto& type_proto) { auto ret_it = std::find_if(std::begin(def_list), std::end(def_list), [&name](const NodeArg* node_arg) { return name == node_arg->Name(); }); if (ret_it == std::end(def_list)) { @@ -1033,7 +765,7 @@ void addGlobalMethods(py::module& m, Environment& env) { } }); #ifdef ENABLE_TRAINING - m.def( + m.def( "register_aten_op_executor", [](const std::string& aten_op_executor_address_str) -> void { size_t aten_op_executor_address_int; ORT_THROW_IF_ERROR(ParseStringWithClassicLocale(aten_op_executor_address_str, aten_op_executor_address_int)); @@ -1321,11 +1053,13 @@ void addOpSchemaSubmodule(py::module& m) { .value("INT", ONNX_NAMESPACE::AttributeProto::INT) .value("STRING", ONNX_NAMESPACE::AttributeProto::STRING) .value("TENSOR", ONNX_NAMESPACE::AttributeProto::TENSOR) + .value("SPARSE_TENSOR", ONNX_NAMESPACE::AttributeProto::SPARSE_TENSOR) .value("GRAPH", ONNX_NAMESPACE::AttributeProto::GRAPH) .value("FLOATS", ONNX_NAMESPACE::AttributeProto::FLOATS) .value("INTS", ONNX_NAMESPACE::AttributeProto::INTS) .value("STRINGS", ONNX_NAMESPACE::AttributeProto::STRINGS) .value("TENSORS", ONNX_NAMESPACE::AttributeProto::TENSORS) + .value("SPARSE_TENSORS", ONNX_NAMESPACE::AttributeProto::SPARSE_TENSORS) .value("GRAPHS", ONNX_NAMESPACE::AttributeProto::GRAPHS); // Keep this binding local to this module @@ -1400,320 +1134,6 @@ void addObjectMethods(py::module& m, Environment& env) { } })); - py::class_ - ortvalue_binding(m, "OrtValue"); - ortvalue_binding - // Factory method to create an OrtValue (Tensor) from the given Numpy object - // The Tensor allocates and manages its own memory (on the specified device) and copies data from the Numpy data buffer - .def_static("ortvalue_from_numpy", [](py::object& array_on_cpu, OrtDevice& device) { - if (!IsNumericNumpyArray(array_on_cpu)) { - throw std::runtime_error("Creation of OrtValues is currently only supported from non-string numpy arrays"); - } - - auto ml_value = std::make_unique(); - - // The tensor's memory is allocated on the CPU - if (GetDeviceName(device) == CPU) { - // InputDeflist is null because OrtValue creation is not tied to a specific model - // Likewise, there is no need to specify the name (as the name was previously used to lookup the def list) - - CreateGenericMLValue(nullptr, GetAllocator(), "", array_on_cpu, ml_value.get(), true); - } else if (GetDeviceName(device) == CUDA) { - // The tensor's memory is allocated on CUDA - -#ifdef USE_CUDA - if (!IsCudaDeviceIdValid(logging::LoggingManager::DefaultLogger(), device.Id())) { - throw std::runtime_error("The provided device id doesn't match any available GPUs on the machine."); - } - - // InputDeflist is null because OrtValue creation is not tied to a specific model - // Likewise, there is no need to specify the name (as the name was previously used to lookup the def list) - // TODO: Add check to ensure that string arrays are not passed - we currently don't support string tensors in CUDA - CreateGenericMLValue(nullptr, GetCudaAllocator(device.Id()), "", array_on_cpu, ml_value.get(), true, false, CpuToCudaMemCpy); -#elif USE_ROCM - if (!IsRocmDeviceIdValid(logging::LoggingManager::DefaultLogger(), device.Id())) { - throw std::runtime_error("The provided device id doesn't match any available GPUs on the machine."); - } - - // InputDeflist is null because OrtValue creation is not tied to a specific model - // Likewise, there is no need to specify the name (as the name was previously used to lookup the def list) - // TODO: Add check to ensure that string arrays are not passed - we currently don't support string tensors in CUDA - CreateGenericMLValue(nullptr, GetRocmAllocator(device.Id()), "", array_on_cpu, ml_value.get(), true, false, CpuToRocmMemCpy); - -#else - throw std::runtime_error( - "Can't allocate memory on the CUDA device using this package of OnnxRuntime. " - "Please use the CUDA package of OnnxRuntime to use this feature."); -#endif - } else { - throw std::runtime_error("Unsupported device: Cannot place the OrtValue on this device"); - } - - return ml_value; - }) - - // Factory method to create an OrtValue (Tensor) from the given shape and element type with memory on the specified device - // The memory is left uninitialized - .def_static("ortvalue_from_shape_and_type", [](std::vector& shape, py::object& element_type, OrtDevice& device) { - PyArray_Descr* dtype; - if (!PyArray_DescrConverter(element_type.ptr(), &dtype)) { - throw std::runtime_error("Not a valid numpy type"); - } - - int type_num = dtype->type_num; - Py_DECREF(dtype); - - if (!IsNumericNumpyType(type_num)) { - throw std::runtime_error("Creation of OrtValues is currently only supported from non-string numpy arrays"); - } - - auto ml_value = std::make_unique(); - - std::unique_ptr tensor; - // The tensor's memory is allocated on the CPU - if (GetDeviceName(device) == CPU) { - tensor = std::make_unique(NumpyTypeToOnnxRuntimeType(type_num), shape, GetAllocator()); - } else if (GetDeviceName(device) == CUDA) { - // The tensor's memory is allocated on CUDA -#ifdef USE_CUDA - if (!IsCudaDeviceIdValid(logging::LoggingManager::DefaultLogger(), device.Id())) { - throw std::runtime_error("The provided device id doesn't match any available GPUs on the machine."); - } - - tensor = std::make_unique(NumpyTypeToOnnxRuntimeType(type_num), shape, GetCudaAllocator(device.Id())); -#else - throw std::runtime_error( - "Can't allocate memory on the CUDA device using this package of OnnxRuntime. " - "Please use the CUDA package of OnnxRuntime to use this feature."); -#endif - } else { - throw std::runtime_error("Unsupported device: Cannot place the OrtValue on this device"); - } - - auto ml_tensor = DataTypeImpl::GetType(); - ml_value->Init(tensor.release(), - ml_tensor, - ml_tensor->GetDeleteFunc()); - - return ml_value; - }) - .def("data_ptr", [](OrtValue* ml_value) -> int64_t { - // TODO: Assumes that the OrtValue is a Tensor, make this generic to handle non-Tensors - ORT_ENFORCE(ml_value->IsTensor(), "Only OrtValues that are Tensors are currently supported"); - - auto* tensor = ml_value->GetMutable(); - - if (tensor->Shape().Size() == 0) { - return 0; - } - - // Should cover x86 and x64 platforms - return reinterpret_cast(tensor->MutableDataRaw()); - }) - .def("device_name", [](OrtValue* ml_value) -> std::string { - // TODO: Assumes that the OrtValue is a Tensor, make this generic to handle non-Tensors - ORT_ENFORCE(ml_value->IsTensor(), "Only OrtValues that are Tensors are currently supported"); - - return std::string(GetDeviceName(ml_value->Get().Location().device)); - }) - .def("shape", [](OrtValue* ml_value) -> py::list { - // TODO: Assumes that the OrtValue is a Tensor, make this generic to handle non-Tensors - ORT_ENFORCE(ml_value->IsTensor(), "Only OrtValues that are Tensors are currently supported"); - - py::list shape_arr; - const auto& dims = ml_value->Get().Shape().GetDims(); - - for (auto dim : dims) { - // For sequence tensors - we would append a list of dims to the outermost list - // For now only tensors are supported in OrtValue - shape_arr.append(dim); - } - - return shape_arr; - }) - .def("data_type", [](OrtValue* ml_value) -> std::string { - // TODO: Assumes that the OrtValue is a Tensor, make this generic to handle non-Tensors - ORT_ENFORCE(ml_value->IsTensor(), "Only OrtValues that are Tensors are currently supported"); - - // Currently only "tensor" OrtValues are supported - std::ostringstream ostr; - ostr << "tensor"; - ostr << "("; - ostr << DataTypeImpl::ToString(ml_value->Get().DataType()); - ostr << ")"; - - return ostr.str(); - }) - .def("is_tensor", [](OrtValue* ml_value) -> bool { - return ml_value->IsTensor(); - }) - .def("numpy", [](OrtValue* ml_value) -> py::object { - ORT_ENFORCE(ml_value->IsTensor(), "Only OrtValues that are Tensors are convertible to Numpy objects"); - - py::object obj; - -#ifdef USE_CUDA - GetPyObjFromTensor(ml_value->Get(), obj, nullptr, GetCudaToHostMemCpyFunction()); -#elif USE_ROCM - GetPyObjFromTensor(ml_value->Get(), obj, nullptr, GetRocmToHostMemCpyFunction()); -#else - GetPyObjFromTensor(ml_value->Get(), obj, nullptr, nullptr); -#endif - return obj; - }) -#ifdef ENABLE_TRAINING - .def("to_dlpack", [](OrtValue* ort_value) -> py::object { - return ToDlpack(*ort_value); - }) - .def_static("from_dlpack", [](py::object data, bool is_bool_tensor = false) { - return FromDlpack(data, is_bool_tensor); - }) -#endif - ; - - py::class_ session_io_binding(m, "SessionIOBinding"); - session_io_binding - .def(py::init([](PyInferenceSession* sess) { - auto sess_io_binding = std::make_unique(sess->GetSessionHandle()); - return sess_io_binding; - })) - .def("bind_input", [](SessionIOBinding* io_binding, const std::string& name, py::object& arr_on_cpu) -> void { - InferenceSession* sess = io_binding->GetInferenceSession(); - auto px = sess->GetModelInputs(); - if (!px.first.IsOK() || !px.second) { - throw std::runtime_error("Either failed to get model inputs from the session object or the input def list was null"); - } - - // For now, limit binding support to only non-string Tensors - // TODO: Support non-tensors - const auto& def_list = *px.second; - onnx::TypeProto type_proto; - if (!CheckIfTensor(def_list, name, type_proto)) { - throw std::runtime_error("Only binding Tensors is currently supported"); - } - - ORT_ENFORCE(type_proto.tensor_type().has_elem_type()); - if (type_proto.tensor_type().elem_type() == onnx::TensorProto::STRING) { - throw std::runtime_error("Only binding non-string Tensors is currently supported"); - } - - OrtValue ml_value; - // Set the parameter `accept_only_numpy_array` to `true` (we only support binding Tensors) - CreateGenericMLValue(px.second, GetAllocator(), name, arr_on_cpu, &ml_value, true); - - auto status = io_binding->Get()->BindInput(name, ml_value); - if (!status.IsOK()) { - throw std::runtime_error("Error when bind input: " + status.ErrorMessage()); - } - }) - .def("bind_input", [](SessionIOBinding* io_binding, const std::string& name, const OrtDevice& device, py::object& element_type, std::vector& shape, int64_t data_ptr) -> void { - ORT_ENFORCE(data_ptr != 0, "Pointer to data memory is not valid"); - - PyArray_Descr* dtype; - if (!PyArray_DescrConverter(element_type.ptr(), &dtype)) { - throw std::runtime_error("Not a valid numpy type"); - } - int type_num = dtype->type_num; - Py_DECREF(dtype); - - OrtMemoryInfo info(GetDeviceName(device), OrtDeviceAllocator, device, device.Id()); - std::unique_ptr p_tensor = - std::make_unique(NumpyTypeToOnnxRuntimeType(type_num), shape, reinterpret_cast(data_ptr), info); - - OrtValue ml_value; - ml_value.Init(p_tensor.release(), - DataTypeImpl::GetType(), - DataTypeImpl::GetType()->GetDeleteFunc()); - - auto status = io_binding->Get()->BindInput(name, ml_value); - if (!status.IsOK()) { - throw std::runtime_error("Error when binding input: " + status.ErrorMessage()); - } - }) - .def("bind_ortvalue_input", [](SessionIOBinding* io_binding, const std::string& name, OrtValue& ml_value) -> void { - auto status = io_binding->Get()->BindInput(name, ml_value); - if (!status.IsOK()) { - throw std::runtime_error("Error when binding input: " + status.ErrorMessage()); - } - }) - .def("bind_output", [](SessionIOBinding* io_binding, const std::string& name, const OrtDevice& device, py::object& element_type, std::vector& shape, int64_t data_ptr) -> void { - ORT_ENFORCE(data_ptr != 0, "Pointer to data memory is not valid"); - - InferenceSession* sess = io_binding->GetInferenceSession(); - auto px = sess->GetModelOutputs(); - if (!px.first.IsOK() || !px.second) { - throw std::runtime_error("Either failed to get model inputs from the session object or the input def list was null"); - } - - // For now, limit binding support to only non-string Tensors - // TODO: Support non-tensors - const auto& def_list = *px.second; - onnx::TypeProto type_proto; - if (!CheckIfTensor(def_list, name, type_proto)) { - throw std::runtime_error("Only binding Tensors is currently supported"); - } - - ORT_ENFORCE(type_proto.tensor_type().has_elem_type()); - if (type_proto.tensor_type().elem_type() == onnx::TensorProto::STRING) { - throw std::runtime_error("Only binding non-string Tensors is currently supported"); - } - - PyArray_Descr* dtype; - if (!PyArray_DescrConverter(element_type.ptr(), &dtype)) { - throw std::runtime_error("Not a valid numpy type"); - } - int type_num = dtype->type_num; - Py_DECREF(dtype); - - OrtMemoryInfo info(GetDeviceName(device), OrtDeviceAllocator, device, device.Id()); - - std::unique_ptr p_tensor = std::make_unique(NumpyTypeToOnnxRuntimeType(type_num), shape, reinterpret_cast(data_ptr), info); - - OrtValue ml_value; - ml_value.Init(p_tensor.release(), - DataTypeImpl::GetType(), - DataTypeImpl::GetType()->GetDeleteFunc()); - - auto status = io_binding->Get()->BindOutput(name, ml_value); - if (!status.IsOK()) { - throw std::runtime_error("Error when binding output: " + status.ErrorMessage()); - } - }) - .def("bind_output", [](SessionIOBinding* io_binding, const std::string& name, const OrtDevice& device) -> void { - auto status = io_binding->Get()->BindOutput(name, device); - if (!status.IsOK()) { - throw std::runtime_error("Error when binding output: " + status.ErrorMessage()); - } - }) - .def("bind_ortvalue_output", [](SessionIOBinding* io_binding, const std::string& name, OrtValue& ml_value) -> void { - auto status = io_binding->Get()->BindOutput(name, ml_value); - if (!status.IsOK()) { - throw std::runtime_error("Error when binding output: " + status.ErrorMessage()); - } - }) - .def("clear_binding_inputs", [](SessionIOBinding* io_binding) -> void { - io_binding->Get()->ClearInputs(); - }) - .def("clear_binding_outputs", [](SessionIOBinding* io_binding) -> void { - io_binding->Get()->ClearOutputs(); - }) - .def("get_outputs", [](SessionIOBinding* io_binding) -> std::vector& { - return io_binding->Get()->GetOutputs(); - }) - .def("copy_outputs_to_cpu", [](SessionIOBinding* io_binding) -> std::vector { - const std::vector& outputs = io_binding->Get()->GetOutputs(); - std::vector rfetch; - rfetch.reserve(outputs.size()); - for (const auto& _ : outputs) { - if (_.IsTensor()) { - AddTensorAsPyObj(_, rfetch, &io_binding->GetInferenceSession()->GetDataTransferManager(), nullptr); - } else { - AddNonTensorAsPyObj(_, rfetch, &io_binding->GetInferenceSession()->GetDataTransferManager(), nullptr); - } - } - return rfetch; - }); - py::class_ sess(m, "SessionOptions", R"pbdoc(Configuration information for a session.)pbdoc"); sess @@ -1833,7 +1253,7 @@ Applies to session load, initialization, etc. Default is 0.)pbdoc") R"pbdoc(Set a single session configuration entry as a pair of strings.)pbdoc") .def( "get_session_config_entry", - [](PySessionOptions* options, const char* config_key) -> std::string { + [](const PySessionOptions* options, const char* config_key) -> std::string { const std::string key(config_key); std::string value; if (!options->config_options.TryGetConfigEntry(key, value)) @@ -1844,8 +1264,7 @@ Applies to session load, initialization, etc. Default is 0.)pbdoc") R"pbdoc(Get a single session configuration value using the given configuration key.)pbdoc") .def( "register_custom_ops_library", - [](PySessionOptions* options, const char* library_path) - -> void { + [](PySessionOptions* options, const char* library_path) -> void { #if !defined(ORT_MINIMAL_BUILD) || defined(ORT_MINIMAL_BUILD_CUSTOM_OPS) // We need to pass in an `OrtSessionOptions` instance because the exported method in the shared library expects that // Once we have access to the `OrtCustomOpDomains` within the passed in `OrtSessionOptions` instance, we place it @@ -1872,7 +1291,7 @@ Applies to session load, initialization, etc. Default is 0.)pbdoc") // The user needs to ensure that the python OrtValue being provided as an overriding initializer // is not destructed as long as any session that uses the provided OrtValue initializer is still in scope // This is no different than the native APIs - OrtValue* ml_value = ml_value_pyobject.attr(PYTHON_ORTVALUE_NATIVE_OBJECT_ATTR).cast(); + const OrtValue* ml_value = ml_value_pyobject.attr(PYTHON_ORTVALUE_NATIVE_OBJECT_ATTR).cast(); options->AddInitializer(name, ml_value); }); @@ -2050,42 +1469,56 @@ including arg name, arg type (contains both type and shape).)pbdoc") } return rfetch; }) - .def("end_profiling", [](PyInferenceSession* sess) -> std::string { + .def("end_profiling", [](const PyInferenceSession* sess) -> std::string { return sess->GetSessionHandle()->EndProfiling(); }) .def_property_readonly("get_profiling_start_time_ns", [](const PyInferenceSession* sess) -> uint64_t { return sess->GetSessionHandle()->GetProfiling().GetStartTimeNs(); }) - .def("get_providers", [](PyInferenceSession* sess) -> const std::vector& { - return sess->GetSessionHandle()->GetRegisteredProviderTypes(); - }) - .def("get_provider_options", [](const PyInferenceSession* sess) -> const ProviderOptionsMap& { - return sess->GetSessionHandle()->GetAllProviderOptions(); - }) - .def_property_readonly("session_options", [](PyInferenceSession* sess) -> const PySessionOptions& { - const auto& session_options = sess->GetSessionHandle()->GetSessionOptions(); - return static_cast(session_options); - }) - .def_property_readonly("inputs_meta", [](const PyInferenceSession* sess) -> const std::vector& { - auto res = sess->GetSessionHandle()->GetModelInputs(); - OrtPybindThrowIfError(res.first); - return *(res.second); - }) - .def_property_readonly("outputs_meta", [](const PyInferenceSession* sess) -> const std::vector& { - auto res = sess->GetSessionHandle()->GetModelOutputs(); - OrtPybindThrowIfError(res.first); - return *(res.second); - }) - .def_property_readonly("overridable_initializers", [](const PyInferenceSession* sess) -> const std::vector& { - auto res = sess->GetSessionHandle()->GetOverridableInitializers(); - OrtPybindThrowIfError(res.first); - return *(res.second); - }) - .def_property_readonly("model_meta", [](const PyInferenceSession* sess) -> const onnxruntime::ModelMetadata& { - auto res = sess->GetSessionHandle()->GetModelMetadata(); - OrtPybindThrowIfError(res.first); - return *(res.second); - }) + .def( + "get_providers", [](const PyInferenceSession* sess) -> const std::vector& { + return sess->GetSessionHandle()->GetRegisteredProviderTypes(); + }, + py::return_value_policy::reference_internal) + .def( + "get_provider_options", [](const PyInferenceSession* sess) -> const ProviderOptionsMap& { + return sess->GetSessionHandle()->GetAllProviderOptions(); + }, + py::return_value_policy::reference_internal) + .def_property_readonly( + "session_options", [](const PyInferenceSession* sess) -> const PySessionOptions& { + const auto& session_options = sess->GetSessionHandle()->GetSessionOptions(); + return static_cast(session_options); + }, + py::return_value_policy::reference_internal) + .def_property_readonly( + "inputs_meta", [](const PyInferenceSession* sess) -> const std::vector& { + auto res = sess->GetSessionHandle()->GetModelInputs(); + OrtPybindThrowIfError(res.first); + return *(res.second); + }, + py::return_value_policy::reference_internal) + .def_property_readonly( + "outputs_meta", [](const PyInferenceSession* sess) -> const std::vector& { + auto res = sess->GetSessionHandle()->GetModelOutputs(); + OrtPybindThrowIfError(res.first); + return *(res.second); + }, + py::return_value_policy::reference_internal) + .def_property_readonly( + "overridable_initializers", [](const PyInferenceSession* sess) -> const std::vector& { + auto res = sess->GetSessionHandle()->GetOverridableInitializers(); + OrtPybindThrowIfError(res.first); + return *(res.second); + }, + py::return_value_policy::reference_internal) + .def_property_readonly( + "model_meta", [](const PyInferenceSession* sess) -> const onnxruntime::ModelMetadata& { + auto res = sess->GetSessionHandle()->GetModelMetadata(); + OrtPybindThrowIfError(res.first); + return *(res.second); + }, + py::return_value_policy::reference_internal) .def("run_with_iobinding", [](PyInferenceSession* sess, SessionIOBinding& io_binding, RunOptions* run_options = nullptr) -> void { Status status; if (!run_options) @@ -2172,6 +1605,8 @@ PYBIND11_MODULE(onnxruntime_pybind11_state, m) { addGlobalMethods(m, env); addObjectMethods(m, env); + addOrtValueMethods(m); + addIoBindingMethods(m); #if !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD) || defined(ORT_MINIMAL_BUILD_CUSTOM_OPS) Ort::SessionOptions tmp_options; diff --git a/onnxruntime/python/onnxruntime_pybind_state_common.cc b/onnxruntime/python/onnxruntime_pybind_state_common.cc index 78f80aea72..a4b26de0de 100644 --- a/onnxruntime/python/onnxruntime_pybind_state_common.cc +++ b/onnxruntime/python/onnxruntime_pybind_state_common.cc @@ -1,27 +1,48 @@ #include "onnxruntime_pybind_exceptions.h" #include "onnxruntime_pybind_state_common.h" +#include "core/framework/arena_extend_strategy.h" + namespace onnxruntime { namespace python { namespace py = pybind11; const std::string onnxruntime::python::SessionObjectInitializer::default_logger_id = "Default"; -void ThrowIfPyErrOccured() { - if (PyErr_Occurred()) { - PyObject *ptype, *pvalue, *ptraceback; - PyErr_Fetch(&ptype, &pvalue, &ptraceback); +#ifdef USE_OPENVINO +// TODO remove deprecated global config +std::string openvino_device_type; +#endif - PyObject* pStr = PyObject_Str(ptype); - std::string sType = py::reinterpret_borrow(pStr); - Py_XDECREF(pStr); - pStr = PyObject_Str(pvalue); - sType += ": "; - sType += py::reinterpret_borrow(pStr); - Py_XDECREF(pStr); - throw Fail(sType); - } -} +#ifdef USE_NUPHAR +// TODO remove deprecated global config +std::string nuphar_settings; +#endif + + +// TODO remove deprecated global config +OrtDevice::DeviceId cuda_device_id = 0; +// TODO remove deprecated global config +size_t gpu_mem_limit = std::numeric_limits::max(); + +#if defined(USE_CUDA) || defined(USE_ROCM) +#ifdef USE_CUDA +// TODO remove deprecated global config +OrtCudnnConvAlgoSearch cudnn_conv_algo_search = OrtCudnnConvAlgoSearch::EXHAUSTIVE; +// TODO remove deprecated global config +bool do_copy_in_default_stream = true; +onnxruntime::CUDAExecutionProviderExternalAllocatorInfo external_allocator_info{}; +#endif + +#ifdef USE_ROCM +#include "core/providers/rocm/rocm_execution_provider.h" +#include "core/providers/rocm/rocm_allocator.h" +onnxruntime::ROCMExecutionProviderExternalAllocatorInfo external_allocator_info{}; +#endif + +// TODO remove deprecated global config +onnxruntime::ArenaExtendStrategy arena_extend_strategy = onnxruntime::ArenaExtendStrategy::kNextPowerOfTwo; +#endif #ifdef ENABLE_TRAINING diff --git a/onnxruntime/python/onnxruntime_pybind_state_common.h b/onnxruntime/python/onnxruntime_pybind_state_common.h index 72100bd780..8dd547bbb5 100644 --- a/onnxruntime/python/onnxruntime_pybind_state_common.h +++ b/onnxruntime/python/onnxruntime_pybind_state_common.h @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +#pragma once + #include "core/common/logging/logging.h" #include "core/common/logging/sinks/cerr_sink.h" #include "core/framework/allocator.h" @@ -12,8 +14,196 @@ #include "core/dlpack/dlpack_converter.h" #endif +// execution provider factory creator headers +struct OrtStatus { + OrtErrorCode code; + char msg[1]; // a null-terminated string +}; + +#define BACKEND_DEVICE BACKEND_PROC BACKEND_DNNL BACKEND_OPENVINO BACKEND_NUPHAR BACKEND_OPENBLAS BACKEND_MIGRAPHX BACKEND_ACL BACKEND_ARMNN BACKEND_DML +#include "core/session/onnxruntime_cxx_api.h" +#include "core/providers/providers.h" +#include "core/providers/cpu/cpu_execution_provider.h" +#include "core/providers/cpu/cpu_provider_factory_creator.h" + +#if defined(USE_CUDA) || defined(USE_ROCM) +#define BACKEND_PROC "GPU" +#else +#define BACKEND_PROC "CPU" +#endif + +#if _OPENMP +#define BACKEND_OPENMP "-OPENMP" +#else +#define BACKEND_OPENMP "" +#endif + +#if USE_DNNL +#define BACKEND_DNNL "-DNNL" +#else +#define BACKEND_DNNL "" +#endif + +#if USE_MIGRAPHX +#define BACKEND_MIGRAPHX "-MIGRAPHX" +#else +#define BACKEND_MIGRAPHX "" +#endif + +#ifdef USE_OPENVINO +#if OPENVINO_CONFIG_CPU_FP32 +#define BACKEND_OPENVINO "-OPENVINO_CPU_FP32" + +#elif OPENVINO_CONFIG_GPU_FP32 +#define BACKEND_OPENVINO "-OPENVINO_GPU_FP32" + +#elif OPENVINO_CONFIG_GPU_FP16 +#define BACKEND_OPENVINO "-OPENVINO_GPU_FP16" + +#elif OPENVINO_CONFIG_MYRIAD +#define BACKEND_OPENVINO "-OPENVINO_MYRIAD" + +#elif OPENVINO_CONFIG_VAD_M +#define BACKEND_OPENVINO "-OPENVINO_VAD_M" + +#elif OPENVINO_CONFIG_VAD_F +#define BACKEND_OPENVINO "-OPENVINO_VAD_F" + +#elif OPENVINO_CONFIG_MULTI +#define BACKEND_OPENVINO "-OPENVINO_MULTI" + +#elif OPENVINO_CONFIG_HETERO +#define BACKEND_OPENVINO "-OPENVINO_HETERO" +#endif +#else +#define BACKEND_OPENVINO "" +#endif + +#ifdef USE_NUPHAR +#define BACKEND_NUPHAR "-NUPHAR" +#else +#define BACKEND_NUPHAR "" +#endif + +#if USE_VITISAI +#define BACKEND_VITISAI "-VITISAI" +#include "core/providers/vitisai/vitisai_execution_provider.h" +#else +#define BACKEND_VITISAI "" +#endif + +#if USE_OPENBLAS +#define BACKEND_OPENBLAS "-OPENBLAS" +#else +#define BACKEND_OPENBLAS "" +#endif + +#if USE_ACL +#define BACKEND_ACL "-ACL" +#else +#define BACKEND_ACL "" +#endif + +#if USE_ARMNN +#define BACKEND_ARMNN "-ARMNN" +#else +#define BACKEND_ARMNN "" +#endif + +#if USE_DML +#define BACKEND_DML "-DML" +#else +#define BACKEND_DML "" +#endif + +#ifdef USE_CUDA +#include "core/providers/cuda/cuda_provider_factory.h" +#include "core/providers/cuda/cuda_execution_provider_info.h" +#endif +#ifdef USE_TENSORRT +#include "core/providers/tensorrt/tensorrt_provider_factory.h" +#endif +#ifdef USE_MIGRAPHX +#include "core/providers/migraphx/migraphx_provider_factory.h" +#endif +#ifdef USE_OPENVINO +#include "core/providers/openvino/openvino_provider_factory.h" +// TODO remove deprecated global config +namespace onnxruntime { +ProviderInfo_OpenVINO* GetProviderInfo_OpenVINO(); +namespace python { +extern std::string openvino_device_type; +} +} +#endif +#ifdef USE_NUPHAR +#include "core/providers/nuphar/nuphar_provider_factory.h" +// TODO remove deprecated global config namespace onnxruntime { namespace python { +extern std::string nuphar_settings; +} +} +#endif +#ifdef USE_VITISAI +#include "core/providers/vitisai/vitisai_provider_factory.h" +#endif +#ifdef USE_ACL +#include "core/providers/acl/acl_provider_factory.h" +#endif +#ifdef USE_ARMNN +#include "core/providers/armnn/armnn_provider_factory.h" +#endif +#ifdef USE_DML +#include "core/providers/dml/dml_provider_factory.h" +#endif + +#if defined(USE_CUDA) || defined(USE_ROCM) +#ifdef USE_CUDA +namespace onnxruntime { +ProviderInfo_CUDA* GetProviderInfo_CUDA(); +namespace python { +// TODO remove deprecated global config +extern OrtCudnnConvAlgoSearch cudnn_conv_algo_search; +// TODO remove deprecated global config +extern bool do_copy_in_default_stream; +extern onnxruntime::CUDAExecutionProviderExternalAllocatorInfo external_allocator_info; +} // namespace python +} // namespace onnxruntime +#endif + +#ifdef USE_ROCM +#include "core/providers/rocm/rocm_execution_provider.h" +#include "core/providers/rocm/rocm_allocator.h" +#include "core/providers/rocm/rocm_provider_factory_creator.h" +namespace onnxruntime { +namespace python { +extern onnxruntime::ROCMExecutionProviderExternalAllocatorInfo external_allocator_info; +} +} +#endif + +// TODO remove deprecated global config +namespace onnxruntime { +namespace python { +extern onnxruntime::ArenaExtendStrategy arena_extend_strategy; +} +} // namespace onnxruntime +#endif + + +#include "core/providers/dnnl/dnnl_provider_factory.h" +#include "core/providers/shared_library/provider_host_api.h" + + +namespace onnxruntime { +namespace python { + +// TODO remove deprecated global config +extern OrtDevice::DeviceId cuda_device_id; +// TODO remove deprecated global config +extern size_t gpu_mem_limit; + #if !defined(ORT_MINIMAL_BUILD) || defined(ORT_MINIMAL_BUILD_CUSTOM_OPS) struct CustomOpLibrary { @@ -139,6 +329,20 @@ void InitializeSession(InferenceSession* sess, // Checks if PyErrOccured, fetches status and throws. void ThrowIfPyErrOccured(); +void addOrtValueMethods(pybind11::module& m); + +void addIoBindingMethods(pybind11::module& m); + +const char* GetDeviceName(const OrtDevice& device); + +bool IsCudaDeviceIdValid(const onnxruntime::logging::Logger& logger, int id); + +AllocatorPtr GetCudaAllocator(OrtDevice::DeviceId id); + +bool CheckIfTensor(const std::vector& def_list, + const std::string& name, + /*out*/ ONNX_NAMESPACE::TypeProto& type_proto); + #ifdef ENABLE_TRAINING namespace py = pybind11; From c487824a31fe11cd96c7fc1c6c0dbefa609c4009 Mon Sep 17 00:00:00 2001 From: Hariharan Seshadri Date: Wed, 26 May 2021 10:27:19 -0700 Subject: [PATCH 11/47] Fix bug in Einsum implementation (#7822) --- .../einsum_typed_compute_processor.cc | 17 +- .../test/providers/cpu/math/einsum_test.cc | 378 +++++++++++++++++- 2 files changed, 389 insertions(+), 6 deletions(-) diff --git a/onnxruntime/core/providers/cpu/math/einsum_utils/einsum_typed_compute_processor.cc b/onnxruntime/core/providers/cpu/math/einsum_utils/einsum_typed_compute_processor.cc index 71af7adda2..ce14eaef90 100644 --- a/onnxruntime/core/providers/cpu/math/einsum_utils/einsum_typed_compute_processor.cc +++ b/onnxruntime/core/providers/cpu/math/einsum_utils/einsum_typed_compute_processor.cc @@ -148,7 +148,8 @@ std::unique_ptr EinsumTypedComputeProcessor::PairwiseOperandProcess(c bool has_left_dim = left_dim > 1; // non-trivial dimension (dim_value != 1) bool has_right_dim = right_dim > 1; // non-trivial dimension (dim_value != 1) - if (reduce_dims_iter < reduce_dims_size && reduce_dims[reduce_dims_iter] == i) { // This dimension is to be reduced after this pair-wise operation + if (reduce_dims_iter < reduce_dims_size && reduce_dims[reduce_dims_iter] == i) { + // This dimension is to be reduced after this pair-wise operation ++reduce_dims_iter; if (has_left_dim && has_right_dim) { // Both inputs have non-trivial dim values along this dimension // Both the left and right operands have non-trivial dimension value along this axis @@ -156,12 +157,20 @@ std::unique_ptr EinsumTypedComputeProcessor::PairwiseOperandProcess(c ORT_ENFORCE(left_dim == right_dim, "Einsum op: Input dimensions must be equal along an axis to be reduced across all inputs"); reduced_size *= left_dim; - } else if (has_left_dim) { // if it is only in one of left and right, we can reduce right away + } else if (has_left_dim) { // if the dim to be reduced is only in one of left and right, we can reduce right away + const Tensor& tensor_to_be_reduced = current_left ? *current_left : left; + const std::vector& tensor_to_be_reduced_dims = + current_left ? current_left->Shape().GetDims() : left_dims; + current_left = EinsumOp::ReduceSum( - left, left_dims, {i}, allocator_, tp_, einsum_ep_assets_, device_reduce_sum_func_); + tensor_to_be_reduced, tensor_to_be_reduced_dims, {i}, allocator_, tp_, einsum_ep_assets_, device_reduce_sum_func_); } else if (has_right_dim) { + const Tensor& tensor_to_be_reduced = current_right ? *current_right : right; + const std::vector& tensor_to_be_reduced_dims = + current_right ? current_right->Shape().GetDims() : right_dims; + current_right = EinsumOp::ReduceSum( - right, right_dims, {i}, allocator_, tp_, einsum_ep_assets_, device_reduce_sum_func_); + tensor_to_be_reduced, tensor_to_be_reduced_dims, {i}, allocator_, tp_, einsum_ep_assets_, device_reduce_sum_func_); } } else { // This dimension is not reduced (i.e.) it appears in the output after processing these 2 operands // Both the left and right operands have non-trivial dimension value along this axis diff --git a/onnxruntime/test/providers/cpu/math/einsum_test.cc b/onnxruntime/test/providers/cpu/math/einsum_test.cc index 25688375f2..9d1bf63b60 100644 --- a/onnxruntime/test/providers/cpu/math/einsum_test.cc +++ b/onnxruntime/test/providers/cpu/math/einsum_test.cc @@ -10,7 +10,7 @@ namespace onnxruntime { namespace test { -// Tests are aplit up "theme-wise" (i.e.) each kind of operation Einsum can be used for +// Tests are split up "theme-wise" (i.e.) each kind of operation Einsum can be used for // Within each theme we test "explicit" and "implicit" versions of the Einsum equation (wherever possible) // Some operations are not possible with implicit notation (reordering, reduction, etc.) @@ -540,7 +540,8 @@ TEST(Einsum, ImplicitEinsumAsTensorContraction) { test.Run(); } -// Test each theme for half support +// Theme: Half support + TEST(Einsum, ExplicitEinsumAsIdentity_1D_input_Half) { if (!HasCudaEnvironment(600)) { return; @@ -714,5 +715,378 @@ TEST(Einsum, ExplicitEinsumAsTensorContraction_Half) { test.Run(); } +// Theme: Tests involving MatMul(s) interleaved with Transpose(s) +// for two and three inputs (most common use-case of Einsum operator) + +struct EinsumTestCase { + std::string equation; + std::vector shape; + std::vector expected; + EinsumTestCase(const std::string& eq, const std::vector& sh, const std::vector& exp) : equation(eq), shape(sh), expected(exp) {} +}; + +TEST(Einsum, EinsumTransposeMatMulTwoInputsTestSuite) { + std::vector test_cases{ + EinsumTestCase("abc,cd->abc", std::vector{2, 2, 2}, std::vector{0.f, 5.f, 2.f, 15.f, 4.f, 25.f, 6.f, 35.f}), + EinsumTestCase("abc,cd->abd", std::vector{2, 2, 2}, std::vector{2.f, 3.f, 6.f, 11.f, 10.f, 19.f, 14.f, 27.f}), + EinsumTestCase("abc,cd->acd", std::vector{2, 2, 2}, std::vector{0.f, 2.f, 8.f, 12.f, 0.f, 10.f, 24.f, 36.f}), + EinsumTestCase("abc,dc->abd", std::vector{2, 2, 2}, std::vector{1.f, 3.f, 3.f, 13.f, 5.f, 23.f, 7.f, 33.f}), + EinsumTestCase("abc,dc->abc", std::vector{2, 2, 2}, std::vector{0.f, 4.f, 4.f, 12.f, 8.f, 20.f, 12.f, 28.f}), + EinsumTestCase("abc,dc->acd", std::vector{2, 2, 2}, std::vector{0.f, 4.f, 4.f, 12.f, 0.f, 20.f, 12.f, 36.f}), + EinsumTestCase("acb,cd->acd", std::vector{2, 2, 2}, std::vector{0.f, 1.f, 10.f, 15.f, 0.f, 9.f, 26.f, 39.f}), + EinsumTestCase("acb,cd->abc", std::vector{2, 2, 2}, std::vector{0.f, 10.f, 1.f, 15.f, 4.f, 30.f, 5.f, 35.f}), + EinsumTestCase("acb,cd->abd", std::vector{2, 2, 2}, std::vector{4.f, 6.f, 6.f, 10.f, 12.f, 22.f, 14.f, 26.f}), + EinsumTestCase("acb,dc->acd", std::vector{2, 2, 2}, std::vector{0.f, 2.f, 5.f, 15.f, 0.f, 18.f, 13.f, 39.f}), + EinsumTestCase("acb,dc->abd", std::vector{2, 2, 2}, std::vector{2.f, 6.f, 3.f, 11.f, 6.f, 26.f, 7.f, 31.f}), + EinsumTestCase("acb,dc->abc", std::vector{2, 2, 2}, std::vector{0.f, 8.f, 2.f, 12.f, 8.f, 24.f, 10.f, 28.f}), + EinsumTestCase("bac,cd->bac", std::vector{2, 2, 2}, std::vector{0.f, 5.f, 2.f, 15.f, 4.f, 25.f, 6.f, 35.f}), + EinsumTestCase("bac,cd->bad", std::vector{2, 2, 2}, std::vector{2.f, 3.f, 6.f, 11.f, 10.f, 19.f, 14.f, 27.f}), + EinsumTestCase("bac,cd->bcd", std::vector{2, 2, 2}, std::vector{0.f, 2.f, 8.f, 12.f, 0.f, 10.f, 24.f, 36.f}), + EinsumTestCase("bac,dc->bad", std::vector{2, 2, 2}, std::vector{1.f, 3.f, 3.f, 13.f, 5.f, 23.f, 7.f, 33.f}), + EinsumTestCase("bac,dc->bac", std::vector{2, 2, 2}, std::vector{0.f, 4.f, 4.f, 12.f, 8.f, 20.f, 12.f, 28.f}), + EinsumTestCase("bac,dc->bcd", std::vector{2, 2, 2}, std::vector{0.f, 4.f, 4.f, 12.f, 0.f, 20.f, 12.f, 36.f}), + EinsumTestCase("bca,cd->bcd", std::vector{2, 2, 2}, std::vector{0.f, 1.f, 10.f, 15.f, 0.f, 9.f, 26.f, 39.f}), + EinsumTestCase("bca,cd->bac", std::vector{2, 2, 2}, std::vector{0.f, 10.f, 1.f, 15.f, 4.f, 30.f, 5.f, 35.f}), + EinsumTestCase("bca,cd->bad", std::vector{2, 2, 2}, std::vector{4.f, 6.f, 6.f, 10.f, 12.f, 22.f, 14.f, 26.f}), + EinsumTestCase("bca,dc->bcd", std::vector{2, 2, 2}, std::vector{0.f, 2.f, 5.f, 15.f, 0.f, 18.f, 13.f, 39.f}), + EinsumTestCase("bca,dc->bad", std::vector{2, 2, 2}, std::vector{2.f, 6.f, 3.f, 11.f, 6.f, 26.f, 7.f, 31.f}), + EinsumTestCase("bca,dc->bac", std::vector{2, 2, 2}, std::vector{0.f, 8.f, 2.f, 12.f, 8.f, 24.f, 10.f, 28.f}), + EinsumTestCase("cab,cd->cad", std::vector{2, 2, 2}, std::vector{0.f, 1.f, 0.f, 5.f, 18.f, 27.f, 26.f, 39.f}), + EinsumTestCase("cab,cd->cbd", std::vector{2, 2, 2}, std::vector{0.f, 2.f, 0.f, 4.f, 20.f, 30.f, 24.f, 36.f}), + EinsumTestCase("cab,dc->cad", std::vector{2, 2, 2}, std::vector{0.f, 2.f, 0.f, 10.f, 9.f, 27.f, 13.f, 39.f}), + EinsumTestCase("cab,dc->cbd", std::vector{2, 2, 2}, std::vector{0.f, 4.f, 0.f, 8.f, 10.f, 30.f, 12.f, 36.f}), + EinsumTestCase("cba,cd->cbd", std::vector{2, 2, 2}, std::vector{0.f, 1.f, 0.f, 5.f, 18.f, 27.f, 26.f, 39.f}), + EinsumTestCase("cba,cd->cad", std::vector{2, 2, 2}, std::vector{0.f, 2.f, 0.f, 4.f, 20.f, 30.f, 24.f, 36.f}), + EinsumTestCase("cba,dc->cbd", std::vector{2, 2, 2}, std::vector{0.f, 2.f, 0.f, 10.f, 9.f, 27.f, 13.f, 39.f}), + EinsumTestCase("cba,dc->cad", std::vector{2, 2, 2}, std::vector{0.f, 4.f, 0.f, 8.f, 10.f, 30.f, 12.f, 36.f})}; + + std::vector m1{0.f, 1.f, 2.f, 3.f, 4.f, 5.f, 6.f, 7.f}; + std::vector m2{0.f, 1.f, 2.f, 3.f}; + for (const auto& tst : test_cases) { + OpTester test("Einsum", 12, onnxruntime::kOnnxDomain); + test.AddAttribute("equation", tst.equation); + test.AddInput("x", {2, 2, 2}, m1); + test.AddInput("y", {2, 2}, m2); + test.AddOutput("o", tst.shape, tst.expected); + test.Run(); + } +} + +TEST(Einsum, EinsumTransposeMatMulThreeInputsTestSuite) { + std::vector test_cases_set_1{ + EinsumTestCase("abc,cd,def->abd", std::vector{2, 2, 2}, std::vector{12.f, 66.f, 36.f, 242.f, 60.f, 418.f, 84.f, 594.f}), + EinsumTestCase("abc,cd,def->abe", std::vector{2, 2, 2}, std::vector{29.f, 49.f, 105.f, 173.f, 181.f, 297.f, 257.f, 421.f}), + EinsumTestCase("abc,cd,def->acd", std::vector{2, 2, 2}, std::vector{0.f, 44.f, 48.f, 264.f, 0.f, 220.f, 144.f, 792.f}), + EinsumTestCase("abc,cd,def->ace", std::vector{2, 2, 2}, std::vector{18.f, 26.f, 116.f, 196.f, 90.f, 130.f, 348.f, 588.f}), + EinsumTestCase("abc,cd,dfe->abd", std::vector{2, 2, 2}, std::vector{12.f, 66.f, 36.f, 242.f, 60.f, 418.f, 84.f, 594.f}), + EinsumTestCase("abc,cd,dfe->abf", std::vector{2, 2, 2}, std::vector{29.f, 49.f, 105.f, 173.f, 181.f, 297.f, 257.f, 421.f}), + EinsumTestCase("abc,cd,dfe->acd", std::vector{2, 2, 2}, std::vector{0.f, 44.f, 48.f, 264.f, 0.f, 220.f, 144.f, 792.f}), + EinsumTestCase("abc,cd,dfe->acf", std::vector{2, 2, 2}, std::vector{18.f, 26.f, 116.f, 196.f, 90.f, 130.f, 348.f, 588.f}), + EinsumTestCase("abc,cd,edf->abe", std::vector{2, 2, 2}, std::vector{17.f, 57.f, 61.f, 197.f, 105.f, 337.f, 149.f, 477.f}), + EinsumTestCase("abc,cd,edf->abd", std::vector{2, 2, 2}, std::vector{20.f, 54.f, 60.f, 198.f, 100.f, 342.f, 140.f, 486.f}), + EinsumTestCase("abc,cd,edf->ace", std::vector{2, 2, 2}, std::vector{10.f, 26.f, 68.f, 228.f, 50.f, 130.f, 204.f, 684.f}), + EinsumTestCase("abc,cd,edf->acd", std::vector{2, 2, 2}, std::vector{0.f, 36.f, 80.f, 216.f, 0.f, 180.f, 240.f, 648.f}), + EinsumTestCase("abc,cd,efd->abe", std::vector{2, 2, 2}, std::vector{16.f, 56.f, 56.f, 192.f, 96.f, 328.f, 136.f, 464.f}), + EinsumTestCase("abc,cd,efd->abf", std::vector{2, 2, 2}, std::vector{26.f, 46.f, 90.f, 158.f, 154.f, 270.f, 218.f, 382.f}), + EinsumTestCase("abc,cd,efd->ace", std::vector{2, 2, 2}, std::vector{8.f, 24.f, 64.f, 224.f, 40.f, 120.f, 192.f, 672.f}), + EinsumTestCase("abc,cd,efd->acf", std::vector{2, 2, 2}, std::vector{12.f, 20.f, 104.f, 184.f, 60.f, 100.f, 312.f, 552.f}), + EinsumTestCase("abc,cd,fde->abf", std::vector{2, 2, 2}, std::vector{17.f, 57.f, 61.f, 197.f, 105.f, 337.f, 149.f, 477.f}), + EinsumTestCase("abc,cd,fde->abd", std::vector{2, 2, 2}, std::vector{20.f, 54.f, 60.f, 198.f, 100.f, 342.f, 140.f, 486.f}), + EinsumTestCase("abc,cd,fde->acf", std::vector{2, 2, 2}, std::vector{10.f, 26.f, 68.f, 228.f, 50.f, 130.f, 204.f, 684.f}), + EinsumTestCase("abc,cd,fde->acd", std::vector{2, 2, 2}, std::vector{0.f, 36.f, 80.f, 216.f, 0.f, 180.f, 240.f, 648.f}), + EinsumTestCase("abc,cd,fed->abf", std::vector{2, 2, 2}, std::vector{16.f, 56.f, 56.f, 192.f, 96.f, 328.f, 136.f, 464.f}), + EinsumTestCase("abc,cd,fed->abe", std::vector{2, 2, 2}, std::vector{26.f, 46.f, 90.f, 158.f, 154.f, 270.f, 218.f, 382.f}), + EinsumTestCase("abc,cd,fed->acf", std::vector{2, 2, 2}, std::vector{8.f, 24.f, 64.f, 224.f, 40.f, 120.f, 192.f, 672.f}), + EinsumTestCase("abc,cd,fed->ace", std::vector{2, 2, 2}, std::vector{12.f, 20.f, 104.f, 184.f, 60.f, 100.f, 312.f, 552.f}), + EinsumTestCase("abc,dc,def->abd", std::vector{2, 2, 2}, std::vector{6.f, 66.f, 18.f, 286.f, 30.f, 506.f, 42.f, 726.f}), + EinsumTestCase("abc,dc,def->abe", std::vector{2, 2, 2}, std::vector{28.f, 44.f, 120.f, 184.f, 212.f, 324.f, 304.f, 464.f}), + EinsumTestCase("abc,dc,def->acd", std::vector{2, 2, 2}, std::vector{0.f, 88.f, 24.f, 264.f, 0.f, 440.f, 72.f, 792.f}), + EinsumTestCase("abc,dc,def->ace", std::vector{2, 2, 2}, std::vector{36.f, 52.f, 112.f, 176.f, 180.f, 260.f, 336.f, 528.f}), + EinsumTestCase("abc,dc,dfe->abd", std::vector{2, 2, 2}, std::vector{6.f, 66.f, 18.f, 286.f, 30.f, 506.f, 42.f, 726.f}), + EinsumTestCase("abc,dc,dfe->abf", std::vector{2, 2, 2}, std::vector{28.f, 44.f, 120.f, 184.f, 212.f, 324.f, 304.f, 464.f}), + EinsumTestCase("abc,dc,dfe->acd", std::vector{2, 2, 2}, std::vector{0.f, 88.f, 24.f, 264.f, 0.f, 440.f, 72.f, 792.f}), + EinsumTestCase("abc,dc,dfe->acf", std::vector{2, 2, 2}, std::vector{36.f, 52.f, 112.f, 176.f, 180.f, 260.f, 336.f, 528.f}), + EinsumTestCase("abc,dc,edf->abe", std::vector{2, 2, 2}, std::vector{16.f, 48.f, 68.f, 196.f, 120.f, 344.f, 172.f, 492.f}), + EinsumTestCase("abc,dc,edf->abd", std::vector{2, 2, 2}, std::vector{10.f, 54.f, 30.f, 234.f, 50.f, 414.f, 70.f, 594.f}), + EinsumTestCase("abc,dc,edf->ace", std::vector{2, 2, 2}, std::vector{20.f, 52.f, 64.f, 192.f, 100.f, 260.f, 192.f, 576.f}), + EinsumTestCase("abc,dc,edf->acd", std::vector{2, 2, 2}, std::vector{0.f, 72.f, 40.f, 216.f, 0.f, 360.f, 120.f, 648.f}), + EinsumTestCase("abc,dc,efd->abe", std::vector{2, 2, 2}, std::vector{14.f, 46.f, 58.f, 186.f, 102.f, 326.f, 146.f, 466.f}), + EinsumTestCase("abc,dc,efd->abf", std::vector{2, 2, 2}, std::vector{22.f, 38.f, 90.f, 154.f, 158.f, 270.f, 226.f, 386.f}), + EinsumTestCase("abc,dc,efd->ace", std::vector{2, 2, 2}, std::vector{16.f, 48.f, 56.f, 184.f, 80.f, 240.f, 168.f, 552.f}), + EinsumTestCase("abc,dc,efd->acf", std::vector{2, 2, 2}, std::vector{24.f, 40.f, 88.f, 152.f, 120.f, 200.f, 264.f, 456.f}), + EinsumTestCase("abc,dc,fde->abf", std::vector{2, 2, 2}, std::vector{16.f, 48.f, 68.f, 196.f, 120.f, 344.f, 172.f, 492.f}), + EinsumTestCase("abc,dc,fde->abd", std::vector{2, 2, 2}, std::vector{10.f, 54.f, 30.f, 234.f, 50.f, 414.f, 70.f, 594.f}), + EinsumTestCase("abc,dc,fde->acf", std::vector{2, 2, 2}, std::vector{20.f, 52.f, 64.f, 192.f, 100.f, 260.f, 192.f, 576.f}), + EinsumTestCase("abc,dc,fde->acd", std::vector{2, 2, 2}, std::vector{0.f, 72.f, 40.f, 216.f, 0.f, 360.f, 120.f, 648.f}), + EinsumTestCase("abc,dc,fed->abf", std::vector{2, 2, 2}, std::vector{14.f, 46.f, 58.f, 186.f, 102.f, 326.f, 146.f, 466.f}), + EinsumTestCase("abc,dc,fed->abe", std::vector{2, 2, 2}, std::vector{22.f, 38.f, 90.f, 154.f, 158.f, 270.f, 226.f, 386.f}), + EinsumTestCase("abc,dc,fed->acf", std::vector{2, 2, 2}, std::vector{16.f, 48.f, 56.f, 184.f, 80.f, 240.f, 168.f, 552.f}), + EinsumTestCase("abc,dc,fed->ace", std::vector{2, 2, 2}, std::vector{24.f, 40.f, 88.f, 152.f, 120.f, 200.f, 264.f, 456.f}), + EinsumTestCase("acb,cd,def->acd", std::vector{2, 2, 2}, std::vector{0.f, 22.f, 60.f, 330.f, 0.f, 198.f, 156.f, 858.f}), + EinsumTestCase("acb,cd,def->ace", std::vector{2, 2, 2}, std::vector{9.f, 13.f, 145.f, 245.f, 81.f, 117.f, 377.f, 637.f}), + EinsumTestCase("acb,cd,def->abd", std::vector{2, 2, 2}, std::vector{24.f, 132.f, 36.f, 220.f, 72.f, 484.f, 84.f, 572.f}), + EinsumTestCase("acb,cd,def->abe", std::vector{2, 2, 2}, std::vector{58.f, 98.f, 96.f, 160.f, 210.f, 346.f, 248.f, 408.f}), + EinsumTestCase("acb,cd,dfe->acd", std::vector{2, 2, 2}, std::vector{0.f, 22.f, 60.f, 330.f, 0.f, 198.f, 156.f, 858.f}), + EinsumTestCase("acb,cd,dfe->acf", std::vector{2, 2, 2}, std::vector{9.f, 13.f, 145.f, 245.f, 81.f, 117.f, 377.f, 637.f}), + EinsumTestCase("acb,cd,dfe->abd", std::vector{2, 2, 2}, std::vector{24.f, 132.f, 36.f, 220.f, 72.f, 484.f, 84.f, 572.f}), + EinsumTestCase("acb,cd,dfe->abf", std::vector{2, 2, 2}, std::vector{58.f, 98.f, 96.f, 160.f, 210.f, 346.f, 248.f, 408.f}), + EinsumTestCase("acb,cd,edf->ace", std::vector{2, 2, 2}, std::vector{5.f, 13.f, 85.f, 285.f, 45.f, 117.f, 221.f, 741.f}), + EinsumTestCase("acb,cd,edf->acd", std::vector{2, 2, 2}, std::vector{0.f, 18.f, 100.f, 270.f, 0.f, 162.f, 260.f, 702.f}), + EinsumTestCase("acb,cd,edf->abe", std::vector{2, 2, 2}, std::vector{34.f, 114.f, 56.f, 184.f, 122.f, 394.f, 144.f, 464.f}), + EinsumTestCase("acb,cd,edf->abd", std::vector{2, 2, 2}, std::vector{40.f, 108.f, 60.f, 180.f, 120.f, 396.f, 140.f, 468.f}), + EinsumTestCase("acb,cd,efd->ace", std::vector{2, 2, 2}, std::vector{4.f, 12.f, 80.f, 280.f, 36.f, 108.f, 208.f, 728.f}), + EinsumTestCase("acb,cd,efd->acf", std::vector{2, 2, 2}, std::vector{6.f, 10.f, 130.f, 230.f, 54.f, 90.f, 338.f, 598.f}), + EinsumTestCase("acb,cd,efd->abe", std::vector{2, 2, 2}, std::vector{32.f, 112.f, 52.f, 180.f, 112.f, 384.f, 132.f, 452.f}), + EinsumTestCase("acb,cd,efd->abf", std::vector{2, 2, 2}, std::vector{52.f, 92.f, 84.f, 148.f, 180.f, 316.f, 212.f, 372.f}), + EinsumTestCase("acb,cd,fde->acf", std::vector{2, 2, 2}, std::vector{5.f, 13.f, 85.f, 285.f, 45.f, 117.f, 221.f, 741.f}), + EinsumTestCase("acb,cd,fde->acd", std::vector{2, 2, 2}, std::vector{0.f, 18.f, 100.f, 270.f, 0.f, 162.f, 260.f, 702.f}), + EinsumTestCase("acb,cd,fde->abf", std::vector{2, 2, 2}, std::vector{34.f, 114.f, 56.f, 184.f, 122.f, 394.f, 144.f, 464.f}), + EinsumTestCase("acb,cd,fde->abd", std::vector{2, 2, 2}, std::vector{40.f, 108.f, 60.f, 180.f, 120.f, 396.f, 140.f, 468.f}), + EinsumTestCase("acb,cd,fed->acf", std::vector{2, 2, 2}, std::vector{4.f, 12.f, 80.f, 280.f, 36.f, 108.f, 208.f, 728.f}), + EinsumTestCase("acb,cd,fed->ace", std::vector{2, 2, 2}, std::vector{6.f, 10.f, 130.f, 230.f, 54.f, 90.f, 338.f, 598.f}), + EinsumTestCase("acb,cd,fed->abf", std::vector{2, 2, 2}, std::vector{32.f, 112.f, 52.f, 180.f, 112.f, 384.f, 132.f, 452.f}), + EinsumTestCase("acb,cd,fed->abe", std::vector{2, 2, 2}, std::vector{52.f, 92.f, 84.f, 148.f, 180.f, 316.f, 212.f, 372.f}), + EinsumTestCase("acb,dc,def->acd", std::vector{2, 2, 2}, std::vector{0.f, 44.f, 30.f, 330.f, 0.f, 396.f, 78.f, 858.f}), + EinsumTestCase("acb,dc,def->ace", std::vector{2, 2, 2}, std::vector{18.f, 26.f, 140.f, 220.f, 162.f, 234.f, 364.f, 572.f})}; + + std::vector test_cases_set_2{ + EinsumTestCase("acb,dc,def->abd", std::vector{2, 2, 2}, std::vector{12.f, 132.f, 18.f, 242.f, 36.f, 572.f, 42.f, 682.f}), + EinsumTestCase("acb,dc,def->abe", std::vector{2, 2, 2}, std::vector{56.f, 88.f, 102.f, 158.f, 240.f, 368.f, 286.f, 438.f}), + EinsumTestCase("acb,dc,dfe->acd", std::vector{2, 2, 2}, std::vector{0.f, 44.f, 30.f, 330.f, 0.f, 396.f, 78.f, 858.f}), + EinsumTestCase("acb,dc,dfe->acf", std::vector{2, 2, 2}, std::vector{18.f, 26.f, 140.f, 220.f, 162.f, 234.f, 364.f, 572.f}), + EinsumTestCase("acb,dc,dfe->abd", std::vector{2, 2, 2}, std::vector{12.f, 132.f, 18.f, 242.f, 36.f, 572.f, 42.f, 682.f}), + EinsumTestCase("acb,dc,dfe->abf", std::vector{2, 2, 2}, std::vector{56.f, 88.f, 102.f, 158.f, 240.f, 368.f, 286.f, 438.f}), + EinsumTestCase("acb,dc,edf->ace", std::vector{2, 2, 2}, std::vector{10.f, 26.f, 80.f, 240.f, 90.f, 234.f, 208.f, 624.f}), + EinsumTestCase("acb,dc,edf->acd", std::vector{2, 2, 2}, std::vector{0.f, 36.f, 50.f, 270.f, 0.f, 324.f, 130.f, 702.f}), + EinsumTestCase("acb,dc,edf->abe", std::vector{2, 2, 2}, std::vector{32.f, 96.f, 58.f, 170.f, 136.f, 392.f, 162.f, 466.f}), + EinsumTestCase("acb,dc,edf->abd", std::vector{2, 2, 2}, std::vector{20.f, 108.f, 30.f, 198.f, 60.f, 468.f, 70.f, 558.f}), + EinsumTestCase("acb,dc,efd->ace", std::vector{2, 2, 2}, std::vector{8.f, 24.f, 70.f, 230.f, 72.f, 216.f, 182.f, 598.f}), + EinsumTestCase("acb,dc,efd->acf", std::vector{2, 2, 2}, std::vector{12.f, 20.f, 110.f, 190.f, 108.f, 180.f, 286.f, 494.f}), + EinsumTestCase("acb,dc,efd->abe", std::vector{2, 2, 2}, std::vector{28.f, 92.f, 50.f, 162.f, 116.f, 372.f, 138.f, 442.f}), + EinsumTestCase("acb,dc,efd->abf", std::vector{2, 2, 2}, std::vector{44.f, 76.f, 78.f, 134.f, 180.f, 308.f, 214.f, 366.f}), + EinsumTestCase("acb,dc,fde->acf", std::vector{2, 2, 2}, std::vector{10.f, 26.f, 80.f, 240.f, 90.f, 234.f, 208.f, 624.f}), + EinsumTestCase("acb,dc,fde->acd", std::vector{2, 2, 2}, std::vector{0.f, 36.f, 50.f, 270.f, 0.f, 324.f, 130.f, 702.f}), + EinsumTestCase("acb,dc,fde->abf", std::vector{2, 2, 2}, std::vector{32.f, 96.f, 58.f, 170.f, 136.f, 392.f, 162.f, 466.f}), + EinsumTestCase("acb,dc,fde->abd", std::vector{2, 2, 2}, std::vector{20.f, 108.f, 30.f, 198.f, 60.f, 468.f, 70.f, 558.f}), + EinsumTestCase("acb,dc,fed->acf", std::vector{2, 2, 2}, std::vector{8.f, 24.f, 70.f, 230.f, 72.f, 216.f, 182.f, 598.f}), + EinsumTestCase("acb,dc,fed->ace", std::vector{2, 2, 2}, std::vector{12.f, 20.f, 110.f, 190.f, 108.f, 180.f, 286.f, 494.f}), + EinsumTestCase("acb,dc,fed->abf", std::vector{2, 2, 2}, std::vector{28.f, 92.f, 50.f, 162.f, 116.f, 372.f, 138.f, 442.f}), + EinsumTestCase("acb,dc,fed->abe", std::vector{2, 2, 2}, std::vector{44.f, 76.f, 78.f, 134.f, 180.f, 308.f, 214.f, 366.f}), + EinsumTestCase("bac,cd,def->bad", std::vector{2, 2, 2}, std::vector{12.f, 66.f, 36.f, 242.f, 60.f, 418.f, 84.f, 594.f}), + EinsumTestCase("bac,cd,def->bae", std::vector{2, 2, 2}, std::vector{29.f, 49.f, 105.f, 173.f, 181.f, 297.f, 257.f, 421.f}), + EinsumTestCase("bac,cd,def->bcd", std::vector{2, 2, 2}, std::vector{0.f, 44.f, 48.f, 264.f, 0.f, 220.f, 144.f, 792.f}), + EinsumTestCase("bac,cd,def->bce", std::vector{2, 2, 2}, std::vector{18.f, 26.f, 116.f, 196.f, 90.f, 130.f, 348.f, 588.f}), + EinsumTestCase("bac,cd,dfe->bad", std::vector{2, 2, 2}, std::vector{12.f, 66.f, 36.f, 242.f, 60.f, 418.f, 84.f, 594.f}), + EinsumTestCase("bac,cd,dfe->baf", std::vector{2, 2, 2}, std::vector{29.f, 49.f, 105.f, 173.f, 181.f, 297.f, 257.f, 421.f}), + EinsumTestCase("bac,cd,dfe->bcd", std::vector{2, 2, 2}, std::vector{0.f, 44.f, 48.f, 264.f, 0.f, 220.f, 144.f, 792.f}), + EinsumTestCase("bac,cd,dfe->bcf", std::vector{2, 2, 2}, std::vector{18.f, 26.f, 116.f, 196.f, 90.f, 130.f, 348.f, 588.f}), + EinsumTestCase("bac,cd,edf->bae", std::vector{2, 2, 2}, std::vector{17.f, 57.f, 61.f, 197.f, 105.f, 337.f, 149.f, 477.f}), + EinsumTestCase("bac,cd,edf->bad", std::vector{2, 2, 2}, std::vector{20.f, 54.f, 60.f, 198.f, 100.f, 342.f, 140.f, 486.f}), + EinsumTestCase("bac,cd,edf->bce", std::vector{2, 2, 2}, std::vector{10.f, 26.f, 68.f, 228.f, 50.f, 130.f, 204.f, 684.f}), + EinsumTestCase("bac,cd,edf->bcd", std::vector{2, 2, 2}, std::vector{0.f, 36.f, 80.f, 216.f, 0.f, 180.f, 240.f, 648.f}), + EinsumTestCase("bac,cd,efd->bae", std::vector{2, 2, 2}, std::vector{16.f, 56.f, 56.f, 192.f, 96.f, 328.f, 136.f, 464.f}), + EinsumTestCase("bac,cd,efd->baf", std::vector{2, 2, 2}, std::vector{26.f, 46.f, 90.f, 158.f, 154.f, 270.f, 218.f, 382.f}), + EinsumTestCase("bac,cd,efd->bce", std::vector{2, 2, 2}, std::vector{8.f, 24.f, 64.f, 224.f, 40.f, 120.f, 192.f, 672.f}), + EinsumTestCase("bac,cd,efd->bcf", std::vector{2, 2, 2}, std::vector{12.f, 20.f, 104.f, 184.f, 60.f, 100.f, 312.f, 552.f}), + EinsumTestCase("bac,cd,fde->baf", std::vector{2, 2, 2}, std::vector{17.f, 57.f, 61.f, 197.f, 105.f, 337.f, 149.f, 477.f}), + EinsumTestCase("bac,cd,fde->bad", std::vector{2, 2, 2}, std::vector{20.f, 54.f, 60.f, 198.f, 100.f, 342.f, 140.f, 486.f}), + EinsumTestCase("bac,cd,fde->bcf", std::vector{2, 2, 2}, std::vector{10.f, 26.f, 68.f, 228.f, 50.f, 130.f, 204.f, 684.f}), + EinsumTestCase("bac,cd,fde->bcd", std::vector{2, 2, 2}, std::vector{0.f, 36.f, 80.f, 216.f, 0.f, 180.f, 240.f, 648.f}), + EinsumTestCase("bac,cd,fed->baf", std::vector{2, 2, 2}, std::vector{16.f, 56.f, 56.f, 192.f, 96.f, 328.f, 136.f, 464.f}), + EinsumTestCase("bac,cd,fed->bae", std::vector{2, 2, 2}, std::vector{26.f, 46.f, 90.f, 158.f, 154.f, 270.f, 218.f, 382.f}), + EinsumTestCase("bac,cd,fed->bcf", std::vector{2, 2, 2}, std::vector{8.f, 24.f, 64.f, 224.f, 40.f, 120.f, 192.f, 672.f}), + EinsumTestCase("bac,cd,fed->bce", std::vector{2, 2, 2}, std::vector{12.f, 20.f, 104.f, 184.f, 60.f, 100.f, 312.f, 552.f}), + EinsumTestCase("bac,dc,def->bad", std::vector{2, 2, 2}, std::vector{6.f, 66.f, 18.f, 286.f, 30.f, 506.f, 42.f, 726.f}), + EinsumTestCase("bac,dc,def->bae", std::vector{2, 2, 2}, std::vector{28.f, 44.f, 120.f, 184.f, 212.f, 324.f, 304.f, 464.f}), + EinsumTestCase("bac,dc,def->bcd", std::vector{2, 2, 2}, std::vector{0.f, 88.f, 24.f, 264.f, 0.f, 440.f, 72.f, 792.f}), + EinsumTestCase("bac,dc,def->bce", std::vector{2, 2, 2}, std::vector{36.f, 52.f, 112.f, 176.f, 180.f, 260.f, 336.f, 528.f}), + EinsumTestCase("bac,dc,dfe->bad", std::vector{2, 2, 2}, std::vector{6.f, 66.f, 18.f, 286.f, 30.f, 506.f, 42.f, 726.f}), + EinsumTestCase("bac,dc,dfe->baf", std::vector{2, 2, 2}, std::vector{28.f, 44.f, 120.f, 184.f, 212.f, 324.f, 304.f, 464.f}), + EinsumTestCase("bac,dc,dfe->bcd", std::vector{2, 2, 2}, std::vector{0.f, 88.f, 24.f, 264.f, 0.f, 440.f, 72.f, 792.f}), + EinsumTestCase("bac,dc,dfe->bcf", std::vector{2, 2, 2}, std::vector{36.f, 52.f, 112.f, 176.f, 180.f, 260.f, 336.f, 528.f}), + EinsumTestCase("bac,dc,edf->bae", std::vector{2, 2, 2}, std::vector{16.f, 48.f, 68.f, 196.f, 120.f, 344.f, 172.f, 492.f}), + EinsumTestCase("bac,dc,edf->bad", std::vector{2, 2, 2}, std::vector{10.f, 54.f, 30.f, 234.f, 50.f, 414.f, 70.f, 594.f}), + EinsumTestCase("bac,dc,edf->bce", std::vector{2, 2, 2}, std::vector{20.f, 52.f, 64.f, 192.f, 100.f, 260.f, 192.f, 576.f}), + EinsumTestCase("bac,dc,edf->bcd", std::vector{2, 2, 2}, std::vector{0.f, 72.f, 40.f, 216.f, 0.f, 360.f, 120.f, 648.f}), + EinsumTestCase("bac,dc,efd->bae", std::vector{2, 2, 2}, std::vector{14.f, 46.f, 58.f, 186.f, 102.f, 326.f, 146.f, 466.f}), + EinsumTestCase("bac,dc,efd->baf", std::vector{2, 2, 2}, std::vector{22.f, 38.f, 90.f, 154.f, 158.f, 270.f, 226.f, 386.f}), + EinsumTestCase("bac,dc,efd->bce", std::vector{2, 2, 2}, std::vector{16.f, 48.f, 56.f, 184.f, 80.f, 240.f, 168.f, 552.f}), + EinsumTestCase("bac,dc,efd->bcf", std::vector{2, 2, 2}, std::vector{24.f, 40.f, 88.f, 152.f, 120.f, 200.f, 264.f, 456.f}), + EinsumTestCase("bac,dc,fde->baf", std::vector{2, 2, 2}, std::vector{16.f, 48.f, 68.f, 196.f, 120.f, 344.f, 172.f, 492.f}), + EinsumTestCase("bac,dc,fde->bad", std::vector{2, 2, 2}, std::vector{10.f, 54.f, 30.f, 234.f, 50.f, 414.f, 70.f, 594.f}), + EinsumTestCase("bac,dc,fde->bcf", std::vector{2, 2, 2}, std::vector{20.f, 52.f, 64.f, 192.f, 100.f, 260.f, 192.f, 576.f}), + EinsumTestCase("bac,dc,fde->bcd", std::vector{2, 2, 2}, std::vector{0.f, 72.f, 40.f, 216.f, 0.f, 360.f, 120.f, 648.f}), + EinsumTestCase("bac,dc,fed->baf", std::vector{2, 2, 2}, std::vector{14.f, 46.f, 58.f, 186.f, 102.f, 326.f, 146.f, 466.f}), + EinsumTestCase("bac,dc,fed->bae", std::vector{2, 2, 2}, std::vector{22.f, 38.f, 90.f, 154.f, 158.f, 270.f, 226.f, 386.f}), + EinsumTestCase("bac,dc,fed->bcf", std::vector{2, 2, 2}, std::vector{16.f, 48.f, 56.f, 184.f, 80.f, 240.f, 168.f, 552.f}), + EinsumTestCase("bac,dc,fed->bce", std::vector{2, 2, 2}, std::vector{24.f, 40.f, 88.f, 152.f, 120.f, 200.f, 264.f, 456.f}), + EinsumTestCase("bca,cd,def->bcd", std::vector{2, 2, 2}, std::vector{0.f, 22.f, 60.f, 330.f, 0.f, 198.f, 156.f, 858.f}), + EinsumTestCase("bca,cd,def->bce", std::vector{2, 2, 2}, std::vector{9.f, 13.f, 145.f, 245.f, 81.f, 117.f, 377.f, 637.f}), + EinsumTestCase("bca,cd,def->bad", std::vector{2, 2, 2}, std::vector{24.f, 132.f, 36.f, 220.f, 72.f, 484.f, 84.f, 572.f}), + EinsumTestCase("bca,cd,def->bae", std::vector{2, 2, 2}, std::vector{58.f, 98.f, 96.f, 160.f, 210.f, 346.f, 248.f, 408.f}), + EinsumTestCase("bca,cd,dfe->bcd", std::vector{2, 2, 2}, std::vector{0.f, 22.f, 60.f, 330.f, 0.f, 198.f, 156.f, 858.f}), + EinsumTestCase("bca,cd,dfe->bcf", std::vector{2, 2, 2}, std::vector{9.f, 13.f, 145.f, 245.f, 81.f, 117.f, 377.f, 637.f}), + EinsumTestCase("bca,cd,dfe->bad", std::vector{2, 2, 2}, std::vector{24.f, 132.f, 36.f, 220.f, 72.f, 484.f, 84.f, 572.f}), + EinsumTestCase("bca,cd,dfe->baf", std::vector{2, 2, 2}, std::vector{58.f, 98.f, 96.f, 160.f, 210.f, 346.f, 248.f, 408.f}), + EinsumTestCase("bca,cd,edf->bce", std::vector{2, 2, 2}, std::vector{5.f, 13.f, 85.f, 285.f, 45.f, 117.f, 221.f, 741.f}), + EinsumTestCase("bca,cd,edf->bcd", std::vector{2, 2, 2}, std::vector{0.f, 18.f, 100.f, 270.f, 0.f, 162.f, 260.f, 702.f}), + EinsumTestCase("bca,cd,edf->bae", std::vector{2, 2, 2}, std::vector{34.f, 114.f, 56.f, 184.f, 122.f, 394.f, 144.f, 464.f}), + EinsumTestCase("bca,cd,edf->bad", std::vector{2, 2, 2}, std::vector{40.f, 108.f, 60.f, 180.f, 120.f, 396.f, 140.f, 468.f}), + EinsumTestCase("bca,cd,efd->bce", std::vector{2, 2, 2}, std::vector{4.f, 12.f, 80.f, 280.f, 36.f, 108.f, 208.f, 728.f}), + EinsumTestCase("bca,cd,efd->bcf", std::vector{2, 2, 2}, std::vector{6.f, 10.f, 130.f, 230.f, 54.f, 90.f, 338.f, 598.f}), + EinsumTestCase("bca,cd,efd->bae", std::vector{2, 2, 2}, std::vector{32.f, 112.f, 52.f, 180.f, 112.f, 384.f, 132.f, 452.f}), + EinsumTestCase("bca,cd,efd->baf", std::vector{2, 2, 2}, std::vector{52.f, 92.f, 84.f, 148.f, 180.f, 316.f, 212.f, 372.f}), + EinsumTestCase("bca,cd,fde->bcf", std::vector{2, 2, 2}, std::vector{5.f, 13.f, 85.f, 285.f, 45.f, 117.f, 221.f, 741.f}), + EinsumTestCase("bca,cd,fde->bcd", std::vector{2, 2, 2}, std::vector{0.f, 18.f, 100.f, 270.f, 0.f, 162.f, 260.f, 702.f}), + EinsumTestCase("bca,cd,fde->baf", std::vector{2, 2, 2}, std::vector{34.f, 114.f, 56.f, 184.f, 122.f, 394.f, 144.f, 464.f}), + EinsumTestCase("bca,cd,fde->bad", std::vector{2, 2, 2}, std::vector{40.f, 108.f, 60.f, 180.f, 120.f, 396.f, 140.f, 468.f}), + EinsumTestCase("bca,cd,fed->bcf", std::vector{2, 2, 2}, std::vector{4.f, 12.f, 80.f, 280.f, 36.f, 108.f, 208.f, 728.f}), + EinsumTestCase("bca,cd,fed->bce", std::vector{2, 2, 2}, std::vector{6.f, 10.f, 130.f, 230.f, 54.f, 90.f, 338.f, 598.f}), + EinsumTestCase("bca,cd,fed->baf", std::vector{2, 2, 2}, std::vector{32.f, 112.f, 52.f, 180.f, 112.f, 384.f, 132.f, 452.f}), + EinsumTestCase("bca,cd,fed->bae", std::vector{2, 2, 2}, std::vector{52.f, 92.f, 84.f, 148.f, 180.f, 316.f, 212.f, 372.f}), + EinsumTestCase("bca,dc,def->bcd", std::vector{2, 2, 2}, std::vector{0.f, 44.f, 30.f, 330.f, 0.f, 396.f, 78.f, 858.f}), + EinsumTestCase("bca,dc,def->bce", std::vector{2, 2, 2}, std::vector{18.f, 26.f, 140.f, 220.f, 162.f, 234.f, 364.f, 572.f}), + EinsumTestCase("bca,dc,def->bad", std::vector{2, 2, 2}, std::vector{12.f, 132.f, 18.f, 242.f, 36.f, 572.f, 42.f, 682.f}), + EinsumTestCase("bca,dc,def->bae", std::vector{2, 2, 2}, std::vector{56.f, 88.f, 102.f, 158.f, 240.f, 368.f, 286.f, 438.f}), + EinsumTestCase("bca,dc,dfe->bcd", std::vector{2, 2, 2}, std::vector{0.f, 44.f, 30.f, 330.f, 0.f, 396.f, 78.f, 858.f}), + EinsumTestCase("bca,dc,dfe->bcf", std::vector{2, 2, 2}, std::vector{18.f, 26.f, 140.f, 220.f, 162.f, 234.f, 364.f, 572.f}), + EinsumTestCase("bca,dc,dfe->bad", std::vector{2, 2, 2}, std::vector{12.f, 132.f, 18.f, 242.f, 36.f, 572.f, 42.f, 682.f}), + EinsumTestCase("bca,dc,dfe->baf", std::vector{2, 2, 2}, std::vector{56.f, 88.f, 102.f, 158.f, 240.f, 368.f, 286.f, 438.f}), + EinsumTestCase("bca,dc,edf->bce", std::vector{2, 2, 2}, std::vector{10.f, 26.f, 80.f, 240.f, 90.f, 234.f, 208.f, 624.f}), + EinsumTestCase("bca,dc,edf->bcd", std::vector{2, 2, 2}, std::vector{0.f, 36.f, 50.f, 270.f, 0.f, 324.f, 130.f, 702.f}), + EinsumTestCase("bca,dc,edf->bae", std::vector{2, 2, 2}, std::vector{32.f, 96.f, 58.f, 170.f, 136.f, 392.f, 162.f, 466.f}), + EinsumTestCase("bca,dc,edf->bad", std::vector{2, 2, 2}, std::vector{20.f, 108.f, 30.f, 198.f, 60.f, 468.f, 70.f, 558.f}), + EinsumTestCase("bca,dc,efd->bce", std::vector{2, 2, 2}, std::vector{8.f, 24.f, 70.f, 230.f, 72.f, 216.f, 182.f, 598.f}), + EinsumTestCase("bca,dc,efd->bcf", std::vector{2, 2, 2}, std::vector{12.f, 20.f, 110.f, 190.f, 108.f, 180.f, 286.f, 494.f}), + EinsumTestCase("bca,dc,efd->bae", std::vector{2, 2, 2}, std::vector{28.f, 92.f, 50.f, 162.f, 116.f, 372.f, 138.f, 442.f}), + EinsumTestCase("bca,dc,efd->baf", std::vector{2, 2, 2}, std::vector{44.f, 76.f, 78.f, 134.f, 180.f, 308.f, 214.f, 366.f}), + EinsumTestCase("bca,dc,fde->bcf", std::vector{2, 2, 2}, std::vector{10.f, 26.f, 80.f, 240.f, 90.f, 234.f, 208.f, 624.f}), + EinsumTestCase("bca,dc,fde->bcd", std::vector{2, 2, 2}, std::vector{0.f, 36.f, 50.f, 270.f, 0.f, 324.f, 130.f, 702.f}), + EinsumTestCase("bca,dc,fde->baf", std::vector{2, 2, 2}, std::vector{32.f, 96.f, 58.f, 170.f, 136.f, 392.f, 162.f, 466.f}), + EinsumTestCase("bca,dc,fde->bad", std::vector{2, 2, 2}, std::vector{20.f, 108.f, 30.f, 198.f, 60.f, 468.f, 70.f, 558.f}), + EinsumTestCase("bca,dc,fed->bcf", std::vector{2, 2, 2}, std::vector{8.f, 24.f, 70.f, 230.f, 72.f, 216.f, 182.f, 598.f}), + EinsumTestCase("bca,dc,fed->bce", std::vector{2, 2, 2}, std::vector{12.f, 20.f, 110.f, 190.f, 108.f, 180.f, 286.f, 494.f}), + EinsumTestCase("bca,dc,fed->baf", std::vector{2, 2, 2}, std::vector{28.f, 92.f, 50.f, 162.f, 116.f, 372.f, 138.f, 442.f}), + EinsumTestCase("bca,dc,fed->bae", std::vector{2, 2, 2}, std::vector{44.f, 76.f, 78.f, 134.f, 180.f, 308.f, 214.f, 366.f}), + EinsumTestCase("cab,cd,def->cad", std::vector{2, 2, 2}, std::vector{0.f, 22.f, 0.f, 110.f, 108.f, 594.f, 156.f, 858.f}), + EinsumTestCase("cab,cd,def->cae", std::vector{2, 2, 2}, std::vector{9.f, 13.f, 45.f, 65.f, 261.f, 441.f, 377.f, 637.f}), + EinsumTestCase("cab,cd,def->cbd", std::vector{2, 2, 2}, std::vector{0.f, 44.f, 0.f, 88.f, 120.f, 660.f, 144.f, 792.f}), + EinsumTestCase("cab,cd,def->cbe", std::vector{2, 2, 2}, std::vector{18.f, 26.f, 36.f, 52.f, 290.f, 490.f, 348.f, 588.f}), + EinsumTestCase("cab,cd,dfe->cad", std::vector{2, 2, 2}, std::vector{0.f, 22.f, 0.f, 110.f, 108.f, 594.f, 156.f, 858.f}), + EinsumTestCase("cab,cd,dfe->caf", std::vector{2, 2, 2}, std::vector{9.f, 13.f, 45.f, 65.f, 261.f, 441.f, 377.f, 637.f}), + EinsumTestCase("cab,cd,dfe->cbd", std::vector{2, 2, 2}, std::vector{0.f, 44.f, 0.f, 88.f, 120.f, 660.f, 144.f, 792.f}), + EinsumTestCase("cab,cd,dfe->cbf", std::vector{2, 2, 2}, std::vector{18.f, 26.f, 36.f, 52.f, 290.f, 490.f, 348.f, 588.f}), + EinsumTestCase("cab,cd,edf->cae", std::vector{2, 2, 2}, std::vector{5.f, 13.f, 25.f, 65.f, 153.f, 513.f, 221.f, 741.f}), + EinsumTestCase("cab,cd,edf->cad", std::vector{2, 2, 2}, std::vector{0.f, 18.f, 0.f, 90.f, 180.f, 486.f, 260.f, 702.f}), + EinsumTestCase("cab,cd,edf->cbe", std::vector{2, 2, 2}, std::vector{10.f, 26.f, 20.f, 52.f, 170.f, 570.f, 204.f, 684.f}), + EinsumTestCase("cab,cd,edf->cbd", std::vector{2, 2, 2}, std::vector{0.f, 36.f, 0.f, 72.f, 200.f, 540.f, 240.f, 648.f}), + EinsumTestCase("cab,cd,efd->cae", std::vector{2, 2, 2}, std::vector{4.f, 12.f, 20.f, 60.f, 144.f, 504.f, 208.f, 728.f}), + EinsumTestCase("cab,cd,efd->caf", std::vector{2, 2, 2}, std::vector{6.f, 10.f, 30.f, 50.f, 234.f, 414.f, 338.f, 598.f}), + EinsumTestCase("cab,cd,efd->cbe", std::vector{2, 2, 2}, std::vector{8.f, 24.f, 16.f, 48.f, 160.f, 560.f, 192.f, 672.f}), + EinsumTestCase("cab,cd,efd->cbf", std::vector{2, 2, 2}, std::vector{12.f, 20.f, 24.f, 40.f, 260.f, 460.f, 312.f, 552.f}), + EinsumTestCase("cab,cd,fde->caf", std::vector{2, 2, 2}, std::vector{5.f, 13.f, 25.f, 65.f, 153.f, 513.f, 221.f, 741.f}), + EinsumTestCase("cab,cd,fde->cad", std::vector{2, 2, 2}, std::vector{0.f, 18.f, 0.f, 90.f, 180.f, 486.f, 260.f, 702.f}), + EinsumTestCase("cab,cd,fde->cbf", std::vector{2, 2, 2}, std::vector{10.f, 26.f, 20.f, 52.f, 170.f, 570.f, 204.f, 684.f}), + EinsumTestCase("cab,cd,fde->cbd", std::vector{2, 2, 2}, std::vector{0.f, 36.f, 0.f, 72.f, 200.f, 540.f, 240.f, 648.f}), + EinsumTestCase("cab,cd,fed->caf", std::vector{2, 2, 2}, std::vector{4.f, 12.f, 20.f, 60.f, 144.f, 504.f, 208.f, 728.f}), + EinsumTestCase("cab,cd,fed->cae", std::vector{2, 2, 2}, std::vector{6.f, 10.f, 30.f, 50.f, 234.f, 414.f, 338.f, 598.f}), + EinsumTestCase("cab,cd,fed->cbf", std::vector{2, 2, 2}, std::vector{8.f, 24.f, 16.f, 48.f, 160.f, 560.f, 192.f, 672.f}), + EinsumTestCase("cab,cd,fed->cbe", std::vector{2, 2, 2}, std::vector{12.f, 20.f, 24.f, 40.f, 260.f, 460.f, 312.f, 552.f}), + EinsumTestCase("cab,dc,def->cad", std::vector{2, 2, 2}, std::vector{0.f, 44.f, 0.f, 220.f, 54.f, 594.f, 78.f, 858.f}), + EinsumTestCase("cab,dc,def->cae", std::vector{2, 2, 2}, std::vector{18.f, 26.f, 90.f, 130.f, 252.f, 396.f, 364.f, 572.f}), + EinsumTestCase("cab,dc,def->cbd", std::vector{2, 2, 2}, std::vector{0.f, 88.f, 0.f, 176.f, 60.f, 660.f, 72.f, 792.f})}; + + std::vector test_cases_set_3{ + EinsumTestCase("cab,dc,def->cbe", std::vector{2, 2, 2}, std::vector{36.f, 52.f, 72.f, 104.f, 280.f, 440.f, 336.f, 528.f}), + EinsumTestCase("cab,dc,dfe->cad", std::vector{2, 2, 2}, std::vector{0.f, 44.f, 0.f, 220.f, 54.f, 594.f, 78.f, 858.f}), + EinsumTestCase("cab,dc,dfe->caf", std::vector{2, 2, 2}, std::vector{18.f, 26.f, 90.f, 130.f, 252.f, 396.f, 364.f, 572.f}), + EinsumTestCase("cab,dc,dfe->cbd", std::vector{2, 2, 2}, std::vector{0.f, 88.f, 0.f, 176.f, 60.f, 660.f, 72.f, 792.f}), + EinsumTestCase("cab,dc,dfe->cbf", std::vector{2, 2, 2}, std::vector{36.f, 52.f, 72.f, 104.f, 280.f, 440.f, 336.f, 528.f}), + EinsumTestCase("cab,dc,edf->cae", std::vector{2, 2, 2}, std::vector{10.f, 26.f, 50.f, 130.f, 144.f, 432.f, 208.f, 624.f}), + EinsumTestCase("cab,dc,edf->cad", std::vector{2, 2, 2}, std::vector{0.f, 36.f, 0.f, 180.f, 90.f, 486.f, 130.f, 702.f}), + EinsumTestCase("cab,dc,edf->cbe", std::vector{2, 2, 2}, std::vector{20.f, 52.f, 40.f, 104.f, 160.f, 480.f, 192.f, 576.f}), + EinsumTestCase("cab,dc,edf->cbd", std::vector{2, 2, 2}, std::vector{0.f, 72.f, 0.f, 144.f, 100.f, 540.f, 120.f, 648.f}), + EinsumTestCase("cab,dc,efd->cae", std::vector{2, 2, 2}, std::vector{8.f, 24.f, 40.f, 120.f, 126.f, 414.f, 182.f, 598.f}), + EinsumTestCase("cab,dc,efd->caf", std::vector{2, 2, 2}, std::vector{12.f, 20.f, 60.f, 100.f, 198.f, 342.f, 286.f, 494.f}), + EinsumTestCase("cab,dc,efd->cbe", std::vector{2, 2, 2}, std::vector{16.f, 48.f, 32.f, 96.f, 140.f, 460.f, 168.f, 552.f}), + EinsumTestCase("cab,dc,efd->cbf", std::vector{2, 2, 2}, std::vector{24.f, 40.f, 48.f, 80.f, 220.f, 380.f, 264.f, 456.f}), + EinsumTestCase("cab,dc,fde->caf", std::vector{2, 2, 2}, std::vector{10.f, 26.f, 50.f, 130.f, 144.f, 432.f, 208.f, 624.f}), + EinsumTestCase("cab,dc,fde->cad", std::vector{2, 2, 2}, std::vector{0.f, 36.f, 0.f, 180.f, 90.f, 486.f, 130.f, 702.f}), + EinsumTestCase("cab,dc,fde->cbf", std::vector{2, 2, 2}, std::vector{20.f, 52.f, 40.f, 104.f, 160.f, 480.f, 192.f, 576.f}), + EinsumTestCase("cab,dc,fde->cbd", std::vector{2, 2, 2}, std::vector{0.f, 72.f, 0.f, 144.f, 100.f, 540.f, 120.f, 648.f}), + EinsumTestCase("cab,dc,fed->caf", std::vector{2, 2, 2}, std::vector{8.f, 24.f, 40.f, 120.f, 126.f, 414.f, 182.f, 598.f}), + EinsumTestCase("cab,dc,fed->cae", std::vector{2, 2, 2}, std::vector{12.f, 20.f, 60.f, 100.f, 198.f, 342.f, 286.f, 494.f}), + EinsumTestCase("cab,dc,fed->cbf", std::vector{2, 2, 2}, std::vector{16.f, 48.f, 32.f, 96.f, 140.f, 460.f, 168.f, 552.f}), + EinsumTestCase("cab,dc,fed->cbe", std::vector{2, 2, 2}, std::vector{24.f, 40.f, 48.f, 80.f, 220.f, 380.f, 264.f, 456.f}), + EinsumTestCase("cba,cd,def->cbd", std::vector{2, 2, 2}, std::vector{0.f, 22.f, 0.f, 110.f, 108.f, 594.f, 156.f, 858.f}), + EinsumTestCase("cba,cd,def->cbe", std::vector{2, 2, 2}, std::vector{9.f, 13.f, 45.f, 65.f, 261.f, 441.f, 377.f, 637.f}), + EinsumTestCase("cba,cd,def->cad", std::vector{2, 2, 2}, std::vector{0.f, 44.f, 0.f, 88.f, 120.f, 660.f, 144.f, 792.f}), + EinsumTestCase("cba,cd,def->cae", std::vector{2, 2, 2}, std::vector{18.f, 26.f, 36.f, 52.f, 290.f, 490.f, 348.f, 588.f}), + EinsumTestCase("cba,cd,dfe->cbd", std::vector{2, 2, 2}, std::vector{0.f, 22.f, 0.f, 110.f, 108.f, 594.f, 156.f, 858.f}), + EinsumTestCase("cba,cd,dfe->cbf", std::vector{2, 2, 2}, std::vector{9.f, 13.f, 45.f, 65.f, 261.f, 441.f, 377.f, 637.f}), + EinsumTestCase("cba,cd,dfe->cad", std::vector{2, 2, 2}, std::vector{0.f, 44.f, 0.f, 88.f, 120.f, 660.f, 144.f, 792.f}), + EinsumTestCase("cba,cd,dfe->caf", std::vector{2, 2, 2}, std::vector{18.f, 26.f, 36.f, 52.f, 290.f, 490.f, 348.f, 588.f}), + EinsumTestCase("cba,cd,edf->cbe", std::vector{2, 2, 2}, std::vector{5.f, 13.f, 25.f, 65.f, 153.f, 513.f, 221.f, 741.f}), + EinsumTestCase("cba,cd,edf->cbd", std::vector{2, 2, 2}, std::vector{0.f, 18.f, 0.f, 90.f, 180.f, 486.f, 260.f, 702.f}), + EinsumTestCase("cba,cd,edf->cae", std::vector{2, 2, 2}, std::vector{10.f, 26.f, 20.f, 52.f, 170.f, 570.f, 204.f, 684.f}), + EinsumTestCase("cba,cd,edf->cad", std::vector{2, 2, 2}, std::vector{0.f, 36.f, 0.f, 72.f, 200.f, 540.f, 240.f, 648.f}), + EinsumTestCase("cba,cd,efd->cbe", std::vector{2, 2, 2}, std::vector{4.f, 12.f, 20.f, 60.f, 144.f, 504.f, 208.f, 728.f}), + EinsumTestCase("cba,cd,efd->cbf", std::vector{2, 2, 2}, std::vector{6.f, 10.f, 30.f, 50.f, 234.f, 414.f, 338.f, 598.f}), + EinsumTestCase("cba,cd,efd->cae", std::vector{2, 2, 2}, std::vector{8.f, 24.f, 16.f, 48.f, 160.f, 560.f, 192.f, 672.f}), + EinsumTestCase("cba,cd,efd->caf", std::vector{2, 2, 2}, std::vector{12.f, 20.f, 24.f, 40.f, 260.f, 460.f, 312.f, 552.f}), + EinsumTestCase("cba,cd,fde->cbf", std::vector{2, 2, 2}, std::vector{5.f, 13.f, 25.f, 65.f, 153.f, 513.f, 221.f, 741.f}), + EinsumTestCase("cba,cd,fde->cbd", std::vector{2, 2, 2}, std::vector{0.f, 18.f, 0.f, 90.f, 180.f, 486.f, 260.f, 702.f}), + EinsumTestCase("cba,cd,fde->caf", std::vector{2, 2, 2}, std::vector{10.f, 26.f, 20.f, 52.f, 170.f, 570.f, 204.f, 684.f}), + EinsumTestCase("cba,cd,fde->cad", std::vector{2, 2, 2}, std::vector{0.f, 36.f, 0.f, 72.f, 200.f, 540.f, 240.f, 648.f}), + EinsumTestCase("cba,cd,fed->cbf", std::vector{2, 2, 2}, std::vector{4.f, 12.f, 20.f, 60.f, 144.f, 504.f, 208.f, 728.f}), + EinsumTestCase("cba,cd,fed->cbe", std::vector{2, 2, 2}, std::vector{6.f, 10.f, 30.f, 50.f, 234.f, 414.f, 338.f, 598.f}), + EinsumTestCase("cba,cd,fed->caf", std::vector{2, 2, 2}, std::vector{8.f, 24.f, 16.f, 48.f, 160.f, 560.f, 192.f, 672.f}), + EinsumTestCase("cba,cd,fed->cae", std::vector{2, 2, 2}, std::vector{12.f, 20.f, 24.f, 40.f, 260.f, 460.f, 312.f, 552.f}), + EinsumTestCase("cba,dc,def->cbd", std::vector{2, 2, 2}, std::vector{0.f, 44.f, 0.f, 220.f, 54.f, 594.f, 78.f, 858.f}), + EinsumTestCase("cba,dc,def->cbe", std::vector{2, 2, 2}, std::vector{18.f, 26.f, 90.f, 130.f, 252.f, 396.f, 364.f, 572.f}), + EinsumTestCase("cba,dc,def->cad", std::vector{2, 2, 2}, std::vector{0.f, 88.f, 0.f, 176.f, 60.f, 660.f, 72.f, 792.f}), + EinsumTestCase("cba,dc,def->cae", std::vector{2, 2, 2}, std::vector{36.f, 52.f, 72.f, 104.f, 280.f, 440.f, 336.f, 528.f}), + EinsumTestCase("cba,dc,dfe->cbd", std::vector{2, 2, 2}, std::vector{0.f, 44.f, 0.f, 220.f, 54.f, 594.f, 78.f, 858.f}), + EinsumTestCase("cba,dc,dfe->cbf", std::vector{2, 2, 2}, std::vector{18.f, 26.f, 90.f, 130.f, 252.f, 396.f, 364.f, 572.f}), + EinsumTestCase("cba,dc,dfe->cad", std::vector{2, 2, 2}, std::vector{0.f, 88.f, 0.f, 176.f, 60.f, 660.f, 72.f, 792.f}), + EinsumTestCase("cba,dc,dfe->caf", std::vector{2, 2, 2}, std::vector{36.f, 52.f, 72.f, 104.f, 280.f, 440.f, 336.f, 528.f}), + EinsumTestCase("cba,dc,edf->cbe", std::vector{2, 2, 2}, std::vector{10.f, 26.f, 50.f, 130.f, 144.f, 432.f, 208.f, 624.f}), + EinsumTestCase("cba,dc,edf->cbd", std::vector{2, 2, 2}, std::vector{0.f, 36.f, 0.f, 180.f, 90.f, 486.f, 130.f, 702.f}), + EinsumTestCase("cba,dc,edf->cae", std::vector{2, 2, 2}, std::vector{20.f, 52.f, 40.f, 104.f, 160.f, 480.f, 192.f, 576.f}), + EinsumTestCase("cba,dc,edf->cad", std::vector{2, 2, 2}, std::vector{0.f, 72.f, 0.f, 144.f, 100.f, 540.f, 120.f, 648.f}), + EinsumTestCase("cba,dc,efd->cbe", std::vector{2, 2, 2}, std::vector{8.f, 24.f, 40.f, 120.f, 126.f, 414.f, 182.f, 598.f}), + EinsumTestCase("cba,dc,efd->cbf", std::vector{2, 2, 2}, std::vector{12.f, 20.f, 60.f, 100.f, 198.f, 342.f, 286.f, 494.f}), + EinsumTestCase("cba,dc,efd->cae", std::vector{2, 2, 2}, std::vector{16.f, 48.f, 32.f, 96.f, 140.f, 460.f, 168.f, 552.f}), + EinsumTestCase("cba,dc,efd->caf", std::vector{2, 2, 2}, std::vector{24.f, 40.f, 48.f, 80.f, 220.f, 380.f, 264.f, 456.f}), + EinsumTestCase("cba,dc,fde->cbf", std::vector{2, 2, 2}, std::vector{10.f, 26.f, 50.f, 130.f, 144.f, 432.f, 208.f, 624.f}), + EinsumTestCase("cba,dc,fde->cbd", std::vector{2, 2, 2}, std::vector{0.f, 36.f, 0.f, 180.f, 90.f, 486.f, 130.f, 702.f}), + EinsumTestCase("cba,dc,fde->caf", std::vector{2, 2, 2}, std::vector{20.f, 52.f, 40.f, 104.f, 160.f, 480.f, 192.f, 576.f}), + EinsumTestCase("cba,dc,fde->cad", std::vector{2, 2, 2}, std::vector{0.f, 72.f, 0.f, 144.f, 100.f, 540.f, 120.f, 648.f}), + EinsumTestCase("cba,dc,fed->cbf", std::vector{2, 2, 2}, std::vector{8.f, 24.f, 40.f, 120.f, 126.f, 414.f, 182.f, 598.f}), + EinsumTestCase("cba,dc,fed->cbe", std::vector{2, 2, 2}, std::vector{12.f, 20.f, 60.f, 100.f, 198.f, 342.f, 286.f, 494.f}), + EinsumTestCase("cba,dc,fed->caf", std::vector{2, 2, 2}, std::vector{16.f, 48.f, 32.f, 96.f, 140.f, 460.f, 168.f, 552.f}), + EinsumTestCase("cba,dc,fed->cae", std::vector{2, 2, 2}, std::vector{24.f, 40.f, 48.f, 80.f, 220.f, 380.f, 264.f, 456.f})}; + + auto test_lambda = [](const std::vector& test_cases_set) { + std::vector m1{0.f, 1.f, 2.f, 3.f, 4.f, 5.f, 6.f, 7.f}; + std::vector m2{0.f, 1.f, 2.f, 3.f}; + std::vector m3{0.f, 1.f, 2.f, 3.f, 4.f, 5.f, 6.f, 7.f}; + for (const auto& tst : test_cases_set) { + OpTester test("Einsum", 12, onnxruntime::kOnnxDomain); + test.AddAttribute("equation", tst.equation); + test.AddInput("x", {2, 2, 2}, m1); + test.AddInput("y", {2, 2}, m2); + test.AddInput("z", {2, 2, 2}, m3); + test.AddOutput("o", tst.shape, tst.expected); + test.Run(); + } + }; + + test_lambda(test_cases_set_1); + test_lambda(test_cases_set_2); + test_lambda(test_cases_set_3); + +} // namespace test + } // namespace test } // namespace onnxruntime From c5ea5907c09cc39077749971637852ab424a5c50 Mon Sep 17 00:00:00 2001 From: Thiago Crepaldi Date: Wed, 26 May 2021 14:18:25 -0700 Subject: [PATCH 12/47] Fix permission error for ORTModule lock file (#7814) --- .../python/training/ortmodule/__init__.py | 32 ++++++++++++++++--- .../templates/py-packaging-stage.yml | 16 +++++----- 2 files changed, 35 insertions(+), 13 deletions(-) diff --git a/orttraining/orttraining/python/training/ortmodule/__init__.py b/orttraining/orttraining/python/training/ortmodule/__init__.py index 5bdcd39840..70793d8a36 100644 --- a/orttraining/orttraining/python/training/ortmodule/__init__.py +++ b/orttraining/orttraining/python/training/ortmodule/__init__.py @@ -11,7 +11,29 @@ from packaging import version ################################################################################ ONNX_OPSET_VERSION = 12 MINIMUM_TORCH_VERSION_STR = '1.8.1' -TORCH_CPP_BUILD_DIR = os.path.join(os.path.dirname(__file__),'torch_inline_extensions') + +# Use one of the available directories as Torch CPP extension in the following order: +# 1) Path at listed at TORCH_EXTENSIONS_DIR environment variable +# 2) Default Python package dir +# 3) /.cache +home_dir = os.path.expanduser("~") +python_package_dir = os.path.dirname(__file__) +torch_extensions_dir = os.environ.get('TORCH_EXTENSIONS_DIR') + +TORCH_CPP_BUILD_DIR = os.path.join(python_package_dir,'torch_inline_extensions') +TORCH_CPP_BUILD_DIR_BACKUP = os.path.join(home_dir, '.cache', 'torch_ort_extensions') + +if torch_extensions_dir is not None and os.access(torch_extensions_dir, os.X_OK | os.W_OK): + TORCH_CPP_BUILD_DIR = torch_extensions_dir +elif not os.access(python_package_dir, os.X_OK | os.W_OK): + if os.access(home_dir, os.X_OK | os.W_OK): + TORCH_CPP_BUILD_DIR = TORCH_CPP_BUILD_DIR_BACKUP + else: + extra_message = '' + if torch_extensions_dir: + extra_message = 'or the path pointed by the TORCH_EXTENSIONS_DIR environment variable ' + raise PermissionError('ORTModule could not find a writable directory to cache its internal files.', + f'Make {python_package_dir} or {home_dir} {extra_message}writable and try again.') # Check whether Torch C++ extension compilation was aborted in previous runs if not os.path.exists(TORCH_CPP_BUILD_DIR): @@ -19,19 +41,19 @@ if not os.path.exists(TORCH_CPP_BUILD_DIR): elif os.path.exists(os.path.join(TORCH_CPP_BUILD_DIR,'lock')): print("WARNING: ORTModule detected PyTorch CPP extension's lock file during initialization, " "which can cause unexpected hangs. " - f"Delete {os.path.join(TORCH_CPP_BUILD_DIR,'lock')} to prevent unexpected behavior.") + f"Delete {os.path.join(TORCH_CPP_BUILD_DIR,'lock')} if a hang occurs.") -# Verify proper PyTorch is installed before proceding to ONNX Runtime initializetion +# Verify proper PyTorch is installed before proceding to ONNX Runtime initialization try: import torch torch_version = version.parse(torch.__version__.split('+')[0]) minimum_torch_version = version.parse(MINIMUM_TORCH_VERSION_STR) if torch_version < minimum_torch_version: raise RuntimeError( - f'ONNXRuntime ORTModule frontend requires PyTorch version greater or equal to {MINIMUM_TORCH_VERSION_STR}, ' + f'ONNX Runtime ORTModule frontend requires PyTorch version greater or equal to {MINIMUM_TORCH_VERSION_STR}, ' f'but version {torch.__version__} was found instead.') except: - raise(f'PyTorch {MINIMUM_TORCH_VERSION_STR} must be installed in order to run ONNXRuntime ORTModule frontend!') + raise(f'PyTorch {MINIMUM_TORCH_VERSION_STR} must be installed in order to run ONNX Runtime ORTModule frontend!') # ORTModule must be loaded only after all validation passes from .ortmodule import ORTModule diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml index 4b6db23260..bb121d6b88 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml @@ -119,7 +119,7 @@ stages: - task: CmdLine@2 displayName: 'Build Python Documentation' - condition: ne(variables['PythonVersion'], '3.9') # tensorflow not available on python 3.9 + condition: and(succeeded(), ne(variables['PythonVersion'], '3.9')) # tensorflow not available on python 3.9 inputs: script: | mkdir -p $HOME/.onnx @@ -137,7 +137,7 @@ stages: - task: CopyFiles@2 displayName: 'Copy Python Documentation to: $(Build.ArtifactStagingDirectory)' - condition: ne(variables['PythonVersion'], '3.9') # tensorflow not available on python 3.9 + condition: and(succeeded(), ne(variables['PythonVersion'], '3.9')) # tensorflow not available on python 3.9 inputs: SourceFolder: '$(Build.BinariesDirectory)/docs/inference/html' Contents: '**' @@ -431,7 +431,7 @@ stages: - task: CmdLine@2 displayName: 'Build Python Documentation' - condition: ne(variables['PythonVersion'], '3.9') # tensorflow not available on python 3.9 + condition: and(succeeded(), ne(variables['PythonVersion'], '3.9')) # tensorflow not available on python 3.9 inputs: script: | mkdir -p $HOME/.onnx @@ -447,7 +447,7 @@ stages: - task: CopyFiles@2 displayName: 'Copy Python Documentation to: $(Build.ArtifactStagingDirectory)' - condition: ne(variables['PythonVersion'], '3.9') # tensorflow not available on python 3.9 + condition: and(succeeded(), ne(variables['PythonVersion'], '3.9')) # tensorflow not available on python 3.9 inputs: SourceFolder: '$(Build.BinariesDirectory)/docs/training/html' Contents: '**' @@ -588,7 +588,7 @@ stages: - task: CmdLine@2 displayName: 'Build Python Documentation' - condition: ne(variables['PythonVersion'], '3.9') # tensorflow not available on python 3.9 + condition: and(succeeded(), ne(variables['PythonVersion'], '3.9')) # tensorflow not available on python 3.9 inputs: script: | mkdir -p $HOME/.onnx @@ -606,7 +606,7 @@ stages: - task: CopyFiles@2 displayName: 'Copy Python Documentation to: $(Build.ArtifactStagingDirectory)' - condition: ne(variables['PythonVersion'], '3.9') # tensorflow not available on python 3.9 + condition: and(succeeded(), ne(variables['PythonVersion'], '3.9')) # tensorflow not available on python 3.9 inputs: SourceFolder: '$(Build.BinariesDirectory)/docs/training/html' Contents: '**' @@ -761,7 +761,7 @@ stages: - task: CmdLine@2 displayName: 'Build Python Documentation' - condition: ne(variables['PythonVersion'], '3.9') # tensorflow not available on python 3.9 + condition: and(succeeded(), ne(variables['PythonVersion'], '3.9')) # tensorflow not available on python 3.9 inputs: script: | mkdir -p $HOME/.onnx @@ -779,7 +779,7 @@ stages: - task: CopyFiles@2 displayName: 'Copy Python Documentation to: $(Build.ArtifactStagingDirectory)' - condition: ne(variables['PythonVersion'], '3.9') # tensorflow not available on python 3.9 + condition: and(succeeded(), ne(variables['PythonVersion'], '3.9')) # tensorflow not available on python 3.9 inputs: SourceFolder: '$(Build.BinariesDirectory)/docs/training/html' Contents: '**' From 331f20428c6c94f6c485ab5210a9bf8dc9391f04 Mon Sep 17 00:00:00 2001 From: Yulong Wang Date: Wed, 26 May 2021 15:46:50 -0700 Subject: [PATCH 13/47] [js/web] only apply max thread number when it's omitted (#7834) --- js/web/lib/backend-wasm.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/js/web/lib/backend-wasm.ts b/js/web/lib/backend-wasm.ts index 9c97d47c42..35d34b27fb 100644 --- a/js/web/lib/backend-wasm.ts +++ b/js/web/lib/backend-wasm.ts @@ -20,11 +20,10 @@ export const initializeFlags = (): void => { env.wasm.initTimeout = 0; } - if (typeof env.wasm.numThreads !== 'number' || !Number.isInteger(env.wasm.numThreads) || env.wasm.numThreads < 0) { + if (typeof env.wasm.numThreads !== 'number' || !Number.isInteger(env.wasm.numThreads) || env.wasm.numThreads <= 0) { const numCpuLogicalCores = typeof navigator === 'undefined' ? cpus().length : navigator.hardwareConcurrency; - env.wasm.numThreads = Math.ceil((numCpuLogicalCores || 1) / 2); + env.wasm.numThreads = Math.min(4, Math.ceil((numCpuLogicalCores || 1) / 2)); } - env.wasm.numThreads = Math.min(4, env.wasm.numThreads); }; class OnnxruntimeWebAssemblyBackend implements Backend { From afca89dce6d852dd8faabb5c53e6a9ff739de141 Mon Sep 17 00:00:00 2001 From: Guoyu Wang <62914304+gwang-msft@users.noreply.github.com> Date: Wed, 26 May 2021 16:08:57 -0700 Subject: [PATCH 14/47] fix boost download url (#7843) --- cgmanifests/cgmanifest.json | 16 ++++++++-------- server/get_boost.cmake | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cgmanifests/cgmanifest.json b/cgmanifests/cgmanifest.json index 91374bf86d..e4cf9747c0 100644 --- a/cgmanifests/cgmanifest.json +++ b/cgmanifests/cgmanifest.json @@ -171,7 +171,7 @@ "Other": { "Name": "Boost", "Version": "1.69.0", - "DownloadUrl": "http://dl.bintray.com/boostorg/release/1.69.0/source/boost_1_69_0.tar.bz2" + "DownloadUrl": "https://boostorg.jfrog.io/artifactory/main/release/1.69.0/source/boost_1_69_0.tar.bz2" } } }, @@ -462,14 +462,14 @@ }, { "component": { - "type": "git", - "git": { - "commitHash": "e1e11e0d555c08bec08a6c7773aa777dfcaae9da", - "repositoryUrl": "https://github.com/dmlc/dlpack.git" - }, - "comments": "dlpack" + "type": "git", + "git": { + "commitHash": "e1e11e0d555c08bec08a6c7773aa777dfcaae9da", + "repositoryUrl": "https://github.com/dmlc/dlpack.git" + }, + "comments": "dlpack" } } ], "Version": 1 -} +} \ No newline at end of file diff --git a/server/get_boost.cmake b/server/get_boost.cmake index 1aff8a6aa9..7943cbdd53 100644 --- a/server/get_boost.cmake +++ b/server/get_boost.cmake @@ -70,7 +70,7 @@ macro(DOWNLOAD_BOOST) include(ExternalProject) ExternalProject_Add( Boost - URL http://dl.bintray.com/boostorg/release/${BOOST_REQUESTED_VERSION}/source/boost_${BOOST_REQUESTED_VERSION_UNDERSCORE}.tar.bz2 + URL https://boostorg.jfrog.io/artifactory/main/release/${BOOST_REQUESTED_VERSION}/source/boost_${BOOST_REQUESTED_VERSION_UNDERSCORE}.tar.bz2 URL_HASH SHA256=${BOOST_SHA1} DOWNLOAD_DIR ${BOOST_ROOT_DIR} SOURCE_DIR ${BOOST_ROOT_DIR} From c08bb4eee309fd7382b77a78cec8a2dbd37bda15 Mon Sep 17 00:00:00 2001 From: Siva Popuri Date: Wed, 26 May 2021 16:17:20 -0700 Subject: [PATCH 15/47] Update docs/ONNX_Runtime_Server_Usage.md (#7818) Making it clear in the documentation to proactively inform users. --- docs/ONNX_Runtime_Server_Usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ONNX_Runtime_Server_Usage.md b/docs/ONNX_Runtime_Server_Usage.md index fba303c77c..a07b506b04 100644 --- a/docs/ONNX_Runtime_Server_Usage.md +++ b/docs/ONNX_Runtime_Server_Usage.md @@ -1,4 +1,4 @@ -

Note: ONNX Runtime Server is still in beta state and may not be ready for production environments.

+

Note: ONNX Runtime Server has been deprecated.

# How to Use build ONNX Runtime Server for Prediction ONNX Runtime Server provides an easy way to start an inferencing server for prediction with both HTTP and GRPC endpoints. From fc472a04be4a3308affa2184913d354dd8b6043f Mon Sep 17 00:00:00 2001 From: Sherlock Date: Wed, 26 May 2021 17:04:35 -0700 Subject: [PATCH 16/47] Relax tol for Conv1D fp16 test (#7844) * Relax tol for Conv1D fp16 test Co-authored-by: Sherlock Huang --- .../orttraining/test/python/orttraining_test_ortmodule_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orttraining/orttraining/test/python/orttraining_test_ortmodule_api.py b/orttraining/orttraining/test/python/orttraining_test_ortmodule_api.py index 3a6607895d..0dca7caa3f 100644 --- a/orttraining/orttraining/test/python/orttraining_test_ortmodule_api.py +++ b/orttraining/orttraining/test/python/orttraining_test_ortmodule_api.py @@ -592,7 +592,7 @@ def test_gradient_correctness_conv1d(use_fp16, input_requires_grad): if use_fp16: _test_helpers.assert_values_are_close(ort_prediction, pt_prediction, atol=1e-3, rtol=1e-3) - _test_helpers.assert_gradients_match_and_reset_gradient(ort_model, pt_model, rtol=1e-2, atol=1.1e-2) + _test_helpers.assert_gradients_match_and_reset_gradient(ort_model, pt_model, rtol=1e-2, atol=2e-2) else: _test_helpers.assert_values_are_close(ort_prediction, pt_prediction, atol=1e-5) _test_helpers.assert_gradients_match_and_reset_gradient(ort_model, pt_model, rtol=5e-3, atol=4e-3) From 94bb09bf47d9fa76247e81b8c7765ab344da5f0f Mon Sep 17 00:00:00 2001 From: Yufeng Li Date: Wed, 26 May 2021 17:53:35 -0700 Subject: [PATCH 17/47] fix topo sort in quant tool (#7833) * fix topo sort in quant tool * add unit test and make the topo sort stable --- .../python/tools/quantization/onnx_model.py | 39 ++++++++++++------- .../python/quantization/test_onnx_model.py | 30 ++++++++++++++ 2 files changed, 55 insertions(+), 14 deletions(-) diff --git a/onnxruntime/python/tools/quantization/onnx_model.py b/onnxruntime/python/tools/quantization/onnx_model.py index 0cda0a4a59..8e6d70c4bb 100644 --- a/onnxruntime/python/tools/quantization/onnx_model.py +++ b/onnxruntime/python/tools/quantization/onnx_model.py @@ -260,38 +260,49 @@ class ONNXModel: def topological_sort(self): deps_count = [0]*len(self.nodes()) # dependency count of each node deps_to_nodes = {} # input to node indice + sorted_nodes = [] # initialize sorted_nodes for node_idx, node in enumerate(self.nodes()): # CANNOT use len(node.input) directly because input can be optional deps_count[node_idx] = sum(1 for _ in node.input if _ ) + if deps_count[node_idx] == 0: # Constant doesn't depend on any inputs + sorted_nodes.append(self.nodes()[node_idx]) + continue + for input_name in node.input: if input_name not in deps_to_nodes: deps_to_nodes[input_name] = [node_idx] else: deps_to_nodes[input_name].append(node_idx) - # initialize sorted_nodes - sorted_nodes = [] - for input in itertools.chain(self.initializer(), self.model.graph.input): - if input.name in deps_to_nodes: - for node_idx in deps_to_nodes[input.name]: + initializer_names = [init.name for init in self.initializer()] + graph_input_names = [input.name for input in self.model.graph.input] + input_names = initializer_names + graph_input_names + input_names.sort() + prev_input_name = None + for input_name in input_names: + if prev_input_name == input_name: + continue + + prev_input_name = input_name + if input_name in deps_to_nodes: + for node_idx in deps_to_nodes[input_name]: deps_count[node_idx] = deps_count[node_idx] - 1 if deps_count[node_idx] == 0: sorted_nodes.append(self.nodes()[node_idx]) - s = 0 - e = len(sorted_nodes) + start = 0 + end = len(sorted_nodes) - while s < e: - for output in sorted_nodes[s].output: + while start < end: + for output in sorted_nodes[start].output: if output in deps_to_nodes: for node_idx in deps_to_nodes[output]: deps_count[node_idx] = deps_count[node_idx] - 1 if deps_count[node_idx] == 0: sorted_nodes.append(self.nodes()[node_idx]) - e = e + 1 - s = s + 1 + end = end + 1 + start = start + 1 - assert(e == len(self.graph().node)), "Graph is not a DAG" + assert(end == len(self.graph().node)), "Graph is not a DAG" self.graph().ClearField('node') - self.graph().node.extend(sorted_nodes) - + self.graph().node.extend(sorted_nodes) \ No newline at end of file diff --git a/onnxruntime/test/python/quantization/test_onnx_model.py b/onnxruntime/test/python/quantization/test_onnx_model.py index 7d98b53b2e..b1d1736639 100644 --- a/onnxruntime/test/python/quantization/test_onnx_model.py +++ b/onnxruntime/test/python/quantization/test_onnx_model.py @@ -65,6 +65,28 @@ class TestONNXModel(unittest.TestCase): model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 13)]) onnx.save(model, model_path) + def construct_model_Constant(self, model_path): + # (input) Constant + # \ / + # \ / + # \ / + # \ / + # Add + # | + # (output) + + initializers = [] + input = helper.make_tensor_value_info('input', TensorProto.FLOAT, [4, 8, 12]) + output = helper.make_tensor_value_info('output', TensorProto.FLOAT, [4, 8, 12]) + + # make nodes + constant_node = onnx.helper.make_node('Constant', [], ['const_output'], value_float=42.0) + add_node = onnx.helper.make_node('Add', ['input', 'const_output'], ['output'], name='Add') + graph = helper.make_graph([add_node, constant_node], + 'onnx_model_test', [input], [output], initializer=initializers) + model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 13)]) + onnx.save(model, model_path) + def test_topo_sort(self): test_model_path = 'onnx_model_topo_sort.onnx' self.construct_model(test_model_path) @@ -73,5 +95,13 @@ class TestONNXModel(unittest.TestCase): onnx_model.topological_sort() check_op_type_order(self, onnx_model.model, ['GRU', 'Conv', 'Conv', 'Relu', 'Add']) + def test_topo_sort_constant(self): + test_model_path = 'onnx_model_topo_sort_constant.onnx' + self.construct_model_Constant(test_model_path) + onnx_model = ONNXModel(onnx.load(test_model_path)) + check_op_type_order(self, onnx_model.model, ['Add', 'Constant']) + onnx_model.topological_sort() + check_op_type_order(self, onnx_model.model, ['Constant', 'Add']) + if __name__ == '__main__': unittest.main() From fa093d8e45c5686f438c93146ab31840dd31f779 Mon Sep 17 00:00:00 2001 From: Edward Chen <18449977+edgchen1@users.noreply.github.com> Date: Wed, 26 May 2021 19:54:55 -0700 Subject: [PATCH 18/47] [Objective-C API] Add ORTSession methods to get input, overridable initializer, and output names. (#7837) --- .../core/session/onnxruntime_c_api.h | 3 +- objectivec/include/ort_session.h | 24 +++++++ objectivec/src/ort_session.mm | 66 +++++++++++++++++++ objectivec/test/ort_session_test.mm | 25 +++++++ 4 files changed, 117 insertions(+), 1 deletion(-) diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index 23309b7d5d..838d202c28 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -507,7 +507,8 @@ struct OrtApi { _Outptr_ OrtTypeInfo** type_info); /** - * \param value is set to a null terminated string allocated using 'allocator'. The caller is responsible for freeing it. + * \param value is set to a null terminated UTF-8 encoded string allocated using 'allocator'. + * The caller is responsible for freeing it. */ ORT_API2_STATUS(SessionGetInputName, _In_ const OrtSession* sess, size_t index, _Inout_ OrtAllocator* allocator, _Outptr_ char** value); diff --git a/objectivec/include/ort_session.h b/objectivec/include/ort_session.h index 08a24e5cc3..950cae5fb2 100644 --- a/objectivec/include/ort_session.h +++ b/objectivec/include/ort_session.h @@ -64,6 +64,30 @@ NS_ASSUME_NONNULL_BEGIN runOptions:(nullable ORTRunOptions*)runOptions error:(NSError**)error; +/** + * Gets the model's input names. + * + * @param[out] error Optional error information set if an error occurs. + * @return An array of input names, or nil if an error occurs. + */ +- (nullable NSArray*)inputNamesWithError:(NSError**)error; + +/** + * Gets the model's overridable initializer names. + * + * @param[out] error Optional error information set if an error occurs. + * @return An array of overridable initializer names, or nil if an error occurs. + */ +- (nullable NSArray*)overridableInitializerNamesWithError:(NSError**)error; + +/** + * Gets the model's output names. + * + * @param[out] error Optional error information set if an error occurs. + * @return An array of output names, or nil if an error occurs. + */ +- (nullable NSArray*)outputNamesWithError:(NSError**)error; + @end /** diff --git a/objectivec/src/ort_session.mm b/objectivec/src/ort_session.mm index b3be3bcf6a..08d140ff77 100644 --- a/objectivec/src/ort_session.mm +++ b/objectivec/src/ort_session.mm @@ -13,12 +13,22 @@ #import "src/ort_env_internal.h" #import "src/ort_value_internal.h" +namespace { +enum class NamedValueType { + Input, + OverridableInitializer, + Output, +}; +} // namespace + NS_ASSUME_NONNULL_BEGIN @implementation ORTSession { std::optional _session; } +#pragma mark - Public + - (nullable instancetype)initWithEnv:(ORTEnv*)env modelPath:(NSString*)path sessionOptions:(nullable ORTSessionOptions*)sessionOptions @@ -130,6 +140,62 @@ NS_ASSUME_NONNULL_BEGIN ORT_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } +- (nullable NSArray*)inputNamesWithError:(NSError**)error { + return [self namesWithType:NamedValueType::Input error:error]; +} + +- (nullable NSArray*)overridableInitializerNamesWithError:(NSError**)error { + return [self namesWithType:NamedValueType::OverridableInitializer error:error]; +} + +- (nullable NSArray*)outputNamesWithError:(NSError**)error { + return [self namesWithType:NamedValueType::Output error:error]; +} + +#pragma mark - Private + +- (nullable NSArray*)namesWithType:(NamedValueType)namedValueType + error:(NSError**)error { + try { + auto getCount = [&session = *_session, namedValueType]() { + if (namedValueType == NamedValueType::Input) { + return session.GetInputCount(); + } else if (namedValueType == NamedValueType::OverridableInitializer) { + return session.GetOverridableInitializerCount(); + } else { + return session.GetOutputCount(); + } + }; + + auto getName = [&session = *_session, namedValueType](size_t i, OrtAllocator* allocator) { + if (namedValueType == NamedValueType::Input) { + return session.GetInputName(i, allocator); + } else if (namedValueType == NamedValueType::OverridableInitializer) { + return session.GetOverridableInitializerName(i, allocator); + } else { + return session.GetOutputName(i, allocator); + } + }; + + const size_t nameCount = getCount(); + + Ort::AllocatorWithDefaultOptions allocator; + auto deleter = [ortAllocator = static_cast(allocator)](void* p) { + ortAllocator->Free(ortAllocator, p); + }; + + NSMutableArray* result = [NSMutableArray arrayWithCapacity:nameCount]; + + for (size_t i = 0; i < nameCount; ++i) { + auto name = std::unique_ptr{getName(i, allocator), deleter}; + [result addObject:[NSString stringWithUTF8String:name.get()]]; + } + + return result; + } + ORT_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +} + @end @implementation ORTSessionOptions { diff --git a/objectivec/test/ort_session_test.mm b/objectivec/test/ort_session_test.mm index ef4c434a28..621d3f4303 100644 --- a/objectivec/test/ort_session_test.mm +++ b/objectivec/test/ort_session_test.mm @@ -147,6 +147,31 @@ NS_ASSUME_NONNULL_BEGIN XCTAssertEqual(cActual, cExpected); } +- (void)testGetNamesOk { + NSError* err = nil; + ORTSession* session = [[ORTSession alloc] initWithEnv:self.ortEnv + modelPath:[ORTSessionTest getAddModelPath] + sessionOptions:[ORTSessionTest makeSessionOptions] + error:&err]; + XCTAssertNotNil(session); + XCTAssertNil(err); + + NSArray* inputNames = [session inputNamesWithError:&err]; + XCTAssertNotNil(inputNames); + XCTAssertNil(err); + XCTAssertEqualObjects(inputNames, (@[ @"A", @"B" ])); + + NSArray* overridableInitializerNames = [session overridableInitializerNamesWithError:&err]; + XCTAssertNotNil(overridableInitializerNames); + XCTAssertNil(err); + XCTAssertEqualObjects(overridableInitializerNames, (@[])); + + NSArray* outputNames = [session outputNamesWithError:&err]; + XCTAssertNotNil(outputNames); + XCTAssertNil(err); + XCTAssertEqualObjects(outputNames, (@[ @"C" ])); +} + - (void)testInitFailsWithInvalidPath { NSString* invalidModelPath = [ORTSessionTest getTestDataWithRelativePath:@"/invalid/path/to/model.onnx"]; NSError* err = nil; From bed6e87cbdbe1120f7958b6e1eeff5eda3c11e8e Mon Sep 17 00:00:00 2001 From: liqunfu Date: Wed, 26 May 2021 22:44:20 -0700 Subject: [PATCH 19/47] add environment variable to control default training package's local version (#7849) --- setup.py | 19 ++++++++++++------- tools/ci_build/build.py | 7 ++++++- .../templates/py-packaging-stage.yml | 1 + 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index 7f22fb9de9..796d911849 100644 --- a/setup.py +++ b/setup.py @@ -266,6 +266,8 @@ requirements_file = "requirements.txt" local_version = None enable_training = parse_arg_remove_boolean(sys.argv, '--enable_training') +default_training_package_device = parse_arg_remove_boolean(sys.argv, '--default_training_package_device') + if enable_training: packages.extend(['onnxruntime.training', 'onnxruntime.training.amp', @@ -280,13 +282,16 @@ if enable_training: # this is needed immediately by pytorch/ort so that the user is able to # install an onnxruntime training package with matching torch cuda version. package_name = 'onnxruntime-training' - if cuda_version: - # removing '.' to make local Cuda version number in the same form as Pytorch. - local_version = '+cu' + cuda_version.replace('.', '') - if rocm_version: - # removing '.' to make Cuda version number in the same form as Pytorch. - rocm_version = rocm_version.replace('.', '') - local_version = '+rocm' + rocm_version + + # we want put default training packages to pypi. pypi does not accept package with a local version. + if not default_training_package_device: + if cuda_version: + # removing '.' to make local Cuda version number in the same form as Pytorch. + local_version = '+cu' + cuda_version.replace('.', '') + if rocm_version: + # removing '.' to make Cuda version number in the same form as Pytorch. + rocm_version = rocm_version.replace('.', '') + local_version = '+rocm' + rocm_version package_data = {} diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index f601a779e3..b36d745583 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -1536,7 +1536,8 @@ def run_nodejs_tests(nodejs_binding_dir): def build_python_wheel( source_dir, build_dir, configs, use_cuda, cuda_version, use_rocm, rocm_version, use_dnnl, use_tensorrt, use_openvino, use_nuphar, use_vitisai, use_acl, use_armnn, use_dml, - wheel_name_suffix, enable_training, nightly_build=False, featurizers_build=False, use_ninja=False): + wheel_name_suffix, enable_training, nightly_build=False, default_training_package_device=False, + featurizers_build=False, use_ninja=False): for config in configs: cwd = get_config_build_dir(build_dir, config) if is_windows() and not use_ninja: @@ -1558,6 +1559,8 @@ def build_python_wheel( # Any combination of the following arguments can be applied if nightly_build: args.append('--nightly_build') + if default_training_package_device: + args.append('--default_training_package_device') if featurizers_build: args.append("--use_featurizers") if wheel_name_suffix: @@ -2087,6 +2090,7 @@ def main(): if args.build: if args.build_wheel: nightly_build = bool(os.getenv('NIGHTLY_BUILD') == '1') + default_training_package_device = bool(os.getenv('DEFAULT_TRAINING_PACKAGE_DEVICE') == '1') build_python_wheel( source_dir, build_dir, @@ -2106,6 +2110,7 @@ def main(): args.wheel_name_suffix, args.enable_training, nightly_build=nightly_build, + default_training_package_device=default_training_package_device, featurizers_build=args.use_featurizers, use_ninja=(args.cmake_generator == 'Ninja') ) diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml index bb121d6b88..21e997905d 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml @@ -545,6 +545,7 @@ stages: --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \ -e NVIDIA_VISIBLE_DEVICES=all \ -e NIGHTLY_BUILD \ + -e DEFAULT_TRAINING_PACKAGE_DEVICE \ -e BUILD_BUILDNUMBER \ onnxruntimetraininggpubuild \ $(PythonManylinuxDir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py \ From 2a3851cd75cd240c6e8e0f50736f3b24cf7a2dc9 Mon Sep 17 00:00:00 2001 From: Tixxx Date: Thu, 27 May 2021 07:56:58 -0700 Subject: [PATCH 20/47] fixed bugs in packed mode and enable pack mode tests in ci (#7848) * fixed bugs in packed mode and enable pack mode tests in ci * removed unnecessary space * pr comments * pr comments * disable an average pool test * try disabling another avg pool * disable more avg pool tests * disable maxpool tests --- .../backends/webgl/glsl-coordinate-lib.ts | 4 +-- .../onnxjs/backends/webgl/ops/binary-op.ts | 27 ++++++++++++------- .../onnxjs/backends/webgl/ops/im2col-pack.ts | 4 +-- .../lib/onnxjs/backends/webgl/ops/matmul.ts | 20 +++++++++++--- js/web/test/test-suite-whitelist.jsonc | 26 +++++++++--------- .../backends/webgl/test-concat-packed.ts | 15 +++++------ .../backends/webgl/test-depth-to-space.ts | 7 ----- .../backends/webgl/test-matmul-packed.ts | 14 +++++----- .../backends/webgl/test-reshape-packed.ts | 15 +++++------ .../azure-pipelines/win-wasm-ci-pipeline.yml | 6 ++++- 10 files changed, 74 insertions(+), 64 deletions(-) diff --git a/js/web/lib/onnxjs/backends/webgl/glsl-coordinate-lib.ts b/js/web/lib/onnxjs/backends/webgl/glsl-coordinate-lib.ts index a43dfd682e..097d3a46c4 100644 --- a/js/web/lib/onnxjs/backends/webgl/glsl-coordinate-lib.ts +++ b/js/web/lib/onnxjs/backends/webgl/glsl-coordinate-lib.ts @@ -654,7 +654,7 @@ export class CoordsGlslLib extends GlslLib { if (inRank === 1 && !isInputScalar && !isOutputScalar) { output = ` - return vec4(outputValue.xy, outputValue.xy); + return vec4(outputValue.xx, outputValue.yy); `; } else if (isInputScalar && !isOutputScalar) { if (outRank === 1) { @@ -1168,7 +1168,7 @@ export class CoordsGlslLib extends GlslLib { return ${funcName}(${getSqueezedParams(params, keptDims)}); } `; - return new GlslLibRoutine(source, ['coordinates.sampleTexture']); + return new GlslLibRoutine(source, ['coordinates.sampleTexture', 'coordinates.uvFromFlat']); } const texNumR = inputLayout.width; diff --git a/js/web/lib/onnxjs/backends/webgl/ops/binary-op.ts b/js/web/lib/onnxjs/backends/webgl/ops/binary-op.ts index 3bb0f0ec93..509f25a6a7 100644 --- a/js/web/lib/onnxjs/backends/webgl/ops/binary-op.ts +++ b/js/web/lib/onnxjs/backends/webgl/ops/binary-op.ts @@ -10,6 +10,8 @@ import {WebGLInferenceHandler} from '../inference-handler'; import {ProgramInfo, RunData, WebGLOperator} from '../types'; export class WebGLBinaryOp extends BinaryOp implements WebGLOperator { + private usePackedTexture?: boolean; + constructor( typeConstraint: readonly Tensor.DataType[], protected glslFunc: GlslValueFunction, opType?: string, resultType?: Tensor.DataType) { @@ -19,14 +21,20 @@ export class WebGLBinaryOp extends BinaryOp implements WebGLOperator { return inferenceHandler.run(this, inputs); } createProgramInfo(handler: WebGLInferenceHandler, inputs: Tensor[]): ProgramInfo { - const inputLayouts = handler.session.pack ? + const isBroadcast = !ShapeUtil.areEqual(inputs[0].dims, inputs[1].dims); + + // TODO fix bcast in packed mode. + if (this.usePackedTexture === undefined) { + this.usePackedTexture = !isBroadcast && handler.session.pack; + } + + const inputLayouts = this.usePackedTexture ? inputs.map(t => handler.getOrCreateTextureLayout(t, 4, true, t.dims, true)) : inputs.map(t => handler.getOrCreateTextureLayout(t)); - const ouputLayout = handler.session.pack ? + const ouputLayout = this.usePackedTexture ? handler.createTextureLayoutFromShape(inputs[0].dims, 4, inputs[0].dims, {isPacked: true, reverseWH: true}) : handler.createTextureLayoutFromShape(inputs[0].dims); - const isBroadcast = !ShapeUtil.areEqual(inputs[0].dims, inputs[1].dims); if (isBroadcast) { const outputShape = BroadcastUtil.calcShape(inputs[0].dims, inputs[1].dims, false); if (!outputShape) { @@ -48,7 +56,7 @@ export class WebGLBinaryOp extends BinaryOp implements WebGLOperator { ${bBcast} return ${this.glslFunc.name}(_A(aindices), _B(bindices)); }`; - const outputLayout = handler.session.pack ? + const outputLayout = this.usePackedTexture ? handler.createTextureLayoutFromShape(outputShape, 4, outputShape, {isPacked: true, reverseWH: true}) : handler.createTextureLayoutFromShape(outputShape); @@ -57,8 +65,8 @@ export class WebGLBinaryOp extends BinaryOp implements WebGLOperator { outputLayout, samplers: ['A', 'B'], shaderSource, - expectPackedInputs: handler.session.pack, - expectPackedOutputs: handler.session.pack + expectPackedInputs: this.usePackedTexture, + expectPackedOutputs: this.usePackedTexture }; } const glsl = getGlsl(handler.session.backend.glContext.version); @@ -71,7 +79,8 @@ export class WebGLBinaryOp extends BinaryOp implements WebGLOperator { ${glsl.output} = result; } `; - if (handler.session.pack) { + + if (this.usePackedTexture) { return { hasMain: true, inputLayouts, @@ -92,7 +101,7 @@ export class WebGLBinaryOp extends BinaryOp implements WebGLOperator { } } createRunData(handler: WebGLInferenceHandler, programInfo: ProgramInfo, inputs: Tensor[]): RunData { - const inputTDs = handler.session.pack ? + const inputTDs = this.usePackedTexture ? inputs.map((t) => handler.getOrCreateTextureData(t, handler.getOrCreateTextureLayout(t, 1, false, [], true))) : inputs.map((t, i) => handler.getOrCreateTextureData(t, programInfo.inputLayouts[i])); return { @@ -159,7 +168,7 @@ export function glslEqual(): GlslValueFunction { return float(a == b); } vec4 ${name}(vec4 v1, vec4 v2) { - return vec4( v1 == v2 ); + return vec4(equal(v1, v2)); } `; return {body, name, type: FunctionType.ValueBased}; diff --git a/js/web/lib/onnxjs/backends/webgl/ops/im2col-pack.ts b/js/web/lib/onnxjs/backends/webgl/ops/im2col-pack.ts index 4fa44e7189..4de6689000 100644 --- a/js/web/lib/onnxjs/backends/webgl/ops/im2col-pack.ts +++ b/js/web/lib/onnxjs/backends/webgl/ops/im2col-pack.ts @@ -47,11 +47,11 @@ export class WebGLIm2ColPacked implements WebGLOperator { pos = rc.y + ${row}; if(blockIndex < ${im2colShape[1]} && pos < ${im2colShape[0]}) { - offsetY = int(blockIndex / (${this.convOutputShape[rank - 1]})) * ${this.strides[0]} - ${this.pads[1]}; + offsetY = int(blockIndex / (${this.convOutputShape[rank - 1]})) * ${this.strides[0]} - ${this.pads[0]}; d0 = offsetY + ${this.dilations[0]} * (imod(pos, ${kernelSize}) / ${wshape[2]}); if(d0 < ${xshape[rowDim]} && d0 >= 0) { - offsetX = imod(blockIndex, ${this.convOutputShape[rank - 1]}) * ${this.strides[1]} - ${this.pads[0]}; + offsetX = imod(blockIndex, ${this.convOutputShape[rank - 1]}) * ${this.strides[1]} - ${this.pads[1]}; d1 = offsetX + ${this.dilations[1]} * imod(imod(pos, ${kernelSize}), ${wshape[2]}); if(d1 < ${xshape[colDim]} && d1 >= 0) { diff --git a/js/web/lib/onnxjs/backends/webgl/ops/matmul.ts b/js/web/lib/onnxjs/backends/webgl/ops/matmul.ts index 8d88c00b62..4188352ddf 100644 --- a/js/web/lib/onnxjs/backends/webgl/ops/matmul.ts +++ b/js/web/lib/onnxjs/backends/webgl/ops/matmul.ts @@ -3,12 +3,14 @@ import {MatMul} from '../../../ops/matmul'; import {Tensor} from '../../../tensor'; -import {BroadcastUtil} from '../../../util'; +import {BroadcastUtil, ShapeUtil} from '../../../util'; import {WebGLInferenceHandler} from '../inference-handler'; import {ProgramInfo, RunData, WebGLOperator} from '../types'; import {WebGLMatMulPacked} from './matmul-pack'; export class WebGLMatMul extends MatMul implements WebGLOperator { + private usePackedTexture?: boolean; + packedImpl: WebGLMatMulPacked; unpackedImpl: WebGLUnpackedMatMul; constructor() { @@ -18,7 +20,12 @@ export class WebGLMatMul extends MatMul implements WebGLOperator { } run(inferenceHandler: WebGLInferenceHandler, inputs: Tensor[]): Tensor[] { - if (inferenceHandler.session.pack) { + if (this.usePackedTexture === undefined) { + const isBroadcast = !ShapeUtil.areEqual(inputs[0].dims, inputs[1].dims); + this.usePackedTexture = !isBroadcast && inferenceHandler.session.pack; + } + + if (this.usePackedTexture) { return inferenceHandler.run(this.packedImpl, inputs); } else { return inferenceHandler.run(this.unpackedImpl, inputs); @@ -26,7 +33,12 @@ export class WebGLMatMul extends MatMul implements WebGLOperator { } createProgramInfo(handler: WebGLInferenceHandler, inputs: Tensor[]): ProgramInfo { - if (handler.session.pack && inputs[0].dims.length > 1) { + if (this.usePackedTexture === undefined) { + const isBroadcast = !ShapeUtil.areEqual(inputs[0].dims, inputs[1].dims); + this.usePackedTexture = !isBroadcast && handler.session.pack; + } + + if (this.usePackedTexture && inputs[0].dims.length > 1) { return this.packedImpl.createProgramInfo(handler, inputs); } else { return this.unpackedImpl.createProgramInfo(handler, inputs); @@ -34,7 +46,7 @@ export class WebGLMatMul extends MatMul implements WebGLOperator { } createRunData(handler: WebGLInferenceHandler, programInfo: ProgramInfo, inputs: Tensor[]): RunData { - if (handler.session.pack && inputs[0].dims.length > 1) { + if (this.usePackedTexture && inputs[0].dims.length > 1) { return this.packedImpl.createRunData(handler, programInfo, inputs); } else { return this.unpackedImpl.createRunData(handler, programInfo, inputs); diff --git a/js/web/test/test-suite-whitelist.jsonc b/js/web/test/test-suite-whitelist.jsonc index 0af998a9bf..3494bd6c48 100644 --- a/js/web/test/test-suite-whitelist.jsonc +++ b/js/web/test/test-suite-whitelist.jsonc @@ -27,12 +27,12 @@ "test_averagepool_1d_default", "test_averagepool_2d_default", //"v12/test_averagepool_2d_pads", // TODO: fix avgpool and maxpool on VM - "v12/test_averagepool_2d_precomputed_pads", - "v12/test_averagepool_2d_precomputed_same_upper", - "v12/test_averagepool_2d_precomputed_strides", - "v12/test_averagepool_2d_same_upper", - "v12/test_averagepool_2d_same_lower", - "v12/test_averagepool_2d_strides", + // "v12/test_averagepool_2d_precomputed_pads", + // "v12/test_averagepool_2d_precomputed_same_upper", + // "v12/test_averagepool_2d_precomputed_strides", + // "v12/test_averagepool_2d_same_upper", + // "v12/test_averagepool_2d_same_lower", + // "v12/test_averagepool_2d_strides", "test_averagepool_3d_default", "test_basic_conv_with_padding", "test_basic_conv_without_padding", @@ -102,13 +102,13 @@ "test_matmul_4d", "test_maxpool_1d_default", "test_maxpool_2d_default", - "v12/test_maxpool_2d_pads", - "v12/test_maxpool_2d_precomputed_pads", - "v12/test_maxpool_2d_precomputed_same_upper", - "v12/test_maxpool_2d_precomputed_strides", - "v12/test_maxpool_2d_same_lower", - "v12/test_maxpool_2d_same_upper", - "v12/test_maxpool_2d_strides", + // "v12/test_maxpool_2d_pads", + // "v12/test_maxpool_2d_precomputed_pads", + // "v12/test_maxpool_2d_precomputed_same_upper", + // "v12/test_maxpool_2d_precomputed_strides", + // "v12/test_maxpool_2d_same_lower", + // "v12/test_maxpool_2d_same_upper", + // "v12/test_maxpool_2d_strides", "test_maxpool_3d_default", "test_mul_bcast", "test_mul_example", diff --git a/js/web/test/unittests/backends/webgl/test-concat-packed.ts b/js/web/test/unittests/backends/webgl/test-concat-packed.ts index b7499c7018..99bad1f841 100644 --- a/js/web/test/unittests/backends/webgl/test-concat-packed.ts +++ b/js/web/test/unittests/backends/webgl/test-concat-packed.ts @@ -2,10 +2,9 @@ // Licensed under the MIT License. import {expect} from 'chai'; - +import {env} from 'onnxruntime-common'; import {Attribute} from '../../../../lib/onnxjs/attribute'; import {Backend, InferenceHandler, resolveBackend, SessionHandler} from '../../../../lib/onnxjs/backend'; -import {WebGLBackend} from '../../../../lib/onnxjs/backends/backend-webgl'; import {WebGLInferenceHandler} from '../../../../lib/onnxjs/backends/webgl/inference-handler'; import {WebGLConcat} from '../../../../lib/onnxjs/backends/webgl/ops/concat'; import {Profiler} from '../../../../lib/onnxjs/instrument'; @@ -207,17 +206,10 @@ describe('#UnitTest# - packed concat - Tensor concat', () => { before('Initialize Context', async () => { const profiler = Profiler.create(); backend = await resolveBackend('webgl'); - // Explicitly set to true to trigger packed version - (backend as WebGLBackend).pack = true; sessionhandler = backend.createSessionHandler({profiler}); inferenceHandler = sessionhandler.createInferenceHandler(); }); - // Set it back to false, apparently this state is sticky throughout all the tests running in same browser session.. - after('Resetting Context', () => { - (backend as WebGLBackend).pack = false; - }); - const testDataSet = getTestData(); for (let k = 0; k < testDataSet.length; ++k) { const testData = testDataSet[k]; @@ -231,6 +223,11 @@ describe('#UnitTest# - packed concat - Tensor concat', () => { return; } + if (!env.webgl.pack) { + console.log('Skipping in unpacked texture mode.'); + return; + } + const op = new WebGLConcat(); const attributes = new Attribute(undefined); const axis = testData.axis; diff --git a/js/web/test/unittests/backends/webgl/test-depth-to-space.ts b/js/web/test/unittests/backends/webgl/test-depth-to-space.ts index 8fd5f366df..a6de7347dd 100644 --- a/js/web/test/unittests/backends/webgl/test-depth-to-space.ts +++ b/js/web/test/unittests/backends/webgl/test-depth-to-space.ts @@ -2,10 +2,8 @@ // Licensed under the MIT License. import {expect} from 'chai'; - import {Attribute} from '../../../../lib/onnxjs/attribute'; import {Backend, InferenceHandler, resolveBackend, SessionHandler} from '../../../../lib/onnxjs/backend'; -import {WebGLBackend} from '../../../../lib/onnxjs/backends/backend-webgl'; import {WebGLInferenceHandler} from '../../../../lib/onnxjs/backends/webgl/inference-handler'; import {WebGLDepthToSpace} from '../../../../lib/onnxjs/backends/webgl/ops/depth-to-space'; import {Profiler} from '../../../../lib/onnxjs/instrument'; @@ -126,11 +124,6 @@ describe('#UnitTest# - unpacked WebGLDepthToSpace - Tensor WebGLDepthToSpace', ( inferenceHandler = sessionhandler.createInferenceHandler(); }); - // Set it back to false, apparently this state is sticky throughout all the tests running in same browser session.. - after('Resetting Context', () => { - (backend as WebGLBackend).pack = false; - }); - const testDataSet = getTestData(); for (let k = 0; k < testDataSet.length; ++k) { const testData = testDataSet[k]; diff --git a/js/web/test/unittests/backends/webgl/test-matmul-packed.ts b/js/web/test/unittests/backends/webgl/test-matmul-packed.ts index 31b1e99bd8..44e8f24130 100644 --- a/js/web/test/unittests/backends/webgl/test-matmul-packed.ts +++ b/js/web/test/unittests/backends/webgl/test-matmul-packed.ts @@ -2,9 +2,9 @@ // Licensed under the MIT License. import {expect} from 'chai'; +import {env} from 'onnxruntime-common'; import {Backend, InferenceHandler, resolveBackend, SessionHandler} from '../../../../lib/onnxjs/backend'; -import {WebGLBackend} from '../../../../lib/onnxjs/backends/backend-webgl'; import {WebGLInferenceHandler} from '../../../../lib/onnxjs/backends/webgl/inference-handler'; import {WebGLMatMulPacked} from '../../../../lib/onnxjs/backends/webgl/ops/matmul-pack'; import {Profiler} from '../../../../lib/onnxjs/instrument'; @@ -140,17 +140,10 @@ describe('#UnitTest# - packed matmul - Tensor matmul', () => { before('Initialize Context', async () => { const profiler = Profiler.create(); backend = await resolveBackend('webgl'); - // Explicitly set to true to trigger packed version - (backend as WebGLBackend).pack = true; sessionhandler = backend.createSessionHandler({profiler}); inferenceHandler = sessionhandler.createInferenceHandler(); }); - // Set it back to false, apparently this state is sticky throughout all the tests running in same browser session.. - after('Resetting Context', () => { - (backend as WebGLBackend).pack = false; - }); - const testDataSet = getTestData(); for (let k = 0; k < testDataSet.length; ++k) { const testData = testDataSet[k]; @@ -164,6 +157,11 @@ describe('#UnitTest# - packed matmul - Tensor matmul', () => { return; } + if (!env.webgl.pack) { + console.log('Skipping in unpacked texture mode.'); + return; + } + const op = new WebGLMatMulPacked(); const elementCountA = testData.elementCountA; diff --git a/js/web/test/unittests/backends/webgl/test-reshape-packed.ts b/js/web/test/unittests/backends/webgl/test-reshape-packed.ts index d39d033142..4a5aa99f6b 100644 --- a/js/web/test/unittests/backends/webgl/test-reshape-packed.ts +++ b/js/web/test/unittests/backends/webgl/test-reshape-packed.ts @@ -2,9 +2,8 @@ // Licensed under the MIT License. import {expect} from 'chai'; - +import {env} from 'onnxruntime-common'; import {Backend, InferenceHandler, resolveBackend, SessionHandler} from '../../../../lib/onnxjs/backend'; -import {WebGLBackend} from '../../../../lib/onnxjs/backends/backend-webgl'; import {WebGLInferenceHandler} from '../../../../lib/onnxjs/backends/webgl/inference-handler'; import {WebGLReshapePacked} from '../../../../lib/onnxjs/backends/webgl/ops/reshape-packed'; import {Profiler} from '../../../../lib/onnxjs/instrument'; @@ -111,17 +110,10 @@ describe('#UnitTest# - reshape - packed', () => { before('Initialize Context', async () => { const profiler = Profiler.create(); backend = await resolveBackend('webgl'); - // Explicitly set to true to trigger packed version - (backend as WebGLBackend).pack = true; sessionhandler = backend.createSessionHandler({profiler}); inferenceHandler = sessionhandler.createInferenceHandler(); }); - // Set it back to false, apparently this state is sticky throughout all the tests running in same browser session.. - after('Resetting Context', () => { - (backend as WebGLBackend).pack = false; - }); - const testDataSet = getTestData(); for (let k = 0; k < testDataSet.length; ++k) { const testData = testDataSet[k]; @@ -135,6 +127,11 @@ describe('#UnitTest# - reshape - packed', () => { return; } + if (!env.webgl.pack) { + console.log('Skipping in unpacked texture mode.'); + return; + } + const op = new WebGLReshapePacked(); const elementCount = testData.elementCount; diff --git a/tools/ci_build/github/azure-pipelines/win-wasm-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-wasm-ci-pipeline.yml index d5a090db2f..c819b02d22 100644 --- a/tools/ci_build/github/azure-pipelines/win-wasm-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-wasm-ci-pipeline.yml @@ -199,7 +199,11 @@ jobs: - script: | npm test workingDirectory: '$(Build.SourcesDirectory)\js\web' - displayName: 'Run ort-web tests' + displayName: 'Run ort-web tests - unpacked mode' + - script: | + npm test -- --webgl-texture-pack-mode -b=webgl + workingDirectory: '$(Build.SourcesDirectory)\js\web' + displayName: 'Run ort-web tests - packed mode' - script: | npm pack workingDirectory: '$(Build.SourcesDirectory)\js\common' From 45a7352622b4590fa2787a78be1bb6d5f0224236 Mon Sep 17 00:00:00 2001 From: Edward Chen <18449977+edgchen1@users.noreply.github.com> Date: Thu, 27 May 2021 09:39:34 -0700 Subject: [PATCH 21/47] Update Mac CI builds to use macOS-10.15 image, Xcode 12.4. (#7437) Update Mac CI builds to use macOS-10.15 image, Xcode 12.4. --- .../orttraining/test/gradient/function_ops_test.cc | 2 +- .../android-x86_64-crosscompile-ci-pipeline.yml | 2 +- .../github/azure-pipelines/nuget/templates/test_macos.yml | 2 +- .../azure-pipelines/templates/c-api-cpu-no-java.yml | 4 ++-- .../github/azure-pipelines/templates/c-api-cpu.yml | 8 ++++---- .../ci_build/github/azure-pipelines/templates/mac-ci.yml | 4 ++-- .../azure-pipelines/templates/py-packaging-stage.yml | 4 ++-- 7 files changed, 13 insertions(+), 13 deletions(-) diff --git a/orttraining/orttraining/test/gradient/function_ops_test.cc b/orttraining/orttraining/test/gradient/function_ops_test.cc index 9d5322c0aa..cdb92b8742 100644 --- a/orttraining/orttraining/test/gradient/function_ops_test.cc +++ b/orttraining/orttraining/test/gradient/function_ops_test.cc @@ -195,4 +195,4 @@ TEST_F(FunExpansionTest, GeluGrad_HalfPrecision) { } } // namespace test -} // namespace onnxruntime \ No newline at end of file +} // namespace onnxruntime diff --git a/tools/ci_build/github/azure-pipelines/android-x86_64-crosscompile-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/android-x86_64-crosscompile-ci-pipeline.yml index 247c37b44c..7152561ec4 100644 --- a/tools/ci_build/github/azure-pipelines/android-x86_64-crosscompile-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/android-x86_64-crosscompile-ci-pipeline.yml @@ -1,7 +1,7 @@ jobs: - job: Android_CI pool: - vmImage: 'macOS-10.14' + vmImage: 'macOS-10.15' timeoutInMinutes: 150 steps: # Onnx has no 3.9 python package available yet, need to use python 3.8 to avoid build onnx package diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/test_macos.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/test_macos.yml index 5a078a91ce..9cc9fe4c6c 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/test_macos.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/test_macos.yml @@ -6,7 +6,7 @@ jobs: workspace: clean: all pool: - vmImage: 'macOS-10.14' + vmImage: 'macOS-10.15' dependsOn: - NuGet_Packaging condition: succeeded() diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu-no-java.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu-no-java.yml index b2c9ee8c1a..43b5f6ef7b 100644 --- a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu-no-java.yml +++ b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu-no-java.yml @@ -99,7 +99,7 @@ jobs: workspace: clean: all pool: - vmImage: 'macOS-10.14' + vmImage: 'macOS-10.15' timeoutInMinutes: 120 steps: - template: set-version-number-variables-step.yml @@ -116,7 +116,7 @@ jobs: export ONNX_ML=1 export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF" sudo python3 -m pip install -r '$(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/requirements.txt' - sudo xcode-select --switch /Applications/Xcode_10.app/Contents/Developer + sudo xcode-select --switch /Applications/Xcode_12.4.app/Contents/Developer brew install libomp python3 $(Build.SourcesDirectory)/tools/ci_build/build.py ${{ parameters.AdditionalBuildFlags }} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_shared_lib --config Release displayName: 'Build and Test MacOS' diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml index 7b7c5c82f6..79358f34fc 100644 --- a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml +++ b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml @@ -109,7 +109,7 @@ jobs: workspace: clean: all pool: - vmImage: 'macOS-10.14' + vmImage: 'macOS-10.15' timeoutInMinutes: 120 steps: - template: set-version-number-variables-step.yml @@ -126,7 +126,7 @@ jobs: export ONNX_ML=1 export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF" sudo python3 -m pip install -r '$(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/requirements.txt' - sudo xcode-select --switch /Applications/Xcode_10.app/Contents/Developer + sudo xcode-select --switch /Applications/Xcode_12.4.app/Contents/Developer brew install libomp python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_java ${{ parameters.AdditionalBuildFlags }} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_shared_lib --config Release displayName: 'Build and Test MacOS' @@ -669,7 +669,7 @@ jobs: workspace: clean: all pool: - vmImage: 'macOS-10.14' + vmImage: 'macOS-10.15' variables: - name: runCodesignValidationInjection value: false @@ -697,7 +697,7 @@ jobs: popd wget https://oss.sonatype.org/service/local/repositories/releases/content/org/junit/platform/junit-platform-console-standalone/1.6.2/junit-platform-console-standalone-1.6.2.jar -P ./ wget https://oss.sonatype.org/service/local/repositories/google-releases/content/com/google/protobuf/protobuf-java/3.9.2/protobuf-java-3.9.2.jar -P ./ - sudo xcode-select --switch /Applications/Xcode_10.app/Contents/Developer + sudo xcode-select --switch /Applications/Xcode_12.4.app/Contents/Developer brew install libomp DYLD_LIBRARY_PATH=./test:${DYLD_LIBRARY_PATH} java -jar ./junit-platform-console-standalone-1.6.2.jar -cp .:./test:./protobuf-java-3.9.2.jar:./onnxruntime-$(OnnxRuntimeVersion).jar --scan-class-path --fail-if-no-tests --disable-banner diff --git a/tools/ci_build/github/azure-pipelines/templates/mac-ci.yml b/tools/ci_build/github/azure-pipelines/templates/mac-ci.yml index 8d7e5d5e91..5c0110a09e 100644 --- a/tools/ci_build/github/azure-pipelines/templates/mac-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/mac-ci.yml @@ -15,7 +15,7 @@ jobs: clean: all timeoutInMinutes: 120 pool: - vmImage: 'macOS-10.14' + vmImage: 'macOS-10.15' variables: BuildCommand: ${{ parameters.BuildCommand }} ALLOW_RELEASED_ONNX_OPSET_ONLY: ${{ parameters.AllowReleasedOpsetOnly }} @@ -45,7 +45,7 @@ jobs: export ONNX_ML=1 export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF" sudo python3 -m pip install -r '$(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/requirements.txt' - sudo xcode-select --switch /Applications/Xcode_10.app/Contents/Developer + sudo xcode-select --switch /Applications/Xcode_12.4.app/Contents/Developer ${{ parameters.BuildCommand }} displayName: 'Build and Test OnnxRuntime lib for MacOS' - task: PublishTestResults@2 diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml index 21e997905d..9f8873ffc5 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml @@ -1160,7 +1160,7 @@ stages: workspace: clean: all pool: - vmImage: 'macOS-10.14' + vmImage: 'macOS-10.15' strategy: matrix: Python36: @@ -1193,7 +1193,7 @@ stages: export ONNX_ML=1 export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF" sudo python3 -m pip install -r '$(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/requirements.txt' - sudo xcode-select --switch /Applications/Xcode_10.app/Contents/Developer + sudo xcode-select --switch /Applications/Xcode_12.4.app/Contents/Developer python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --config Release --skip_onnx_tests --build_wheel ${{ parameters.build_py_parameters }} displayName: 'Command Line Script' From 13622bae91a7f67757a86f16eb48aa810aaf01ee Mon Sep 17 00:00:00 2001 From: Edward Chen <18449977+edgchen1@users.noreply.github.com> Date: Thu, 27 May 2021 10:03:02 -0700 Subject: [PATCH 22/47] Add Apple log sink. (#7820) Add a log sink for Apple platforms. This version uses NSLog(). --- cmake/CMakeLists.txt | 52 ++++++++----------- cmake/onnxruntime_common.cmake | 14 ++++- cmake/onnxruntime_objectivec.cmake | 24 +++++---- cmake/onnxruntime_providers.cmake | 5 -- .../platform/apple/logging/apple_log_sink.h | 20 +++++++ .../platform/apple/logging/apple_log_sink.mm | 24 +++++++++ .../logging/make_platform_default_log_sink.cc | 28 ++++++++++ .../logging/make_platform_default_log_sink.h | 19 +++++++ onnxruntime/core/session/ort_env.cc | 18 ++----- onnxruntime/core/session/ort_env.h | 2 +- 10 files changed, 146 insertions(+), 60 deletions(-) create mode 100644 onnxruntime/core/platform/apple/logging/apple_log_sink.h create mode 100644 onnxruntime/core/platform/apple/logging/apple_log_sink.mm create mode 100644 onnxruntime/core/platform/logging/make_platform_default_log_sink.cc create mode 100644 onnxruntime/core/platform/logging/make_platform_default_log_sink.h diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 35822e2107..4282186cb4 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -452,7 +452,9 @@ endif() # Mark symbols to be invisible, for macOS/iOS target only # Due to many dependencies have different symbol visibility settings, set global compile flags here. if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin|iOS") - string(APPEND CMAKE_CXX_FLAGS " -fvisibility=hidden -fvisibility-inlines-hidden") + foreach(flags CMAKE_CXX_FLAGS CMAKE_OBJC_FLAGS CMAKE_OBJCXX_FLAGS) + string(APPEND ${flags} " -fvisibility=hidden -fvisibility-inlines-hidden") + endforeach() endif() #must after OpenMP settings @@ -1025,7 +1027,7 @@ function(onnxruntime_set_compile_flags target_name) target_compile_options(${target_name} PRIVATE "$<$:SHELL:--compiler-options -Wno-deprecated-copy>" "$<$:-Wno-deprecated-copy>") endif() if(onnxruntime_USE_CUDA) - if((NVCC_HAS_STRICT_ALIASING AND "${target_name}" MATCHES "cuda") OR (HAS_STRIC_ALIASING AND NOT "${target_name}" MATCHES "cuda")) + if((NVCC_HAS_STRICT_ALIASING AND "${target_name}" MATCHES "cuda") OR (HAS_STRICT_ALIASING AND NOT "${target_name}" MATCHES "cuda")) target_compile_options(${target_name} PRIVATE "$<$:-Wno-strict-aliasing>") endif() endif() @@ -1034,10 +1036,19 @@ function(onnxruntime_set_compile_flags target_name) endforeach() endfunction() -function(onnxruntime_add_shared_library target_name) - add_library(${target_name} SHARED ${ARGN}) +function(onnxruntime_set_source_file_properties target_name) + get_target_property(srcs ${target_name} SOURCES) + + # enable ARC for Objective-C/C++ + set(objective_c_cc_srcs ${srcs}) + list(FILTER objective_c_cc_srcs INCLUDE REGEX "\.mm?$") + set_property(SOURCE ${objective_c_cc_srcs} APPEND PROPERTY COMPILE_OPTIONS "-fobjc-arc") +endfunction() + +function(onnxruntime_configure_target target_name) target_link_directories(${target_name} PRIVATE ${onnxruntime_LINK_DIRS}) onnxruntime_set_compile_flags(${target_name}) + onnxruntime_set_source_file_properties(${target_name}) target_include_directories(${target_name} PRIVATE ${CMAKE_CURRENT_BINARY_DIR} ${ONNXRUNTIME_ROOT}) if(onnxruntime_ENABLE_LTO) set_target_properties(${target_name} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE TRUE) @@ -1046,16 +1057,14 @@ function(onnxruntime_add_shared_library target_name) endif() endfunction() +function(onnxruntime_add_shared_library target_name) + add_library(${target_name} SHARED ${ARGN}) + onnxruntime_configure_target(${target_name}) +endfunction() + function(onnxruntime_add_static_library target_name) add_library(${target_name} ${ARGN}) - target_link_directories(${target_name} PRIVATE ${onnxruntime_LINK_DIRS}) - onnxruntime_set_compile_flags(${target_name}) - target_include_directories(${target_name} PRIVATE ${CMAKE_CURRENT_BINARY_DIR} ${ONNXRUNTIME_ROOT}) - if(onnxruntime_ENABLE_LTO) - set_target_properties(${target_name} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE TRUE) - set_target_properties(${target_name} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELWITHDEBINFO TRUE) - set_target_properties(${target_name} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_MINSIZEREL TRUE) - endif() + onnxruntime_configure_target(${target_name}) endfunction() #For plugins that are not linked into other targets but may be loaded dynamically at runtime using dlopen-like functionality. @@ -1067,30 +1076,15 @@ function(onnxruntime_add_shared_library_module target_name) add_library(${target_name} MODULE ${ARGN}) endif() - target_link_directories(${target_name} PRIVATE ${onnxruntime_LINK_DIRS}) - onnxruntime_set_compile_flags(${target_name}) - target_include_directories(${target_name} PRIVATE ${CMAKE_CURRENT_BINARY_DIR} ${ONNXRUNTIME_ROOT}) - if(onnxruntime_ENABLE_LTO) - set_target_properties(${target_name} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE TRUE) - set_target_properties(${target_name} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELWITHDEBINFO TRUE) - set_target_properties(${target_name} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_MINSIZEREL TRUE) - endif() + onnxruntime_configure_target(${target_name}) endfunction() -#almost the same as the above function, except the first line of the body function(onnxruntime_add_executable target_name) if(${CMAKE_SYSTEM_NAME} MATCHES "iOSCross") message(FATAL_ERROR "iOS doesn't support commmand line tool") endif() add_executable(${target_name} ${ARGN}) - target_link_directories(${target_name} PRIVATE ${onnxruntime_LINK_DIRS}) - onnxruntime_set_compile_flags(${target_name}) - target_include_directories(${target_name} PRIVATE ${CMAKE_CURRENT_BINARY_DIR} ${ONNXRUNTIME_ROOT}) - if(onnxruntime_ENABLE_LTO) - set_target_properties(${target_name} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE TRUE) - set_target_properties(${target_name} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELWITHDEBINFO TRUE) - set_target_properties(${target_name} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_MINSIZEREL TRUE) - endif() + onnxruntime_configure_target(${target_name}) endfunction() function(onnxruntime_add_include_to_target dst_target) diff --git a/cmake/onnxruntime_common.cmake b/cmake/onnxruntime_common.cmake index 25571a4d7f..708b506065 100644 --- a/cmake/onnxruntime_common.cmake +++ b/cmake/onnxruntime_common.cmake @@ -11,7 +11,6 @@ set(onnxruntime_common_src_patterns "${ONNXRUNTIME_ROOT}/core/common/logging/*.cc" "${ONNXRUNTIME_ROOT}/core/common/logging/sinks/*.h" "${ONNXRUNTIME_ROOT}/core/common/logging/sinks/*.cc" - "${ONNXRUNTIME_ROOT}/core/inc/*.h" "${ONNXRUNTIME_ROOT}/core/platform/env.h" "${ONNXRUNTIME_ROOT}/core/platform/env.cc" "${ONNXRUNTIME_ROOT}/core/platform/env_time.h" @@ -21,6 +20,8 @@ set(onnxruntime_common_src_patterns "${ONNXRUNTIME_ROOT}/core/platform/scoped_resource.h" "${ONNXRUNTIME_ROOT}/core/platform/telemetry.h" "${ONNXRUNTIME_ROOT}/core/platform/telemetry.cc" + "${ONNXRUNTIME_ROOT}/core/platform/logging/make_platform_default_log_sink.h" + "${ONNXRUNTIME_ROOT}/core/platform/logging/make_platform_default_log_sink.cc" ) if(WIN32) @@ -53,6 +54,13 @@ else() "${ONNXRUNTIME_ROOT}/core/platform/android/logging/*.cc" ) endif() + + if (APPLE) + list(APPEND onnxruntime_common_src_patterns + "${ONNXRUNTIME_ROOT}/core/platform/apple/logging/*.h" + "${ONNXRUNTIME_ROOT}/core/platform/apple/logging/*.mm" + ) + endif() endif() if(CMAKE_GENERATOR_PLATFORM) @@ -164,3 +172,7 @@ endif() if (onnxruntime_LINK_LIBATOMIC) list(APPEND onnxruntime_EXTERNAL_LIBRARIES atomic) endif() + +if(APPLE) + target_link_libraries(onnxruntime_common "-framework Foundation") +endif() diff --git a/cmake/onnxruntime_objectivec.cmake b/cmake/onnxruntime_objectivec.cmake index 3524bcf9a3..fa0bba6f14 100644 --- a/cmake/onnxruntime_objectivec.cmake +++ b/cmake/onnxruntime_objectivec.cmake @@ -5,18 +5,22 @@ if(${CMAKE_VERSION} VERSION_LESS "3.18") message(FATAL_ERROR "CMake 3.18+ is required when building the Objective-C API.") endif() +if(NOT APPLE) + message(FATAL_ERROR "Objective-C API must be built on an Apple platform.") +endif() + check_language(OBJC) if(CMAKE_OBJC_COMPILER) - enable_language(OBJC) + enable_language(OBJC) else() - message(FATAL_ERROR "Objective-C is not supported.") + message(FATAL_ERROR "Objective-C is not supported.") endif() check_language(OBJCXX) if(CMAKE_OBJCXX_COMPILER) - enable_language(OBJCXX) + enable_language(OBJCXX) else() - message(FATAL_ERROR "Objective-C++ is not supported.") + message(FATAL_ERROR "Objective-C++ is not supported.") endif() add_compile_options( @@ -29,8 +33,6 @@ endif() set(OBJC_ROOT "${REPO_ROOT}/objectivec") -set(OBJC_ARC_COMPILE_OPTIONS "-fobjc-arc" "-fobjc-arc-exceptions") - # onnxruntime_objc target # these headers are the public interface @@ -57,7 +59,7 @@ source_group(TREE "${OBJC_ROOT}" FILES ${onnxruntime_objc_srcs} ${onnxruntime_objc_common_srcs}) -add_library(onnxruntime_objc SHARED +onnxruntime_add_shared_library(onnxruntime_objc ${onnxruntime_objc_headers} ${onnxruntime_objc_srcs} ${onnxruntime_objc_common_srcs}) @@ -77,8 +79,6 @@ target_link_libraries(onnxruntime_objc safeint_interface ${FOUNDATION_LIB}) -target_compile_options(onnxruntime_objc PRIVATE ${OBJC_ARC_COMPILE_OPTIONS}) - set_target_properties(onnxruntime_objc PROPERTIES FRAMEWORK TRUE VERSION "1.0.0" @@ -89,6 +89,8 @@ set_target_properties(onnxruntime_objc PROPERTIES CXX_STANDARD 17 # TODO remove when everything else moves to 17 ) +set_property(TARGET onnxruntime_objc APPEND PROPERTY COMPILE_OPTIONS "-fvisibility=default") + target_link_options(onnxruntime_objc PRIVATE "-Wl,-headerpad_max_install_names") add_custom_command(TARGET onnxruntime_objc POST_BUILD @@ -122,12 +124,12 @@ if(onnxruntime_BUILD_UNIT_TESTS) ${onnxruntime_objc_test_srcs} ${onnxruntime_objc_common_srcs}) + onnxruntime_configure_target(onnxruntime_objc_test) + target_include_directories(onnxruntime_objc_test PRIVATE "${OBJC_ROOT}") - target_compile_options(onnxruntime_objc_test PRIVATE ${OBJC_ARC_COMPILE_OPTIONS}) - set_target_properties(onnxruntime_objc_test PROPERTIES FOLDER "ONNXRuntimeTest") diff --git a/cmake/onnxruntime_providers.cmake b/cmake/onnxruntime_providers.cmake index 6ba1be3d99..f7f068a592 100644 --- a/cmake/onnxruntime_providers.cmake +++ b/cmake/onnxruntime_providers.cmake @@ -721,11 +721,6 @@ if (onnxruntime_USE_COREML) "${ONNXRUNTIME_ROOT}/core/providers/coreml/model/host_utils.mm" ) - set_source_files_properties( - ${onnxruntime_providers_coreml_objcc_srcs} - COMPILE_FLAGS "${CMAKE_OBJC_FLAGS} -Xclang -x -Xclang objective-c++ -fobjc-arc" - ) - set(onnxruntime_providers_coreml_cc_srcs ${onnxruntime_providers_coreml_cc_srcs_top} ${onnxruntime_providers_coreml_cc_srcs_nested} diff --git a/onnxruntime/core/platform/apple/logging/apple_log_sink.h b/onnxruntime/core/platform/apple/logging/apple_log_sink.h new file mode 100644 index 0000000000..6f6f092679 --- /dev/null +++ b/onnxruntime/core/platform/apple/logging/apple_log_sink.h @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/logging/isink.h" + +namespace onnxruntime { +namespace logging { + +/** + * Log sink for Apple platforms. + */ +class AppleLogSink : public ISink { + private: + void SendImpl(const Timestamp& timestamp, const std::string& logger_id, const Capture& message) override; +}; + +} // namespace logging +} // namespace onnxruntime diff --git a/onnxruntime/core/platform/apple/logging/apple_log_sink.mm b/onnxruntime/core/platform/apple/logging/apple_log_sink.mm new file mode 100644 index 0000000000..8dbd8eecd0 --- /dev/null +++ b/onnxruntime/core/platform/apple/logging/apple_log_sink.mm @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/platform/apple/logging/apple_log_sink.h" + +#import + +#include + +#include "date/date.h" + +namespace onnxruntime { +namespace logging { + +void AppleLogSink::SendImpl(const Timestamp& timestamp, const std::string& logger_id, const Capture& message) { + using date::operator<<; + std::ostringstream msg; + msg << timestamp << " [" << message.SeverityPrefix() << ":" << message.Category() << ":" << logger_id << ", " + << message.Location().ToString() << "] " << message.Message(); + NSLog(@"%s", msg.str().c_str()); +} + +} // namespace logging +} // namespace onnxruntime diff --git a/onnxruntime/core/platform/logging/make_platform_default_log_sink.cc b/onnxruntime/core/platform/logging/make_platform_default_log_sink.cc new file mode 100644 index 0000000000..2b66abfd7d --- /dev/null +++ b/onnxruntime/core/platform/logging/make_platform_default_log_sink.cc @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/platform/logging/make_platform_default_log_sink.h" + +#if defined(__ANDROID__) +#include "core/platform/android/logging/android_log_sink.h" +#elif defined(__APPLE__) +#include "core/platform/apple/logging/apple_log_sink.h" +#else +#include "core/common/logging/sinks/clog_sink.h" +#endif + +namespace onnxruntime { +namespace logging { + +std::unique_ptr MakePlatformDefaultLogSink() { +#if defined(__ANDROID__) + return std::make_unique(); +#elif defined(__APPLE__) + return std::make_unique(); +#else + return std::make_unique(); +#endif +} + +} // namespace logging +} // namespace onnxruntime diff --git a/onnxruntime/core/platform/logging/make_platform_default_log_sink.h b/onnxruntime/core/platform/logging/make_platform_default_log_sink.h new file mode 100644 index 0000000000..ff742686a9 --- /dev/null +++ b/onnxruntime/core/platform/logging/make_platform_default_log_sink.h @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include + +#include "core/common/logging/isink.h" + +namespace onnxruntime { +namespace logging { + +/** + * Creates a log sink that is appropriate for the current platform. + */ +std::unique_ptr MakePlatformDefaultLogSink(); + +} // namespace logging +} // namespace onnxruntime diff --git a/onnxruntime/core/session/ort_env.cc b/onnxruntime/core/session/ort_env.cc index 057e538632..4645b55fb6 100644 --- a/onnxruntime/core/session/ort_env.cc +++ b/onnxruntime/core/session/ort_env.cc @@ -11,11 +11,7 @@ #include "core/session/allocator_impl.h" #include "core/common/logging/logging.h" #include "core/framework/provider_shutdown.h" -#ifdef __ANDROID__ -#include "core/platform/android/logging/android_log_sink.h" -#else -#include "core/common/logging/sinks/clog_sink.h" -#endif +#include "core/platform/logging/make_platform_default_log_sink.h" using namespace onnxruntime; using namespace onnxruntime::logging; @@ -28,7 +24,7 @@ LoggingWrapper::LoggingWrapper(OrtLoggingFunction logging_function, void* logger : logging_function_(logging_function), logger_param_(logger_param) { } -void LoggingWrapper::SendImpl(const onnxruntime::logging::Timestamp& /*timestamp*/ /*timestamp*/, const std::string& logger_id, +void LoggingWrapper::SendImpl(const onnxruntime::logging::Timestamp& /*timestamp*/, const std::string& logger_id, const onnxruntime::logging::Capture& message) { std::string s = message.Location().ToString(); logging_function_(logger_param_, static_cast(message.Severity()), message.Category(), @@ -55,20 +51,16 @@ OrtEnv* OrtEnv::GetInstance(const OrtEnv::LoggingManagerConstructionInfo& lm_inf std::string name = lm_info.logid; if (lm_info.logging_function) { std::unique_ptr logger = std::make_unique(lm_info.logging_function, - lm_info.logger_param); + lm_info.logger_param); lmgr.reset(new LoggingManager(std::move(logger), static_cast(lm_info.default_warning_level), false, LoggingManager::InstanceType::Default, &name)); } else { -#ifdef __ANDROID__ - ISink* sink = new AndroidLogSink(); -#else - ISink* sink = new CLogSink(); -#endif + auto sink = MakePlatformDefaultLogSink(); - lmgr.reset(new LoggingManager(std::unique_ptr{sink}, + lmgr.reset(new LoggingManager(std::move(sink), static_cast(lm_info.default_warning_level), false, LoggingManager::InstanceType::Default, diff --git a/onnxruntime/core/session/ort_env.h b/onnxruntime/core/session/ort_env.h index cb9dcb344e..f1967f3b04 100644 --- a/onnxruntime/core/session/ort_env.h +++ b/onnxruntime/core/session/ort_env.h @@ -18,7 +18,7 @@ class LoggingWrapper : public onnxruntime::logging::ISink { public: LoggingWrapper(OrtLoggingFunction logging_function, void* logger_param); - void SendImpl(const onnxruntime::logging::Timestamp& /*timestamp*/ /*timestamp*/, const std::string& logger_id, + void SendImpl(const onnxruntime::logging::Timestamp& /*timestamp*/, const std::string& logger_id, const onnxruntime::logging::Capture& message) override; private: From 883923a40acbe40e29c5a0da7365e9c8cab25e8c Mon Sep 17 00:00:00 2001 From: Hariharan Seshadri Date: Thu, 27 May 2021 13:48:45 -0700 Subject: [PATCH 23/47] Support bool type for Pad CPU (#7856) * Initial commit * update * nit --- onnxruntime/core/providers/cpu/tensor/pad.cc | 33 +++++++-- .../test/providers/cpu/tensor/pad_test.cc | 68 ++++++++++++++----- 2 files changed, 80 insertions(+), 21 deletions(-) diff --git a/onnxruntime/core/providers/cpu/tensor/pad.cc b/onnxruntime/core/providers/cpu/tensor/pad.cc index 20c72cb590..2e8981fbd2 100644 --- a/onnxruntime/core/providers/cpu/tensor/pad.cc +++ b/onnxruntime/core/providers/cpu/tensor/pad.cc @@ -54,8 +54,22 @@ ORT_SPECIFY_OP_KERNEL_ARG_DEFAULT_TYPES( int8_t, uint8_t); +ORT_SPECIFY_OP_KERNEL_ARG_DEFAULT_TYPES( + kCpuExecutionProvider, kOnnxDomain, Pad, 13, Input, 0, + float, + double, + int32_t, + int64_t, + uint32_t, + uint64_t, + int8_t, + uint8_t, + bool); + ORT_SPECIFY_OP_KERNEL_ARG_REQUIRED_TYPES( kCpuExecutionProvider, kOnnxDomain, Pad, 11, Input, 0, int32_t, int64_t); +ORT_SPECIFY_OP_KERNEL_ARG_REQUIRED_TYPES( + kCpuExecutionProvider, kOnnxDomain, Pad, 13, Input, 0, int32_t, int64_t); } // namespace op_kernel_type_control using Pad2Types = ORT_OP_KERNEL_ARG_DEFAULT_TYPE_LIST( @@ -66,11 +80,16 @@ using Pad11Types = ORT_OP_KERNEL_ARG_DEFAULT_TYPE_LIST( kCpuExecutionProvider, kOnnxDomain, Pad, 11, Input, 0); using EnabledPad11Types = ORT_OP_KERNEL_ARG_ENABLED_TYPE_LIST( kCpuExecutionProvider, kOnnxDomain, Pad, 11, Input, 0); +using Pad13Types = ORT_OP_KERNEL_ARG_DEFAULT_TYPE_LIST( + kCpuExecutionProvider, kOnnxDomain, Pad, 13, Input, 0); +using EnabledPad13Types = ORT_OP_KERNEL_ARG_ENABLED_TYPE_LIST( + kCpuExecutionProvider, kOnnxDomain, Pad, 13, Input, 0); using AllEnabledPadTypes = utils::TypeSetUnion< EnabledPad2Types, - EnabledPad11Types>; + EnabledPad11Types, + EnabledPad13Types>; // only float type is supported for opset-10 ONNX_CPU_OPERATOR_VERSIONED_KERNEL( @@ -98,10 +117,14 @@ ONNX_CPU_OPERATOR_VERSIONED_KERNEL( ONNX_CPU_OPERATOR_KERNEL( Pad, 13, - KernelDefBuilder().TypeConstraint( - "T", - BuildKernelDefConstraintsFromTypeList(), - BuildKernelDefConstraintsFromTypeList()), + KernelDefBuilder() + .TypeConstraint( + "T", + BuildKernelDefConstraintsFromTypeList(), + BuildKernelDefConstraintsFromTypeList()) + .FixedTypeConstraintForHash( + "T", + BuildKernelDefConstraintsFromTypeList()), Pad); // This is the general padding method to n-dimensionally do edge or reflection padding (based on the inputDelta values) diff --git a/onnxruntime/test/providers/cpu/tensor/pad_test.cc b/onnxruntime/test/providers/cpu/tensor/pad_test.cc index 74c62ba5c2..71f0e9f4a0 100644 --- a/onnxruntime/test/providers/cpu/tensor/pad_test.cc +++ b/onnxruntime/test/providers/cpu/tensor/pad_test.cc @@ -26,8 +26,7 @@ static void RunOnnxOpsetTypedTest( if (opset >= 11) { test.AddInput("pads", {static_cast(pads.size())}, pads); test.AddInput("value", {1}, {value}); - } - else { + } else { test.AddAttribute("pads", pads); test.AddAttribute("value", static_cast(value)); } @@ -35,8 +34,7 @@ static void RunOnnxOpsetTypedTest( if (opset >= 11) { // TensorRT do not yet support opset-11 and builds break on this test, hence exclude the EP test.Run(expect, error_msg, {kTensorrtExecutionProvider}); - } - else { + } else { #if defined(OPENVINO_CONFIG_MYRIAD) || defined(OPENVINO_CONFIG_VAD_M) test.Run(expect, error_msg, {kOpenVINOExecutionProvider}); #else @@ -56,7 +54,7 @@ static void RunAllOpsetAllDomainPadTests( std::string mode = "constant", OpTester::ExpectResult expect = OpTester::ExpectResult::kExpectSuccess, const std::string& error_msg = "") { - // ONNX domain opset-11 is the only one to support all data types + // Test opset-11 and opset-13 kernels of Pad RunOnnxOpsetTypedTest(input_dims, input, pads, @@ -64,8 +62,16 @@ static void RunAllOpsetAllDomainPadTests( output_dims, output, mode, expect, error_msg); + + RunOnnxOpsetTypedTest(input_dims, + input, + pads, + value, + output_dims, + output, + mode, expect, error_msg); } -template<> +template <> void RunAllOpsetAllDomainPadTests<>( const std::vector& input_dims, const std::vector& input, @@ -76,7 +82,7 @@ void RunAllOpsetAllDomainPadTests<>( std::string mode, OpTester::ExpectResult expect, const std::string& error_msg) { - // ONNX domain supports double type + // Test opset-10, opset-11 and opset-13 kernels of Pad (for double type) RunOnnxOpsetTypedTest(input_dims, input, pads, @@ -84,6 +90,7 @@ void RunAllOpsetAllDomainPadTests<>( output_dims, output, mode, expect, error_msg); + RunOnnxOpsetTypedTest(input_dims, input, pads, @@ -91,9 +98,18 @@ void RunAllOpsetAllDomainPadTests<>( output_dims, output, mode, expect, error_msg); + + RunOnnxOpsetTypedTest(input_dims, + input, + pads, + value, + output_dims, + output, + mode, expect, error_msg); } + // There is only support for float type for MSDomain kernel in ORT -template<> +template <> void RunAllOpsetAllDomainPadTests<>( const std::vector& input_dims, const std::vector& input, @@ -104,6 +120,7 @@ void RunAllOpsetAllDomainPadTests<>( std::string mode, OpTester::ExpectResult expect, const std::string& error_msg) { + // Test opset-10, opset-11 and opset-13 kernels of Pad (for float type) RunOnnxOpsetTypedTest(input_dims, input, pads, @@ -111,6 +128,7 @@ void RunAllOpsetAllDomainPadTests<>( output_dims, output, mode, expect, error_msg); + RunOnnxOpsetTypedTest(input_dims, input, pads, @@ -119,6 +137,14 @@ void RunAllOpsetAllDomainPadTests<>( output, mode, expect, error_msg); + RunOnnxOpsetTypedTest(input_dims, + input, + pads, + value, + output_dims, + output, + mode, expect, error_msg); + #ifndef DISABLE_CONTRIB_OPS // MSFT domain opset-1 (contrib op) @@ -129,7 +155,7 @@ void RunAllOpsetAllDomainPadTests<>( test3.AddInput("value", {1}, {value}); test3.AddOutput("output", output_dims, output); //TensorRT does not support pads as an input - test3.Run(expect, error_msg, {kTensorrtExecutionProvider,kOpenVINOExecutionProvider}); + test3.Run(expect, error_msg, {kTensorrtExecutionProvider, kOpenVINOExecutionProvider}); #endif } @@ -679,19 +705,19 @@ TYPED_TEST(PadOpTest, Pad_Constant_DimWithZeroInput) { {T(1), T(1), T(1), T(1), T(1), T(1), T(1), T(1)}); } // Added output shape verification b/w the output shape generated by operator specific ONNX inference and -// the output shape generated by operator specific ORT implementation. After adding this verification, +// the output shape generated by operator specific ORT implementation. After adding this verification, // this test logs warning as validation fails for 2 data types out of 8 data types i.e. Float and Double. // Reason: // Pad ORT implementation output shape does not match with Pad ONNX inference function output shape. -// -// For Float and Double this test gets executed for 2 different opset version, 10 and 11. Specifically this -// test is failing for opset version 10. -// Investigation Analysis: Different ONNX inference class/method gets executed per opset version. Main difference b/w the 2 +// +// For Float and Double this test gets executed for 2 different opset version, 10 and 11. Specifically this +// test is failing for opset version 10. +// Investigation Analysis: Different ONNX inference class/method gets executed per opset version. Main difference b/w the 2 // pad operator ONNX inference class/method is: // Older Pad operator ONNX inference: Accepts "pads and values" as attribute. // Newer Pad operator ONNX inference: Accetps "pads and values" as input. -// For newer version, "pads & values" fields have not been added as initializer, thus instead of shape -// inference, rank inference gets triggered. Whereas, in older version shape inference gets executed +// For newer version, "pads & values" fields have not been added as initializer, thus instead of shape +// inference, rank inference gets triggered. Whereas, in older version shape inference gets executed // as "pads & values" fields have been added as attribute. // In order to remove the warning, shape inference methods needs to be fixed. @@ -743,5 +769,15 @@ TYPED_TEST(PadOpTest, Pad_Reflect_DimWithZeroInput) { "Cannot use 'reflect' mode to pad dimension with a value of 0. Input shape:{0,2,1}"); } +TEST(PadOpTest, BoolType) { + OpTester test("Pad", 13); + test.AddAttribute("mode", "constant"); + test.AddInput("data", {3, 2}, {true, false, true, false, true, false}); + test.AddInput("pads", {4}, {0, 2, 0, 0}); + test.AddInput("value", {1}, {true}); + test.AddOutput("output", {3, 4}, {true, true, true, false, true, true, true, false, true, true, true, false}); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); +} + } // namespace test } // namespace onnxruntime From 738021971738d99ca28cef2fe4e8e0af29cd09e4 Mon Sep 17 00:00:00 2001 From: Hariharan Seshadri Date: Thu, 27 May 2021 14:01:08 -0700 Subject: [PATCH 24/47] Fix bug in Transpose CUDA kernel (#7329) --- .../core/providers/cuda/tensor/transpose.cc | 19 ++- .../providers/cuda/tensor/transpose_impl.cu | 136 ++++++++++++++---- .../providers/cuda/tensor/transpose_impl.h | 26 +++- .../core/providers/rocm/tensor/transpose.cc | 57 +++++--- .../providers/cpu/tensor/transpose_test.cc | 14 +- 5 files changed, 190 insertions(+), 62 deletions(-) diff --git a/onnxruntime/core/providers/cuda/tensor/transpose.cc b/onnxruntime/core/providers/cuda/tensor/transpose.cc index 33bf3ce05e..e429a1eefb 100644 --- a/onnxruntime/core/providers/cuda/tensor/transpose.cc +++ b/onnxruntime/core/providers/cuda/tensor/transpose.cc @@ -166,13 +166,26 @@ Status Transpose::DoTranspose(const cudaDeviceProp& prop, if (CanDoTranspose3D(new_rank, new_input_dims, new_permutations)) { return Transpose3DImpl(stream, element_size, input_shape, tmp_input_strides, input.DataRaw(), output.MutableDataRaw(), output.Shape().Size()); - } else if (CanDoTranspose4D(prop, element_size, new_rank, new_input_dims, new_permutations)) { + } else if (CanDoTranspose4DParallelizeMultipleElementsPerThreadInInnermostDim( + prop, element_size, new_rank, new_input_dims, new_permutations)) { TArray tmp_output_strides(new_rank); for (auto i = 0; i < new_rank; i++) { tmp_output_strides[i] = new_output_strides[new_permutations[i]]; } - return Transpose4DImpl(stream, element_size, input_shape, tmp_input_strides, input.DataRaw(), - tmp_output_strides, output.MutableDataRaw(), gsl::narrow(output.Shape().Size())); + return Transpose4DParallelizeMultipleElementsPerThreadInInnermostDim( + stream, element_size, input_shape, tmp_input_strides, input.DataRaw(), + tmp_output_strides, output.MutableDataRaw(), gsl::narrow(output.Shape().Size())); + } else if (CanDoTranspose4DParallelizeOneElementPerThread( + prop, element_size, new_rank, new_input_dims, new_permutations)) { + // Trying to see if we can still do (best effort) more optimized transposing + // for the 4-D case before falling back to the generic case + TArray tmp_output_strides(new_rank); + for (auto i = 0; i < new_rank; i++) { + tmp_output_strides[i] = new_output_strides[new_permutations[i]]; + } + return Transpose4DParallelizeOneElementPerThread( + stream, element_size, input_shape, tmp_input_strides, input.DataRaw(), + tmp_output_strides, output.MutableDataRaw(), gsl::narrow(output.Shape().Size())); } // General cases diff --git a/onnxruntime/core/providers/cuda/tensor/transpose_impl.cu b/onnxruntime/core/providers/cuda/tensor/transpose_impl.cu index 10611c9cd9..006dce292f 100644 --- a/onnxruntime/core/providers/cuda/tensor/transpose_impl.cu +++ b/onnxruntime/core/providers/cuda/tensor/transpose_impl.cu @@ -80,9 +80,10 @@ Status Transpose3DImpl(cudaStream_t stream, size_t element_size, } template -__global__ void Transpose4DKernel(const TArray input_strides, const void* input_data, - const TArray output_strides, void* output_data, - CUDA_LONG N) { +__global__ void Transpose4DKernelParallelizeMultipleElementsPerThreadInInnermostDim( + const TArray input_strides, const void* input_data, + const TArray output_strides, void* output_data, + CUDA_LONG N) { // output coordinates will be: blockIdx.y, blockIdx.x, threadIdx.y, threadIdx.x CUDA_LONG input_index = (blockIdx.y * input_strides[0] + blockIdx.x * input_strides[1] + @@ -104,59 +105,69 @@ __global__ void Transpose4DKernel(const TArray input_strides, const voi } } -bool CanDoTranspose4D(const cudaDeviceProp& prop, - size_t element_size, - int32_t rank, - const std::vector& input_dims, - const std::vector& permutations) { +bool CanDoTranspose4DParallelizeMultipleElementsPerThreadInInnermostDim(const cudaDeviceProp& prop, + size_t element_size, + int32_t rank, + const std::vector& input_dims, + const std::vector& permutations) { if (rank == 4 && // the permutations is not on the last dimension. - permutations[rank - 1] == (rank - 1)) { - // The block size will be set based on the last two dimensions of 4D tensor. + permutations[3] == 3) { + // The block size will be set based on the outer-most two dimensions of 4D tensor. // the number threads per block will be calculated as below. unsigned int num_elements_per_thread = 4 * sizeof(int) / static_cast(element_size); // int4 is used in the kernel to access data. - int64_t num_elements_in_last_two_dimensions = input_dims[rank - 2] * input_dims[rank - 1]; + int64_t num_elements_in_last_two_dimensions = input_dims[2] * input_dims[3]; int64_t num_threads_per_block = num_elements_in_last_two_dimensions / num_elements_per_thread; if (((num_elements_in_last_two_dimensions & (num_elements_per_thread - 1)) == 0) && num_threads_per_block <= prop.maxThreadsPerBlock && num_threads_per_block >= prop.warpSize && - // num_threads_per_block must be aligned with warp size: 32 - ((num_threads_per_block & (prop.warpSize - 1)) == 0)) { + // num_threads_per_block must be a multiple of warp size (32) + ((num_threads_per_block & (prop.warpSize - 1)) == 0) && + // input_dims[3] must be a multiple of `num_elements_per_thread` + ((input_dims[3] % num_elements_per_thread) == 0)) { return true; } } return false; } -Status Transpose4DImpl(cudaStream_t stream, size_t element_size, const TArray& input_shape, const TArray& input_strides, const void* input_data, - const TArray& output_strides, void* output_data, int N) { +Status Transpose4DParallelizeMultipleElementsPerThreadInInnermostDim( + cudaStream_t stream, size_t element_size, + const TArray& input_shape, const TArray& input_strides, + const void* input_data, const TArray& output_strides, + void* output_data, int N) { unsigned int num_elements_per_thread = 4 * sizeof(int) / static_cast(element_size); // int4 is used in the kernel to access data. dim3 block_size(static_cast(input_shape[3] / num_elements_per_thread), static_cast(input_shape[2])); dim3 grid_size(static_cast(input_shape[1]), static_cast(input_shape[0])); switch (element_size) { case sizeof(int8_t): - Transpose4DKernel<<>>( - input_strides, input_data, - output_strides, output_data, N / num_elements_per_thread); + Transpose4DKernelParallelizeMultipleElementsPerThreadInInnermostDim + <<>>( + input_strides, input_data, + output_strides, output_data, N / num_elements_per_thread); break; case sizeof(int16_t): - Transpose4DKernel<<>>( - input_strides, input_data, - output_strides, output_data, N / num_elements_per_thread); + Transpose4DKernelParallelizeMultipleElementsPerThreadInInnermostDim + <<>>( + input_strides, input_data, + output_strides, output_data, N / num_elements_per_thread); break; case sizeof(int32_t): - Transpose4DKernel<<>>( - input_strides, input_data, - output_strides, output_data, N / num_elements_per_thread); + Transpose4DKernelParallelizeMultipleElementsPerThreadInInnermostDim + <<>>( + input_strides, input_data, + output_strides, output_data, N / num_elements_per_thread); break; case sizeof(int64_t): - Transpose4DKernel<<>>( - input_strides, input_data, - output_strides, output_data, N / num_elements_per_thread); + Transpose4DKernelParallelizeMultipleElementsPerThreadInInnermostDim + <<>>( + input_strides, input_data, + output_strides, output_data, N / num_elements_per_thread); break; default: + // User will not hit this as this kernel is for fixed element size tensors only return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Type not supported for transpose on CUDA. Element size was ", element_size); } @@ -164,6 +175,77 @@ Status Transpose4DImpl(cudaStream_t stream, size_t element_size, const TArray input_strides, const int8_t* input_data, + const TArray output_strides, int8_t* output_data, + size_t element_size, + CUDA_LONG N) { + CUDA_LONG input_index = blockIdx.y * input_strides[0] + + blockIdx.x * input_strides[1] + + threadIdx.y * input_strides[2] + + threadIdx.x * input_strides[3]; + + CUDA_LONG output_index = blockIdx.y * output_strides[0] + + blockIdx.x * output_strides[1] + + threadIdx.y * output_strides[2] + + threadIdx.x * output_strides[3]; + + if (input_index < N && output_index < N) { + const int8_t* input_data_to_be_copied = input_data + (input_index * element_size); + int8_t* output_data_to_be_copied = output_data + (output_index * element_size); + + // copy over the bytes + for (size_t iter = 0; iter < element_size; ++iter) { + *output_data_to_be_copied++ = *input_data_to_be_copied++; + } + } +} + +bool CanDoTranspose4DParallelizeOneElementPerThread(const cudaDeviceProp& prop, + size_t element_size, + int32_t rank, + const std::vector& input_dims, + const std::vector& permutations) { + if (rank == 4) { + // The block size will be set based on the outer-most two dimensions of 4D tensor. + // the number threads per block will be calculated as below. + int64_t number_of_threads_per_block = input_dims[2] * input_dims[3]; + + if (number_of_threads_per_block <= prop.maxThreadsPerBlock && + number_of_threads_per_block >= prop.warpSize && + // num_threads_per_block must be a multiple of warp size (32) + ((number_of_threads_per_block & (prop.warpSize - 1)) == 0)) { + return true; + } + } + return false; +} + +Status Transpose4DParallelizeOneElementPerThread( + cudaStream_t stream, size_t element_size, + const TArray& input_shape, const TArray& input_strides, + const void* input_data, const TArray& output_strides, + void* output_data, int N) { + if (element_size != sizeof(int8_t) && + element_size != sizeof(int16_t) && + element_size != sizeof(int32_t) && + element_size != sizeof(int64_t)) { + // User will not hit this as this kernel is for fixed element size tensors only + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Type not supported for transpose on CUDA. Element size was ", + element_size); + } + + dim3 block_size(static_cast(input_shape[3]), static_cast(input_shape[2])); + dim3 grid_size(static_cast(input_shape[1]), static_cast(input_shape[0])); + + Transpose4DKernelParallelizeOneElementPerThread<<>>( + input_strides, reinterpret_cast(input_data), + output_strides, reinterpret_cast(output_data), + element_size, N); + + return Status::OK(); +} + template __global__ void TransposeKernel(int32_t shape_rank, const TArray input_strides, const T* input_data, const TArray output_strides, T* output_data, CUDA_LONG N) { diff --git a/onnxruntime/core/providers/cuda/tensor/transpose_impl.h b/onnxruntime/core/providers/cuda/tensor/transpose_impl.h index 1a4d469776..a9184d2a16 100644 --- a/onnxruntime/core/providers/cuda/tensor/transpose_impl.h +++ b/onnxruntime/core/providers/cuda/tensor/transpose_impl.h @@ -11,13 +11,25 @@ namespace cuda { bool CanDoTranspose3D(int32_t rank, const std::vector& input_dims, const std::vector& permutations); Status Transpose3DImpl(cudaStream_t stream, size_t element_size, const TArray& input_shape, const TArray& input_strides, const void* input_data, void* output_data, int64_t N); -bool CanDoTranspose4D(const cudaDeviceProp& prop, - size_t element_size, - int32_t rank, - const std::vector& input_dims, - const std::vector& permutations); -Status Transpose4DImpl(cudaStream_t stream, size_t element_size, const TArray& input_shape, const TArray& input_strides, const void* input_data, - const TArray& output_strides, void* output_data, int N); + +bool CanDoTranspose4DParallelizeMultipleElementsPerThreadInInnermostDim(const cudaDeviceProp& prop, + size_t element_size, + int32_t rank, + const std::vector& input_dims, + const std::vector& permutations); +Status Transpose4DParallelizeMultipleElementsPerThreadInInnermostDim(cudaStream_t stream, size_t element_size, const TArray& input_shape, + const TArray& input_strides, const void* input_data, + const TArray& output_strides, void* output_data, int N); + +bool CanDoTranspose4DParallelizeOneElementPerThread(const cudaDeviceProp& prop, + size_t element_size, + int32_t rank, + const std::vector& input_dims, + const std::vector& permutations); +Status Transpose4DParallelizeOneElementPerThread(cudaStream_t stream, size_t element_size, const TArray& input_shape, + const TArray& input_strides, const void* input_data, + const TArray& output_strides, void* output_data, int N); + Status TransposeImpl(cudaStream_t stream, size_t element_size, int32_t shape_rank, const TArray& input_strides, const void* input_data, const TArray& fdm_output_strides, void* output_data, int N); } // namespace cuda diff --git a/onnxruntime/core/providers/rocm/tensor/transpose.cc b/onnxruntime/core/providers/rocm/tensor/transpose.cc index 38b2a9cef1..61e1147abe 100644 --- a/onnxruntime/core/providers/rocm/tensor/transpose.cc +++ b/onnxruntime/core/providers/rocm/tensor/transpose.cc @@ -62,16 +62,16 @@ Status TransposeWithRocblas(hipStream_t stream, rocblas_handle rocblas_handle, c HipT* output_data = reinterpret_cast(output.MutableData()); ROCBLAS_RETURN_IF_ERROR( rocblasTransposeHelper(stream, - rocblas_handle, - rocblas_operation_transpose, rocblas_operation_transpose, M, N, - &one, - input_data, - N, - &zero, - input_data, - N, - output_data, - M)); + rocblas_handle, + rocblas_operation_transpose, rocblas_operation_transpose, M, N, + &one, + input_data, + N, + &zero, + input_data, + N, + output_data, + M)); return Status::OK(); } @@ -128,25 +128,25 @@ Status Transpose::DoTranspose(const hipDeviceProp_t& prop, new_permutations[j] -= 1; } } - for (auto j = i+1; j < new_rank; j++) { - new_permutations[j-1] = new_permutations[j]; + for (auto j = i + 1; j < new_rank; j++) { + new_permutations[j - 1] = new_permutations[j]; } // update input dims new_input_dims[prev] *= new_input_dims[curr]; new_input_dims[curr] = 1; - for (auto j = static_cast(curr+1); j < new_rank; j++) { - new_input_dims[j-1] = new_input_dims[j]; + for (auto j = static_cast(curr + 1); j < new_rank; j++) { + new_input_dims[j - 1] = new_input_dims[j]; } - new_input_dims[new_rank-1] = 1; + new_input_dims[new_rank - 1] = 1; // update output dims - new_output_dims[i-1] *= new_output_dims[i]; + new_output_dims[i - 1] *= new_output_dims[i]; new_output_dims[i] = 1; - for (auto j = i+1; j < new_rank; j++) { - new_output_dims[j-1] = new_output_dims[j]; + for (auto j = i + 1; j < new_rank; j++) { + new_output_dims[j - 1] = new_output_dims[j]; } - new_output_dims[new_rank-1] = 1; + new_output_dims[new_rank - 1] = 1; new_rank--; } @@ -166,13 +166,26 @@ Status Transpose::DoTranspose(const hipDeviceProp_t& prop, if (CanDoTranspose3D(new_rank, new_input_dims, new_permutations)) { return Transpose3DImpl(stream, element_size, input_shape, tmp_input_strides, input.DataRaw(), output.MutableDataRaw(), output.Shape().Size()); - } else if (CanDoTranspose4D(prop, element_size, new_rank, new_input_dims, new_permutations)) { + } else if (CanDoTranspose4DParallelizeMultipleElementsPerThreadInInnermostDim( + prop, element_size, new_rank, new_input_dims, new_permutations)) { TArray tmp_output_strides(new_rank); for (auto i = 0; i < new_rank; i++) { tmp_output_strides[i] = new_output_strides[new_permutations[i]]; } - return Transpose4DImpl(stream, element_size, input_shape, tmp_input_strides, input.DataRaw(), - tmp_output_strides, output.MutableDataRaw(), output.Shape().Size()); + return Transpose4DParallelizeMultipleElementsPerThreadInInnermostDim( + stream, element_size, input_shape, tmp_input_strides, input.DataRaw(), + tmp_output_strides, output.MutableDataRaw(), gsl::narrow(output.Shape().Size())); + } else if (CanDoTranspose4DParallelizeOneElementPerThread( + prop, element_size, new_rank, new_input_dims, new_permutations)) { + // Trying to see if we can still do (best effort) more optimized transposing + // for the 4-D case before falling back to the generic case + TArray tmp_output_strides(new_rank); + for (auto i = 0; i < new_rank; i++) { + tmp_output_strides[i] = new_output_strides[new_permutations[i]]; + } + return Transpose4DParallelizeOneElementPerThread( + stream, element_size, input_shape, tmp_input_strides, input.DataRaw(), + tmp_output_strides, output.MutableDataRaw(), gsl::narrow(output.Shape().Size())); } // General cases diff --git a/onnxruntime/test/providers/cpu/tensor/transpose_test.cc b/onnxruntime/test/providers/cpu/tensor/transpose_test.cc index b971d85072..515fa120c6 100644 --- a/onnxruntime/test/providers/cpu/tensor/transpose_test.cc +++ b/onnxruntime/test/providers/cpu/tensor/transpose_test.cc @@ -590,26 +590,34 @@ static void TestTranspose( test.CompareWithCPU(kGpuExecutionProvider, error_tolerance); } -TEST(TransposeOpTest, Transpose0213) { +TEST(TransposeOpTest, Transpose0213) { // Will trigger Transpose4DParallelizeMultipleElementsPerThreadInInnermostDim() const std::vector X_dims{64, 128, 16, 64}; const std::vector perm{0, 2, 1, 3}; const std::vector Y_dims{64, 16, 128, 64}; TestTranspose(perm, X_dims, Y_dims); } -TEST(TransposeOpTest, Transpose0231) { +TEST(TransposeOpTest, Transpose0213_V2) { // Will trigger Transpose4DParallelizeOneElementPerThread() + const std::vector X_dims{64, 128, 64, 2}; + const std::vector perm{0, 2, 1, 3}; + const std::vector Y_dims{64, 64, 128, 2}; + TestTranspose(perm, X_dims, Y_dims); +} + +TEST(TransposeOpTest, Transpose0231) { // Will trigger Transpose3DImpl() because of "flattening" of dims 2 and 3 into one dim const std::vector X_dims{64, 128, 16, 64}; const std::vector perm{0, 2, 3, 1}; const std::vector Y_dims{64, 16, 64, 128}; TestTranspose(perm, X_dims, Y_dims); } -TEST(TransposeOpTest, Transpose0312) { +TEST(TransposeOpTest, Transpose0312) { // Will trigger Transpose3DImpl() because of "flattening" of dims 1 and 2 into one dim const std::vector X_dims{64, 16, 64, 128}; const std::vector perm{0, 3, 1, 2}; const std::vector Y_dims{64, 128, 16, 64}; TestTranspose(perm, X_dims, Y_dims); } + #endif } // namespace test From ccdedf1b2e0dca5e64eb0016efccaef2d6922bae Mon Sep 17 00:00:00 2001 From: Yulong Wang Date: Thu, 27 May 2021 14:51:57 -0700 Subject: [PATCH 25/47] [js] update documents (#7852) * [js] update documents * escape double quotes * update operators.md * resolve comments --- js/README.md | 177 +++++++++++++++--- js/web/README.md | 42 ++--- js/web/docs/operators.md | 174 +++++++++++++++++ js/web/package.json | 1 + js/web/script/generate-operator-md.ts | 105 +++++++++++ .../azure-pipelines/win-wasm-ci-pipeline.yml | 10 +- 6 files changed, 451 insertions(+), 58 deletions(-) create mode 100644 js/web/docs/operators.md create mode 100644 js/web/script/generate-operator-md.ts diff --git a/js/README.md b/js/README.md index cbc7fb588f..177cba8b1e 100644 --- a/js/README.md +++ b/js/README.md @@ -7,7 +7,7 @@ This directory contains multiple NPM projects: - [onnxruntime-web](#onnxruntime-web) - [onnxruntime-react-native](#onnxruntime-react-native) -### Development +## Development This folder contains a `.vscode` folder for Visual Studio Code workspace configs. Using VSCode to open this folder will allow code-formatting and linting features on typescript and C/C++ source code inside this folder. Following files @@ -19,20 +19,45 @@ are used for code-formatting and linting features for developers: - .eslintrc.js - .clang-format -#### Using VSCode: +Please follow the steps described below to setup development environment. -1. in `/js`, run: - > npm ci -2. use VSCode to open folder `/js` -3. install VSCode extension if not installed yet: - - Clang-Format - - ESLint +### Prerequisites -To populate typescript type declarations, in each projects, run `npm ci`. +- Node.js (14.0+): https://nodejs.org/ - (Optional) Use nvm ([Windows](https://github.com/coreybutler/nvm-windows) / [Mac/Linux](https://github.com/creationix/nvm)) to install Node.js -#### Run code formatter and linter manually +- Python (2.7 or 3.6+): https://www.python.org/downloads/ -in `/js`, use `npm run lint` to run ESLint , and use `npm run format` to run clang-format. + - python should be added to the PATH environment variable + +- Visual Studio Code: https://code.visualstudio.com/ + + - **required** extension: [ESLint](https://marketplace.visualstudio.com/items?itemName=dbaeumer.vscode-eslint) + - **required** extension: [Clang-Format](https://marketplace.visualstudio.com/items?itemName=xaver.clang-format) + - **required** extension: [Debugger for Chrome](https://marketplace.visualstudio.com/items?itemName=msjsdiag.debugger-for-chrome) + +- Chrome or Edge Browser + +### Setup TypeScript development environment + +In `/js`, run: + +``` +npm ci +``` + +This will install Clang-format and ESLint for code-formatting and linting features. This is a one-time setup unless a `git clean` is performed or folder `/js/node_modules` is removed manually. + +### Using VSCode: + +Use VSCode to open folder `/js`. + +Make sure to open the correct folder to allow VSCode to load workspace configuration. Otherwise typescript and code formatter may not work as expected. + +To populate typescript type declarations, in each project folder, run `npm ci`. + +### Run code formatter and linter manually + +In `/js`, use `npm run lint` to run ESLint , and use `npm run format` to run clang-format. ## onnxruntime-common @@ -112,10 +137,6 @@ It should be able to consumed by from projects that uses NPM packages (through a This project is a library for running ONNX models on browsers. It is the successor of [ONNX.js](https://github.com/Microsoft/onnxjs). -### Requirements - -Node.js v12+ (recommended v14+) - ### Build 1. Install NPM packages @@ -124,9 +145,9 @@ Node.js v12+ (recommended v14+) 2. in `/js/common/`, run `npm ci`. 3. in `/js/web/`, run `npm ci`. -2. ~~Follow [instructions](https://www.onnxruntime.ai/docs/how-to/build.html#apis-and-language-bindings) for building ONNX Runtime WebAssembly. (TODO: document is not ready. we are working on it.)~~ +2. ~~Follow [instructions](https://www.onnxruntime.ai/docs/how-to/build.html#apis-and-language-bindings) for building ONNX Runtime WebAssembly. (TODO: document is not ready. we are working on it. Please see steps described as below.)~~ - in `/`, run either of the following commands to build WebAssembly: + in `/`, run one of the following commands to build WebAssembly: ```sh # In windows, use 'build' to replace './build.sh' @@ -134,11 +155,16 @@ Node.js v12+ (recommended v14+) # The following command build debug. ./build.sh --build_wasm + # The following command build debug with debug info. + ./build.sh --build_wasm --skip_tests --enable_wasm_debug_info + # The following command build release. ./build.sh --config Release --build_wasm --skip_tests --disable_wasm_exception_catching --disable_rtti ``` - To build with multi-thread support, append flag ` --enable_wasm_threads` to the command. Make sure to build both single-thread and multi-thread before next step. + To build with multi-thread support, append flag `--enable_wasm_threads` to the command. Make sure to build both single-thread and multi-thread before next step. + + NOTE: You can also find latest build artifacts on [Windows WebAssembly CI Pipeline](https://dev.azure.com/onnxruntime/onnxruntime/_build?definitionId=161&_a=summary&repositoryFilter=1&branchFilter=4%2C4%2C4%2C4%2C4%2C4). Choose any build for master branch, download artifacts "Release_ort-wasm" and "Release_ort-wasm-threaded" and unzip. 3. Copy following files from build output folder to `/js/web/dist/`: @@ -156,9 +182,91 @@ Node.js v12+ (recommended v14+) npm run build ``` +### Test + +We use command `npm test` (test runner) and `npm run test:e2e` (E2E test) for tests in ONNXRuntime Web. + +#### test runner + +In folder `/js/web`, + +- Run `npm test -- --help` for a full CLI instruction. +- Run `npm test -- --debug` to run one or more test cases. + +There are multiple levels of tests for ONNXRuntime Web: + +- unit test: tests for individual components written in TypeScript. Launch unit test by: + ``` + npm test -- unittest + ``` +- model test: run a single model. The model folder should contains one .onnx model file and one or more folders for test cases, each folder contains several input*\*.pb and output*\*.pb as test data. Launch model test by: + ``` + npm test -- model + ``` +- op test: test a single operator. An op test is described in a `.jsonc` file which specify the operator type, its attributes and one or more test case(s), each includes a list of expected input tensor(s) and output tensor(s). The `.jsonc` file is located at `/js/web/test/data/ops`. Launch op test by: + + ``` + npm test -- op + ``` + +- suite test: suite test includes unit test, a list of model tests and op tests. Launch suite test by: + ``` + npm test + ``` + +#### E2E test + +E2E test is for testing end-to-end package consuming. In this test, NPM packages for `onnxruntime-common` and `onnxruntime-web` are generated and a clean folder is used for installing packages. Then a simple mocha test is performed to make sure package can be consumed correctly. + +To launch E2E test: + +``` +npm run test:e2e +``` + +### Debugging + +#### Debugging TypeScript on Desktop/Chrome + +To debug the code from test-runner on Chrome: + +- Launch `npm test -- --debug`. It opens an instance of Chrome browser. +- In the open Chrome browser, click the `DEBUG` button on the top-right of the page. +- In VSCode, click [side bar]->Run and Debug->select [Attach to Chrome]->click [Start Debugging] to attach. +- put breakpoints in source code, and Refresh the page to reload. + +#### Debugging TypeScript on iOS/Safari + +To debug on an Apple iOS device, please refer to the following steps: + +- install [ + RemoteDebug iOS WebKit Adapter](https://github.com/RemoteDebug/remotedebug-ios-webkit-adapter) by following its instructions. +- launch the adapter in commandline: `remotedebug_ios_webkit_adapter --port=9000`. +- in VSCode, select debug configuration `Remote Browser via Webkit Adaptor`. +- follow the steps above to debug. + +#### Debugging TypeScript on Android/Chrome + +To debug on an Android device, please refer to the following steps: + +- Install [Android SDK Platform Tools](https://developer.android.com/studio/releases/platform-tools) and make sure `adb` is ready to use. +- Follow instructions in [Remote Debugging on Android](https://developer.chrome.com/devtools/docs/remote-debugging-legacy) to launch `adb`. Make sure to use port 9000 so that the existing debug configuration works. +- in VSCode, select debug configuration `Remote Browser via Webkit Adaptor`. +- follow the steps above to debug. + +#### Debugging C/C++ for ONNX Runtime WebAssembly + +To debug C/C++ code for ONNX Runtime WebAssembly, you need to build ONNX Runtime with debug info (see [Build](#Build-2)). + +Currently debugging C/C++ code in WebAssembly is not supported in VSCode yet. Please follow [this instruction](https://developer.chrome.com/blog/wasm-debugging-2020/) to debug in browser devtool using extension [C/C++ DevTools Support (DWARF)](https://chrome.google.com/webstore/detail/cc%20%20-devtools-support-dwa/pdcpmagijalfljmkmjngeonclgbbannb). + +### Generating Document + +Use command `npm run build:doc` to generate the latest documents. + ### Distribution -It should be able to consumed by both from projects that uses NPM packages (through a Node.js folder structure of `node_modules` folder that generated by `npm install onnxruntime-web`) and from a CDN service that serves a `.min.js` file and one or multiple `.wasm` file(s). +It should be able to consumed by both from projects that uses NPM packages (through a Node.js folder structure of `node_modules` folder that generated by `npm install onnxruntime-web`) and from a CDN service that serves a `ort.min.js` file and one or multiple `.wasm` file(s). ## onnxruntime-react-native @@ -192,6 +300,7 @@ This project provides an ONNX Runtime React Native JavaScript library to run ONN 1. Set up an Android build environment referring to [instruction](https://www.onnxruntime.ai/docs/how-to/build.html#android) 2. In ``, run this python script to build ONNX Runtime Android archive file. In windows, this requires admin account to build. If an app uses a fixed set of models, refer to [instruction](https://www.onnxruntime.ai/docs/how-to/build.html#android) and build a mobile version package + ```python python tools/ci_build/github/android/build_aar_package.py js/react_native/scripts/aar_build_settings.json --config MinSizeRel --android_sdk_path --android_ndk_path --build_dir ``` @@ -199,6 +308,7 @@ This project provides an ONNX Runtime React Native JavaScript library to run ONN 3. This generates `onnxruntime-mobile-.aar` in `/aar_out/MinSizeRel/com/microsoft/onnxruntime/onnxruntime-mobile/`. Copy `aar` file into `/js/react_native/android/libs` and rename it as `onnxruntime.aar` 4. To verify, open Android Emulator and run this command from `/js/react_native/android` + ```sh adb shell am instrument -w ai.onnxruntime.react_native.test/androidx.test.runner.AndroidJUnitRunner ``` @@ -206,40 +316,45 @@ This project provides an ONNX Runtime React Native JavaScript library to run ONN 3. Build iOS ONNX Runtime package 1. Set up iOS build environment referring to [instruction](https://www.onnxruntime.ai/docs/how-to/build.html#ios). - + 2. Build ONNX Runtime library for iOS from `` using this command, + ```sh ./build.sh --config MinSizeRel --use_xcode --ios --ios_sysroot iphoneos --osx_arch arm64 --apple_deploy_target 11 ``` + Copy `/build/iOS/MinSizeRel/MinSizeRel-iphoneos/libonnxruntime..dylib` file into `/js/react_native/ios/Libraries/onnxruntime/lib/iphoneos` 3. Clean up the previous build and build ONNX Runtime library for iOS Simulator from `` + ```sh ./build.sh --config MinSizeRel --use_xcode --ios --ios_sysroot iphonesimulator --osx_arch x86_64 --apple_deploy_target 11 ``` + Copy `/build/iOS/MinSizeRel/MinSizeRel-iphonesimulator/libonnxruntime..dylib` file into `/js/react_native/ios/Libraries/onnxruntime/lib/iphonesimulator` - + 4. Edit `onnxruntime-react-native.iphoneos.podspec` and `onnxruntime-react-native.iphonesimulator.podsepc` in `/js/react_native` to change a version of ONNX Runtime library. 5. Copy ONNX Runtime header files + ```sh cp /include/onnxruntime/core/session/*.h /js/react_native/ios/Libraries/onnxruntime/include ``` 6. To verify, open iOS Simulator and run this command from `/js/react_native/ios`. Change a destination to specify a running iOS Simulator. - ```sh - pod install - export ONNXRUNTIME_VERSION=; xcodebuild test -workspace OnnxruntimeModule.xcworkspace -scheme OnnxruntimeModuleTest -destination 'platform=iOS Simulator,name=iPhone 11,OS=14.5' - ``` + ```sh + pod install + export ONNXRUNTIME_VERSION=; xcodebuild test -workspace OnnxruntimeModule.xcworkspace -scheme OnnxruntimeModuleTest -destination 'platform=iOS Simulator,name=iPhone 11,OS=14.5' + ``` 4. Update a version in `package.json` to align with ONNX Runtime version. 5. Test an example for Android and iOS. In Windows, open Android Emulator first. From `/js/react_native` - ```sh - yarn bootstrap - yarn example ios - yarn example android - ``` + ```sh + yarn bootstrap + yarn example ios + yarn example android + ``` ### NPM Packaging @@ -251,7 +366,7 @@ This project provides an ONNX Runtime React Native JavaScript library to run ONN 4. Run `npm publish --dry-run` to see how it's going to be published -5. Run `npm publish ` to publish to npmjs +5. Run `npm publish ` to publish to npmjs. If it's for a dev, add flag `--tag dev`. ### Distribution diff --git a/js/web/README.md b/js/web/README.md index a59c84db11..731f87cbb2 100644 --- a/js/web/README.md +++ b/js/web/README.md @@ -6,7 +6,7 @@ ONNX Runtime Web has adopted WebAssembly and WebGL technologies for providing an ### Why ONNX models -The [Open Neural Network Exchange](http://onnx.ai/) (ONNX) is an open standard for representing machine learning models. The biggest advantage of ONNX is that it allows interoperability across different open source AI frameworks, which itself offers more flexibility for AI frameworks adoption. See [Getting ONNX Models](#Getting-ONNX-models). +The [Open Neural Network Exchange](http://onnx.ai/) (ONNX) is an open standard for representing machine learning models. The biggest advantage of ONNX is that it allows interoperability across different open source AI frameworks, which itself offers more flexibility for AI frameworks adoption. ### Why ONNX Runtime Web @@ -22,35 +22,25 @@ Refer to [ONNX Runtime JavaScript examples](https://github.com/microsoft/onnxrun ## Documents -### Developers +### Developement -Refer to [Using VSCode](../README.md#Using-VSCode) for setting up development environment. +Refer to the following links for development information: -For information about building ONNX Runtime Web development, please check [Build](../README.md#build-2). - -### Getting ONNX models - -You can get ONNX models easily in multiple ways: - -- Choose a pre-trained ONNX model from the [ONNX Model Zoo](https://github.com/onnx/models) -- Convert models from mainstream frameworks, e.g. PyTorch, TensorFlow and Keras, by following [ONNX tutorials](https://github.com/onnx/tutorials) -- Use your data to generate a customized ONNX model from [Azure Custom Vision service](https://docs.microsoft.com/en-us/azure/cognitive-services/Custom-Vision-Service/home) -- [Train a custom model in AzureML](https://github.com/Azure/MachineLearningNotebooks/tree/master/training) and save it in the ONNX format - -Learn more about ONNX - -- [ONNX website](http://onnx.ai/) -- [ONNX on GitHub](https://github.com/onnx/onnx) +- [Development](../README.md#Development) +- [Build](../README.md#Build-2) +- [Test](../README.md#Test) +- [Debugging](../README.md#Debugging) +- [Generating Document](../README.md#Generating-Document) ### Compatibility -| OS/Browser | Chrome | Edge | Safari | Electron | -| :--------------: | :----------------: | :----------------: | :----------------: | :----------------: | -| Windows 10 | :heavy_check_mark: | :heavy_check_mark: | - | :heavy_check_mark: | -| macOS | :heavy_check_mark: | - | :heavy_check_mark: | :heavy_check_mark: | -| Ubuntu LTS 18.04 | :heavy_check_mark: | - | - | :heavy_check_mark: | -| iOS | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | - | -| Android | :heavy_check_mark: | - | - | - | +| OS/Browser | Chrome | Edge | Safari | Electron | Node.js | +| :--------------: | :---------: | :---------: | :----: | :---------: | :-----: | +| Windows 10 | wasm, webgl | wasm, webgl | - | wasm, webgl | wasm | +| macOS | wasm | - | wasm | wasm | wasm | +| Ubuntu LTS 18.04 | wasm | - | - | wasm | wasm | +| iOS | wasm | wasm | wasm | - | - | +| Android | wasm | - | - | - | - | ### Operators @@ -60,7 +50,7 @@ ONNX Runtime Web currently support all operators in [ai.onnx](https://github.com #### WebGL backend -ONNX Runtime Web currently supports most operators in [ai.onnx](https://github.com/onnx/onnx/blob/rel-1.2.3/docs/Operators.md) operator set v7 (opset v7). See [operators.md](./docs/operators.md) for a complete, detailed list of which ONNX operators are supported by WebGL backend. +ONNX Runtime Web currently supports a subset of operators in [ai.onnx](https://github.com/onnx/onnx/blob/master/docs/Operators.md) operator set. See [operators.md](./docs/operators.md) for a complete, detailed list of which ONNX operators are supported by WebGL backend. ## License diff --git a/js/web/docs/operators.md b/js/web/docs/operators.md new file mode 100644 index 0000000000..212937df99 --- /dev/null +++ b/js/web/docs/operators.md @@ -0,0 +1,174 @@ +## Operators Support Table + +The following table shows [ai.onnx](https://github.com/onnx/onnx/blob/master/docs/Operators.md) operators from which onnx opset version are currently supported by onnxjs. For example, `4-6, 8+` means ONNX Runtime Web currently support opset version 4 to 6, 8 and above. + +See [Compatibility](../README.md#Compatibility) for a list of the supported platforms. + +*This file is automatically generated from the def files via [this script](../script/generate-operator-md.ts). Do not modify directly.* + +| Operator | WebGl Backend | +|:--------:|:-------------:| +| [Abs](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Abs) | [6-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Abs-6), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Abs-13) | +| [Acos](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Acos) | [7+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Acos-7) | +| [Acosh](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Acosh) | | +| [Add](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Add) | [7-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Add-7), [13](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Add-13), [14+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Add-14) | +| [And](https://github.com/onnx/onnx/blob/master/docs/Operators.md#And) | [7+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#And-7) | +| [ArgMax](https://github.com/onnx/onnx/blob/master/docs/Operators.md#ArgMax) | | +| [ArgMin](https://github.com/onnx/onnx/blob/master/docs/Operators.md#ArgMin) | | +| [Asin](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Asin) | [7+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Asin-7) | +| [Asinh](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Asinh) | | +| [Atan](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Atan) | [7+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Atan-7) | +| [Atanh](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Atanh) | | +| [AveragePool](https://github.com/onnx/onnx/blob/master/docs/Operators.md#AveragePool) | [7-9](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#AveragePool-7), [10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#AveragePool-10) | +| [BatchNormalization](https://github.com/onnx/onnx/blob/master/docs/Operators.md#BatchNormalization) | [7-8](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#BatchNormalization-7), [9-13](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#BatchNormalization-9), [14+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#BatchNormalization-14) | +| [BitShift](https://github.com/onnx/onnx/blob/master/docs/Operators.md#BitShift) | | +| [Cast](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Cast) | | +| [Ceil](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Ceil) | [6-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Ceil-6), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Ceil-13) | +| [Celu](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Celu) | | +| [Clip](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Clip) | [6-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Clip-6) | +| [Compress](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Compress) | | +| [Concat](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Concat) | [4-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Concat-4), [11-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Concat-11), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Concat-13) | +| [ConcatFromSequence](https://github.com/onnx/onnx/blob/master/docs/Operators.md#ConcatFromSequence) | | +| [Constant](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Constant) | | +| [ConstantOfShape](https://github.com/onnx/onnx/blob/master/docs/Operators.md#ConstantOfShape) | | +| [Conv](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Conv) | [1-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Conv-1), [11+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Conv-11) | +| [ConvInteger](https://github.com/onnx/onnx/blob/master/docs/Operators.md#ConvInteger) | | +| [ConvTranspose](https://github.com/onnx/onnx/blob/master/docs/Operators.md#ConvTranspose) | | +| [Cos](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Cos) | [7+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Cos-7) | +| [Cosh](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Cosh) | | +| [CumSum](https://github.com/onnx/onnx/blob/master/docs/Operators.md#CumSum) | | +| [DepthToSpace](https://github.com/onnx/onnx/blob/master/docs/Operators.md#DepthToSpace) | [1-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#DepthToSpace-1), [11-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#DepthToSpace-11), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#DepthToSpace-13) | +| [DequantizeLinear](https://github.com/onnx/onnx/blob/master/docs/Operators.md#DequantizeLinear) | | +| [Det](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Det) | | +| [Div](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Div) | [7-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Div-7), [13](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Div-13), [14+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Div-14) | +| [Dropout](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Dropout) | [7-9](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Dropout-7), [10-11](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Dropout-10), [12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Dropout-12), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Dropout-13) | +| [DynamicQuantizeLinear](https://github.com/onnx/onnx/blob/master/docs/Operators.md#DynamicQuantizeLinear) | | +| [Einsum](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Einsum) | | +| [Elu](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Elu) | [6+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Elu-6) | +| [Equal](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Equal) | [7-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Equal-7), [11-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Equal-11), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Equal-13) | +| [Erf](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Erf) | | +| [Exp](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Exp) | [6-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Exp-6), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Exp-13) | +| [Expand](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Expand) | | +| [EyeLike](https://github.com/onnx/onnx/blob/master/docs/Operators.md#EyeLike) | | +| [Flatten](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Flatten) | [1-8](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Flatten-1), [9-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Flatten-9), [11-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Flatten-11), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Flatten-13) | +| [Floor](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Floor) | [6-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Floor-6), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Floor-13) | +| [GRU](https://github.com/onnx/onnx/blob/master/docs/Operators.md#GRU) | | +| [Gather](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Gather) | [1-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Gather-1), [11-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Gather-11), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Gather-13) | +| [GatherElements](https://github.com/onnx/onnx/blob/master/docs/Operators.md#GatherElements) | | +| [GatherND](https://github.com/onnx/onnx/blob/master/docs/Operators.md#GatherND) | | +| [Gemm](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Gemm) | [7-8](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Gemm-7), [9-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Gemm-9), [11-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Gemm-11), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Gemm-13) | +| [GlobalAveragePool](https://github.com/onnx/onnx/blob/master/docs/Operators.md#GlobalAveragePool) | [1+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#GlobalAveragePool-1) | +| [GlobalLpPool](https://github.com/onnx/onnx/blob/master/docs/Operators.md#GlobalLpPool) | | +| [GlobalMaxPool](https://github.com/onnx/onnx/blob/master/docs/Operators.md#GlobalMaxPool) | [1+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#GlobalMaxPool-1) | +| [Greater](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Greater) | [7-8](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Greater-7), [9-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Greater-9), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Greater-13) | +| [GreaterOrEqual](https://github.com/onnx/onnx/blob/master/docs/Operators.md#GreaterOrEqual) | | +| [HardSigmoid](https://github.com/onnx/onnx/blob/master/docs/Operators.md#HardSigmoid) | | +| [HardSwish](https://github.com/onnx/onnx/blob/master/docs/Operators.md#HardSwish) | | +| [Hardmax](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Hardmax) | | +| [Identity](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Identity) | [1-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Identity-1), [13](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Identity-13), [14+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Identity-14) | +| [If](https://github.com/onnx/onnx/blob/master/docs/Operators.md#If) | | +| [InstanceNormalization](https://github.com/onnx/onnx/blob/master/docs/Operators.md#InstanceNormalization) | [6+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#InstanceNormalization-6) | +| [IsInf](https://github.com/onnx/onnx/blob/master/docs/Operators.md#IsInf) | | +| [IsNaN](https://github.com/onnx/onnx/blob/master/docs/Operators.md#IsNaN) | | +| [LRN](https://github.com/onnx/onnx/blob/master/docs/Operators.md#LRN) | | +| [LSTM](https://github.com/onnx/onnx/blob/master/docs/Operators.md#LSTM) | | +| [LeakyRelu](https://github.com/onnx/onnx/blob/master/docs/Operators.md#LeakyRelu) | [6+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#LeakyRelu-6) | +| [Less](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Less) | [7-8](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Less-7), [9-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Less-9), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Less-13) | +| [LessOrEqual](https://github.com/onnx/onnx/blob/master/docs/Operators.md#LessOrEqual) | | +| [Log](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Log) | [6-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Log-6), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Log-13) | +| [LogSoftmax](https://github.com/onnx/onnx/blob/master/docs/Operators.md#LogSoftmax) | | +| [Loop](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Loop) | | +| [LpNormalization](https://github.com/onnx/onnx/blob/master/docs/Operators.md#LpNormalization) | | +| [LpPool](https://github.com/onnx/onnx/blob/master/docs/Operators.md#LpPool) | | +| [MatMul](https://github.com/onnx/onnx/blob/master/docs/Operators.md#MatMul) | [1-8](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#MatMul-1), [9-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#MatMul-9), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#MatMul-13) | +| [MatMulInteger](https://github.com/onnx/onnx/blob/master/docs/Operators.md#MatMulInteger) | | +| [Max](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Max) | | +| [MaxPool](https://github.com/onnx/onnx/blob/master/docs/Operators.md#MaxPool) | [1-7](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#MaxPool-1), [8-9](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#MaxPool-8) | +| [MaxRoiPool](https://github.com/onnx/onnx/blob/master/docs/Operators.md#MaxRoiPool) | | +| [MaxUnpool](https://github.com/onnx/onnx/blob/master/docs/Operators.md#MaxUnpool) | | +| [Mean](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Mean) | | +| [MeanVarianceNormalization](https://github.com/onnx/onnx/blob/master/docs/Operators.md#MeanVarianceNormalization) | | +| [Min](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Min) | | +| [Mod](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Mod) | | +| [Mul](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Mul) | [7-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Mul-7), [13](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Mul-13), [14+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Mul-14) | +| [Multinomial](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Multinomial) | | +| [Neg](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Neg) | [6-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Neg-6), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Neg-13) | +| [NegativeLogLikelihoodLoss](https://github.com/onnx/onnx/blob/master/docs/Operators.md#NegativeLogLikelihoodLoss) | | +| [NonMaxSuppression](https://github.com/onnx/onnx/blob/master/docs/Operators.md#NonMaxSuppression) | | +| [NonZero](https://github.com/onnx/onnx/blob/master/docs/Operators.md#NonZero) | | +| [Not](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Not) | [1+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Not-1) | +| [OneHot](https://github.com/onnx/onnx/blob/master/docs/Operators.md#OneHot) | | +| [Or](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Or) | [7+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Or-7) | +| [PRelu](https://github.com/onnx/onnx/blob/master/docs/Operators.md#PRelu) | [7-8](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#PRelu-7), [9+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#PRelu-9) | +| [Pad](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Pad) | [2-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Pad-2) | +| [Pow](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Pow) | [7-11](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Pow-7), [12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Pow-12), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Pow-13) | +| [QLinearConv](https://github.com/onnx/onnx/blob/master/docs/Operators.md#QLinearConv) | | +| [QLinearMatMul](https://github.com/onnx/onnx/blob/master/docs/Operators.md#QLinearMatMul) | | +| [QuantizeLinear](https://github.com/onnx/onnx/blob/master/docs/Operators.md#QuantizeLinear) | | +| [RNN](https://github.com/onnx/onnx/blob/master/docs/Operators.md#RNN) | | +| [RandomNormal](https://github.com/onnx/onnx/blob/master/docs/Operators.md#RandomNormal) | | +| [RandomNormalLike](https://github.com/onnx/onnx/blob/master/docs/Operators.md#RandomNormalLike) | | +| [RandomUniform](https://github.com/onnx/onnx/blob/master/docs/Operators.md#RandomUniform) | | +| [RandomUniformLike](https://github.com/onnx/onnx/blob/master/docs/Operators.md#RandomUniformLike) | | +| [Range](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Range) | | +| [Reciprocal](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Reciprocal) | | +| [ReduceL1](https://github.com/onnx/onnx/blob/master/docs/Operators.md#ReduceL1) | | +| [ReduceL2](https://github.com/onnx/onnx/blob/master/docs/Operators.md#ReduceL2) | | +| [ReduceLogSum](https://github.com/onnx/onnx/blob/master/docs/Operators.md#ReduceLogSum) | [1-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceLogSum-1), [11-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceLogSum-11), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceLogSum-13) | +| [ReduceLogSumExp](https://github.com/onnx/onnx/blob/master/docs/Operators.md#ReduceLogSumExp) | | +| [ReduceMax](https://github.com/onnx/onnx/blob/master/docs/Operators.md#ReduceMax) | [1-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceMax-1), [11](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceMax-11), [12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceMax-12), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceMax-13) | +| [ReduceMean](https://github.com/onnx/onnx/blob/master/docs/Operators.md#ReduceMean) | [1-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceMean-1), [11-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceMean-11), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceMean-13) | +| [ReduceMin](https://github.com/onnx/onnx/blob/master/docs/Operators.md#ReduceMin) | [1-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceMin-1), [11](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceMin-11), [12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceMin-12), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceMin-13) | +| [ReduceProd](https://github.com/onnx/onnx/blob/master/docs/Operators.md#ReduceProd) | [1-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceProd-1), [11-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceProd-11), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceProd-13) | +| [ReduceSum](https://github.com/onnx/onnx/blob/master/docs/Operators.md#ReduceSum) | [1-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceSum-1), [11-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceSum-11), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceSum-13) | +| [ReduceSumSquare](https://github.com/onnx/onnx/blob/master/docs/Operators.md#ReduceSumSquare) | [1-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceSumSquare-1), [11-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceSumSquare-11), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceSumSquare-13) | +| [Relu](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Relu) | [6-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Relu-6), [13](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Relu-13), [14+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Relu-14) | +| [Reshape](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Reshape) | [5-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Reshape-5), [13](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Reshape-13), [14+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Reshape-14) | +| [Resize](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Resize) | [10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Resize-10), [11-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Resize-11), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Resize-13) | +| [ReverseSequence](https://github.com/onnx/onnx/blob/master/docs/Operators.md#ReverseSequence) | | +| [RoiAlign](https://github.com/onnx/onnx/blob/master/docs/Operators.md#RoiAlign) | | +| [Round](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Round) | | +| [Scan](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Scan) | | +| [Scatter](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Scatter) | | +| [ScatterElements](https://github.com/onnx/onnx/blob/master/docs/Operators.md#ScatterElements) | | +| [ScatterND](https://github.com/onnx/onnx/blob/master/docs/Operators.md#ScatterND) | | +| [Selu](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Selu) | | +| [SequenceAt](https://github.com/onnx/onnx/blob/master/docs/Operators.md#SequenceAt) | | +| [SequenceConstruct](https://github.com/onnx/onnx/blob/master/docs/Operators.md#SequenceConstruct) | | +| [SequenceEmpty](https://github.com/onnx/onnx/blob/master/docs/Operators.md#SequenceEmpty) | | +| [SequenceErase](https://github.com/onnx/onnx/blob/master/docs/Operators.md#SequenceErase) | | +| [SequenceInsert](https://github.com/onnx/onnx/blob/master/docs/Operators.md#SequenceInsert) | | +| [SequenceLength](https://github.com/onnx/onnx/blob/master/docs/Operators.md#SequenceLength) | | +| [Shape](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Shape) | | +| [Shrink](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Shrink) | | +| [Sigmoid](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Sigmoid) | [6-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Sigmoid-6), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Sigmoid-13) | +| [Sign](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Sign) | | +| [Sin](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Sin) | [7+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Sin-7) | +| [Sinh](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Sinh) | | +| [Size](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Size) | | +| [Slice](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Slice) | [1-9](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Slice-1), [10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Slice-10), [11-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Slice-11), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Slice-13) | +| [Softmax](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Softmax) | [1-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Softmax-1), [11-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Softmax-11), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Softmax-13) | +| [SoftmaxCrossEntropyLoss](https://github.com/onnx/onnx/blob/master/docs/Operators.md#SoftmaxCrossEntropyLoss) | | +| [Softplus](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Softplus) | | +| [Softsign](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Softsign) | | +| [SpaceToDepth](https://github.com/onnx/onnx/blob/master/docs/Operators.md#SpaceToDepth) | | +| [Split](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Split) | [2-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Split-2), [11-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Split-11), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Split-13) | +| [SplitToSequence](https://github.com/onnx/onnx/blob/master/docs/Operators.md#SplitToSequence) | | +| [Sqrt](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Sqrt) | [6-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Sqrt-6), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Sqrt-13) | +| [Squeeze](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Squeeze) | [1-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Squeeze-1), [11-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Squeeze-11), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Squeeze-13) | +| [StringNormalizer](https://github.com/onnx/onnx/blob/master/docs/Operators.md#StringNormalizer) | | +| [Sub](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Sub) | [7-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Sub-7), [13](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Sub-13), [14+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Sub-14) | +| [Sum](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Sum) | [6-7](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Sum-6), [8-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Sum-8), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Sum-13) | +| [Tan](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Tan) | [7+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Tan-7) | +| [Tanh](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Tanh) | [6-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Tanh-6), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Tanh-13) | +| [TfIdfVectorizer](https://github.com/onnx/onnx/blob/master/docs/Operators.md#TfIdfVectorizer) | | +| [ThresholdedRelu](https://github.com/onnx/onnx/blob/master/docs/Operators.md#ThresholdedRelu) | | +| [Tile](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Tile) | [6-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Tile-6), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Tile-13) | +| [TopK](https://github.com/onnx/onnx/blob/master/docs/Operators.md#TopK) | | +| [Transpose](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Transpose) | [1-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Transpose-1), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Transpose-13) | +| [Trilu](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Trilu) | | +| [Unique](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Unique) | | +| [Unsqueeze](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Unsqueeze) | [1-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Unsqueeze-1), [11-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Unsqueeze-11), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Unsqueeze-13) | +| [Upsample](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Upsample) | [7-8](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Upsample-7), [9](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Upsample-9) | +| [Where](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Where) | | +| [Xor](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Xor) | [7+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Xor-7) | diff --git a/js/web/package.json b/js/web/package.json index 662261261e..0c3b0e1de3 100644 --- a/js/web/package.json +++ b/js/web/package.json @@ -16,6 +16,7 @@ "scripts": { "prepare": "tsc", "build": "node ./script/build", + "build:doc": "node ./script/generate-operator-md", "test": "node ./script/prepare-test-data && node ./script/test-runner-cli", "test:e2e": "node ./test/e2e/run", "prepack": "node ./script/prepack" diff --git a/js/web/script/generate-operator-md.ts b/js/web/script/generate-operator-md.ts new file mode 100644 index 0000000000..ac0e1e6a81 --- /dev/null +++ b/js/web/script/generate-operator-md.ts @@ -0,0 +1,105 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import * as assert from 'assert'; +import * as fs from 'fs'; +import {EOL} from 'os'; +import * as path from 'path'; + +import {Attribute} from '../lib/onnxjs/attribute'; +import {WEBGL_OP_RESOLVE_RULES} from '../lib/onnxjs/backends/webgl/op-resolve-rules'; +import {Operator} from '../lib/onnxjs/operators'; +import {OpSet, resolveOperator} from '../lib/onnxjs/opset'; + +function checkSupport(type: string, range: [number, number], rules: readonly OpSet.ResolveRule[]) { + const node = {name: '', opType: type, inputs: [], outputs: [], attributes: new Attribute(undefined)}; + for (let i = range[0]; i <= range[1]; i++) { + try { + resolveOperator(node, [{domain: '', version: i}], rules); + } catch (_e) { + return false; + } + } + return true; +} + +function formatDesc(opType: string, range: [number, number], support: boolean, last: boolean) { + let versionDesc = ''; + if (support) { + versionDesc = last ? `${range[0]}+` : range[0] === range[1] ? `${range[0]}` : `${range[0]}-${range[1]}`; + versionDesc = `[${versionDesc}](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#${opType}-${range[0]})`; + } + return versionDesc; +} +function dummyOpConstructor(): Operator { + return {} as any as Operator; +} + +const ops = new Map>(); +const webglCheckOnlyRules = + WEBGL_OP_RESOLVE_RULES.map(rule => [rule[0], rule[1], rule[2], dummyOpConstructor] as OpSet.ResolveRule); + +fs.readFileSync(path.join(__dirname, '../../../cmake/external/onnx/onnx/defs/operator_sets.h'), 'utf8') + .split(/\r?\n/) + .forEach(line => { + const matcher = /class ONNX_OPERATOR_SET_SCHEMA_CLASS_NAME\(\s*(\w+),\s*(\d+),\s*(\w+)\)/; + const matches = matcher.exec(line); + if (matches) { + const opset = matches[1]; + const version = Number.parseInt(matches[2], 10); + const opType = matches[3]; + + let currentSet = ops.get(opset); + if (currentSet === undefined) { + currentSet = new Map(); + ops.set(opset, currentSet); + } + + let currentOp = currentSet.get(opType); + if (currentOp === undefined) { + currentOp = []; + currentSet.set(opType, currentOp); + } + + currentOp.push(version); + } + }); + +const opsets = Array.from(ops.keys()); +assert.ok(opsets.length === 1 && opsets[0] === 'Onnx'); + +const onnxOpset = ops.get(opsets[0])!; +const opTypes = Array.from(onnxOpset.keys()).sort(); + +const doc = fs.createWriteStream(path.join(__dirname, '../docs/operators.md')); +doc.write(`## Operators Support Table${EOL}${EOL}`); +doc.write(`The following table shows [ai.onnx](https://github.com/onnx/onnx/blob/master/docs/Operators.md)\ + operators from which onnx opset version are currently supported by onnxjs. For example, \`4-6, 8+\` means\ + ONNX Runtime Web currently support opset version 4 to 6, 8 and above.${EOL}${EOL}`); +doc.write(`See [Compatibility](../README.md#Compatibility) for a list of the supported platforms.${EOL}${EOL}`); +doc.write(`*This file is automatically generated from the\ + def files via [this script](../script/generate-operator-md.ts).\ + Do not modify directly.*${EOL}${EOL}`); +doc.write(`| Operator | WebGl Backend |${EOL}`); +doc.write(`|:--------:|:-------------:|${EOL}`); + +let VERSION_MAX = 0; +onnxOpset.forEach(versions => { + versions.forEach(version => VERSION_MAX = Math.max(VERSION_MAX, version)); +}); + +for (const type of opTypes) { + const versions = onnxOpset.get(type)!.sort((a, b) => a - b); + + const webgl: string[] = []; + for (let i = 0; i < versions.length; i++) { + const last = i === versions.length - 1; + const versionRange: [number, number] = [versions[i], last ? VERSION_MAX : versions[i + 1] - 1]; + + webgl.push(formatDesc(type, versionRange, checkSupport(type, versionRange, webglCheckOnlyRules), last)); + } + + doc.write(`| [${type}](https://github.com/onnx/onnx/blob/master/docs/Operators.md#${type}) | ${ + webgl.filter(d => d.length > 0).join(', ')} |${EOL}`); +} +doc.end(); diff --git a/tools/ci_build/github/azure-pipelines/win-wasm-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-wasm-ci-pipeline.yml index c819b02d22..a3121b4f44 100644 --- a/tools/ci_build/github/azure-pipelines/win-wasm-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-wasm-ci-pipeline.yml @@ -189,9 +189,17 @@ jobs: workingDirectory: '$(Build.SourcesDirectory)\js' displayName: 'Clang-format' - script: | - node -e "a=require('child_process').execSync('git ls-files -m').toString();if(a)throw new Error('Following source files are not formatted:\n'+a)" + node -e "a=require('child_process').execSync('git ls-files -m').toString();if(a)throw new Error('Following source files are not formatted: (did you run \"npm run format\"?)\n'+a)" workingDirectory: '$(Build.SourcesDirectory)\js' displayName: 'Check unformatted files' + - script: | + npm run build:doc + workingDirectory: '$(Build.SourcesDirectory)\js\web' + displayName: 'Generating documents' + - script: | + node -e "a=require('child_process').execSync('git ls-files -m').toString();if(a)throw new Error('Following documents are not up-to-date: (did you run \"npm run build:doc\"?)\n'+a)" + workingDirectory: '$(Build.SourcesDirectory)\js\web' + displayName: 'Check out of dated documents' - script: | npm run build workingDirectory: '$(Build.SourcesDirectory)\js\web' From 8140e3fde5c22d8fc95a50fa2f840fa15238daba Mon Sep 17 00:00:00 2001 From: Chen Fu Date: Thu, 27 May 2021 15:05:04 -0700 Subject: [PATCH 26/47] Make requantize a qgemm post processor (#7850) Description: Change requantize interface so it can be processed block by block. This enable as to make requantize to be a post processor of QGEMM. Motivation and Context Previous changes show we improve performance by parallelize batch gemm. Unfortunately we could not parallelize the batch gemm in quantize_linear_matmul due to the requantize operation at the end of each gemm. By changing requantize to be a qgemm post processor, we now can parallelize the batch operation. Co-authored-by: Chen Fu --- onnxruntime/core/mlas/inc/mlas.h | 70 ++++- onnxruntime/core/mlas/lib/qlgavgpool.cpp | 13 +- onnxruntime/core/mlas/lib/quantize.cpp | 264 ++++++++---------- .../cpu/math/quantize_linear_matmul.cc | 63 +++-- .../core/providers/cpu/nn/qlinearconv.cc | 11 +- 5 files changed, 227 insertions(+), 194 deletions(-) diff --git a/onnxruntime/core/mlas/inc/mlas.h b/onnxruntime/core/mlas/inc/mlas.h index ecceb64f18..e9f8e44446 100644 --- a/onnxruntime/core/mlas/inc/mlas.h +++ b/onnxruntime/core/mlas/inc/mlas.h @@ -963,19 +963,83 @@ MlasQuantizeLinear( OutputType ZeroPoint ); +/** + * @brief Requantize a block of the intermediate buffer to the output buffer, + * optionally adding the supplied bias + * + * @param Input Input matrix + * @param InputLeadingDimension Input matrix leading dimension + * @param Output Output matrix + * @param OutputLeadingDimension Output matrix leading dimension + * @param Bias Optional bias vector, to be added + to the input before quantization + * @param Scale Quantization scale + * @param PerColumnScale true if scale is per-column + * @param ZeroPoint quantization zero point value + * @param StartM + * @param StartN + * @param CountM + * @param CountN + * @return +*/ void MLASCALL MlasRequantizeOutput( const int32_t* Input, + size_t InputLeadingDimension, uint8_t* Output, + size_t OutputLeadingDimension, const int32_t* Bias, - size_t M, - size_t N, const float* Scale, bool PerColumnScale, - uint8_t ZeroPoint + uint8_t ZeroPoint, + size_t StartM, + size_t StartN, + size_t CountM, + size_t CountN ); +class MLAS_QGEMM_REQUANT_OUTPUT_PROCESSOR : public MLAS_QGEMM_OUTPUT_PROCESSOR +{ + public: + MLAS_QGEMM_REQUANT_OUTPUT_PROCESSOR( + uint8_t* Output, + size_t OutputLeadingDimension, + const int32_t* Bias, + const float* Scale, + bool PerColumnScale, + uint8_t ZeroPoint) + : Output_(Output), + OutputLeadingDimension_(OutputLeadingDimension), + Bias_(Bias), + Scale_(Scale), + PerColumnScale_(PerColumnScale), + ZeroPoint_(ZeroPoint) + { + } + + void Process(const int32_t* C, + size_t StartM, + size_t StartN, + size_t CountM, + size_t CountN, + size_t ldc) const override + { + MlasRequantizeOutput(C, ldc, Output_, OutputLeadingDimension_, Bias_, Scale_, + PerColumnScale_, ZeroPoint_, StartM, StartN, CountM, CountN); + } + + + private: + uint8_t* Output_; + size_t OutputLeadingDimension_; + const int32_t* Bias_; + const float* Scale_; + bool PerColumnScale_; + uint8_t ZeroPoint_; +}; + + void MLASCALL MlasFindMinMaxElement( diff --git a/onnxruntime/core/mlas/lib/qlgavgpool.cpp b/onnxruntime/core/mlas/lib/qlgavgpool.cpp index 81345dfd97..d8972eecbf 100644 --- a/onnxruntime/core/mlas/lib/qlgavgpool.cpp +++ b/onnxruntime/core/mlas/lib/qlgavgpool.cpp @@ -121,7 +121,9 @@ MlasQLinearGlobalAveragePoolNchw( int32x2_t vacc = vadd_s32(vget_high_s32(vacc_lo), vget_low_s32(vacc_lo)); *sum_buffer++ = vget_lane_s32(vpadd_s32(vacc, vacc), 0); } - MlasRequantizeOutput(AccumulateBuffer, Output, nullptr, 1, Channels, &scale, false, static_cast(ZeroPointOutput)); + + MlasRequantizeOutput(AccumulateBuffer, Channels, Output, Channels, nullptr, &scale, false, + static_cast(ZeroPointOutput), 0, 0, 1, Channels); } MLAS_FORCEINLINE @@ -256,7 +258,8 @@ MlasQLinearGlobalAveragePoolNhwcSingleBatch( vst1q_s32(acc + 4, vacc_hi); } } - MlasRequantizeOutput(AccumulateBuffer, Output, nullptr, 1, Channels, &Scale, false, Output_zero_point); + MlasRequantizeOutput(AccumulateBuffer, Channels, Output, Channels, nullptr, &Scale, false, + Output_zero_point, 0, 0, 1, Channels); } #elif defined(MLAS_SSE2_INTRINSICS) @@ -323,7 +326,8 @@ MlasQLinearGlobalAveragePoolNchw( vsums = _mm_add_epi32(vsums, vshuf); *sum_buffer++ = _mm_cvtsi128_si32(vsums); } - MlasRequantizeOutput(AccumulateBuffer, Output, nullptr, 1, Channels, &scale, false, static_cast(ZeroPointOutput)); + MlasRequantizeOutput(AccumulateBuffer, Channels, Output, Channels, nullptr, &scale, false, + static_cast(ZeroPointOutput), 0, 0, 1, Channels); } MLAS_FORCEINLINE @@ -515,7 +519,8 @@ MlasQLinearGlobalAveragePoolNhwcSingleBatch( _mm_storeu_si128(((__m128i*)acc) + 1, vacc_hi); } } - MlasRequantizeOutput(AccumulateBuffer, Output, nullptr, 1, Channels, &Scale, false, Output_zero_point); + MlasRequantizeOutput(AccumulateBuffer, Channels, Output, Channels, nullptr, &Scale, false, + Output_zero_point, 0, 0, 1, Channels); } #else diff --git a/onnxruntime/core/mlas/lib/quantize.cpp b/onnxruntime/core/mlas/lib/quantize.cpp index facb060218..01a5529fb6 100644 --- a/onnxruntime/core/mlas/lib/quantize.cpp +++ b/onnxruntime/core/mlas/lib/quantize.cpp @@ -356,65 +356,46 @@ void MLASCALL MlasRequantizeOutput( const int32_t* Input, + size_t InputLeadingDimension, uint8_t* Output, + size_t OutputLeadingDimension, const int32_t* Bias, - size_t M, - size_t N, const float* Scale, bool PerColumnScale, - uint8_t ZeroPoint + uint8_t ZeroPoint, + size_t StartM, + size_t StartN, + size_t CountM, + size_t CountN ) -/*++ - -Routine Description: - - This routine requantizes the intermediate buffer to the output buffer - optionally adding the supplied bias. - -Arguments: - - Input - Supplies the input matrix. - - Output - Supplies the output matrix. - - Bias - Supplies the optional bias vector to be added to the input buffer - before requantization. - - Buffer - Supplies the output matrix. - - M - Supplies the number of elements of the bias vector and the number of - rows in the output matrix. - - N - Supplies the number of columns of the output matrix. - - Scale - Supplies the quantization scale. - - PerColumnScale - Supplies true if the quantization scale has per-column - values, else false if a single quantization scale applies to the - entire matrix. - - ZeroPoint - Supplies the quantization zero point value. - -Return Value: - - None. - ---*/ { const __m128 PerMatrixScaleVector = PerColumnScale ? _mm_setzero_ps() : _mm_load1_ps(Scale); const __m128 MinimumValueVector = _mm_set1_ps(float(0 - ZeroPoint)); const __m128 MaximumValueVector = _mm_set1_ps(float(255 - ZeroPoint)); const __m128i ZeroPointVector = _mm_set1_epi32(ZeroPoint); + if (nullptr != Bias) { + Bias += StartN; + } + if (PerColumnScale) { + Scale += StartN; + } + + Input += StartM * InputLeadingDimension + StartN; + Output += StartM * OutputLeadingDimension + StartN; + // // Step through each row of the output matrix. // - while (M-- > 0) { + while (CountM-- > 0) { const int32_t* bias = Bias; const float* scale = PerColumnScale ? Scale : nullptr; - size_t n = N; + size_t n = CountN; + + auto* RowInput = Input; + auto* RowOutput = Output; // // Process 16 columns of the matrices at a time. @@ -426,11 +407,11 @@ Return Value: // Load the input data and optionally add the per-column bias. // - __m128i IntegerVector0 = _mm_loadu_si128((const __m128i *)&Input[0]); - __m128i IntegerVector1 = _mm_loadu_si128((const __m128i *)&Input[4]); - __m128i IntegerVector2 = _mm_loadu_si128((const __m128i *)&Input[8]); - __m128i IntegerVector3 = _mm_loadu_si128((const __m128i *)&Input[12]); - Input += 16; + __m128i IntegerVector0 = _mm_loadu_si128((const __m128i*)&RowInput[0]); + __m128i IntegerVector1 = _mm_loadu_si128((const __m128i*)&RowInput[4]); + __m128i IntegerVector2 = _mm_loadu_si128((const __m128i*)&RowInput[8]); + __m128i IntegerVector3 = _mm_loadu_si128((const __m128i*)&RowInput[12]); + RowInput += 16; if (bias != nullptr) { IntegerVector0 = _mm_add_epi32(IntegerVector0, _mm_loadu_si128((const __m128i *)&bias[0])); @@ -491,8 +472,8 @@ Return Value: __m128i ByteVector = _mm_packus_epi16(WordVector0, WordVector1); - _mm_storeu_si128((__m128i*)Output, ByteVector); - Output += 16; + _mm_storeu_si128((__m128i*)RowOutput, ByteVector); + RowOutput += 16; n -= 16; } @@ -511,8 +492,8 @@ Return Value: if (n >= 4) { - IntegerVector = _mm_loadu_si128((const __m128i*)&Input[0]); - Input += 4; + IntegerVector = _mm_loadu_si128((const __m128i*)&RowInput[0]); + RowInput += 4; if (bias != nullptr) { IntegerVector = _mm_add_epi32(IntegerVector, _mm_loadu_si128((const __m128i*)&bias[0])); @@ -521,7 +502,7 @@ Return Value: } else { - int32_t IntegerValue = *Input++; + int32_t IntegerValue = *RowInput++; if (bias != nullptr) { IntegerValue += *bias++; @@ -567,19 +548,23 @@ Return Value: if (n >= 4) { - *reinterpret_cast(Output) = OutputValue; - Output += 4; + *reinterpret_cast(RowOutput) = OutputValue; + RowOutput += 4; n -= 4; } else { - *Output = uint8_t(OutputValue); - Output += 1; + *RowOutput = uint8_t(OutputValue); + RowOutput += 1; n -= 1; } } + + // Next Row + Input += InputLeadingDimension; + Output += OutputLeadingDimension; } } @@ -589,63 +574,44 @@ void MLASCALL MlasRequantizeOutput( const int32_t* Input, + size_t InputLeadingDimension, uint8_t* Output, + size_t OutputLeadingDimension, const int32_t* Bias, - size_t M, - size_t N, const float* Scale, bool PerColumnScale, - uint8_t ZeroPoint + uint8_t ZeroPoint, + size_t StartM, + size_t StartN, + size_t CountM, + size_t CountN ) -/*++ - -Routine Description: - - This routine requantizes the intermediate buffer to the output buffer - optionally adding the supplied bias. - -Arguments: - - Input - Supplies the input matrix. - - Output - Supplies the output matrix. - - Bias - Supplies the optional bias vector to be added to the input buffer - before requantization. - - Buffer - Supplies the output matrix. - - M - Supplies the number of elements of the bias vector and the number of - rows in the output matrix. - - N - Supplies the number of columns of the output matrix. - - Scale - Supplies the quantization scale. - - PerColumnScale - Supplies true if the quantization scale has per-column - values, else false if a single quantization scale applies to the - entire matrix. - - ZeroPoint - Supplies the quantization zero point value. - -Return Value: - - None. - ---*/ { const float32x4_t PerMatrixScaleVector = PerColumnScale ? vdupq_n_f32(0) : vld1q_dup_f32(Scale); const int16x8_t ZeroPointVector = vdupq_n_s16(ZeroPoint); + if (nullptr != Bias) { + Bias += StartN; + } + if (PerColumnScale) { + Scale += StartN; + } + + Input += StartM * InputLeadingDimension + StartN; + Output += StartM * OutputLeadingDimension + StartN; + // // Step through each row of the output matrix. // - while (M-- > 0) { + while (CountM-- > 0) { const int32_t* bias = Bias; const float* scale = PerColumnScale ? Scale : nullptr; - size_t n = N; + size_t n = CountN; + + auto* RowInput = Input; + auto* RowOutput = Output; // // Process 16 columns of the matrices at a time. @@ -659,11 +625,11 @@ Return Value: int32x4x4_t IntegerVector; - IntegerVector.val[0] = vld1q_s32(&Input[0]); - IntegerVector.val[1] = vld1q_s32(&Input[4]); - IntegerVector.val[2] = vld1q_s32(&Input[8]); - IntegerVector.val[3] = vld1q_s32(&Input[12]); - Input += 16; + IntegerVector.val[0] = vld1q_s32(&RowInput[0]); + IntegerVector.val[1] = vld1q_s32(&RowInput[4]); + IntegerVector.val[2] = vld1q_s32(&RowInput[8]); + IntegerVector.val[3] = vld1q_s32(&RowInput[12]); + RowInput += 16; if (bias != nullptr) { IntegerVector.val[0] = vaddq_s32(IntegerVector.val[0], vld1q_s32(&bias[0])); @@ -731,8 +697,8 @@ Return Value: WordVector.val[0] = vqaddq_s16(WordVector.val[0], ZeroPointVector); WordVector.val[1] = vqaddq_s16(WordVector.val[1], ZeroPointVector); - vst1q_u8(Output, vqmovun_high_s16(vqmovun_s16(WordVector.val[0]), WordVector.val[1])); - Output += 16; + vst1q_u8(RowOutput, vqmovun_high_s16(vqmovun_s16(WordVector.val[0]), WordVector.val[1])); + RowOutput += 16; n -= 16; } @@ -751,8 +717,8 @@ Return Value: if (n >= 4) { - IntegerVector = vld1q_s32(&Input[0]); - Input += 4; + IntegerVector = vld1q_s32(&RowInput[0]); + RowInput += 4; if (bias != nullptr) { IntegerVector = vaddq_s32(IntegerVector, vld1q_s32(&bias[0])); @@ -761,8 +727,8 @@ Return Value: } else { - IntegerVector = vld1q_dup_s32(Input); - Input += 1; + IntegerVector = vld1q_dup_s32(RowInput); + RowInput += 1; if (bias != nullptr) { IntegerVector = vaddq_s32(IntegerVector, vld1q_dup_s32(bias)); @@ -813,19 +779,24 @@ Return Value: if (n >= 4) { - vst1q_lane_u32(reinterpret_cast(Output), vreinterpretq_u32_u8(ByteVector), 0); - Output += 4; + vst1q_lane_u32(reinterpret_cast(RowOutput), + vreinterpretq_u32_u8(ByteVector), 0); + RowOutput += 4; n -= 4; } else { - vst1q_lane_u8(Output, ByteVector, 0); - Output += 1; + vst1q_lane_u8(RowOutput, ByteVector, 0); + RowOutput += 1; n -= 1; } } + + // Next Row + Input += InputLeadingDimension; + Output += OutputLeadingDimension; } } @@ -835,68 +806,49 @@ void MLASCALL MlasRequantizeOutput( const int32_t* Input, + size_t InputLeadingDimension, uint8_t* Output, + size_t OutputLeadingDimension, const int32_t* Bias, - size_t M, - size_t N, const float* Scale, bool PerColumnScale, - uint8_t ZeroPoint + uint8_t ZeroPoint, + size_t StartM, + size_t StartN, + size_t CountM, + size_t CountN ) -/*++ - -Routine Description: - - This routine requantizes the intermediate buffer to the output buffer - optionally adding the supplied bias. - -Arguments: - - Input - Supplies the input matrix. - - Output - Supplies the output matrix. - - Bias - Supplies the optional bias vector to be added to the input buffer - before requantization. - - Buffer - Supplies the output matrix. - - M - Supplies the number of elements of the bias vector and the number of - rows in the output matrix. - - N - Supplies the number of columns of the output matrix. - - Scale - Supplies the quantization scale. - - PerColumnScale - Supplies true if the quantization scale has per-column - values, else false if a single quantization scale applies to the - entire matrix. - - ZeroPoint - Supplies the quantization zero point value. - -Return Value: - - None. - ---*/ { const float PerMatrixScaleValue = PerColumnScale ? 0.0f : *Scale; const float MinimumValue = float(0 - ZeroPoint); const float MaximumValue = float(255 - ZeroPoint); + if (nullptr != Bias) { + Bias += StartN; + } + if (PerColumnScale) { + Scale += StartN; + } + + Input += StartM * InputLeadingDimension + StartN; + Output += StartM * OutputLeadingDimension + StartN; + // // Step through each row of the output matrix. // - while (M-- > 0) { + while (CountM-- > 0) { const int32_t* bias = Bias; const float* scale = Scale; - size_t n = N; + size_t n = CountN; + + auto* RowInput = Input; + auto* RowOutput = Output; while (n > 0) { - int32_t IntegerValue = *Input++; + int32_t IntegerValue = *RowInput++; if (bias != nullptr) { IntegerValue += *bias++; @@ -920,10 +872,14 @@ Return Value: IntegerValue = int32_t(MlasBitsOfFp32(FloatValue + MLAS_ROUNDING_BIAS_MAGIC)) - MLAS_ROUNDING_BIAS_MAGIC_BITS; - *Output++ = uint8_t(IntegerValue + ZeroPoint); + *RowOutput++ = uint8_t(IntegerValue + ZeroPoint); n -= 1; } + + // Next Row + Input += InputLeadingDimension; + Output += OutputLeadingDimension; } } diff --git a/onnxruntime/core/providers/cpu/math/quantize_linear_matmul.cc b/onnxruntime/core/providers/cpu/math/quantize_linear_matmul.cc index 60068885b6..6e5b780a4d 100644 --- a/onnxruntime/core/providers/cpu/math/quantize_linear_matmul.cc +++ b/onnxruntime/core/providers/cpu/math/quantize_linear_matmul.cc @@ -78,47 +78,52 @@ Status QLinearMatMul::Compute(OpKernelContext* ctx) const { output_scales[i] = (a_scale_data * b_scale_data[i] / y_scale_data); } - AllocatorPtr alloc; - ORT_RETURN_IF_ERROR(ctx->GetTempSpaceAllocator(&alloc)); - auto gemm_output_data = alloc->Alloc(SafeInt(sizeof(int32_t)) * - static_cast(helper.M()) * static_cast(helper.N())); - BufferUniquePtr gemm_output_buffer(gemm_output_data, BufferDeleter(alloc)); - auto* gemm_output = static_cast(gemm_output_buffer.get()); - + const size_t num_gemms = helper.OutputOffsets().size(); MLAS_GEMM_U8X8_SHAPE_PARAMS gemm_shape; gemm_shape.M = static_cast(helper.M()); gemm_shape.N = static_cast(helper.N()); gemm_shape.K = static_cast(helper.K()); gemm_shape.BIsSigned = b_is_signed; - MLAS_GEMM_U8X8_DATA_PARAMS gemm_params; - gemm_params.lda = gemm_shape.K; - gemm_params.ZeroPointA = *a_offset->template Data(); - gemm_params.ldb = gemm_shape.N; - gemm_params.C = gemm_output; - gemm_params.ldc = gemm_shape.N; - gemm_params.BIsPacked = bool(packed_b_); - gemm_params.PerColumnZeroPoints = !IsScalarOr1ElementVector(b_offset); + AllocatorPtr alloc; + ORT_RETURN_IF_ERROR(ctx->GetTempSpaceAllocator(&alloc)); + auto gemm_output_data = alloc->Alloc(SafeInt(gemm_shape.M) * + gemm_shape.N * sizeof(int32_t) * num_gemms); + BufferUniquePtr gemm_output_buffer(gemm_output_data, BufferDeleter(alloc)); + auto* gemm_output = static_cast(gemm_output_buffer.get()); + + + std::vector gemm_params(num_gemms); + std::vector requant_procs; + requant_procs.reserve(num_gemms); auto b_zp_data = static_cast(b_offset->DataRaw()); - for (size_t i = 0; i < helper.OutputOffsets().size(); i++) { - gemm_params.A = a->template Data() + helper.LeftOffsets()[i]; - gemm_params.B = b_data + helper.RightOffsets()[i]; - gemm_params.ZeroPointB = b_zp_data + helper.RightZeroPointOffsets()[i]; + for (size_t i = 0; i < num_gemms; i++) { + gemm_params[i].A = a->template Data() + helper.LeftOffsets()[i]; + gemm_params[i].lda = gemm_shape.K; + gemm_params[i].ZeroPointA = *a_offset->template Data(); - MlasGemm(gemm_shape, gemm_params, ctx->GetOperatorThreadPool()); + gemm_params[i].B = b_data + helper.RightOffsets()[i]; + gemm_params[i].ldb = gemm_shape.N; + gemm_params[i].BIsPacked = bool(packed_b_); + gemm_params[i].ZeroPointB = b_zp_data + helper.RightZeroPointOffsets()[i]; - //TODO!! consider making this a post processor, so that we can parallize this loop - MlasRequantizeOutput(gemm_output, - y->template MutableData() + helper.OutputOffsets()[i], - nullptr, - static_cast(helper.M()), - static_cast(helper.N()), - output_scales.data() + helper.RightScaleOffsets()[i], - output_scales.size() > 1, - *y_offset->template Data()); + gemm_params[i].C = gemm_output + (gemm_shape.M * gemm_shape.N * i); + gemm_params[i].ldc = gemm_shape.N; + + gemm_params[i].PerColumnZeroPoints = !IsScalarOr1ElementVector(b_offset); + + requant_procs.emplace_back(y->template MutableData() + helper.OutputOffsets()[i], + static_cast(helper.N()), + nullptr, + output_scales.data() + helper.RightScaleOffsets()[i], + output_scales.size() > 1, + *y_offset->template Data()); + gemm_params[i].OutputProcessor = &(requant_procs[i]); } + MlasGemmBatch(gemm_shape, gemm_params.data(), num_gemms, ctx->GetOperatorThreadPool()); + return Status::OK(); } diff --git a/onnxruntime/core/providers/cpu/nn/qlinearconv.cc b/onnxruntime/core/providers/cpu/nn/qlinearconv.cc index f5fe4bf4fe..882138d50b 100644 --- a/onnxruntime/core/providers/cpu/nn/qlinearconv.cc +++ b/onnxruntime/core/providers/cpu/nn/qlinearconv.cc @@ -590,13 +590,16 @@ Status QLinearConv::Compute(OpKernelContext* context) const { MlasRequantizeOutput( worker_gemm_output, - worker_requantize_output, - Bdata, - static_cast(output_count), static_cast(M), + worker_requantize_output, + static_cast(M), + Bdata, output_scales.data(), output_scales.size() > 1, - Y_zero_point_value); + Y_zero_point_value, + 0,0, + static_cast(output_count), + static_cast(M)); }; concurrency::ThreadPool::TrySimpleParallelFor(thread_pool, thread_count, conv_worker); From ddf4aaaae1c6ad214f146a2cbd8d5b4bec818013 Mon Sep 17 00:00:00 2001 From: baijumeswani Date: Thu, 27 May 2021 16:11:37 -0700 Subject: [PATCH 27/47] Resolve issue with wrapped ORTModule load_state_dict (#7847) * Encapsulate children modules inside a ModuleAccessor object to prevent erroneuos iteration over children while loading the state dictionary * Add named_models, models, apply methods, change ModuleAccessor to ModuleMetadata and modify unit tests * Change ModuleMetadata module getter logic, raise NotImplementedError for add_modules * Add comment explaining why overriding _load_from_state_dict method is needed --- .../python/training/ortmodule/_utils.py | 7 ++ .../python/training/ortmodule/ortmodule.py | 114 +++++++++++++----- .../python/orttraining_test_ortmodule_api.py | 42 +++++-- 3 files changed, 127 insertions(+), 36 deletions(-) diff --git a/orttraining/orttraining/python/training/ortmodule/_utils.py b/orttraining/orttraining/python/training/ortmodule/_utils.py index 98d553bcac..751c5f1a46 100644 --- a/orttraining/orttraining/python/training/ortmodule/_utils.py +++ b/orttraining/orttraining/python/training/ortmodule/_utils.py @@ -97,3 +97,10 @@ def _create_iobinding(io_binding, inputs, model, device): for value_info in model.graph.output: io_binding.bind_output(value_info.name, device.type, device_id=get_device_index(device)) + +class _PytorchModuleMetadata(): + """Encapsulates modules and allows easy access as required""" + + def __init__(self, original_module, flattened_module): + self.original_module = original_module + self.flattened_module = flattened_module diff --git a/orttraining/orttraining/python/training/ortmodule/ortmodule.py b/orttraining/orttraining/python/training/ortmodule/ortmodule.py index 62d1c7ee46..bfdc1c5631 100644 --- a/orttraining/orttraining/python/training/ortmodule/ortmodule.py +++ b/orttraining/orttraining/python/training/ortmodule/ortmodule.py @@ -5,12 +5,13 @@ from . import _io from ._graph_execution_manager_factory import GraphExecutionManagerFactory +from ._utils import _PytorchModuleMetadata from onnxruntime.training import register_custom_ops_pytorch_exporter import functools import torch -from typing import Iterator, Optional, Tuple, TypeVar +from typing import Iterator, Optional, Tuple, TypeVar, Set, Callable # Needed to override PyTorch methods T = TypeVar('T', bound='Module') @@ -51,12 +52,11 @@ class ORTModule(torch.nn.Module): register_custom_ops_pytorch_exporter.register_custom_op(is_ortmodule=True) # User module is wrapped to use its initializers and save computed gradients - self._original_module = module + # along with the module that flattens both input and output of the user module + # inside _PytorchModuleMetadata + self._module_metadata = _PytorchModuleMetadata(module, _io._FlattenedModule(module)) - # Get the module that flattens both input and output - self._flattened_module = _io._FlattenedModule(self._original_module) - - self._execution_manager = GraphExecutionManagerFactory(self._flattened_module) + self._execution_manager = GraphExecutionManagerFactory(self._module_metadata.flattened_module) # IMPORTANT: DO NOT add code here # This declaration is for automatic document generation purposes only @@ -65,57 +65,82 @@ class ORTModule(torch.nn.Module): '''Dummy documentation for forward method''' ... + def _apply(self, fn): + """Override original method to delegate execution to the flattened PyTorch user module""" + + # Delegation must happen to _flattened_module since methods depend on + # _apply to recursively apply the internal setting changes + self._module_metadata.flattened_module._apply(fn) + return self + + def apply(self: T, fn: Callable[['Module'], None]) -> T: + """Override original method to delegate execution to the flattened PyTorch user module""" + + # Delegation must happen to _flattened_module since methods depend on + # apply to recursively apply the internal setting changes + self._module_metadata.flattened_module.apply(fn) + return self + def _is_training(self): - return self._flattened_module.training and torch.is_grad_enabled() + return self.training and torch.is_grad_enabled() + + def train(self: T, mode: bool = True) -> T: + """Override original method to delegate execution to the flattened PyTorch user module""" + + # Since _modules is empty, the task needs to be delegated to _module.flattened_module.train + # which will recursively update the original_module + self.training = mode + self._module_metadata.flattened_module.train(mode) + return self def state_dict(self, destination=None, prefix='', keep_vars=False): - """Override original method to delegate execution to the base module""" + """Override original method to delegate execution to the original PyTorch user module""" # Override the state_dict() method so that the state dict key names - # do not contain the _flattened_module._original_module prefix - return self._original_module.state_dict( + # do not contain the flattened_module._original_module prefix + return self._module_metadata.original_module.state_dict( destination=destination, prefix=prefix, keep_vars=keep_vars) def load_state_dict(self, state_dict: 'OrderedDict[str, Tensor]', strict: bool = True): - """Override original method to delegate execution to the base module""" + """Override original method to delegate execution to the original PyTorch user module""" # Override the load_state_dict() method so that the loaded state dict - # key names does not need to contain the _flattened_module._original_module prefix - return self._original_module.load_state_dict( + # key names does not need to contain the _module.flattened_module._original_module prefix + return self._module_metadata.original_module.load_state_dict( state_dict, strict=strict) def register_buffer(self, name: str, tensor: Optional[torch.Tensor], persistent: bool = True) -> None: - """Override original method to delegate execution to the base module""" - self._original_module.register_buffer(name, tensor, persistent=persistent) + """Override original method to delegate execution to the original PyTorch user module""" + self._module_metadata.original_module.register_buffer(name, tensor, persistent=persistent) def register_parameter(self, name: str, param: Optional[torch.nn.Parameter]) -> None: - """Override original method to delegate execution to the base module""" - self._original_module.register_parameter(name, param) + """Override original method to delegate execution to the original PyTorch user module""" + self._module_metadata.original_module.register_parameter(name, param) def get_parameter(self, target: str) -> torch.nn.Parameter: - """Override original method to delegate execution to the base module""" - return self._original_module.get_parameter(target) + """Override original method to delegate execution to the original PyTorch user module""" + return self._module_metadata.original_module.get_parameter(target) def get_buffer(self, target: str) -> torch.Tensor: - """Override original method to delegate execution to the base module""" - return self._original_module.get_buffer(target) + """Override original method to delegate execution to the original PyTorch user module""" + return self._module_metadata.original_module.get_buffer(target) def parameters(self, recurse: bool = True) -> Iterator[torch.nn.Parameter]: - """Override original method to delegate execution to the base module""" - yield from self._original_module.parameters(recurse=recurse) + """Override original method to delegate execution to the original PyTorch user module""" + yield from self._module_metadata.original_module.parameters(recurse=recurse) def named_parameters(self, prefix: str = '', recurse: bool = True) -> Iterator[Tuple[str, torch.nn.Parameter]]: - """Override original method to delegate execution to the base module""" - yield from self._original_module.named_parameters(prefix=prefix, recurse=recurse) + """Override original method to delegate execution to the original PyTorch user module""" + yield from self._module_metadata.original_module.named_parameters(prefix=prefix, recurse=recurse) def buffers(self, recurse: bool = True) -> Iterator[torch.Tensor]: - """Override original method to delegate execution to the base module""" - yield from self._original_module.buffers(recurse=recurse) + """Override original method to delegate execution to the original PyTorch user module""" + yield from self._module_metadata.original_module.buffers(recurse=recurse) def named_buffers(self, prefix: str = '', recurse: bool = True) -> Iterator[Tuple[str, torch.Tensor]]: - """Override original method to delegate execution to the base module""" - yield from self._original_module.named_buffers(prefix=prefix, recurse=recurse) + """Override original method to delegate execution to the original PyTorch user module""" + yield from self._module_metadata.original_module.named_buffers(prefix=prefix, recurse=recurse) def _replicate_for_data_parallel(self): """Raises a NotImplementedError exception since ORTModule is not compatible with torch.nn.DataParallel @@ -135,3 +160,34 @@ class ORTModule(torch.nn.Module): raise NotImplementedError("ORTModule is not compatible with torch.nn.DataParallel. " "Please use torch.nn.parallel.DistributedDataParallel instead.") + + def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, + missing_keys, unexpected_keys, error_msgs): + """Override original method to delegate execution to the original PyTorch user module""" + + # PyTorch load_state_dict implementation does not recursively call load_state_dict on its sub-modules. + # Instead, it creates a recursive function and invokes _load_from_state_dict on all child modules. + # For the scenario where an ORTModule is a sub-module of another module, loading of the state + # dictionary requires the _load_from_state_dict to be overridden to prevent an error. + self._module_metadata.original_module._load_from_state_dict(state_dict, prefix, local_metadata, strict, + missing_keys, unexpected_keys, error_msgs) + + def named_children(self) -> Iterator[Tuple[str, 'Module']]: + """Override original method to delegate execution to the original PyTorch user module""" + + yield from self._module_metadata.original_module.named_children() + + def modules(self) -> Iterator['Module']: + """Override original method to delegate execution to the original PyTorch user module""" + + yield from self._module_metadata.original_module.modules() + + def named_modules(self, memo: Optional[Set['Module']] = None, prefix: str = ''): + """Override original method to delegate execution to the original PyTorch user module""" + + yield from self._module_metadata.original_module.named_modules(memo, prefix) + + def add_module(self, name: str, module: Optional['Module']) -> None: + """Raises a NotImplementedError exception since ORTModule does not support adding modules to it""" + + raise NotImplementedError("ORTModule does not support adding modules to it.") diff --git a/orttraining/orttraining/test/python/orttraining_test_ortmodule_api.py b/orttraining/orttraining/test/python/orttraining_test_ortmodule_api.py index 0dca7caa3f..b44d169e9b 100644 --- a/orttraining/orttraining/test/python/orttraining_test_ortmodule_api.py +++ b/orttraining/orttraining/test/python/orttraining_test_ortmodule_api.py @@ -1666,26 +1666,26 @@ def test_model_initializer_requires_grad_changes_from_one_forward_to_next(): model.fc1.requires_grad_(True) model = ORTModule(model) x = torch.randn(N, D_in, device=device) - assert model._original_module.fc1.weight.grad is None - assert model._original_module.fc1.bias.grad is None + assert model._module_metadata.original_module.fc1.weight.grad is None + assert model._module_metadata.original_module.fc1.bias.grad is None # Make sure no exception is raised output = model(x) loss = torch.sum(output) loss.backward() training_session1 = model._execution_manager(model._is_training())._execution_agent - weight_grad_2 = model._original_module.fc1.weight.grad - bias_grad_2 = model._original_module.fc1.bias.grad + weight_grad_2 = model._module_metadata.original_module.fc1.weight.grad + bias_grad_2 = model._module_metadata.original_module.fc1.bias.grad assert weight_grad_2 is not None assert bias_grad_2 is not None - model._original_module.fc1.requires_grad_(False) + model._module_metadata.original_module.fc1.requires_grad_(False) output = model(x) loss = torch.sum(output) loss.backward() training_session2 = model._execution_manager(model._is_training())._execution_agent - weight_grad_3 = model._original_module.fc1.weight.grad - bias_grad_3 = model._original_module.fc1.bias.grad + weight_grad_3 = model._module_metadata.original_module.fc1.weight.grad + bias_grad_3 = model._module_metadata.original_module.fc1.bias.grad assert training_session1 != training_session2 assert torch.equal(weight_grad_2, weight_grad_3) @@ -2619,3 +2619,31 @@ def test_unused_parameters_does_not_unnecssarily_reinitilize(model): {}) assert not training_manager._reinitialize_graph_builder(input_info) + +def test_load_state_dict_for_wrapped_ortmodule(): + class WrapperModule(torch.nn.Module): + def __init__(self, ortmodule): + super(WrapperModule, self).__init__() + self._ortmodule = ortmodule + + def forward(self, x): + return self._ortmodule(x) + + device = 'cuda' + N, D_in, H, D_out = 64, 784, 500, 10 + model = NeuralNetSinglePositionalArgument(D_in, H, D_out).to(device) + model = ORTModule(copy.deepcopy(model)) + wrapper_module = WrapperModule(model) + x = torch.randn(N, D_in, device=device) + _ = wrapper_module(x) + + state_dict1 = wrapper_module.state_dict() + list(next(iter(state_dict1.items())))[1] += 10 + wrapper_module.load_state_dict(state_dict1) + state_dict2 = wrapper_module.state_dict() + + assert state_dict1 + assert len(state_dict1.keys()) == len(state_dict2.keys()) + for param_name, param_value in state_dict1.items(): + assert param_name in state_dict2 + assert torch.equal(param_value, state_dict2[param_name]) From 63df683040e28329f74b1adad78235fba24f47be Mon Sep 17 00:00:00 2001 From: Scott McKay Date: Fri, 28 May 2021 09:32:13 +1000 Subject: [PATCH 28/47] Fix path used in check for cudnn library (#7786) * There are separate paths for CUDA and CUDNN as they are not guaranteed to be in the same location on a Windows machine. Use the CUDNN path when looking for the CUDNN library. * Refine check --- onnxruntime/python/_pybind_state.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/onnxruntime/python/_pybind_state.py b/onnxruntime/python/_pybind_state.py index 2e3d2c8a58..e76d402681 100644 --- a/onnxruntime/python/_pybind_state.py +++ b/onnxruntime/python/_pybind_state.py @@ -33,8 +33,16 @@ if platform.system() == "Windows": raise ImportError(f"CUDA Toolkit {cuda_version_major}.x not installed on the machine.") cuda_bin_dir = os.path.join(os.environ[cuda_env_variable], "bin") - if not os.path.isfile(os.path.join(cuda_bin_dir, f"cudnn64_{version_info.cudnn_version}.dll")): - raise ImportError(f"cuDNN {version_info.cudnn_version} not installed in {cuda_bin_dir}.") + + # prefer CUDNN_HOME if set. fallback to the CUDA install directory (would have required user to manually + # copy the cudnn dll there + cudnn_path = os.environ["CUDNN_HOME"] if "CUDNN_HOME" in os.environ else os.environ[cuda_env_variable] + cudnn_bin_dir = os.path.join(cudnn_path, "bin") + + if not os.path.isfile(os.path.join(cudnn_bin_dir, f"cudnn64_{version_info.cudnn_version}.dll")): + raise ImportError(f"cuDNN {version_info.cudnn_version} not installed in {cudnn_bin_dir}. " + f"Set the CUDNN_HOME environment variable to the path of the 'cuda' directory " + f"in your CUDNN installation if necessary.") if sys.version_info >= (3, 8): # Python 3.8 (and later) doesn't search system PATH when loading DLLs, so the CUDA location needs to be From 1f4421fe70fb96f67d2fd5b098b90c171d5ef273 Mon Sep 17 00:00:00 2001 From: Guoyu Wang <62914304+gwang-msft@users.noreply.github.com> Date: Thu, 27 May 2021 17:07:48 -0700 Subject: [PATCH 29/47] Include ORT C/C++ API headers in the ORT Mobile AAR package (#7858) * Add header files of ort c/c++ api to aar package * Move header file selection to cmake based on EP choice --- cmake/onnxruntime.cmake | 47 +++++++++++++------ .../github/android/build_aar_package.py | 33 +++++++++++++ 2 files changed, 65 insertions(+), 15 deletions(-) diff --git a/cmake/onnxruntime.cmake b/cmake/onnxruntime.cmake index b542490cee..ec7f2b06de 100644 --- a/cmake/onnxruntime.cmake +++ b/cmake/onnxruntime.cmake @@ -14,6 +14,25 @@ if (${CMAKE_SYSTEM_NAME} STREQUAL "iOS") set(OUTPUT_STYLE xcode) endif() +# This macro is to get the path of header files for mobile packaging, for iOS and Android +macro(get_mobile_api_headers _HEADERS) + # include both c and cxx api + set(${_HEADERS} + "${REPO_ROOT}/include/onnxruntime/core/session/onnxruntime_c_api.h" + "${REPO_ROOT}/include/onnxruntime/core/session/onnxruntime_cxx_api.h" + "${REPO_ROOT}/include/onnxruntime/core/session/onnxruntime_cxx_inline.h" + ) + + # need to add header files for enabled EPs + foreach(f ${ONNXRUNTIME_PROVIDER_NAMES}) + file(GLOB _provider_headers CONFIGURE_DEPENDS + "${REPO_ROOT}/include/onnxruntime/core/providers/${f}/*.h" + ) + list(APPEND ${_HEADERS} "${_provider_headers}") + unset(_provider_headers) + endforeach() +endmacro() + #If you want to verify if there is any extra line in symbols.txt, run # nm -C -g --defined libonnxruntime.so |grep -v '\sA\s' | cut -f 3 -d ' ' | sort # after build @@ -39,21 +58,7 @@ if(WIN32) "${ONNXRUNTIME_ROOT}/core/dll/onnxruntime.rc" ) elseif(onnxruntime_BUILD_APPLE_FRAMEWORK) - # include both c and cxx api - set(APPLE_FRAMEWORK_HEADERS - "${REPO_ROOT}/include/onnxruntime/core/session/onnxruntime_c_api.h" - "${REPO_ROOT}/include/onnxruntime/core/session/onnxruntime_cxx_api.h" - "${REPO_ROOT}/include/onnxruntime/core/session/onnxruntime_cxx_inline.h" - ) - - # need to add header files for enabled EPs - foreach(f ${ONNXRUNTIME_PROVIDER_NAMES}) - file(GLOB _provider_headers CONFIGURE_DEPENDS - "${REPO_ROOT}/include/onnxruntime/core/providers/${f}/*.h" - ) - list(APPEND APPLE_FRAMEWORK_HEADERS "${_provider_headers}") - unset(_provider_headers) - endforeach() + get_mobile_api_headers(APPLE_FRAMEWORK_HEADERS) # apple framework requires the header file be part of the library onnxruntime_add_shared_library(onnxruntime @@ -132,6 +137,18 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Android" OR (onnxruntime_MINIMAL_BUILD AND UNIX)) endif() endif() +# we need to copy C/C++ API headers to be packed into Android AAR package +if(CMAKE_SYSTEM_NAME STREQUAL "Android" AND onnxruntime_BUILD_JAVA) + get_mobile_api_headers(ANDROID_AAR_HEADERS) + set(ANDROID_HEADERS_DIR ${CMAKE_CURRENT_BINARY_DIR}/android/headers) + file(MAKE_DIRECTORY ${ANDROID_HEADERS_DIR}) + # copy the header files one by one + foreach(h_ ${ANDROID_AAR_HEADERS}) + get_filename_component(HEADER_NAME_ ${h_} NAME) + configure_file(${h_} ${ANDROID_HEADERS_DIR}/${HEADER_NAME_} COPYONLY) + endforeach() +endif() + target_link_libraries(onnxruntime PRIVATE onnxruntime_session ${onnxruntime_libs} diff --git a/tools/ci_build/github/android/build_aar_package.py b/tools/ci_build/github/android/build_aar_package.py index d475a6e44d..5678c5d037 100644 --- a/tools/ci_build/github/android/build_aar_package.py +++ b/tools/ci_build/github/android/build_aar_package.py @@ -72,6 +72,29 @@ def _parse_build_settings(args): return build_settings +# Add ORT C and C++ API headers to the AAR package (in fact a zip file) +# Such that developers using ORT native API can extract libraries and header from AAR package without building ORT +# TODO, see if we can use Gradle to add headers to AAR package directly, which is necessary if we want to +# publish the packagee directly using Gradle in the pipeline +def _add_headers_to_aar(aar_file_path, header_files_path): + import shutil + import tempfile + with tempfile.TemporaryDirectory() as temp_dir: + aar_content = os.path.join(temp_dir, 'aar_content') + shutil.unpack_archive(aar_file_path, aar_content, 'zip') + + # copy necessary header files + shutil.copytree(header_files_path, os.path.join(aar_content, 'headers')) + + # create the zip archive + zip_base_filename = os.path.join(temp_dir, 'aar_with_headers') + zip_filename = zip_base_filename + '.zip' + shutil.make_archive(zip_base_filename, 'zip', root_dir=aar_content) + + # overwrite the existing AAR package + shutil.move(zip_filename, aar_file_path) + + def _build_aar(args): build_settings = _parse_build_settings(args) build_dir = os.path.abspath(args.build_dir) @@ -89,6 +112,7 @@ def _build_aar(args): _base_build_command = [ sys.executable, BUILD_PY, '--config=' + _build_config ] + build_settings['build_params'] + header_files_path = '' # Build binary for each ABI, one by one for abi in build_settings['build_abis']: @@ -116,6 +140,10 @@ def _build_aar(args): os.remove(_target_lib_name) os.symlink(os.path.join(_build_dir, _build_config, lib_name), _target_lib_name) + # we only need to define the header files path once + if not header_files_path: + header_files_path = os.path.join(_build_dir, _build_config, 'android', 'headers') + # The directory to publish final AAR _aar_publish_dir = os.path.join(build_dir, 'aar_out', _build_config) os.makedirs(_aar_publish_dir, exist_ok=True) @@ -139,6 +167,11 @@ def _build_aar(args): # clean, build, and publish to a local directory subprocess.run(_gradle_command + ['clean'], env=_env, shell=_shell, check=True, cwd=JAVA_ROOT) subprocess.run(_gradle_command + ['build'], env=_env, shell=_shell, check=True, cwd=JAVA_ROOT) + + # add C and C++ API headers to the intermediate aar package + aar_file_path = os.path.join(_aar_dir, 'outputs', 'aar', 'onnxruntime-release.aar') + _add_headers_to_aar(aar_file_path, header_files_path) + subprocess.run(_gradle_command + ['publish'], env=_env, shell=_shell, check=True, cwd=JAVA_ROOT) From 71b05f74a2d26379129f69b842b1ea0ae09127f6 Mon Sep 17 00:00:00 2001 From: Tianlei Wu Date: Thu, 27 May 2021 17:16:17 -0700 Subject: [PATCH 30/47] fix duplicated node name (#7865) --- .../python/tools/quantization/onnx_model.py | 1 - .../tools/transformers/fusion_layernorm.py | 2 +- .../python/tools/transformers/onnx_model.py | 44 ++++++++++++------- 3 files changed, 30 insertions(+), 17 deletions(-) diff --git a/onnxruntime/python/tools/quantization/onnx_model.py b/onnxruntime/python/tools/quantization/onnx_model.py index 8e6d70c4bb..dc41b8efbf 100644 --- a/onnxruntime/python/tools/quantization/onnx_model.py +++ b/onnxruntime/python/tools/quantization/onnx_model.py @@ -7,7 +7,6 @@ from pathlib import Path class ONNXModel: def __init__(self, model): self.model = model - self.node_name_counter = {} def nodes(self): return self.model.graph.node diff --git a/onnxruntime/python/tools/transformers/fusion_layernorm.py b/onnxruntime/python/tools/transformers/fusion_layernorm.py index 0aa600aac8..57c110dd64 100644 --- a/onnxruntime/python/tools/transformers/fusion_layernorm.py +++ b/onnxruntime/python/tools/transformers/fusion_layernorm.py @@ -112,7 +112,7 @@ class FusionLayerNormalization(Fusion): inputs=[node.input[0], weight_input, bias_input], outputs=[last_add_node.output[0]], name=self.model.create_node_name("LayerNormalization", - name_prefix="SkipLayerNorm")) + name_prefix="LayerNorm")) normalize_node.attribute.extend([helper.make_attribute("epsilon", float(add_weight))]) self.nodes_to_add.append(normalize_node) self.node_name_to_graph_name[normalize_node.name] = self.this_graph_name diff --git a/onnxruntime/python/tools/transformers/onnx_model.py b/onnxruntime/python/tools/transformers/onnx_model.py index 9244421c4b..45914afcb0 100644 --- a/onnxruntime/python/tools/transformers/onnx_model.py +++ b/onnxruntime/python/tools/transformers/onnx_model.py @@ -3,7 +3,7 @@ # Licensed under the MIT License. #-------------------------------------------------------------------------- -from typing import List, Tuple +from typing import List, Tuple, Dict import logging import os import sys @@ -19,7 +19,7 @@ logger = logging.getLogger(__name__) class OnnxModel: def __init__(self, model): self.model = model - self.node_name_counter = {} + self._node_name_suffix: Dict[str, int] = {} # key is node name prefix, value is the last suffix generated self.shape_infer_helper = None self.all_graphs = None @@ -553,25 +553,39 @@ class OnnxModel: cast_node.attribute.extend([helper.make_attribute("to", int(TensorProto.FLOAT))]) self.add_node(cast_node) - # create a new name for node def create_node_name(self, op_type, name_prefix=None): - if op_type in self.node_name_counter: - self.node_name_counter[op_type] += 1 + """Create a unique node name that starts with a prefix (default is operator type). + The name will not be duplicated with any name that generated or existed in current graphs. + Args: + op_type (str): operator type + name_prefix (str, optional): prefix of node name. Defaults to None. + + Returns: + str: node name + """ + + if name_prefix: + prefix = name_prefix if name_prefix.endswith("_") else (name_prefix + "_") else: - self.node_name_counter[op_type] = 1 + prefix = op_type + "_" - if name_prefix is not None: - full_name = name_prefix + str(self.node_name_counter[op_type]) + suffix: int = 0 + if prefix in self._node_name_suffix: + suffix = self._node_name_suffix[prefix] + 1 else: - full_name = op_type + "_" + str(self.node_name_counter[op_type]) + # Check existed node name only once for a prefix as we assume create_node_name is called for every new node in fusion. + for node in self.nodes(): + if node.name and node.name.startswith(prefix): + try: + index = int(node.name[len(prefix):]) + suffix = max(index + 1, suffix) + except ValueError: + continue - # Check whether the name is taken: - nodes = self.get_nodes_by_op_type(op_type) - for node in nodes: - if node.name == full_name: - raise Exception("Node name already taken:", full_name) + # Record the generated suffix so that we can avoid generating duplicated name. + self._node_name_suffix[prefix] = suffix - return full_name + return prefix + str(suffix) def find_graph_input(self, input_name): for input in self.model.graph.input: From 0255c83dc4b93e3d15b83363a523348b8db38b16 Mon Sep 17 00:00:00 2001 From: Hariharan Seshadri Date: Thu, 27 May 2021 19:32:36 -0700 Subject: [PATCH 31/47] Clean up CPU kernel definition for opset 13 Pad (#7867) --- onnxruntime/core/providers/cpu/tensor/pad.cc | 5 +---- onnxruntime/test/testdata/kernel_def_hashes/onnx.cpu.json | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/onnxruntime/core/providers/cpu/tensor/pad.cc b/onnxruntime/core/providers/cpu/tensor/pad.cc index 2e8981fbd2..450dceb36b 100644 --- a/onnxruntime/core/providers/cpu/tensor/pad.cc +++ b/onnxruntime/core/providers/cpu/tensor/pad.cc @@ -121,10 +121,7 @@ ONNX_CPU_OPERATOR_KERNEL( .TypeConstraint( "T", BuildKernelDefConstraintsFromTypeList(), - BuildKernelDefConstraintsFromTypeList()) - .FixedTypeConstraintForHash( - "T", - BuildKernelDefConstraintsFromTypeList()), + BuildKernelDefConstraintsFromTypeList()), Pad); // This is the general padding method to n-dimensionally do edge or reflection padding (based on the inputDelta values) diff --git a/onnxruntime/test/testdata/kernel_def_hashes/onnx.cpu.json b/onnxruntime/test/testdata/kernel_def_hashes/onnx.cpu.json index 7a42831764..00b4e414a2 100644 --- a/onnxruntime/test/testdata/kernel_def_hashes/onnx.cpu.json +++ b/onnxruntime/test/testdata/kernel_def_hashes/onnx.cpu.json @@ -1461,7 +1461,7 @@ ], [ "Pad ai.onnx CPUExecutionProvider", - 9596174091174553032 + 12904240253005862936 ], [ "Pad ai.onnx CPUExecutionProvider", From 35b49b64c72405c22720d325a0f5f36379b271f5 Mon Sep 17 00:00:00 2001 From: Edward Chen <18449977+edgchen1@users.noreply.github.com> Date: Thu, 27 May 2021 19:35:59 -0700 Subject: [PATCH 32/47] Fix regex to detect Objective-C/C++ (.m/.mm) files. (#7870) --- cmake/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 4282186cb4..e473f4b4fa 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -1041,7 +1041,7 @@ function(onnxruntime_set_source_file_properties target_name) # enable ARC for Objective-C/C++ set(objective_c_cc_srcs ${srcs}) - list(FILTER objective_c_cc_srcs INCLUDE REGEX "\.mm?$") + list(FILTER objective_c_cc_srcs INCLUDE REGEX "\\.mm?$") set_property(SOURCE ${objective_c_cc_srcs} APPEND PROPERTY COMPILE_OPTIONS "-fobjc-arc") endfunction() From ab4b5055c7411c4dc835bfd1ffb91190956291d8 Mon Sep 17 00:00:00 2001 From: Edward Chen <18449977+edgchen1@users.noreply.github.com> Date: Thu, 27 May 2021 19:36:50 -0700 Subject: [PATCH 33/47] [Objective-C API] Fixes from package testing and clean up (#7866) --- cmake/onnxruntime_objectivec.cmake | 46 ++++++------- objectivec/format_objc.sh | 1 - .../{common => src}/assert_arc_enabled.mm | 0 objectivec/src/error_utils.h | 2 +- objectivec/src/ort_enums.mm | 2 +- objectivec/src/ort_enums_internal.h | 2 +- objectivec/src/ort_env.mm | 2 +- objectivec/src/ort_env_internal.h | 2 +- objectivec/src/ort_session.mm | 2 +- objectivec/src/ort_session_internal.h | 2 +- objectivec/src/ort_value.mm | 2 +- objectivec/src/ort_value_internal.h | 2 +- objectivec/test/assert_arc_enabled.mm | 4 ++ objectivec/test/assertion_utils.h | 32 +++++++++ objectivec/test/ort_env_test.mm | 5 +- objectivec/test/ort_session_test.mm | 65 +++++++----------- objectivec/test/ort_value_test.mm | 17 ++--- objectivec/test/testdata/gen_models.sh | 12 ++++ objectivec/test/testdata/single_add.basic.ort | Bin 0 -> 1176 bytes objectivec/test/testdata/single_add.onnx | 2 +- 20 files changed, 116 insertions(+), 86 deletions(-) rename objectivec/{common => src}/assert_arc_enabled.mm (100%) create mode 100644 objectivec/test/assert_arc_enabled.mm create mode 100644 objectivec/test/assertion_utils.h create mode 100755 objectivec/test/testdata/gen_models.sh create mode 100644 objectivec/test/testdata/single_add.basic.ort diff --git a/cmake/onnxruntime_objectivec.cmake b/cmake/onnxruntime_objectivec.cmake index fa0bba6f14..3299f87620 100644 --- a/cmake/onnxruntime_objectivec.cmake +++ b/cmake/onnxruntime_objectivec.cmake @@ -1,26 +1,32 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -if(${CMAKE_VERSION} VERSION_LESS "3.18") - message(FATAL_ERROR "CMake 3.18+ is required when building the Objective-C API.") +if(NOT APPLE) + message(FATAL_ERROR "The Objective-C API must be built on an Apple platform.") endif() -if(NOT APPLE) - message(FATAL_ERROR "Objective-C API must be built on an Apple platform.") +set(ONNXRUNTIME_OBJC_MIN_CMAKE_VERSION "3.18") + +if(CMAKE_VERSION VERSION_LESS ONNXRUNTIME_OBJC_MIN_CMAKE_VERSION) + message(FATAL_ERROR "The Objective-C API requires CMake ${ONNXRUNTIME_OBJC_MIN_CMAKE_VERSION}+.") +endif() + +if(NOT onnxruntime_BUILD_SHARED_LIB) + message(FATAL_ERROR "The Objective-C API requires onnxruntime_BUILD_SHARED_LIB to be enabled.") endif() check_language(OBJC) if(CMAKE_OBJC_COMPILER) - enable_language(OBJC) + enable_language(OBJC) else() - message(FATAL_ERROR "Objective-C is not supported.") + message(FATAL_ERROR "Objective-C is not supported.") endif() check_language(OBJCXX) if(CMAKE_OBJCXX_COMPILER) - enable_language(OBJCXX) + enable_language(OBJCXX) else() - message(FATAL_ERROR "Objective-C++ is not supported.") + message(FATAL_ERROR "Objective-C++ is not supported.") endif() add_compile_options( @@ -45,30 +51,24 @@ set(onnxruntime_objc_headers "${OBJC_ROOT}/include/ort_value.h" ) -file(GLOB onnxruntime_objc_srcs +file(GLOB onnxruntime_objc_srcs CONFIGURE_DEPENDS "${OBJC_ROOT}/src/*.h" "${OBJC_ROOT}/src/*.m" "${OBJC_ROOT}/src/*.mm") -# files common to implementation and test targets -set(onnxruntime_objc_common_srcs - "${OBJC_ROOT}/common/assert_arc_enabled.mm") - source_group(TREE "${OBJC_ROOT}" FILES ${onnxruntime_objc_headers} - ${onnxruntime_objc_srcs} - ${onnxruntime_objc_common_srcs}) + ${onnxruntime_objc_srcs}) onnxruntime_add_shared_library(onnxruntime_objc ${onnxruntime_objc_headers} - ${onnxruntime_objc_srcs} - ${onnxruntime_objc_common_srcs}) + ${onnxruntime_objc_srcs}) target_include_directories(onnxruntime_objc PUBLIC "${OBJC_ROOT}/include" PRIVATE - "${ONNXRUNTIME_ROOT}" + "${ONNXRUNTIME_ROOT}/include/onnxruntime/core/session" "${OBJC_ROOT}") find_library(FOUNDATION_LIB Foundation REQUIRED) @@ -112,7 +112,7 @@ if(onnxruntime_BUILD_UNIT_TESTS) # onnxruntime_objc_test target - file(GLOB onnxruntime_objc_test_srcs + file(GLOB onnxruntime_objc_test_srcs CONFIGURE_DEPENDS "${OBJC_ROOT}/test/*.h" "${OBJC_ROOT}/test/*.m" "${OBJC_ROOT}/test/*.mm") @@ -121,8 +121,7 @@ if(onnxruntime_BUILD_UNIT_TESTS) xctest_add_bundle(onnxruntime_objc_test onnxruntime_objc ${onnxruntime_objc_headers} - ${onnxruntime_objc_test_srcs} - ${onnxruntime_objc_common_srcs}) + ${onnxruntime_objc_test_srcs}) onnxruntime_configure_target(onnxruntime_objc_test) @@ -131,12 +130,13 @@ if(onnxruntime_BUILD_UNIT_TESTS) "${OBJC_ROOT}") set_target_properties(onnxruntime_objc_test PROPERTIES - FOLDER "ONNXRuntimeTest") + FOLDER "ONNXRuntimeTest" + XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED "NO") add_custom_command(TARGET onnxruntime_objc_test POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_directory "${OBJC_ROOT}/test/testdata" - "$/Resources/testdata") + "$/Resources") xctest_add_test(XCTest.onnxruntime_objc_test onnxruntime_objc_test) diff --git a/objectivec/format_objc.sh b/objectivec/format_objc.sh index 059ae0f5db..75ab07127b 100755 --- a/objectivec/format_objc.sh +++ b/objectivec/format_objc.sh @@ -7,4 +7,3 @@ set -e SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" clang-format -i $(find ${SCRIPT_DIR} -name "*.h" -o -name "*.m" -o -name "*.mm") - diff --git a/objectivec/common/assert_arc_enabled.mm b/objectivec/src/assert_arc_enabled.mm similarity index 100% rename from objectivec/common/assert_arc_enabled.mm rename to objectivec/src/assert_arc_enabled.mm diff --git a/objectivec/src/error_utils.h b/objectivec/src/error_utils.h index 7b9fc8997c..8c9663023d 100644 --- a/objectivec/src/error_utils.h +++ b/objectivec/src/error_utils.h @@ -5,7 +5,7 @@ #include -#include "core/session/onnxruntime_cxx_api.h" +#include "onnxruntime_cxx_api.h" NS_ASSUME_NONNULL_BEGIN diff --git a/objectivec/src/ort_enums.mm b/objectivec/src/ort_enums.mm index 6aea7d107a..e172bdbf9d 100644 --- a/objectivec/src/ort_enums.mm +++ b/objectivec/src/ort_enums.mm @@ -5,7 +5,7 @@ #include -#include "core/session/onnxruntime_cxx_api.h" +#include "onnxruntime_cxx_api.h" namespace { diff --git a/objectivec/src/ort_enums_internal.h b/objectivec/src/ort_enums_internal.h index 07f85cc296..322e8cb5c8 100644 --- a/objectivec/src/ort_enums_internal.h +++ b/objectivec/src/ort_enums_internal.h @@ -3,7 +3,7 @@ #import "ort_enums.h" -#include "core/session/onnxruntime_c_api.h" +#include "onnxruntime_c_api.h" OrtLoggingLevel PublicToCAPILoggingLevel(ORTLoggingLevel logging_level); diff --git a/objectivec/src/ort_env.mm b/objectivec/src/ort_env.mm index c218cca79f..fb1f6da962 100644 --- a/objectivec/src/ort_env.mm +++ b/objectivec/src/ort_env.mm @@ -5,7 +5,7 @@ #include -#include "core/session/onnxruntime_cxx_api.h" +#include "onnxruntime_cxx_api.h" #import "src/error_utils.h" #import "src/ort_enums_internal.h" diff --git a/objectivec/src/ort_env_internal.h b/objectivec/src/ort_env_internal.h index e886f24afa..f2bb3b2a5f 100644 --- a/objectivec/src/ort_env_internal.h +++ b/objectivec/src/ort_env_internal.h @@ -3,7 +3,7 @@ #import "ort_env.h" -#include "core/session/onnxruntime_cxx_api.h" +#include "onnxruntime_cxx_api.h" NS_ASSUME_NONNULL_BEGIN diff --git a/objectivec/src/ort_session.mm b/objectivec/src/ort_session.mm index 08d140ff77..b34c20c114 100644 --- a/objectivec/src/ort_session.mm +++ b/objectivec/src/ort_session.mm @@ -6,7 +6,7 @@ #include #include -#include "core/session/onnxruntime_cxx_api.h" +#include "onnxruntime_cxx_api.h" #import "src/error_utils.h" #import "src/ort_enums_internal.h" diff --git a/objectivec/src/ort_session_internal.h b/objectivec/src/ort_session_internal.h index a286e9f00a..c97d1ed766 100644 --- a/objectivec/src/ort_session_internal.h +++ b/objectivec/src/ort_session_internal.h @@ -3,7 +3,7 @@ #import "ort_session.h" -#include "core/session/onnxruntime_cxx_api.h" +#include "onnxruntime_cxx_api.h" NS_ASSUME_NONNULL_BEGIN diff --git a/objectivec/src/ort_value.mm b/objectivec/src/ort_value.mm index cc056c7548..f55659d896 100644 --- a/objectivec/src/ort_value.mm +++ b/objectivec/src/ort_value.mm @@ -7,7 +7,7 @@ #include "safeint/SafeInt.hpp" -#include "core/session/onnxruntime_cxx_api.h" +#include "onnxruntime_cxx_api.h" #import "src/error_utils.h" #import "src/ort_enums_internal.h" diff --git a/objectivec/src/ort_value_internal.h b/objectivec/src/ort_value_internal.h index c417ecf631..8e1b598767 100644 --- a/objectivec/src/ort_value_internal.h +++ b/objectivec/src/ort_value_internal.h @@ -3,7 +3,7 @@ #import "ort_value.h" -#include "core/session/onnxruntime_cxx_api.h" +#include "onnxruntime_cxx_api.h" NS_ASSUME_NONNULL_BEGIN diff --git a/objectivec/test/assert_arc_enabled.mm b/objectivec/test/assert_arc_enabled.mm new file mode 100644 index 0000000000..9aa0badb98 --- /dev/null +++ b/objectivec/test/assert_arc_enabled.mm @@ -0,0 +1,4 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +static_assert(__has_feature(objc_arc), "Objective-C ARC must be enabled."); diff --git a/objectivec/test/assertion_utils.h b/objectivec/test/assertion_utils.h new file mode 100644 index 0000000000..f2b73e6d53 --- /dev/null +++ b/objectivec/test/assertion_utils.h @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#import + +NS_ASSUME_NONNULL_BEGIN + +#define ORTAssertNullableResultSuccessful(result, error) \ + do { \ + XCTAssertNotNil(result, @"Expected non-nil result but got nil. Error: %@", error); \ + XCTAssertNil(error); \ + } while (0) + +#define ORTAssertBoolResultSuccessful(result, error) \ + do { \ + XCTAssertTrue(result, @"Expected true result but got false. Error: %@", error); \ + XCTAssertNil(error); \ + } while (0) + +#define ORTAssertNullableResultUnsuccessful(result, error) \ + do { \ + XCTAssertNil(result); \ + XCTAssertNotNil(error); \ + } while (0) + +#define ORTAssertBoolResultUnsuccessful(result, error) \ + do { \ + XCTAssertFalse(result); \ + XCTAssertNotNil(error); \ + } while (0) + +NS_ASSUME_NONNULL_END diff --git a/objectivec/test/ort_env_test.mm b/objectivec/test/ort_env_test.mm index ff04581fb8..041402c5a0 100644 --- a/objectivec/test/ort_env_test.mm +++ b/objectivec/test/ort_env_test.mm @@ -5,6 +5,8 @@ #import "ort_env.h" +#import "test/assertion_utils.h" + NS_ASSUME_NONNULL_BEGIN @interface ORTEnvTest : XCTestCase @@ -16,8 +18,7 @@ NS_ASSUME_NONNULL_BEGIN NSError* err = nil; ORTEnv* env = [[ORTEnv alloc] initWithLoggingLevel:ORTLoggingLevelWarning error:&err]; - XCTAssertNotNil(env); - XCTAssertNil(err); + ORTAssertNullableResultSuccessful(env, err); } @end diff --git a/objectivec/test/ort_session_test.mm b/objectivec/test/ort_session_test.mm index 621d3f4303..965df062ac 100644 --- a/objectivec/test/ort_session_test.mm +++ b/objectivec/test/ort_session_test.mm @@ -7,6 +7,8 @@ #import "ort_session.h" #import "ort_value.h" +#import "test/assertion_utils.h" + #include NS_ASSUME_NONNULL_BEGIN @@ -24,9 +26,10 @@ NS_ASSUME_NONNULL_BEGIN self.continueAfterFailure = NO; + NSError* err = nil; _ortEnv = [[ORTEnv alloc] initWithLoggingLevel:ORTLoggingLevelWarning - error:nil]; - XCTAssertNotNil(_ortEnv); + error:&err]; + ORTAssertNullableResultSuccessful(_ortEnv, err); } - (void)tearDown { @@ -35,17 +38,14 @@ NS_ASSUME_NONNULL_BEGIN [super tearDown]; } -+ (NSString*)getTestDataWithRelativePath:(NSString*)relativePath { - NSString* testDataDir = [NSString stringWithFormat:@"%@/Contents/Resources/testdata", - [[NSBundle bundleForClass:[ORTSessionTest class]] bundlePath]]; - return [testDataDir stringByAppendingString:relativePath]; -} - // model with an Add op // inputs: A, B // output: C = A + B + (NSString*)getAddModelPath { - return [ORTSessionTest getTestDataWithRelativePath:@"/single_add.onnx"]; + NSBundle* bundle = [NSBundle bundleForClass:[ORTSessionTest class]]; + NSString* path = [bundle pathForResource:@"single_add.basic" + ofType:@"ort"]; + return path; } + (NSMutableData*)dataWithScalarFloat:(float)value { @@ -60,24 +60,21 @@ NS_ASSUME_NONNULL_BEGIN elementType:ORTTensorElementDataTypeFloat shape:shape error:&err]; - XCTAssertNotNil(ortValue); - XCTAssertNil(err); + ORTAssertNullableResultSuccessful(ortValue, err); return ortValue; } + (ORTSessionOptions*)makeSessionOptions { NSError* err = nil; ORTSessionOptions* sessionOptions = [[ORTSessionOptions alloc] initWithError:&err]; - XCTAssertNotNil(sessionOptions); - XCTAssertNil(err); + ORTAssertNullableResultSuccessful(sessionOptions, err); return sessionOptions; } + (ORTRunOptions*)makeRunOptions { NSError* err = nil; ORTRunOptions* runOptions = [[ORTRunOptions alloc] initWithError:&err]; - XCTAssertNotNil(runOptions); - XCTAssertNil(err); + ORTAssertNullableResultSuccessful(runOptions, err); return runOptions; } @@ -95,15 +92,13 @@ NS_ASSUME_NONNULL_BEGIN modelPath:[ORTSessionTest getAddModelPath] sessionOptions:[ORTSessionTest makeSessionOptions] error:&err]; - XCTAssertNotNil(session); - XCTAssertNil(err); + ORTAssertNullableResultSuccessful(session, err); BOOL runResult = [session runWithInputs:@{@"A" : a, @"B" : b} outputs:@{@"C" : c} runOptions:[ORTSessionTest makeRunOptions] error:&err]; - XCTAssertTrue(runResult); - XCTAssertNil(err); + ORTAssertBoolResultSuccessful(runResult, err); const float cExpected = 3.0f; float cActual; @@ -123,23 +118,20 @@ NS_ASSUME_NONNULL_BEGIN modelPath:[ORTSessionTest getAddModelPath] sessionOptions:[ORTSessionTest makeSessionOptions] error:&err]; - XCTAssertNotNil(session); - XCTAssertNil(err); + ORTAssertNullableResultSuccessful(session, err); NSDictionary* outputs = [session runWithInputs:@{@"A" : a, @"B" : b} outputNames:[NSSet setWithArray:@[ @"C" ]] runOptions:[ORTSessionTest makeRunOptions] error:&err]; - XCTAssertNotNil(outputs); - XCTAssertNil(err); + ORTAssertNullableResultSuccessful(outputs, err); ORTValue* cOutput = outputs[@"C"]; XCTAssertNotNil(cOutput); NSData* cData = [cOutput tensorDataWithError:&err]; - XCTAssertNotNil(cData); - XCTAssertNil(err); + ORTAssertNullableResultSuccessful(cData, err); const float cExpected = 3.0f; float cActual; @@ -153,34 +145,29 @@ NS_ASSUME_NONNULL_BEGIN modelPath:[ORTSessionTest getAddModelPath] sessionOptions:[ORTSessionTest makeSessionOptions] error:&err]; - XCTAssertNotNil(session); - XCTAssertNil(err); + ORTAssertNullableResultSuccessful(session, err); NSArray* inputNames = [session inputNamesWithError:&err]; - XCTAssertNotNil(inputNames); - XCTAssertNil(err); + ORTAssertNullableResultSuccessful(inputNames, err); XCTAssertEqualObjects(inputNames, (@[ @"A", @"B" ])); NSArray* overridableInitializerNames = [session overridableInitializerNamesWithError:&err]; - XCTAssertNotNil(overridableInitializerNames); - XCTAssertNil(err); + ORTAssertNullableResultSuccessful(overridableInitializerNames, err); XCTAssertEqualObjects(overridableInitializerNames, (@[])); NSArray* outputNames = [session outputNamesWithError:&err]; - XCTAssertNotNil(outputNames); - XCTAssertNil(err); + ORTAssertNullableResultSuccessful(outputNames, err); XCTAssertEqualObjects(outputNames, (@[ @"C" ])); } - (void)testInitFailsWithInvalidPath { - NSString* invalidModelPath = [ORTSessionTest getTestDataWithRelativePath:@"/invalid/path/to/model.onnx"]; + NSString* invalidModelPath = @"invalid/path/to/model.ort"; NSError* err = nil; ORTSession* session = [[ORTSession alloc] initWithEnv:self.ortEnv modelPath:invalidModelPath sessionOptions:[ORTSessionTest makeSessionOptions] error:&err]; - XCTAssertNil(session); - XCTAssertNotNil(err); + ORTAssertNullableResultUnsuccessful(session, err); } - (void)testRunFailsWithInvalidInput { @@ -195,15 +182,13 @@ NS_ASSUME_NONNULL_BEGIN modelPath:[ORTSessionTest getAddModelPath] sessionOptions:[ORTSessionTest makeSessionOptions] error:&err]; - XCTAssertNotNil(session); - XCTAssertNil(err); + ORTAssertNullableResultSuccessful(session, err); BOOL runResult = [session runWithInputs:@{@"D" : d} outputs:@{@"C" : c} runOptions:[ORTSessionTest makeRunOptions] error:&err]; - XCTAssertFalse(runResult); - XCTAssertNotNil(err); + ORTAssertBoolResultUnsuccessful(runResult, err); } @end diff --git a/objectivec/test/ort_value_test.mm b/objectivec/test/ort_value_test.mm index 09ecff7f64..734ad39095 100644 --- a/objectivec/test/ort_value_test.mm +++ b/objectivec/test/ort_value_test.mm @@ -7,6 +7,8 @@ #include +#import "test/assertion_utils.h" + NS_ASSUME_NONNULL_BEGIN @interface ORTValueTest : XCTestCase @@ -33,8 +35,7 @@ NS_ASSUME_NONNULL_BEGIN elementType:elementType shape:shape error:&err]; - XCTAssertNotNil(ortValue); - XCTAssertNil(err); + ORTAssertNullableResultSuccessful(ortValue, err); auto checkTensorInfo = [&](ORTTensorTypeAndShapeInfo* tensorInfo) { XCTAssertEqual(tensorInfo.elementType, elementType); @@ -42,20 +43,17 @@ NS_ASSUME_NONNULL_BEGIN }; ORTValueTypeInfo* typeInfo = [ortValue typeInfoWithError:&err]; - XCTAssertNotNil(typeInfo); - XCTAssertNil(err); + ORTAssertNullableResultSuccessful(typeInfo, err); XCTAssertEqual(typeInfo.type, ORTValueTypeTensor); XCTAssertNotNil(typeInfo.tensorTypeAndShapeInfo); checkTensorInfo(typeInfo.tensorTypeAndShapeInfo); ORTTensorTypeAndShapeInfo* tensorInfo = [ortValue tensorTypeAndShapeInfoWithError:&err]; - XCTAssertNotNil(tensorInfo); - XCTAssertNil(err); + ORTAssertNullableResultSuccessful(tensorInfo, err); checkTensorInfo(tensorInfo); NSData* actualData = [ortValue tensorDataWithError:&err]; - XCTAssertNotNil(actualData); - XCTAssertNil(err); + ORTAssertNullableResultSuccessful(actualData, err); XCTAssertEqual(actualData.length, sizeof(int32_t)); int32_t actualValue; memcpy(&actualValue, actualData.bytes, sizeof(int32_t)); @@ -73,8 +71,7 @@ NS_ASSUME_NONNULL_BEGIN elementType:ORTTensorElementDataTypeInt32 shape:shape error:&err]; - XCTAssertNil(ortValue); - XCTAssertNotNil(err); + ORTAssertNullableResultUnsuccessful(ortValue, err); } @end diff --git a/objectivec/test/testdata/gen_models.sh b/objectivec/test/testdata/gen_models.sh new file mode 100755 index 0000000000..45b0ce3ae6 --- /dev/null +++ b/objectivec/test/testdata/gen_models.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -e + +# Get directory this script is in +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +cd ${DIR} + +python3 ./single_add_gen.py +python3 -m onnxruntime.tools.convert_onnx_models_to_ort --optimization_level basic . + diff --git a/objectivec/test/testdata/single_add.basic.ort b/objectivec/test/testdata/single_add.basic.ort new file mode 100644 index 0000000000000000000000000000000000000000..7476b9e0d7260b464e5397d6d38bb7977464c50e GIT binary patch literal 1176 zcmZ`(Jx?1!5FH0AIYAOxiG?gcCT6&Id-Pcc^)kR7@;beK53Xi&ER#}~8Pvdhf{eWX zlyee(-C_!Q7MKI(fi2)Q&>RA+4`l?Um>O6Iaf*d|KcUY6*1Xi%TMLLeU=Qb6mT``s5buMv;g;&Kv1K@;u*r4)d-$A(CvPxo&%T3{ zc%BRA#i;q4)4Bsr3h|E$@w)@@zxU&h{wF@O3}bFjIXH{2+_rBuVslr_o|HUPRJOIc zK5hzm*^kI`Fv?i-->HWQ|GPg!Hj9fzy#(-mnA6Q`5jM{j09LyNX9Bo|eF+=^ZGgWA z-tp{QU>biMI*3C*)Ik#O#oMVLcD7Y3-B0wPO7>)KfdBR{eiZB+1TtOVx!A7;x$u7! zc&vxtK}-8_6!rYwM0K?~@YAHFBOQIgkz5S$*}LNxdB4HqY$%M&{{qJgJDd&wCHO-5 kyr2wk4EoRhRc}`%I#f~GGWYnM<@fj Date: Fri, 28 May 2021 08:35:41 -0700 Subject: [PATCH 34/47] [OpenVINO-EP] Adding OpenVINO-EP samples to Msft Repo (#7826) * Added ONNX_OV_EP samples ->Added cpp, python and csharp samples using OpenVINO Execution Provider. Signed-off-by: MaajidKhan --- .../squeezenet_cpp_app.cpp | 384 +++++++ .../squeezenet_classification/synset.txt | 1000 +++++++++++++++++ .../yolov3_object_detection/Label.cs | 91 ++ .../yolov3_object_detection/Prediction.cs | 31 + .../yolov3_object_detection/Program.cs | 173 +++ .../tiny_yolov2_obj_detection_sample.py | 195 ++++ 6 files changed, 1874 insertions(+) create mode 100644 samples/c_cxx/OpenVINO_EP/squeezenet_classification/squeezenet_cpp_app.cpp create mode 100644 samples/c_cxx/OpenVINO_EP/squeezenet_classification/synset.txt create mode 100644 samples/c_sharp/OpenVINO_EP/yolov3_object_detection/Label.cs create mode 100644 samples/c_sharp/OpenVINO_EP/yolov3_object_detection/Prediction.cs create mode 100644 samples/c_sharp/OpenVINO_EP/yolov3_object_detection/Program.cs create mode 100644 samples/python/OpenVINO_EP/tiny_yolo_v2_object_detection/tiny_yolov2_obj_detection_sample.py diff --git a/samples/c_cxx/OpenVINO_EP/squeezenet_classification/squeezenet_cpp_app.cpp b/samples/c_cxx/OpenVINO_EP/squeezenet_classification/squeezenet_cpp_app.cpp new file mode 100644 index 0000000000..f553d16fb8 --- /dev/null +++ b/samples/c_cxx/OpenVINO_EP/squeezenet_classification/squeezenet_cpp_app.cpp @@ -0,0 +1,384 @@ +/* +Copyright (C) 2021, Intel Corporation +SPDX-License-Identifier: Apache-2.0 + +Portions of this software are copyright of their respective authors and released under the MIT license: +- ONNX-Runtime-Inference, Copyright 2020 Lei Mao. For licensing see https://github.com/leimao/ONNX-Runtime-Inference/blob/main/LICENSE.md +*/ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include // To use runtime_error + +template +T vectorProduct(const std::vector& v) +{ + return accumulate(v.begin(), v.end(), 1, std::multiplies()); +} + +/** + * @brief Operator overloading for printing vectors + * @tparam T + * @param os + * @param v + * @return std::ostream& + */ + +template +std::ostream& operator<<(std::ostream& os, const std::vector& v) +{ + os << "["; + for (int i = 0; i < v.size(); ++i) + { + os << v[i]; + if (i != v.size() - 1) + { + os << ", "; + } + } + os << "]"; + return os; +} + +// Function to validate the input image file extension. +bool imageFileExtension(std::string str) +{ + // is empty throw error + if (str.empty()) + throw std::runtime_error("[ ERROR ] The image File path is empty"); + + size_t pos = str.rfind('.'); + if (pos == std::string::npos) + return false; + + std::string ext = str.substr(pos+1); + + if (ext == "jpg" || ext == "jpeg" || ext == "gif" || ext == "png" || ext == "jfif" || + ext == "JPG" || ext == "JPEG" || ext == "GIF" || ext == "PNG" || ext == "JFIF") { + return true; + } + + return false; +} + +// Function to read the labels from the labelFilepath. +std::vector readLabels(std::string& labelFilepath) +{ + std::vector labels; + std::string line; + std::ifstream fp(labelFilepath); + while (std::getline(fp, line)) + { + labels.push_back(line); + } + return labels; +} + +// Function to validate the input model file extension. +bool checkModelExtension(const std::string& filename) +{ + if(filename.empty()) + { + throw std::runtime_error("[ ERROR ] The Model file path is empty"); + } + size_t pos = filename.rfind('.'); + if (pos == std::string::npos) + return false; + std::string ext = filename.substr(pos+1); + if (ext == "onnx") + return true; + return false; +} + +// Function to validate the Label file extension. +bool checkLabelFileExtension(const std::string& filename) +{ + size_t pos = filename.rfind('.'); + if (filename.empty()) + { + throw std::runtime_error("[ ERROR ] The Label file path is empty"); + } + if (pos == std::string::npos) + return false; + std::string ext = filename.substr(pos+1); + if (ext == "txt") { + return true; + } else { + return false; + } +} + +//Handling divide by zero +float division(float num, float den){ + if (den == 0) { + throw std::runtime_error("[ ERROR ] Math error: Attempted to divide by Zero\n"); + } + return (num / den); +} + +void printHelp() { + std::cout << "To run the model, use the following command:\n"; + std::cout << "Example: ./run_squeezenet --use_openvino " << std::endl; + std::cout << "\n To Run using OpenVINO EP.\nExample: ./run_squeezenet --use_openvino squeezenet1.1-7.onnx demo.jpeg synset.txt \n" << std::endl; + std::cout << "\n To Run on Default CPU.\n Example: ./run_squeezenet --use_cpu squeezenet1.1-7.onnx demo.jpeg synset.txt \n" << std::endl; +} + +int main(int argc, char* argv[]) +{ + bool useOPENVINO{true}; + const char* useOPENVINOFlag = "--use_openvino"; + const char* useCPUFlag = "--use_cpu"; + + if(argc == 2) { + std::string option = argv[1]; + if (option == "--help" || option == "-help" || option == "--h" || option == "-h") { + printHelp(); + } + return 0; + } else if(argc != 5) { + std::cout << "[ ERROR ] you have used the wrong command to run your program." << std::endl; + printHelp(); + return 0; + } else if (strcmp(argv[1], useOPENVINOFlag) == 0) { + useOPENVINO = true; + } else if (strcmp(argv[1], useCPUFlag) == 0) { + useOPENVINO = false; + } + + if (useOPENVINO) + { + std::cout << "Inference Execution Provider: OPENVINO" << std::endl; + } + else + { + std::cout << "Inference Execution Provider: CPU" << std::endl; + } + + std::string instanceName{"image-classification-inference"}; + + std::string modelFilepath = argv[2]; // .onnx file + + //validate ModelFilePath + checkModelExtension(modelFilepath); + if(!checkModelExtension(modelFilepath)) { + throw std::runtime_error("[ ERROR ] The ModelFilepath is not correct. Make sure you are setting the path to an onnx model file (.onnx)"); + } + std::string imageFilepath = argv[3]; + + // Validate ImageFilePath + imageFileExtension(imageFilepath); + if(!imageFileExtension(imageFilepath)) { + throw std::runtime_error("[ ERROR ] The imageFilepath doesn't have correct image extension. Choose from jpeg, jpg, gif, png, PNG, jfif"); + } + std::ifstream f(imageFilepath.c_str()); + if(!f.good()) { + throw std::runtime_error("[ ERROR ] The imageFilepath is not set correctly or doesn't exist"); + } + + // Validate LabelFilePath + std::string labelFilepath = argv[4]; + if(!checkLabelFileExtension(labelFilepath)) { + throw std::runtime_error("[ ERROR ] The LabelFilepath is not set correctly and the labels file should end with extension .txt"); + } + + std::vector labels{readLabels(labelFilepath)}; + + Ort::Env env(OrtLoggingLevel::ORT_LOGGING_LEVEL_WARNING, + instanceName.c_str()); + Ort::SessionOptions sessionOptions; + sessionOptions.SetIntraOpNumThreads(1); + + //Appending OpenVINO Execution Provider API + if (useOPENVINO) { + // Using OPENVINO backend + OrtOpenVINOProviderOptions options; + options.device_type = "CPU_FP32"; //Other options are: GPU_FP32, GPU_FP16, MYRIAD_FP16 + std::cout << "OpenVINO device type is set to: " << options.device_type << std::endl; + sessionOptions.AppendExecutionProvider_OpenVINO(options); + } + + // Sets graph optimization level + // Available levels are + // ORT_DISABLE_ALL -> To disable all optimizations + // ORT_ENABLE_BASIC -> To enable basic optimizations (Such as redundant node + // removals) ORT_ENABLE_EXTENDED -> To enable extended optimizations + // (Includes level 1 + more complex optimizations like node fusions) + // ORT_ENABLE_ALL -> To Enable All possible optimizations + sessionOptions.SetGraphOptimizationLevel( + GraphOptimizationLevel::ORT_DISABLE_ALL); + + //Creation: The Ort::Session is created here + Ort::Session session(env, modelFilepath.c_str(), sessionOptions); + + Ort::AllocatorWithDefaultOptions allocator; + + size_t numInputNodes = session.GetInputCount(); + size_t numOutputNodes = session.GetOutputCount(); + + std::cout << "Number of Input Nodes: " << numInputNodes << std::endl; + std::cout << "Number of Output Nodes: " << numOutputNodes << std::endl; + + const char* inputName = session.GetInputName(0, allocator); + std::cout << "Input Name: " << inputName << std::endl; + + Ort::TypeInfo inputTypeInfo = session.GetInputTypeInfo(0); + auto inputTensorInfo = inputTypeInfo.GetTensorTypeAndShapeInfo(); + + ONNXTensorElementDataType inputType = inputTensorInfo.GetElementType(); + std::cout << "Input Type: " << inputType << std::endl; + + std::vector inputDims = inputTensorInfo.GetShape(); + std::cout << "Input Dimensions: " << inputDims << std::endl; + + const char* outputName = session.GetOutputName(0, allocator); + std::cout << "Output Name: " << outputName << std::endl; + + Ort::TypeInfo outputTypeInfo = session.GetOutputTypeInfo(0); + auto outputTensorInfo = outputTypeInfo.GetTensorTypeAndShapeInfo(); + + ONNXTensorElementDataType outputType = outputTensorInfo.GetElementType(); + std::cout << "Output Type: " << outputType << std::endl; + + std::vector outputDims = outputTensorInfo.GetShape(); + std::cout << "Output Dimensions: " << outputDims << std::endl; + //pre-processing the Image + // step 1: Read an image in HWC BGR UINT8 format. + cv::Mat imageBGR = cv::imread(imageFilepath, cv::ImreadModes::IMREAD_COLOR); + + // step 2: Resize the image. + cv::Mat resizedImageBGR, resizedImageRGB, resizedImage, preprocessedImage; + cv::resize(imageBGR, resizedImageBGR, + cv::Size(inputDims.at(2), inputDims.at(3)), + cv::InterpolationFlags::INTER_CUBIC); + + // step 3: Convert the image to HWC RGB UINT8 format. + cv::cvtColor(resizedImageBGR, resizedImageRGB, + cv::ColorConversionCodes::COLOR_BGR2RGB); + // step 4: Convert the image to HWC RGB float format by dividing each pixel by 255. + resizedImageRGB.convertTo(resizedImage, CV_32F, 1.0 / 255); + + // step 5: Split the RGB channels from the image. + cv::Mat channels[3]; + cv::split(resizedImage, channels); + + //step 6: Normalize each channel. + // Normalization per channel + // Normalization parameters obtained from + // https://github.com/onnx/models/tree/master/vision/classification/squeezenet + channels[0] = (channels[0] - 0.485) / 0.229; + channels[1] = (channels[1] - 0.456) / 0.224; + channels[2] = (channels[2] - 0.406) / 0.225; + + //step 7: Merge the RGB channels back to the image. + cv::merge(channels, 3, resizedImage); + + // step 8: Convert the image to CHW RGB float format. + // HWC to CHW + cv::dnn::blobFromImage(resizedImage, preprocessedImage); + + + //Run Inference + + /* To run inference using ONNX Runtime, the user is responsible for creating and managing the + input and output buffers. These buffers could be created and managed via std::vector. + The linear-format input data should be copied to the buffer for ONNX Runtime inference. */ + + size_t inputTensorSize = vectorProduct(inputDims); + std::vector inputTensorValues(inputTensorSize); + inputTensorValues.assign(preprocessedImage.begin(), + preprocessedImage.end()); + + size_t outputTensorSize = vectorProduct(outputDims); + assert(("Output tensor size should equal to the label set size.", + labels.size() == outputTensorSize)); + std::vector outputTensorValues(outputTensorSize); + + + /* Once the buffers were created, they would be used for creating instances of Ort::Value + which is the tensor format for ONNX Runtime. There could be multiple inputs for a neural network, + so we have to prepare an array of Ort::Value instances for inputs and outputs respectively even if + we only have one input and one output. */ + + std::vector inputNames{inputName}; + std::vector outputNames{outputName}; + std::vector inputTensors; + std::vector outputTensors; + + /* + Creating ONNX Runtime inference sessions, querying input and output names, + dimensions, and types are trivial. + Setup inputs & outputs: The input & output tensors are created here. */ + + Ort::MemoryInfo memoryInfo = Ort::MemoryInfo::CreateCpu( + OrtAllocatorType::OrtArenaAllocator, OrtMemType::OrtMemTypeDefault); + inputTensors.push_back(Ort::Value::CreateTensor( + memoryInfo, inputTensorValues.data(), inputTensorSize, inputDims.data(), + inputDims.size())); + outputTensors.push_back(Ort::Value::CreateTensor( + memoryInfo, outputTensorValues.data(), outputTensorSize, + outputDims.data(), outputDims.size())); + + /* To run inference, we provide the run options, an array of input names corresponding to the + inputs in the input tensor, an array of input tensor, number of inputs, an array of output names + corresponding to the the outputs in the output tensor, an array of output tensor, number of outputs. */ + + session.Run(Ort::RunOptions{nullptr}, inputNames.data(), + inputTensors.data(), 1, outputNames.data(), + outputTensors.data(), 1); + + int predId = 0; + float activation = 0; + float maxActivation = std::numeric_limits::lowest(); + float expSum = 0; + /* The inference result could be found in the buffer for the output tensors, + which are usually the buffer from std::vector instances. */ + for (int i = 0; i < labels.size(); i++) { + activation = outputTensorValues.at(i); + expSum += std::exp(activation); + if (activation > maxActivation) + { + predId = i; + maxActivation = activation; + } + } + std::cout << "Predicted Label ID: " << predId << std::endl; + std::cout << "Predicted Label: " << labels.at(predId) << std::endl; + float result; + try { + result = division(std::exp(maxActivation), expSum); + std::cout << "Uncalibrated Confidence: " << result << std::endl; + } + catch (std::runtime_error& e) { + std::cout << "Exception occurred" << std::endl << e.what(); + } + + // Measure latency + int numTests{100}; + std::chrono::steady_clock::time_point begin = + std::chrono::steady_clock::now(); + + //Run: Running the session is done in the Run() method: + for (int i = 0; i < numTests; i++) { + session.Run(Ort::RunOptions{nullptr}, inputNames.data(), + inputTensors.data(), 1, outputNames.data(), + outputTensors.data(), 1); + } + std::chrono::steady_clock::time_point end = + std::chrono::steady_clock::now(); + std::cout << "Minimum Inference Latency: " + << std::chrono::duration_cast(end - begin).count() / static_cast(numTests) + << " ms" << std::endl; + return 0; +} \ No newline at end of file diff --git a/samples/c_cxx/OpenVINO_EP/squeezenet_classification/synset.txt b/samples/c_cxx/OpenVINO_EP/squeezenet_classification/synset.txt new file mode 100644 index 0000000000..a9e8c7f50d --- /dev/null +++ b/samples/c_cxx/OpenVINO_EP/squeezenet_classification/synset.txt @@ -0,0 +1,1000 @@ +n01440764 tench, Tinca tinca +n01443537 goldfish, Carassius auratus +n01484850 great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias +n01491361 tiger shark, Galeocerdo cuvieri +n01494475 hammerhead, hammerhead shark +n01496331 electric ray, crampfish, numbfish, torpedo +n01498041 stingray +n01514668 cock +n01514859 hen +n01518878 ostrich, Struthio camelus +n01530575 brambling, Fringilla montifringilla +n01531178 goldfinch, Carduelis carduelis +n01532829 house finch, linnet, Carpodacus mexicanus +n01534433 junco, snowbird +n01537544 indigo bunting, indigo finch, indigo bird, Passerina cyanea +n01558993 robin, American robin, Turdus migratorius +n01560419 bulbul +n01580077 jay +n01582220 magpie +n01592084 chickadee +n01601694 water ouzel, dipper +n01608432 kite +n01614925 bald eagle, American eagle, Haliaeetus leucocephalus +n01616318 vulture +n01622779 great grey owl, great gray owl, Strix nebulosa +n01629819 European fire salamander, Salamandra salamandra +n01630670 common newt, Triturus vulgaris +n01631663 eft +n01632458 spotted salamander, Ambystoma maculatum +n01632777 axolotl, mud puppy, Ambystoma mexicanum +n01641577 bullfrog, Rana catesbeiana +n01644373 tree frog, tree-frog +n01644900 tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui +n01664065 loggerhead, loggerhead turtle, Caretta caretta +n01665541 leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea +n01667114 mud turtle +n01667778 terrapin +n01669191 box turtle, box tortoise +n01675722 banded gecko +n01677366 common iguana, iguana, Iguana iguana +n01682714 American chameleon, anole, Anolis carolinensis +n01685808 whiptail, whiptail lizard +n01687978 agama +n01688243 frilled lizard, Chlamydosaurus kingi +n01689811 alligator lizard +n01692333 Gila monster, Heloderma suspectum +n01693334 green lizard, Lacerta viridis +n01694178 African chameleon, Chamaeleo chamaeleon +n01695060 Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis +n01697457 African crocodile, Nile crocodile, Crocodylus niloticus +n01698640 American alligator, Alligator mississipiensis +n01704323 triceratops +n01728572 thunder snake, worm snake, Carphophis amoenus +n01728920 ringneck snake, ring-necked snake, ring snake +n01729322 hognose snake, puff adder, sand viper +n01729977 green snake, grass snake +n01734418 king snake, kingsnake +n01735189 garter snake, grass snake +n01737021 water snake +n01739381 vine snake +n01740131 night snake, Hypsiglena torquata +n01742172 boa constrictor, Constrictor constrictor +n01744401 rock python, rock snake, Python sebae +n01748264 Indian cobra, Naja naja +n01749939 green mamba +n01751748 sea snake +n01753488 horned viper, cerastes, sand viper, horned asp, Cerastes cornutus +n01755581 diamondback, diamondback rattlesnake, Crotalus adamanteus +n01756291 sidewinder, horned rattlesnake, Crotalus cerastes +n01768244 trilobite +n01770081 harvestman, daddy longlegs, Phalangium opilio +n01770393 scorpion +n01773157 black and gold garden spider, Argiope aurantia +n01773549 barn spider, Araneus cavaticus +n01773797 garden spider, Aranea diademata +n01774384 black widow, Latrodectus mactans +n01774750 tarantula +n01775062 wolf spider, hunting spider +n01776313 tick +n01784675 centipede +n01795545 black grouse +n01796340 ptarmigan +n01797886 ruffed grouse, partridge, Bonasa umbellus +n01798484 prairie chicken, prairie grouse, prairie fowl +n01806143 peacock +n01806567 quail +n01807496 partridge +n01817953 African grey, African gray, Psittacus erithacus +n01818515 macaw +n01819313 sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita +n01820546 lorikeet +n01824575 coucal +n01828970 bee eater +n01829413 hornbill +n01833805 hummingbird +n01843065 jacamar +n01843383 toucan +n01847000 drake +n01855032 red-breasted merganser, Mergus serrator +n01855672 goose +n01860187 black swan, Cygnus atratus +n01871265 tusker +n01872401 echidna, spiny anteater, anteater +n01873310 platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus +n01877812 wallaby, brush kangaroo +n01882714 koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus +n01883070 wombat +n01910747 jellyfish +n01914609 sea anemone, anemone +n01917289 brain coral +n01924916 flatworm, platyhelminth +n01930112 nematode, nematode worm, roundworm +n01943899 conch +n01944390 snail +n01945685 slug +n01950731 sea slug, nudibranch +n01955084 chiton, coat-of-mail shell, sea cradle, polyplacophore +n01968897 chambered nautilus, pearly nautilus, nautilus +n01978287 Dungeness crab, Cancer magister +n01978455 rock crab, Cancer irroratus +n01980166 fiddler crab +n01981276 king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica +n01983481 American lobster, Northern lobster, Maine lobster, Homarus americanus +n01984695 spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish +n01985128 crayfish, crawfish, crawdad, crawdaddy +n01986214 hermit crab +n01990800 isopod +n02002556 white stork, Ciconia ciconia +n02002724 black stork, Ciconia nigra +n02006656 spoonbill +n02007558 flamingo +n02009229 little blue heron, Egretta caerulea +n02009912 American egret, great white heron, Egretta albus +n02011460 bittern +n02012849 crane +n02013706 limpkin, Aramus pictus +n02017213 European gallinule, Porphyrio porphyrio +n02018207 American coot, marsh hen, mud hen, water hen, Fulica americana +n02018795 bustard +n02025239 ruddy turnstone, Arenaria interpres +n02027492 red-backed sandpiper, dunlin, Erolia alpina +n02028035 redshank, Tringa totanus +n02033041 dowitcher +n02037110 oystercatcher, oyster catcher +n02051845 pelican +n02056570 king penguin, Aptenodytes patagonica +n02058221 albatross, mollymawk +n02066245 grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus +n02071294 killer whale, killer, orca, grampus, sea wolf, Orcinus orca +n02074367 dugong, Dugong dugon +n02077923 sea lion +n02085620 Chihuahua +n02085782 Japanese spaniel +n02085936 Maltese dog, Maltese terrier, Maltese +n02086079 Pekinese, Pekingese, Peke +n02086240 Shih-Tzu +n02086646 Blenheim spaniel +n02086910 papillon +n02087046 toy terrier +n02087394 Rhodesian ridgeback +n02088094 Afghan hound, Afghan +n02088238 basset, basset hound +n02088364 beagle +n02088466 bloodhound, sleuthhound +n02088632 bluetick +n02089078 black-and-tan coonhound +n02089867 Walker hound, Walker foxhound +n02089973 English foxhound +n02090379 redbone +n02090622 borzoi, Russian wolfhound +n02090721 Irish wolfhound +n02091032 Italian greyhound +n02091134 whippet +n02091244 Ibizan hound, Ibizan Podenco +n02091467 Norwegian elkhound, elkhound +n02091635 otterhound, otter hound +n02091831 Saluki, gazelle hound +n02092002 Scottish deerhound, deerhound +n02092339 Weimaraner +n02093256 Staffordshire bullterrier, Staffordshire bull terrier +n02093428 American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier +n02093647 Bedlington terrier +n02093754 Border terrier +n02093859 Kerry blue terrier +n02093991 Irish terrier +n02094114 Norfolk terrier +n02094258 Norwich terrier +n02094433 Yorkshire terrier +n02095314 wire-haired fox terrier +n02095570 Lakeland terrier +n02095889 Sealyham terrier, Sealyham +n02096051 Airedale, Airedale terrier +n02096177 cairn, cairn terrier +n02096294 Australian terrier +n02096437 Dandie Dinmont, Dandie Dinmont terrier +n02096585 Boston bull, Boston terrier +n02097047 miniature schnauzer +n02097130 giant schnauzer +n02097209 standard schnauzer +n02097298 Scotch terrier, Scottish terrier, Scottie +n02097474 Tibetan terrier, chrysanthemum dog +n02097658 silky terrier, Sydney silky +n02098105 soft-coated wheaten terrier +n02098286 West Highland white terrier +n02098413 Lhasa, Lhasa apso +n02099267 flat-coated retriever +n02099429 curly-coated retriever +n02099601 golden retriever +n02099712 Labrador retriever +n02099849 Chesapeake Bay retriever +n02100236 German short-haired pointer +n02100583 vizsla, Hungarian pointer +n02100735 English setter +n02100877 Irish setter, red setter +n02101006 Gordon setter +n02101388 Brittany spaniel +n02101556 clumber, clumber spaniel +n02102040 English springer, English springer spaniel +n02102177 Welsh springer spaniel +n02102318 cocker spaniel, English cocker spaniel, cocker +n02102480 Sussex spaniel +n02102973 Irish water spaniel +n02104029 kuvasz +n02104365 schipperke +n02105056 groenendael +n02105162 malinois +n02105251 briard +n02105412 kelpie +n02105505 komondor +n02105641 Old English sheepdog, bobtail +n02105855 Shetland sheepdog, Shetland sheep dog, Shetland +n02106030 collie +n02106166 Border collie +n02106382 Bouvier des Flandres, Bouviers des Flandres +n02106550 Rottweiler +n02106662 German shepherd, German shepherd dog, German police dog, alsatian +n02107142 Doberman, Doberman pinscher +n02107312 miniature pinscher +n02107574 Greater Swiss Mountain dog +n02107683 Bernese mountain dog +n02107908 Appenzeller +n02108000 EntleBucher +n02108089 boxer +n02108422 bull mastiff +n02108551 Tibetan mastiff +n02108915 French bulldog +n02109047 Great Dane +n02109525 Saint Bernard, St Bernard +n02109961 Eskimo dog, husky +n02110063 malamute, malemute, Alaskan malamute +n02110185 Siberian husky +n02110341 dalmatian, coach dog, carriage dog +n02110627 affenpinscher, monkey pinscher, monkey dog +n02110806 basenji +n02110958 pug, pug-dog +n02111129 Leonberg +n02111277 Newfoundland, Newfoundland dog +n02111500 Great Pyrenees +n02111889 Samoyed, Samoyede +n02112018 Pomeranian +n02112137 chow, chow chow +n02112350 keeshond +n02112706 Brabancon griffon +n02113023 Pembroke, Pembroke Welsh corgi +n02113186 Cardigan, Cardigan Welsh corgi +n02113624 toy poodle +n02113712 miniature poodle +n02113799 standard poodle +n02113978 Mexican hairless +n02114367 timber wolf, grey wolf, gray wolf, Canis lupus +n02114548 white wolf, Arctic wolf, Canis lupus tundrarum +n02114712 red wolf, maned wolf, Canis rufus, Canis niger +n02114855 coyote, prairie wolf, brush wolf, Canis latrans +n02115641 dingo, warrigal, warragal, Canis dingo +n02115913 dhole, Cuon alpinus +n02116738 African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus +n02117135 hyena, hyaena +n02119022 red fox, Vulpes vulpes +n02119789 kit fox, Vulpes macrotis +n02120079 Arctic fox, white fox, Alopex lagopus +n02120505 grey fox, gray fox, Urocyon cinereoargenteus +n02123045 tabby, tabby cat +n02123159 tiger cat +n02123394 Persian cat +n02123597 Siamese cat, Siamese +n02124075 Egyptian cat +n02125311 cougar, puma, catamount, mountain lion, painter, panther, Felis concolor +n02127052 lynx, catamount +n02128385 leopard, Panthera pardus +n02128757 snow leopard, ounce, Panthera uncia +n02128925 jaguar, panther, Panthera onca, Felis onca +n02129165 lion, king of beasts, Panthera leo +n02129604 tiger, Panthera tigris +n02130308 cheetah, chetah, Acinonyx jubatus +n02132136 brown bear, bruin, Ursus arctos +n02133161 American black bear, black bear, Ursus americanus, Euarctos americanus +n02134084 ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus +n02134418 sloth bear, Melursus ursinus, Ursus ursinus +n02137549 mongoose +n02138441 meerkat, mierkat +n02165105 tiger beetle +n02165456 ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle +n02167151 ground beetle, carabid beetle +n02168699 long-horned beetle, longicorn, longicorn beetle +n02169497 leaf beetle, chrysomelid +n02172182 dung beetle +n02174001 rhinoceros beetle +n02177972 weevil +n02190166 fly +n02206856 bee +n02219486 ant, emmet, pismire +n02226429 grasshopper, hopper +n02229544 cricket +n02231487 walking stick, walkingstick, stick insect +n02233338 cockroach, roach +n02236044 mantis, mantid +n02256656 cicada, cicala +n02259212 leafhopper +n02264363 lacewing, lacewing fly +n02268443 dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk +n02268853 damselfly +n02276258 admiral +n02277742 ringlet, ringlet butterfly +n02279972 monarch, monarch butterfly, milkweed butterfly, Danaus plexippus +n02280649 cabbage butterfly +n02281406 sulphur butterfly, sulfur butterfly +n02281787 lycaenid, lycaenid butterfly +n02317335 starfish, sea star +n02319095 sea urchin +n02321529 sea cucumber, holothurian +n02325366 wood rabbit, cottontail, cottontail rabbit +n02326432 hare +n02328150 Angora, Angora rabbit +n02342885 hamster +n02346627 porcupine, hedgehog +n02356798 fox squirrel, eastern fox squirrel, Sciurus niger +n02361337 marmot +n02363005 beaver +n02364673 guinea pig, Cavia cobaya +n02389026 sorrel +n02391049 zebra +n02395406 hog, pig, grunter, squealer, Sus scrofa +n02396427 wild boar, boar, Sus scrofa +n02397096 warthog +n02398521 hippopotamus, hippo, river horse, Hippopotamus amphibius +n02403003 ox +n02408429 water buffalo, water ox, Asiatic buffalo, Bubalus bubalis +n02410509 bison +n02412080 ram, tup +n02415577 bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis +n02417914 ibex, Capra ibex +n02422106 hartebeest +n02422699 impala, Aepyceros melampus +n02423022 gazelle +n02437312 Arabian camel, dromedary, Camelus dromedarius +n02437616 llama +n02441942 weasel +n02442845 mink +n02443114 polecat, fitch, foulmart, foumart, Mustela putorius +n02443484 black-footed ferret, ferret, Mustela nigripes +n02444819 otter +n02445715 skunk, polecat, wood pussy +n02447366 badger +n02454379 armadillo +n02457408 three-toed sloth, ai, Bradypus tridactylus +n02480495 orangutan, orang, orangutang, Pongo pygmaeus +n02480855 gorilla, Gorilla gorilla +n02481823 chimpanzee, chimp, Pan troglodytes +n02483362 gibbon, Hylobates lar +n02483708 siamang, Hylobates syndactylus, Symphalangus syndactylus +n02484975 guenon, guenon monkey +n02486261 patas, hussar monkey, Erythrocebus patas +n02486410 baboon +n02487347 macaque +n02488291 langur +n02488702 colobus, colobus monkey +n02489166 proboscis monkey, Nasalis larvatus +n02490219 marmoset +n02492035 capuchin, ringtail, Cebus capucinus +n02492660 howler monkey, howler +n02493509 titi, titi monkey +n02493793 spider monkey, Ateles geoffroyi +n02494079 squirrel monkey, Saimiri sciureus +n02497673 Madagascar cat, ring-tailed lemur, Lemur catta +n02500267 indri, indris, Indri indri, Indri brevicaudatus +n02504013 Indian elephant, Elephas maximus +n02504458 African elephant, Loxodonta africana +n02509815 lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens +n02510455 giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca +n02514041 barracouta, snoek +n02526121 eel +n02536864 coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch +n02606052 rock beauty, Holocanthus tricolor +n02607072 anemone fish +n02640242 sturgeon +n02641379 gar, garfish, garpike, billfish, Lepisosteus osseus +n02643566 lionfish +n02655020 puffer, pufferfish, blowfish, globefish +n02666196 abacus +n02667093 abaya +n02669723 academic gown, academic robe, judge's robe +n02672831 accordion, piano accordion, squeeze box +n02676566 acoustic guitar +n02687172 aircraft carrier, carrier, flattop, attack aircraft carrier +n02690373 airliner +n02692877 airship, dirigible +n02699494 altar +n02701002 ambulance +n02704792 amphibian, amphibious vehicle +n02708093 analog clock +n02727426 apiary, bee house +n02730930 apron +n02747177 ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin +n02749479 assault rifle, assault gun +n02769748 backpack, back pack, knapsack, packsack, rucksack, haversack +n02776631 bakery, bakeshop, bakehouse +n02777292 balance beam, beam +n02782093 balloon +n02783161 ballpoint, ballpoint pen, ballpen, Biro +n02786058 Band Aid +n02787622 banjo +n02788148 bannister, banister, balustrade, balusters, handrail +n02790996 barbell +n02791124 barber chair +n02791270 barbershop +n02793495 barn +n02794156 barometer +n02795169 barrel, cask +n02797295 barrow, garden cart, lawn cart, wheelbarrow +n02799071 baseball +n02802426 basketball +n02804414 bassinet +n02804610 bassoon +n02807133 bathing cap, swimming cap +n02808304 bath towel +n02808440 bathtub, bathing tub, bath, tub +n02814533 beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon +n02814860 beacon, lighthouse, beacon light, pharos +n02815834 beaker +n02817516 bearskin, busby, shako +n02823428 beer bottle +n02823750 beer glass +n02825657 bell cote, bell cot +n02834397 bib +n02835271 bicycle-built-for-two, tandem bicycle, tandem +n02837789 bikini, two-piece +n02840245 binder, ring-binder +n02841315 binoculars, field glasses, opera glasses +n02843684 birdhouse +n02859443 boathouse +n02860847 bobsled, bobsleigh, bob +n02865351 bolo tie, bolo, bola tie, bola +n02869837 bonnet, poke bonnet +n02870880 bookcase +n02871525 bookshop, bookstore, bookstall +n02877765 bottlecap +n02879718 bow +n02883205 bow tie, bow-tie, bowtie +n02892201 brass, memorial tablet, plaque +n02892767 brassiere, bra, bandeau +n02894605 breakwater, groin, groyne, mole, bulwark, seawall, jetty +n02895154 breastplate, aegis, egis +n02906734 broom +n02909870 bucket, pail +n02910353 buckle +n02916936 bulletproof vest +n02917067 bullet train, bullet +n02927161 butcher shop, meat market +n02930766 cab, hack, taxi, taxicab +n02939185 caldron, cauldron +n02948072 candle, taper, wax light +n02950826 cannon +n02951358 canoe +n02951585 can opener, tin opener +n02963159 cardigan +n02965783 car mirror +n02966193 carousel, carrousel, merry-go-round, roundabout, whirligig +n02966687 carpenter's kit, tool kit +n02971356 carton +n02974003 car wheel +n02977058 cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM +n02978881 cassette +n02979186 cassette player +n02980441 castle +n02981792 catamaran +n02988304 CD player +n02992211 cello, violoncello +n02992529 cellular telephone, cellular phone, cellphone, cell, mobile phone +n02999410 chain +n03000134 chainlink fence +n03000247 chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour +n03000684 chain saw, chainsaw +n03014705 chest +n03016953 chiffonier, commode +n03017168 chime, bell, gong +n03018349 china cabinet, china closet +n03026506 Christmas stocking +n03028079 church, church building +n03032252 cinema, movie theater, movie theatre, movie house, picture palace +n03041632 cleaver, meat cleaver, chopper +n03042490 cliff dwelling +n03045698 cloak +n03047690 clog, geta, patten, sabot +n03062245 cocktail shaker +n03063599 coffee mug +n03063689 coffeepot +n03065424 coil, spiral, volute, whorl, helix +n03075370 combination lock +n03085013 computer keyboard, keypad +n03089624 confectionery, confectionary, candy store +n03095699 container ship, containership, container vessel +n03100240 convertible +n03109150 corkscrew, bottle screw +n03110669 cornet, horn, trumpet, trump +n03124043 cowboy boot +n03124170 cowboy hat, ten-gallon hat +n03125729 cradle +n03126707 crane +n03127747 crash helmet +n03127925 crate +n03131574 crib, cot +n03133878 Crock Pot +n03134739 croquet ball +n03141823 crutch +n03146219 cuirass +n03160309 dam, dike, dyke +n03179701 desk +n03180011 desktop computer +n03187595 dial telephone, dial phone +n03188531 diaper, nappy, napkin +n03196217 digital clock +n03197337 digital watch +n03201208 dining table, board +n03207743 dishrag, dishcloth +n03207941 dishwasher, dish washer, dishwashing machine +n03208938 disk brake, disc brake +n03216828 dock, dockage, docking facility +n03218198 dogsled, dog sled, dog sleigh +n03220513 dome +n03223299 doormat, welcome mat +n03240683 drilling platform, offshore rig +n03249569 drum, membranophone, tympan +n03250847 drumstick +n03255030 dumbbell +n03259280 Dutch oven +n03271574 electric fan, blower +n03272010 electric guitar +n03272562 electric locomotive +n03290653 entertainment center +n03291819 envelope +n03297495 espresso maker +n03314780 face powder +n03325584 feather boa, boa +n03337140 file, file cabinet, filing cabinet +n03344393 fireboat +n03345487 fire engine, fire truck +n03347037 fire screen, fireguard +n03355925 flagpole, flagstaff +n03372029 flute, transverse flute +n03376595 folding chair +n03379051 football helmet +n03384352 forklift +n03388043 fountain +n03388183 fountain pen +n03388549 four-poster +n03393912 freight car +n03394916 French horn, horn +n03400231 frying pan, frypan, skillet +n03404251 fur coat +n03417042 garbage truck, dustcart +n03424325 gasmask, respirator, gas helmet +n03425413 gas pump, gasoline pump, petrol pump, island dispenser +n03443371 goblet +n03444034 go-kart +n03445777 golf ball +n03445924 golfcart, golf cart +n03447447 gondola +n03447721 gong, tam-tam +n03450230 gown +n03452741 grand piano, grand +n03457902 greenhouse, nursery, glasshouse +n03459775 grille, radiator grille +n03461385 grocery store, grocery, food market, market +n03467068 guillotine +n03476684 hair slide +n03476991 hair spray +n03478589 half track +n03481172 hammer +n03482405 hamper +n03483316 hand blower, blow dryer, blow drier, hair dryer, hair drier +n03485407 hand-held computer, hand-held microcomputer +n03485794 handkerchief, hankie, hanky, hankey +n03492542 hard disc, hard disk, fixed disk +n03494278 harmonica, mouth organ, harp, mouth harp +n03495258 harp +n03496892 harvester, reaper +n03498962 hatchet +n03527444 holster +n03529860 home theater, home theatre +n03530642 honeycomb +n03532672 hook, claw +n03534580 hoopskirt, crinoline +n03535780 horizontal bar, high bar +n03538406 horse cart, horse-cart +n03544143 hourglass +n03584254 iPod +n03584829 iron, smoothing iron +n03590841 jack-o'-lantern +n03594734 jean, blue jean, denim +n03594945 jeep, landrover +n03595614 jersey, T-shirt, tee shirt +n03598930 jigsaw puzzle +n03599486 jinrikisha, ricksha, rickshaw +n03602883 joystick +n03617480 kimono +n03623198 knee pad +n03627232 knot +n03630383 lab coat, laboratory coat +n03633091 ladle +n03637318 lampshade, lamp shade +n03642806 laptop, laptop computer +n03649909 lawn mower, mower +n03657121 lens cap, lens cover +n03658185 letter opener, paper knife, paperknife +n03661043 library +n03662601 lifeboat +n03666591 lighter, light, igniter, ignitor +n03670208 limousine, limo +n03673027 liner, ocean liner +n03676483 lipstick, lip rouge +n03680355 Loafer +n03690938 lotion +n03691459 loudspeaker, speaker, speaker unit, loudspeaker system, speaker system +n03692522 loupe, jeweler's loupe +n03697007 lumbermill, sawmill +n03706229 magnetic compass +n03709823 mailbag, postbag +n03710193 mailbox, letter box +n03710637 maillot +n03710721 maillot, tank suit +n03717622 manhole cover +n03720891 maraca +n03721384 marimba, xylophone +n03724870 mask +n03729826 matchstick +n03733131 maypole +n03733281 maze, labyrinth +n03733805 measuring cup +n03742115 medicine chest, medicine cabinet +n03743016 megalith, megalithic structure +n03759954 microphone, mike +n03761084 microwave, microwave oven +n03763968 military uniform +n03764736 milk can +n03769881 minibus +n03770439 miniskirt, mini +n03770679 minivan +n03773504 missile +n03775071 mitten +n03775546 mixing bowl +n03776460 mobile home, manufactured home +n03777568 Model T +n03777754 modem +n03781244 monastery +n03782006 monitor +n03785016 moped +n03786901 mortar +n03787032 mortarboard +n03788195 mosque +n03788365 mosquito net +n03791053 motor scooter, scooter +n03792782 mountain bike, all-terrain bike, off-roader +n03792972 mountain tent +n03793489 mouse, computer mouse +n03794056 mousetrap +n03796401 moving van +n03803284 muzzle +n03804744 nail +n03814639 neck brace +n03814906 necklace +n03825788 nipple +n03832673 notebook, notebook computer +n03837869 obelisk +n03838899 oboe, hautboy, hautbois +n03840681 ocarina, sweet potato +n03841143 odometer, hodometer, mileometer, milometer +n03843555 oil filter +n03854065 organ, pipe organ +n03857828 oscilloscope, scope, cathode-ray oscilloscope, CRO +n03866082 overskirt +n03868242 oxcart +n03868863 oxygen mask +n03871628 packet +n03873416 paddle, boat paddle +n03874293 paddlewheel, paddle wheel +n03874599 padlock +n03876231 paintbrush +n03877472 pajama, pyjama, pj's, jammies +n03877845 palace +n03884397 panpipe, pandean pipe, syrinx +n03887697 paper towel +n03888257 parachute, chute +n03888605 parallel bars, bars +n03891251 park bench +n03891332 parking meter +n03895866 passenger car, coach, carriage +n03899768 patio, terrace +n03902125 pay-phone, pay-station +n03903868 pedestal, plinth, footstall +n03908618 pencil box, pencil case +n03908714 pencil sharpener +n03916031 perfume, essence +n03920288 Petri dish +n03924679 photocopier +n03929660 pick, plectrum, plectron +n03929855 pickelhaube +n03930313 picket fence, paling +n03930630 pickup, pickup truck +n03933933 pier +n03935335 piggy bank, penny bank +n03937543 pill bottle +n03938244 pillow +n03942813 ping-pong ball +n03944341 pinwheel +n03947888 pirate, pirate ship +n03950228 pitcher, ewer +n03954731 plane, carpenter's plane, woodworking plane +n03956157 planetarium +n03958227 plastic bag +n03961711 plate rack +n03967562 plow, plough +n03970156 plunger, plumber's helper +n03976467 Polaroid camera, Polaroid Land camera +n03976657 pole +n03977966 police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria +n03980874 poncho +n03982430 pool table, billiard table, snooker table +n03983396 pop bottle, soda bottle +n03991062 pot, flowerpot +n03992509 potter's wheel +n03995372 power drill +n03998194 prayer rug, prayer mat +n04004767 printer +n04005630 prison, prison house +n04008634 projectile, missile +n04009552 projector +n04019541 puck, hockey puck +n04023962 punching bag, punch bag, punching ball, punchball +n04026417 purse +n04033901 quill, quill pen +n04033995 quilt, comforter, comfort, puff +n04037443 racer, race car, racing car +n04039381 racket, racquet +n04040759 radiator +n04041544 radio, wireless +n04044716 radio telescope, radio reflector +n04049303 rain barrel +n04065272 recreational vehicle, RV, R.V. +n04067472 reel +n04069434 reflex camera +n04070727 refrigerator, icebox +n04074963 remote control, remote +n04081281 restaurant, eating house, eating place, eatery +n04086273 revolver, six-gun, six-shooter +n04090263 rifle +n04099969 rocking chair, rocker +n04111531 rotisserie +n04116512 rubber eraser, rubber, pencil eraser +n04118538 rugby ball +n04118776 rule, ruler +n04120489 running shoe +n04125021 safe +n04127249 safety pin +n04131690 saltshaker, salt shaker +n04133789 sandal +n04136333 sarong +n04141076 sax, saxophone +n04141327 scabbard +n04141975 scale, weighing machine +n04146614 school bus +n04147183 schooner +n04149813 scoreboard +n04152593 screen, CRT screen +n04153751 screw +n04154565 screwdriver +n04162706 seat belt, seatbelt +n04179913 sewing machine +n04192698 shield, buckler +n04200800 shoe shop, shoe-shop, shoe store +n04201297 shoji +n04204238 shopping basket +n04204347 shopping cart +n04208210 shovel +n04209133 shower cap +n04209239 shower curtain +n04228054 ski +n04229816 ski mask +n04235860 sleeping bag +n04238763 slide rule, slipstick +n04239074 sliding door +n04243546 slot, one-armed bandit +n04251144 snorkel +n04252077 snowmobile +n04252225 snowplow, snowplough +n04254120 soap dispenser +n04254680 soccer ball +n04254777 sock +n04258138 solar dish, solar collector, solar furnace +n04259630 sombrero +n04263257 soup bowl +n04264628 space bar +n04265275 space heater +n04266014 space shuttle +n04270147 spatula +n04273569 speedboat +n04275548 spider web, spider's web +n04277352 spindle +n04285008 sports car, sport car +n04286575 spotlight, spot +n04296562 stage +n04310018 steam locomotive +n04311004 steel arch bridge +n04311174 steel drum +n04317175 stethoscope +n04325704 stole +n04326547 stone wall +n04328186 stopwatch, stop watch +n04330267 stove +n04332243 strainer +n04335435 streetcar, tram, tramcar, trolley, trolley car +n04336792 stretcher +n04344873 studio couch, day bed +n04346328 stupa, tope +n04347754 submarine, pigboat, sub, U-boat +n04350905 suit, suit of clothes +n04355338 sundial +n04355933 sunglass +n04356056 sunglasses, dark glasses, shades +n04357314 sunscreen, sunblock, sun blocker +n04366367 suspension bridge +n04367480 swab, swob, mop +n04370456 sweatshirt +n04371430 swimming trunks, bathing trunks +n04371774 swing +n04372370 switch, electric switch, electrical switch +n04376876 syringe +n04380533 table lamp +n04389033 tank, army tank, armored combat vehicle, armoured combat vehicle +n04392985 tape player +n04398044 teapot +n04399382 teddy, teddy bear +n04404412 television, television system +n04409515 tennis ball +n04417672 thatch, thatched roof +n04418357 theater curtain, theatre curtain +n04423845 thimble +n04428191 thresher, thrasher, threshing machine +n04429376 throne +n04435653 tile roof +n04442312 toaster +n04443257 tobacco shop, tobacconist shop, tobacconist +n04447861 toilet seat +n04456115 torch +n04458633 totem pole +n04461696 tow truck, tow car, wrecker +n04462240 toyshop +n04465501 tractor +n04467665 trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi +n04476259 tray +n04479046 trench coat +n04482393 tricycle, trike, velocipede +n04483307 trimaran +n04485082 tripod +n04486054 triumphal arch +n04487081 trolleybus, trolley coach, trackless trolley +n04487394 trombone +n04493381 tub, vat +n04501370 turnstile +n04505470 typewriter keyboard +n04507155 umbrella +n04509417 unicycle, monocycle +n04515003 upright, upright piano +n04517823 vacuum, vacuum cleaner +n04522168 vase +n04523525 vault +n04525038 velvet +n04525305 vending machine +n04532106 vestment +n04532670 viaduct +n04536866 violin, fiddle +n04540053 volleyball +n04542943 waffle iron +n04548280 wall clock +n04548362 wallet, billfold, notecase, pocketbook +n04550184 wardrobe, closet, press +n04552348 warplane, military plane +n04553703 washbasin, handbasin, washbowl, lavabo, wash-hand basin +n04554684 washer, automatic washer, washing machine +n04557648 water bottle +n04560804 water jug +n04562935 water tower +n04579145 whiskey jug +n04579432 whistle +n04584207 wig +n04589890 window screen +n04590129 window shade +n04591157 Windsor tie +n04591713 wine bottle +n04592741 wing +n04596742 wok +n04597913 wooden spoon +n04599235 wool, woolen, woollen +n04604644 worm fence, snake fence, snake-rail fence, Virginia fence +n04606251 wreck +n04612504 yawl +n04613696 yurt +n06359193 web site, website, internet site, site +n06596364 comic book +n06785654 crossword puzzle, crossword +n06794110 street sign +n06874185 traffic light, traffic signal, stoplight +n07248320 book jacket, dust cover, dust jacket, dust wrapper +n07565083 menu +n07579787 plate +n07583066 guacamole +n07584110 consomme +n07590611 hot pot, hotpot +n07613480 trifle +n07614500 ice cream, icecream +n07615774 ice lolly, lolly, lollipop, popsicle +n07684084 French loaf +n07693725 bagel, beigel +n07695742 pretzel +n07697313 cheeseburger +n07697537 hotdog, hot dog, red hot +n07711569 mashed potato +n07714571 head cabbage +n07714990 broccoli +n07715103 cauliflower +n07716358 zucchini, courgette +n07716906 spaghetti squash +n07717410 acorn squash +n07717556 butternut squash +n07718472 cucumber, cuke +n07718747 artichoke, globe artichoke +n07720875 bell pepper +n07730033 cardoon +n07734744 mushroom +n07742313 Granny Smith +n07745940 strawberry +n07747607 orange +n07749582 lemon +n07753113 fig +n07753275 pineapple, ananas +n07753592 banana +n07754684 jackfruit, jak, jack +n07760859 custard apple +n07768694 pomegranate +n07802026 hay +n07831146 carbonara +n07836838 chocolate sauce, chocolate syrup +n07860988 dough +n07871810 meat loaf, meatloaf +n07873807 pizza, pizza pie +n07875152 potpie +n07880968 burrito +n07892512 red wine +n07920052 espresso +n07930864 cup +n07932039 eggnog +n09193705 alp +n09229709 bubble +n09246464 cliff, drop, drop-off +n09256479 coral reef +n09288635 geyser +n09332890 lakeside, lakeshore +n09399592 promontory, headland, head, foreland +n09421951 sandbar, sand bar +n09428293 seashore, coast, seacoast, sea-coast +n09468604 valley, vale +n09472597 volcano +n09835506 ballplayer, baseball player +n10148035 groom, bridegroom +n10565667 scuba diver +n11879895 rapeseed +n11939491 daisy +n12057211 yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum +n12144580 corn +n12267677 acorn +n12620546 hip, rose hip, rosehip +n12768682 buckeye, horse chestnut, conker +n12985857 coral fungus +n12998815 agaric +n13037406 gyromitra +n13040303 stinkhorn, carrion fungus +n13044778 earthstar +n13052670 hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa +n13054560 bolete +n13133613 ear, spike, capitulum +n15075141 toilet tissue, toilet paper, bathroom tissue diff --git a/samples/c_sharp/OpenVINO_EP/yolov3_object_detection/Label.cs b/samples/c_sharp/OpenVINO_EP/yolov3_object_detection/Label.cs new file mode 100644 index 0000000000..0a9a9aebe8 --- /dev/null +++ b/samples/c_sharp/OpenVINO_EP/yolov3_object_detection/Label.cs @@ -0,0 +1,91 @@ +/* +Copyright (C) 2021, Intel Corporation +SPDX-License-Identifier: Apache-2.0 +*/ + +namespace yolov3 +{ + public class LabelMap + { + public static readonly string[] Labels = new[] {"person", + "bicycle", + "car", + "motorcycle", + "airplane", + "bus", + "train", + "truck", + "boat", + "traffic light", + "fire hydrant", + "stop sign", + "parking meter", + "bench", + "bird", + "cat", + "dog", + "horse", + "sheep", + "cow", + "elephant", + "bear", + "zebra", + "giraffe", + "backpack", + "umbrella", + "handbag", + "tie", + "suitcase", + "frisbee", + "skis", + "snowboard", + "sports ball", + "kite", + "baseball bat", + "baseball glove", + "skateboard", + "surfboard", + "tennis racket", + "bottle", + "wine glass", + "cup", + "fork", + "knife", + "spoon", + "bowl", + "banana", + "apple", + "sandwich", + "orange", + "broccoli", + "carrot", + "hot dog", + "pizza", + "donut", + "cake", + "chair", + "couch", + "potted plant", + "bed", + "dining table", + "toilet", + "tv", + "laptop", + "mouse", + "remote", + "keyboard", + "cell phone", + "microwave", + "oven", + "toaster", + "sink", + "refrigerator", + "book", + "clock", + "vase", + "scissors", + "teddy bear", + "hair drier", + "toothbrush"}; + } +} \ No newline at end of file diff --git a/samples/c_sharp/OpenVINO_EP/yolov3_object_detection/Prediction.cs b/samples/c_sharp/OpenVINO_EP/yolov3_object_detection/Prediction.cs new file mode 100644 index 0000000000..8eeedfbe98 --- /dev/null +++ b/samples/c_sharp/OpenVINO_EP/yolov3_object_detection/Prediction.cs @@ -0,0 +1,31 @@ +/* +Copyright (C) 2021, Intel Corporation +SPDX-License-Identifier: Apache-2.0 +*/ + +namespace yolov3 +{ + public class Prediction + { + public Box Box { get; set; } + public string Class { get; set; } + public float Score { get; set; } + } + + public class Box + { + public float Xmin { get; set; } + public float Ymin { get; set; } + public float Xmax { get; set; } + public float Ymax { get; set; } + + public Box(float xmin, float ymin, float xmax, float ymax) + { + Xmin = xmin; + Ymin = ymin; + Xmax = xmax; + Ymax = ymax; + + } + } +} \ No newline at end of file diff --git a/samples/c_sharp/OpenVINO_EP/yolov3_object_detection/Program.cs b/samples/c_sharp/OpenVINO_EP/yolov3_object_detection/Program.cs new file mode 100644 index 0000000000..ce39f0bec4 --- /dev/null +++ b/samples/c_sharp/OpenVINO_EP/yolov3_object_detection/Program.cs @@ -0,0 +1,173 @@ +/* +Copyright (C) 2021, Intel Corporation +SPDX-License-Identifier: Apache-2.0 +*/ + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Microsoft.ML.OnnxRuntime.Tensors; +using Microsoft.ML.OnnxRuntime; +using SixLabors.ImageSharp; +using SixLabors.ImageSharp.PixelFormats; +using SixLabors.ImageSharp.Processing; +using SixLabors.ImageSharp.Formats; +using SixLabors.ImageSharp.Drawing.Processing; +using SixLabors.Fonts; + +namespace yolov3 +{ + class Program + { + static void Main(string[] args) + { + // string is null or empty + if (args == null || args.Length < 3) + { + Console.WriteLine("Usage information: dotnet run model.onnx input.jpg output.jpg"); + return; + } else + { + if(!(File.Exists(args[0]))) + { + Console.WriteLine("Model Path does not exist"); + return; + } + if (!(File.Exists(args[1]))) + { + Console.WriteLine("Input Path does not exist"); + return; + } + } + + // Read paths + string modelFilePath = args[0]; + string imageFilePath = args[1]; + string outImageFilePath = args[2]; + + using Image imageOrg = Image.Load(imageFilePath, out IImageFormat format); + + //Letterbox image + var iw = imageOrg.Width; + var ih = imageOrg.Height; + var w = 416; + var h = 416; + + if ((iw == 0) || (ih == 0)) + { + Console.WriteLine("Math error: Attempted to divide by Zero"); + return; + } + + float width = (float)w / iw; + float height = (float)h / ih; + + float scale = Math.Min(width, height); + + var nw = (int)(iw * scale); + var nh = (int)(ih * scale); + + var pad_dims_w = (w - nw) / 2; + var pad_dims_h = (h - nh) / 2; + + // Resize image using default bicubic sampler + var image = imageOrg.Clone(x => x.Resize((nw), (nh))); + + var clone = new Image(w, h); + clone.Mutate(i => i.Fill(Color.Gray)); + clone.Mutate(o => o.DrawImage(image, new Point(pad_dims_w, pad_dims_h), 1f)); // draw the first one top left + + //Preprocessing image + Tensor input = new DenseTensor(new[] { 1, 3, h, w }); + for (int y = 0; y < clone.Height; y++) + { + Span pixelSpan = clone.GetPixelRowSpan(y); + for (int x = 0; x < clone.Width; x++) + { + input[0, 0, y, x] = pixelSpan[x].B / 255f; + input[0, 1, y, x] = pixelSpan[x].G / 255f; + input[0, 2, y, x] = pixelSpan[x].R / 255f; + } + } + + //Get the Image Shape + var image_shape = new DenseTensor(new[] { 1, 2 }); + image_shape[0, 0] = ih; + image_shape[0, 1] = iw; + + // Setup inputs and outputs + var container = new List(); + container.Add(NamedOnnxValue.CreateFromTensor("input_1", input)); + container.Add(NamedOnnxValue.CreateFromTensor("image_shape", image_shape)); + + // Session Options + SessionOptions options = new SessionOptions(); + options.LogSeverityLevel = OrtLoggingLevel.ORT_LOGGING_LEVEL_INFO; + options.AppendExecutionProvider_OpenVINO(@"MYRIAD_FP16"); + options.AppendExecutionProvider_CPU(1); + + // Run inference + using var session = new InferenceSession(modelFilePath,options); + + using IDisposableReadOnlyCollection results = session.Run(container); + + Console.WriteLine("Inference done"); + + //Post Processing Steps + var resultsArray = results.ToArray(); + Tensor boxes = resultsArray[0].AsTensor(); + Tensor scores = resultsArray[1].AsTensor(); + int[] indices = resultsArray[2].AsTensor().ToArray(); + + var len = indices.Length / 3; + var out_classes = new int[len]; + float[] out_scores = new float[len]; + + var predictions = new List(); + var count = 0; + for (int i = 0; i < indices.Length; i = i + 3) + { + out_classes[count] = indices[i + 1]; + out_scores[count] = scores[indices[i], indices[i + 1], indices[i + 2]]; + predictions.Add(new Prediction + { + Box = new Box(boxes[indices[i], indices[i + 2], 1], + boxes[indices[i], indices[i + 2], 0], + boxes[indices[i], indices[i + 2], 3], + boxes[indices[i], indices[i + 2], 2]), + Class = LabelMap.Labels[out_classes[count]], + Score = out_scores[count] + }); + count++; + } + + // Put boxes, labels and confidence on image and save for viewing + using var outputImage = File.OpenWrite(outImageFilePath); + Font font = SystemFonts.CreateFont("Arial", 16); + foreach (var p in predictions) + { + imageOrg.Mutate(x => + { + x.DrawLines(Color.Red, 2f, new PointF[] { + + new PointF(p.Box.Xmin, p.Box.Ymin), + new PointF(p.Box.Xmax, p.Box.Ymin), + + new PointF(p.Box.Xmax, p.Box.Ymin), + new PointF(p.Box.Xmax, p.Box.Ymax), + + new PointF(p.Box.Xmax, p.Box.Ymax), + new PointF(p.Box.Xmin, p.Box.Ymax), + + new PointF(p.Box.Xmin, p.Box.Ymax), + new PointF(p.Box.Xmin, p.Box.Ymin) + }); + x.DrawText($"{p.Class}, {p.Score:0.00}", font, Color.White, new PointF(p.Box.Xmin, p.Box.Ymin)); + }); + } + imageOrg.Save(outputImage, format); + + } + } +} diff --git a/samples/python/OpenVINO_EP/tiny_yolo_v2_object_detection/tiny_yolov2_obj_detection_sample.py b/samples/python/OpenVINO_EP/tiny_yolo_v2_object_detection/tiny_yolov2_obj_detection_sample.py new file mode 100644 index 0000000000..34e3224b60 --- /dev/null +++ b/samples/python/OpenVINO_EP/tiny_yolo_v2_object_detection/tiny_yolov2_obj_detection_sample.py @@ -0,0 +1,195 @@ +''' +Copyright (C) 2021, Intel Corporation +SPDX-License-Identifier: Apache-2.0 +''' + +import numpy as np +import onnxruntime as rt +import cv2 +import time +import os + +def sigmoid(x, derivative=False): + return x*(1-x) if derivative else 1/(1+np.exp(-x)) + +def softmax(x): + scoreMatExp = np.exp(np.asarray(x)) + return scoreMatExp / scoreMatExp.sum(0) + +def checkModelExtension(fp): + # Split the extension from the path and normalise it to lowercase. + ext = os.path.splitext(fp)[-1].lower() + + # Now we can simply use != to check for inequality, no need for wildcards. + if(ext != ".onnx"): + raise Exception(fp, "is an unknown file format. Use the model ending with .onnx format") + + if not os.path.exists(fp): + raise Exception("[ ERROR ] Path of the onnx model file is Invalid") + +def checkVideoFileExtension(fp): + # Split the extension from the path and normalise it to lowercase. + ext = os.path.splitext(fp)[-1].lower() + # Now we can simply use != to check for inequality, no need for wildcards. + + if(ext == ".mp4" or ext == ".avi" or ext == ".mov"): + pass + else: + raise Exception(fp, "is an unknown file format. Use the video file ending with .mp4 or .avi or .mov formats") + + if not os.path.exists(fp): + raise Exception("[ ERROR ] Path of the video file is Invalid") + +# color look up table for different classes for object detection sample +clut = [(0,0,0),(255,0,0),(255,0,255),(0,0,255),(0,255,0),(0,255,128), + (128,255,0),(128,128,0),(0,128,255),(128,0,128), + (255,0,128),(128,0,255),(255,128,128),(128,255,128),(255,255,0), + (255,128,128),(128,128,255),(255,128,128),(128,255,128),(128,255,128)] + +# 20 labels that the tiny-yolov2 model can do the object_detection on +label = ["aeroplane","bicycle","bird","boat","bottle", + "bus","car","cat","chair","cow","diningtable", + "dog","horse","motorbike","person","pottedplant", + "sheep","sofa","train","tvmonitor"] + +model_file_path = "tiny_yolo_v2_zoo_model.onnx" +# TODO: You need to modify the path to the input onnx model based on where it is located on your device after downloading it from ONNX Model zoo. + +# Validate model file path +checkModelExtension(model_file_path) + +# Load the model +sess = rt.InferenceSession(model_file_path) + +# Get the input name of the model +input_name = sess.get_inputs()[0].name + +device = 'CPU_FP32' +# Set OpenVINO as the Execution provider to infer this model +sess.set_providers(['OpenVINOExecutionProvider'], [{'device_type' : device}]) +''' +other 'device_type' options are: (Any hardware target can be assigned if you have the access to it) + +'CPU_FP32', 'GPU_FP32', 'GPU_FP16', 'MYRIAD_FP16', 'VAD-M_FP16', 'VAD-F_FP32', +'HETERO:MYRIAD,CPU', 'MULTI:MYRIAD,GPU,CPU' + +''' + +#Path to video file has to be provided +video_file_path = "sample_demo_video.mp4" +# TODO: You need to specify the path to your own sample video based on where it is located on your device. + +#validate video file input path +checkVideoFileExtension(video_file_path) + +#Path to video file has to be provided +cap = cv2.VideoCapture(video_file_path) + +# capturing different metrics of the image from the video +fps = cap.get(cv2.CAP_PROP_FPS) +width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) +height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) +x_scale = float(width)/416.0 #In the document of tino-yolo-v2, input shape of this network is (1,3,416,416). +y_scale = float(height)/416.0 + +# writing the inferencing output as a video to the local disk +fourcc = cv2.VideoWriter_fourcc(*'XVID') +output_video_name = device + "_output.avi" +output_video = cv2.VideoWriter(output_video_name,fourcc, float(17.0), (640,360)) + +# capturing one frame at a time from the video feed and performing the inference +i = 0 +while cap.isOpened(): + l_start = time.time() + ret, frame = cap.read() + if not ret: + break + initial_w = cap.get(3) + initial_h = cap.get(4) + + # preprocessing the input frame and reshaping it. + #In the document of tino-yolo-v2, input shape of this network is (1,3,416,416). so we resize the model frame w.r.t that size. + in_frame = cv2.resize(frame, (416, 416)) + X = np.asarray(in_frame) + X = X.astype(np.float32) + X = X.transpose(2,0,1) + # Reshaping the input array to align with the input shape of the model + X = X.reshape(1,3,416,416) + + start = time.time() + #Running the session by passing in the input data of the model + out = sess.run(None, {input_name: X}) + end = time.time() + inference_time = end - start + out = out[0][0] + + numClasses = 20 + anchors = [1.08, 1.19, 3.42, 4.41, 6.63, 11.38, 9.42, 5.11, 16.62, 10.52] + + existingLabels = {l: [] for l in label} + + #Inside this loop we compute the bounding box b for grid cell (cy, cx) + for cy in range(0,13): + for cx in range(0,13): + for b in range(0,5): + # First we read the tx, ty, width(tw), and height(th) for the bounding box from the out array, as well as the confidence score + channel = b*(numClasses+5) + tx = out[channel ][cy][cx] + ty = out[channel+1][cy][cx] + tw = out[channel+2][cy][cx] + th = out[channel+3][cy][cx] + tc = out[channel+4][cy][cx] + + x = (float(cx) + sigmoid(tx))*32 + y = (float(cy) + sigmoid(ty))*32 + + w = np.exp(tw) * 32 * anchors[2*b ] + h = np.exp(th) * 32 * anchors[2*b+1] + + #calculating the confidence score + confidence = sigmoid(tc) # The confidence value for the bounding box is given by tc + + classes = np.zeros(numClasses) + for c in range(0,numClasses): + classes[c] = out[channel + 5 +c][cy][cx] + # we take the softmax to turn the array into a probability distribution. And then we pick the class with the largest score as the winner. + classes = softmax(classes) + detectedClass = classes.argmax() + + # Now we can compute the final score for this bounding box and we only want to keep the ones whose combined score is over a certain threshold + if 0.45< classes[detectedClass]*confidence: + color =clut[detectedClass] + x = (x - w/2)*x_scale + y = (y - h/2)*y_scale + w *= x_scale + h *= y_scale + + labelX = int((x+x+w)/2) + labelY = int((y+y+h)/2) + addLabel = True + labThreshold = 40 + for point in existingLabels[label[detectedClass]]: + if labelX < point[0] + labThreshold and labelX > point[0] - labThreshold and \ + labelY < point[1] + labThreshold and labelY > point[1] - labThreshold: + addLabel = False + #Adding class labels to the output of the frame and also drawing a rectangular bounding box around the object detected. + if addLabel: + cv2.rectangle(frame, (int(x),int(y)),(int(x+w),int(y+h)),color,2) + cv2.rectangle(frame, (int(x),int(y-13)),(int(x)+9*len(label[detectedClass]),int(y)),color,-1) + cv2.putText(frame,label[detectedClass],(int(x)+2,int(y)-3),cv2.FONT_HERSHEY_COMPLEX,0.4,(255,255,255),1) + existingLabels[label[detectedClass]].append((labelX,labelY)) + print('{} detected in frame {}'.format(label[detectedClass],i)) + output_video.write(frame) + cv2.putText(frame,device,(10,20),cv2.FONT_HERSHEY_COMPLEX,0.5,(255,255,255),1) + cv2.putText(frame,'FPS: {}'.format(1.0/inference_time),(10,40),cv2.FONT_HERSHEY_COMPLEX,0.5,(255,255,255),1) + cv2.imshow('frame',frame) + + #Press 'q' to quit the process + if cv2.waitKey(1) & 0xFF == ord('q'): + break + print('Processed Frame {}'.format(i)) + i += 1 + l_end = time.time() + print('Loop Time = {}'.format(l_end - l_start)) +output_video.release() +cv2.destroyAllWindows() \ No newline at end of file From 5a63904aa9d089a47b313d52d5f95c7318c54d87 Mon Sep 17 00:00:00 2001 From: Ryan Hill <38674843+RyanUnderhill@users.noreply.github.com> Date: Fri, 28 May 2021 13:22:45 -0700 Subject: [PATCH 35/47] Remove some templated versions of functions that are no longer needed (#7868) * Switch to non template version of function --- .../core/framework/kernel_def_builder.h | 25 ++----------------- .../core/framework/kernel_def_builder.cc | 2 +- .../src/AbiCustomRegistry.cpp | 6 ++--- .../migraphx/migraphx_execution_provider.cc | 4 +-- .../rknpu/rknpu_execution_provider.cc | 4 +-- .../providers/rocm/reduction/reduction_ops.cc | 2 +- .../providers/rocm/rocm_execution_provider.cc | 4 +-- .../training_ops/cpu/communication/recv.cc | 6 ++--- .../training_ops/cpu/communication/send.cc | 6 ++--- .../rocm/reduction/reduction_ops.cc | 2 +- 10 files changed, 20 insertions(+), 41 deletions(-) diff --git a/include/onnxruntime/core/framework/kernel_def_builder.h b/include/onnxruntime/core/framework/kernel_def_builder.h index f01405e8d4..d02f19c591 100644 --- a/include/onnxruntime/core/framework/kernel_def_builder.h +++ b/include/onnxruntime/core/framework/kernel_def_builder.h @@ -302,12 +302,6 @@ class KernelDefBuilder { Specify that this kernel requires an input arg in certain memory type (instead of the default, device memory). */ - template - KernelDefBuilder& InputMemoryType(int input_index) { - kernel_def_->input_memory_type_args_.insert(std::make_pair(input_index, T)); - return *this; - } - KernelDefBuilder& InputMemoryType(OrtMemType type, int input_index) { kernel_def_->input_memory_type_args_.insert(std::make_pair(input_index, type)); return *this; @@ -317,14 +311,6 @@ class KernelDefBuilder { Specify that this kernel requires input arguments in certain memory type (instead of the default, device memory). */ - template - KernelDefBuilder& InputMemoryType(const std::vector& input_indexes) { - for (auto input_index : input_indexes) { - kernel_def_->input_memory_type_args_.insert(std::make_pair(input_index, T)); - } - return *this; - } - KernelDefBuilder& InputMemoryType(OrtMemType type, const std::vector& input_indexes) { for (auto input_index : input_indexes) { kernel_def_->input_memory_type_args_.insert(std::make_pair(input_index, type)); @@ -336,12 +322,6 @@ class KernelDefBuilder { Specify that this kernel provides an output arg in certain memory type (instead of the default, device memory). */ - template - KernelDefBuilder& OutputMemoryType(int output_index) { - kernel_def_->output_memory_type_args_.insert(std::make_pair(output_index, T)); - return *this; - } - KernelDefBuilder& OutputMemoryType(OrtMemType type, int output_index) { kernel_def_->output_memory_type_args_.insert(std::make_pair(output_index, type)); return *this; @@ -351,10 +331,9 @@ class KernelDefBuilder { Specify that this kernel provides an output arguments in certain memory type (instead of the default, device memory). */ - template - KernelDefBuilder& OutputMemoryType(const std::vector& output_indexes) { + KernelDefBuilder& OutputMemoryType(OrtMemType type, const std::vector& output_indexes) { for (auto output_index : output_indexes) { - kernel_def_->output_memory_type_args_.insert(std::make_pair(output_index, T)); + kernel_def_->output_memory_type_args_.insert(std::make_pair(output_index, type)); } return *this; } diff --git a/onnxruntime/core/framework/kernel_def_builder.cc b/onnxruntime/core/framework/kernel_def_builder.cc index 27af2efa41..a1162fd520 100644 --- a/onnxruntime/core/framework/kernel_def_builder.cc +++ b/onnxruntime/core/framework/kernel_def_builder.cc @@ -76,7 +76,7 @@ void KernelDef::CalculateHash() { // TODO: Tell user why it has conflicts // TODO: Investigate why IsConflict() was not triggered when there were duplicate Tile CUDA -// kernels registered. Removing `InputMemoryType(1)` in the kernel definition +// kernels registered. Removing `InputMemoryType(OrtMemTypeCPUInput, 1)` in the kernel definition // triggered the conflict. bool KernelDef::IsConflict(const KernelDef& other) const { if (op_name_ != other.OpName() || provider_type_ != other.Provider()) diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/AbiCustomRegistry.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/AbiCustomRegistry.cpp index 888110d477..b15c84963d 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/AbiCustomRegistry.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/AbiCustomRegistry.cpp @@ -387,11 +387,11 @@ HRESULT STDMETHODCALLTYPE AbiCustomRegistry::RegisterOperatorKernel( std::string_view name(opKernel->name); if (name == "MemcpyToHost") { - builder.OutputMemoryType<::OrtMemType::OrtMemTypeCPUOutput>(0); + builder.OutputMemoryType(::OrtMemType::OrtMemTypeCPUOutput, 0); } else if (name == "MemcpyFromHost") { - builder.InputMemoryType<::OrtMemType::OrtMemTypeCPUInput>(0); + builder.InputMemoryType(::OrtMemType::OrtMemTypeCPUInput, 0); } std::vector constantCpuInputCapture; @@ -399,7 +399,7 @@ HRESULT STDMETHODCALLTYPE AbiCustomRegistry::RegisterOperatorKernel( for (uint32_t inputIndex : constantCpuInputCapture) { - builder.InputMemoryType<::OrtMemType::OrtMemTypeCPUInput>(inputIndex); + builder.InputMemoryType(::OrtMemType::OrtMemTypeCPUInput, inputIndex); } if (canAliasFirstInput) diff --git a/onnxruntime/core/providers/migraphx/migraphx_execution_provider.cc b/onnxruntime/core/providers/migraphx/migraphx_execution_provider.cc index 36022e770c..0f0e3a6d26 100644 --- a/onnxruntime/core/providers/migraphx/migraphx_execution_provider.cc +++ b/onnxruntime/core/providers/migraphx/migraphx_execution_provider.cc @@ -42,7 +42,7 @@ ONNX_OPERATOR_KERNEL_EX( 1, kMIGraphXExecutionProvider, KernelDefBuilder() - .InputMemoryType(0) + .InputMemoryType(OrtMemTypeCPUInput, 0) .ExecQueueId(kHipStreamCopyIn) .TypeConstraint("T", DataTypeImpl::AllFixedSizeTensorTypes()), Memcpy); @@ -53,7 +53,7 @@ ONNX_OPERATOR_KERNEL_EX( 1, kMIGraphXExecutionProvider, KernelDefBuilder() - .OutputMemoryType(0) + .OutputMemoryType(OrtMemTypeCPUOutput, 0) .ExecQueueId(kHipStreamCopyOut) .TypeConstraint("T", DataTypeImpl::AllFixedSizeTensorTypes()), Memcpy); diff --git a/onnxruntime/core/providers/rknpu/rknpu_execution_provider.cc b/onnxruntime/core/providers/rknpu/rknpu_execution_provider.cc index bb19aa12d0..2942d1267d 100644 --- a/onnxruntime/core/providers/rknpu/rknpu_execution_provider.cc +++ b/onnxruntime/core/providers/rknpu/rknpu_execution_provider.cc @@ -524,7 +524,7 @@ ONNX_OPERATOR_KERNEL_EX( 1, kRknpuExecutionProvider, KernelDefBuilder() - .InputMemoryType(0) + .InputMemoryType(OrtMemTypeCPUInput, 0) .TypeConstraint("T", DataTypeImpl::AllFixedSizeTensorTypes()), Memcpy); @@ -534,7 +534,7 @@ ONNX_OPERATOR_KERNEL_EX( 1, kRknpuExecutionProvider, KernelDefBuilder() - .OutputMemoryType(0) + .OutputMemoryType(OrtMemTypeCPUOutput, 0) .TypeConstraint("T", DataTypeImpl::AllFixedSizeTensorTypes()), Memcpy); diff --git a/onnxruntime/core/providers/rocm/reduction/reduction_ops.cc b/onnxruntime/core/providers/rocm/reduction/reduction_ops.cc index 907e9404d8..8c43b2e5b3 100644 --- a/onnxruntime/core/providers/rocm/reduction/reduction_ops.cc +++ b/onnxruntime/core/providers/rocm/reduction/reduction_ops.cc @@ -122,7 +122,7 @@ namespace rocm { T, \ kRocmExecutionProvider, \ KernelDefBuilder() \ - .InputMemoryType(1) \ + .InputMemoryType(OrtMemTypeCPUInput, 1) \ .TypeConstraint("T", DataTypeImpl::GetTensorType()), \ name); diff --git a/onnxruntime/core/providers/rocm/rocm_execution_provider.cc b/onnxruntime/core/providers/rocm/rocm_execution_provider.cc index ba74470fbe..8aef431266 100644 --- a/onnxruntime/core/providers/rocm/rocm_execution_provider.cc +++ b/onnxruntime/core/providers/rocm/rocm_execution_provider.cc @@ -40,7 +40,7 @@ ONNX_OPERATOR_KERNEL_EX( 1, kRocmExecutionProvider, KernelDefBuilder() - .InputMemoryType(0) + .InputMemoryType(OrtMemTypeCPUInput, 0) .ExecQueueId(kHipStreamCopyIn) .TypeConstraint("T", DataTypeImpl::AllFixedSizeTensorTypes()), Memcpy); @@ -51,7 +51,7 @@ ONNX_OPERATOR_KERNEL_EX( 1, kRocmExecutionProvider, KernelDefBuilder() - .OutputMemoryType(0) + .OutputMemoryType(OrtMemTypeCPUOutput, 0) .ExecQueueId(kHipStreamCopyOut) .TypeConstraint("T", DataTypeImpl::AllFixedSizeTensorTypes()), Memcpy); diff --git a/orttraining/orttraining/training_ops/cpu/communication/recv.cc b/orttraining/orttraining/training_ops/cpu/communication/recv.cc index 9cdf127037..960b7a6349 100644 --- a/orttraining/orttraining/training_ops/cpu/communication/recv.cc +++ b/orttraining/orttraining/training_ops/cpu/communication/recv.cc @@ -51,9 +51,9 @@ ONNX_OPERATOR_KERNEL_EX( 1, kCpuExecutionProvider, KernelDefBuilder() - .InputMemoryType(0) /* CPU variable */ - .InputMemoryType(1) /* CPU variable */ - .OutputMemoryType(0) /* CPU variable */ + .InputMemoryType(OrtMemTypeDefault, 0) /* CPU variable */ + .InputMemoryType(OrtMemTypeDefault, 1) /* CPU variable */ + .OutputMemoryType(OrtMemTypeDefault, 0) /* CPU variable */ .TypeConstraint("TBool", DataTypeImpl::GetTensorType()) .TypeConstraint("TInt64", DataTypeImpl::GetTensorType()) .TypeConstraint("V", DataTypeImpl::AllFixedSizeTensorTypes()), diff --git a/orttraining/orttraining/training_ops/cpu/communication/send.cc b/orttraining/orttraining/training_ops/cpu/communication/send.cc index d88698842b..cab7d86b44 100644 --- a/orttraining/orttraining/training_ops/cpu/communication/send.cc +++ b/orttraining/orttraining/training_ops/cpu/communication/send.cc @@ -17,9 +17,9 @@ ONNX_OPERATOR_KERNEL_EX( 1, kCpuExecutionProvider, KernelDefBuilder() - .InputMemoryType(0) /* CPU variable */ - .InputMemoryType(1) /* CPU variable */ - .OutputMemoryType(0) /* CPU variable */ + .InputMemoryType(OrtMemTypeDefault, 0) /* CPU variable */ + .InputMemoryType(OrtMemTypeDefault, 1) /* CPU variable */ + .OutputMemoryType(OrtMemTypeDefault, 0) /* CPU variable */ .TypeConstraint("TBool", DataTypeImpl::GetTensorType()) .TypeConstraint("TInt64", DataTypeImpl::GetTensorType()) .TypeConstraint("V", DataTypeImpl::AllFixedSizeTensorTypes()), diff --git a/orttraining/orttraining/training_ops/rocm/reduction/reduction_ops.cc b/orttraining/orttraining/training_ops/rocm/reduction/reduction_ops.cc index c628efb013..6742ed4c1f 100644 --- a/orttraining/orttraining/training_ops/rocm/reduction/reduction_ops.cc +++ b/orttraining/orttraining/training_ops/rocm/reduction/reduction_ops.cc @@ -22,7 +22,7 @@ namespace rocm { T, \ kRocmExecutionProvider, \ KernelDefBuilder() \ - .InputMemoryType(1) \ + .InputMemoryType(OrtMemTypeCPUInput, 1) \ .TypeConstraint("T", DataTypeImpl::GetTensorType()), \ name); From 4dd724ef1ad5d15a9bc56541e494dbbc03deca4b Mon Sep 17 00:00:00 2001 From: "Gao, Chun" <53756924+chun137@users.noreply.github.com> Date: Sat, 29 May 2021 07:29:58 +0800 Subject: [PATCH 36/47] Enable WebAssembly SIMD build (#7839) Add a build switch "--enable_wasm_simd" to enable WebAssembly SIMD build --- cmake/CMakeLists.txt | 4 ++++ cmake/onnxruntime_mlas.cmake | 12 +++++++++--- cmake/onnxruntime_webassembly.cmake | 18 ++++++++++++++---- onnxruntime/core/mlas/inc/mlas.h | 2 +- .../mlas/lib/wasm_simd/SgemvKernelWasmSimd.cpp | 4 +--- tools/ci_build/build.py | 2 ++ 6 files changed, 31 insertions(+), 11 deletions(-) diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index e473f4b4fa..7e8367ae3d 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -1700,6 +1700,10 @@ if (onnxruntime_BUILD_CSHARP) endif() if (onnxruntime_BUILD_WEBASSEMBLY) + if (onnxruntime_ENABLE_WEBASSEMBLY_SIMD) + string(APPEND CMAKE_CXX_FLAGS " -msimd128") + endif() + if (onnxruntime_ENABLE_WEBASSEMBLY_EXCEPTION_CATCHING) string(APPEND CMAKE_CXX_FLAGS " -s DISABLE_EXCEPTION_CATCHING=0") endif() diff --git a/cmake/onnxruntime_mlas.cmake b/cmake/onnxruntime_mlas.cmake index 6ee1db9ac0..b15d1b78e5 100644 --- a/cmake/onnxruntime_mlas.cmake +++ b/cmake/onnxruntime_mlas.cmake @@ -25,9 +25,15 @@ set(mlas_common_srcs ) if (onnxruntime_BUILD_WEBASSEMBLY) - file(GLOB_RECURSE mlas_platform_srcs - "${ONNXRUNTIME_ROOT}/core/mlas/lib/wasm/*.cpp" - ) + if (onnxruntime_ENABLE_WEBASSEMBLY_SIMD) + file(GLOB_RECURSE mlas_platform_srcs + "${ONNXRUNTIME_ROOT}/core/mlas/lib/wasm_simd/*.cpp" + ) + else() + file(GLOB_RECURSE mlas_platform_srcs + "${ONNXRUNTIME_ROOT}/core/mlas/lib/wasm/*.cpp" + ) + endif() elseif(MSVC) if(onnxruntime_target_platform STREQUAL "ARM64") set(mlas_platform_preprocess_srcs diff --git a/cmake/onnxruntime_webassembly.cmake b/cmake/onnxruntime_webassembly.cmake index 52d6926274..23e7d758f2 100644 --- a/cmake/onnxruntime_webassembly.cmake +++ b/cmake/onnxruntime_webassembly.cmake @@ -60,9 +60,19 @@ else() endif() if (onnxruntime_ENABLE_WEBASSEMBLY_THREADS) - set_property(TARGET onnxruntime_webassembly APPEND_STRING PROPERTY LINK_FLAGS " -s EXPORT_NAME=ortWasmThreaded -s USE_PTHREADS=1") - set_target_properties(onnxruntime_webassembly PROPERTIES OUTPUT_NAME "ort-wasm-threaded") + if (onnxruntime_ENABLE_WEBASSEMBLY_SIMD) + set_property(TARGET onnxruntime_webassembly APPEND_STRING PROPERTY LINK_FLAGS " -s EXPORT_NAME=ortWasmSimdThreaded -s USE_PTHREADS=1") + set_target_properties(onnxruntime_webassembly PROPERTIES OUTPUT_NAME "ort-wasm-simd-threaded") + else() + set_property(TARGET onnxruntime_webassembly APPEND_STRING PROPERTY LINK_FLAGS " -s EXPORT_NAME=ortWasmThreaded -s USE_PTHREADS=1") + set_target_properties(onnxruntime_webassembly PROPERTIES OUTPUT_NAME "ort-wasm-threaded") + endif() else() - set_property(TARGET onnxruntime_webassembly APPEND_STRING PROPERTY LINK_FLAGS " -s EXPORT_NAME=ortWasm") - set_target_properties(onnxruntime_webassembly PROPERTIES OUTPUT_NAME "ort-wasm") + if (onnxruntime_ENABLE_WEBASSEMBLY_SIMD) + set_property(TARGET onnxruntime_webassembly APPEND_STRING PROPERTY LINK_FLAGS " -s EXPORT_NAME=ortWasmSimd") + set_target_properties(onnxruntime_webassembly PROPERTIES OUTPUT_NAME "ort-wasm-simd") + else() + set_property(TARGET onnxruntime_webassembly APPEND_STRING PROPERTY LINK_FLAGS " -s EXPORT_NAME=ortWasm") + set_target_properties(onnxruntime_webassembly PROPERTIES OUTPUT_NAME "ort-wasm") + endif() endif() diff --git a/onnxruntime/core/mlas/inc/mlas.h b/onnxruntime/core/mlas/inc/mlas.h index e9f8e44446..bec4cdf479 100644 --- a/onnxruntime/core/mlas/inc/mlas.h +++ b/onnxruntime/core/mlas/inc/mlas.h @@ -621,7 +621,7 @@ enum MLAS_CONV_ALGORITHM { MlasConvAlgorithmGemmDirect, MlasConvAlgorithmExpandThenGemm, MlasConvAlgorithmExpandThenGemmSegmented, -#if defined(MLAS_TARGET_WASM) +#if defined(MLAS_TARGET_WASM_SCALAR) MlasConvAlgorithmDepthwise, #endif }; diff --git a/onnxruntime/core/mlas/lib/wasm_simd/SgemvKernelWasmSimd.cpp b/onnxruntime/core/mlas/lib/wasm_simd/SgemvKernelWasmSimd.cpp index 2fde6b1e4a..a46efd4093 100644 --- a/onnxruntime/core/mlas/lib/wasm_simd/SgemvKernelWasmSimd.cpp +++ b/onnxruntime/core/mlas/lib/wasm_simd/SgemvKernelWasmSimd.cpp @@ -17,7 +17,7 @@ Abstract: #include "mlasi.h" -size_t +void MLASCALL MlasGemvFloatKernel( const float* A, @@ -155,6 +155,4 @@ Return Value: B += ldb; A++; } - - return 0; } diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index b36d745583..ec19f79f51 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -336,6 +336,7 @@ def parse_arguments(): # WebAssembly build parser.add_argument("--build_wasm", action='store_true', help="Build for WebAssembly") + parser.add_argument("--enable_wasm_simd", action='store_true', help="Enable WebAssembly SIMD") parser.add_argument( "--disable_wasm_exception_catching", action='store_true', help="Disable exception catching in WebAssembly.") @@ -740,6 +741,7 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home "-Donnxruntime_ENABLE_MEMORY_PROFILE=" + ("ON" if args.enable_memory_profile else "OFF"), "-Donnxruntime_ENABLE_CUDA_LINE_NUMBER_INFO=" + ("ON" if args.enable_cuda_line_info else "OFF"), "-Donnxruntime_BUILD_WEBASSEMBLY=" + ("ON" if args.build_wasm else "OFF"), + "-Donnxruntime_ENABLE_WEBASSEMBLY_SIMD=" + ("ON" if args.enable_wasm_simd else "OFF"), "-Donnxruntime_ENABLE_WEBASSEMBLY_EXCEPTION_CATCHING=" + ("OFF" if args.disable_wasm_exception_catching else "ON"), "-Donnxruntime_ENABLE_WEBASSEMBLY_THREADS=" + ("ON" if args.enable_wasm_threads else "OFF"), From 451fcb7df1185d3a6d51fed9fa28c14622b9420c Mon Sep 17 00:00:00 2001 From: RandySheriffH <48490400+RandySheriffH@users.noreply.github.com> Date: Fri, 28 May 2021 18:00:06 -0700 Subject: [PATCH 37/47] Add sequence support for identity on GPU (#7810) * Add sequence supprot for identity on GPU * implement TensorSeq in provider interface * fix definition err * Add new interface to TensorSeq * fix comments * fix comments * fix mac warning * move TensorSeq forward declaration * add TensorSeq header * remove declaration * fix minor format * fix minor format * define TensorSeq as struct Co-authored-by: RandySheriffH --- include/onnxruntime/core/framework/ml_value.h | 2 +- onnxruntime/core/framework/TensorSeq.h | 6 ++ .../core/framework/provider_bridge_ort.cc | 12 +++ onnxruntime/core/framework/utils.cc | 69 ++++++++++---- .../core/providers/cuda/tensor/identity_op.cc | 2 +- .../core/providers/cuda/tensor/identity_op.h | 93 +++++++++++++------ .../providers/shared_library/provider_api.h | 1 + .../provider_bridge_provider.cc | 2 + .../shared_library/provider_interfaces.h | 32 +++++++ 9 files changed, 173 insertions(+), 46 deletions(-) diff --git a/include/onnxruntime/core/framework/ml_value.h b/include/onnxruntime/core/framework/ml_value.h index 96d6ee88a1..1217235df5 100644 --- a/include/onnxruntime/core/framework/ml_value.h +++ b/include/onnxruntime/core/framework/ml_value.h @@ -10,11 +10,11 @@ #include "core/framework/allocator.h" #include "core/framework/data_types.h" #include "core/framework/tensor.h" +#include "core/framework/TensorSeq.h" #endif namespace onnxruntime { class SparseTensor; -class TensorSeq; } // namespace onnxruntime /** diff --git a/onnxruntime/core/framework/TensorSeq.h b/onnxruntime/core/framework/TensorSeq.h index a17a0866c8..8f69f6f972 100644 --- a/onnxruntime/core/framework/TensorSeq.h +++ b/onnxruntime/core/framework/TensorSeq.h @@ -62,6 +62,12 @@ class TensorSeq { return tensors_[i]; } + void Add(Tensor&& tensor) { + ORT_ENFORCE(IsSameDataType(tensor), + "TensorSeq: tensor to be added has a different data type."); + tensors_.push_back(std::move(tensor)); + } + private: // A sequence must be associated with only one data type and all tensors in the seq must be of that type // One other alternative of storing the data type of a seq is to templatize the TensorSeq class. diff --git a/onnxruntime/core/framework/provider_bridge_ort.cc b/onnxruntime/core/framework/provider_bridge_ort.cc index 155b8c8b6e..8dc938a055 100644 --- a/onnxruntime/core/framework/provider_bridge_ort.cc +++ b/onnxruntime/core/framework/provider_bridge_ort.cc @@ -19,6 +19,7 @@ #include "core/session/ort_apis.h" #include "core/util/math.h" #include "core/framework/tensorprotoutils.h" +#include "core/framework/TensorSeq.h" #include "core/framework/fallback_cpu_capability.h" #include "core/framework/random_generator.h" @@ -485,6 +486,7 @@ struct ProviderHostImpl : ProviderHost { // DataTypeImpl (wrapped) MLDataType DataTypeImpl__GetType_Tensor() override { return DataTypeImpl::GetType(); } + MLDataType DataTypeImpl__GetType_TensorSeq () override { return DataTypeImpl::GetType(); } MLDataType DataTypeImpl__GetType_bool() override { return DataTypeImpl::GetType(); } MLDataType DataTypeImpl__GetType_int8() override { return DataTypeImpl::GetType(); } MLDataType DataTypeImpl__GetType_uint8() override { return DataTypeImpl::GetType(); } @@ -651,8 +653,11 @@ struct ProviderHostImpl : ProviderHost { // OpKernelContext (wrapped) const Tensor* OpKernelContext__Input_Tensor(const OpKernelContext* p, int index) override { return p->Input(index); } + const TensorSeq* OpKernelContext__Input_TensorSeq(const OpKernelContext* p, int index) override { return p->Input(index); } const Tensor& OpKernelContext__RequiredInput_Tensor(const OpKernelContext* p, int index) override { return p->RequiredInput(index); } + MLDataType OpKernelContext__InputType(const OpKernelContext* p, int index) override { return p->InputType(index); } Tensor* OpKernelContext__Output_Tensor(OpKernelContext* p, int index) override { return p->Output(index); } + TensorSeq* OpKernelContext__Output_TensorSeq(OpKernelContext* p, int index) override { return p->Output(index); } Tensor* OpKernelContext__Output(OpKernelContext* p, int index, const TensorShape& shape) override { return p->Output(index, shape); } Tensor& OpKernelContext__RequiredOutput(OpKernelContext* p, int index, const TensorShape& shape) override { return p->RequiredOutput(index, shape); } int OpKernelContext__InputCount(const OpKernelContext* p) override { return p->InputCount(); } @@ -749,6 +754,13 @@ struct ProviderHostImpl : ProviderHost { int32_t Tensor__GetElementType(const Tensor* p) override { return p->GetElementType(); } MLDataType Tensor__DataType(const Tensor* p) override { return p->DataType(); } + // TensorSeq(wrapped) + MLDataType TensorSeq__DataType(const TensorSeq* p) noexcept override { return p->DataType(); } + void TensorSeq__SetType(TensorSeq* p, MLDataType data_type) override { p->SetType(data_type); } + size_t TensorSeq__Size(const TensorSeq* p) noexcept override { return p->Size(); } + const Tensor& TensorSeq__Get(const TensorSeq* p, size_t i) override { return p->Get(i); } + void TensorSeq__Add(TensorSeq* p, Tensor&& tensor) override { p->Add(std::move(tensor)); } + // AllocatorManager (direct) void AllocatorManager__InsertAllocator(AllocatorManager* p, AllocatorPtr allocator) override { p->AllocatorManager::InsertAllocator(allocator); } AllocatorPtr AllocatorManager__GetAllocator(const AllocatorManager* p, int id, OrtMemType mem_type) override { return p->AllocatorManager::GetAllocator(id, mem_type); }; diff --git a/onnxruntime/core/framework/utils.cc b/onnxruntime/core/framework/utils.cc index ec3c651adf..648a3be1c8 100644 --- a/onnxruntime/core/framework/utils.cc +++ b/onnxruntime/core/framework/utils.cc @@ -18,6 +18,7 @@ #include "core/framework/sequential_executor.h" #include "core/framework/tensorprotoutils.h" #include "core/mlas/inc/mlas.h" +#include "core/framework/TensorSeq.h" #ifdef ENABLE_TRAINING #include "core/framework/orttraining_partial_executor.h" #endif @@ -111,19 +112,33 @@ bool ProviderIsCpuBased(const std::string& provider_type) { } static common::Status AllocateHelper(const AllocatorPtr& allocator, - const Tensor& fetched_tensor, OrtValue& output_mlvalue) { + const OrtValue& source_mlvalue, + OrtValue& target_mlvalue) { if (!allocator) { - return Status(common::ONNXRUNTIME, common::FAIL, "invalid allocator"); + return Status(common::ONNXRUNTIME, common::FAIL, "invalid allocator."); } + if (source_mlvalue.IsTensor()) { - std::unique_ptr p_tensor = std::make_unique(fetched_tensor.DataType(), - fetched_tensor.Shape(), - allocator); - auto ml_tensor = DataTypeImpl::GetType(); - output_mlvalue.Init(p_tensor.release(), - ml_tensor, - ml_tensor->GetDeleteFunc()); + const Tensor& source_tensor = source_mlvalue.Get(); + std::unique_ptr target_tensor = std::make_unique(source_tensor.DataType(), + source_tensor.Shape(), + allocator); + auto ml_tensor = DataTypeImpl::GetType(); + target_mlvalue.Init(target_tensor.release(), ml_tensor, ml_tensor->GetDeleteFunc()); + } else if (source_mlvalue.IsTensorSequence()) { + const TensorSeq& source_tensor_seq = source_mlvalue.Get(); + auto target_tensor_seq = std::make_unique(source_tensor_seq.DataType()); + std::vector tensors; + for (auto iter = source_tensor_seq.begin(); iter != source_tensor_seq.end(); ++iter) { + tensors.emplace_back(iter->DataType(), onnxruntime::TensorShape(iter->Shape()), allocator); + } + target_tensor_seq->SetElements(std::move(tensors)); + auto ml_tensor_seq = DataTypeImpl::GetType(); + target_mlvalue.Init(target_tensor_seq.release(), ml_tensor_seq, ml_tensor_seq->GetDeleteFunc()); + } else { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Unsupported OrtValue type."); + } return Status::OK(); } @@ -159,22 +174,40 @@ static Status BatchOrCopyMLValue(const SessionState& session_state, return Status::OK(); } - auto& source_tensor = source_mlvalue.Get(); if (!target_mlvalue.IsAllocated()) { auto allocator = session_state.GetAllocator(copy_info.target_device); ORT_ENFORCE(allocator != nullptr, "Failed to find allocator for device ", copy_info.target_device.ToString()); - - ORT_RETURN_IF_ERROR(utils::AllocateHelper(allocator, source_tensor, target_mlvalue)); + ORT_RETURN_IF_ERROR(utils::AllocateHelper(allocator, source_mlvalue, target_mlvalue)); } - Tensor* p_output_tensor = target_mlvalue.GetMutable(); - - if (copy_pairs != nullptr) { - copy_pairs->push_back({source_tensor, *p_output_tensor, 0}); + if (source_mlvalue.IsTensor()) { + const Tensor& source_tensor = source_mlvalue.Get(); + Tensor& target_tensor = *target_mlvalue.GetMutable(); + if (copy_pairs != nullptr) { + copy_pairs->push_back({source_tensor, target_tensor, 0}); + } else { + ORT_RETURN_IF_ERROR(session_state.GetDataTransferMgr().CopyTensor(source_tensor, target_tensor)); + } + } else if (source_mlvalue.IsTensorSequence()) { + const TensorSeq& source_tensor_seq = source_mlvalue.Get(); + const TensorSeq& target_tensor_seq = target_mlvalue.Get(); + ORT_ENFORCE(source_tensor_seq.Size() == target_tensor_seq.Size(), + "source and target tensor sequence have different number of elements."); + auto source_iter = source_tensor_seq.begin(); + auto target_iter = target_tensor_seq.begin(); + while (source_iter != source_tensor_seq.end() && + target_iter != target_tensor_seq.end()) { + if (copy_pairs != nullptr) { + copy_pairs->push_back({*source_iter, const_cast(*target_iter), 0}); + } else { + ORT_RETURN_IF_ERROR(session_state.GetDataTransferMgr().CopyTensor(*source_iter, const_cast(*target_iter))); + } + ++source_iter; + ++target_iter; + }//while } else { - ORT_RETURN_IF_ERROR(session_state.GetDataTransferMgr().CopyTensor(source_tensor, *p_output_tensor)); + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Unsupported OrtValue type to copy between device."); } - return Status::OK(); } diff --git a/onnxruntime/core/providers/cuda/tensor/identity_op.cc b/onnxruntime/core/providers/cuda/tensor/identity_op.cc index 9281e3a42b..563bb45643 100644 --- a/onnxruntime/core/providers/cuda/tensor/identity_op.cc +++ b/onnxruntime/core/providers/cuda/tensor/identity_op.cc @@ -57,7 +57,7 @@ ONNX_OPERATOR_KERNEL_EX( 14, kCudaExecutionProvider, (*KernelDefBuilder::Create()) - .TypeConstraint("V", DataTypeImpl::AllFixedSizeTensorTypes()) + .TypeConstraint("V", DataTypeImpl::AllTensorAndSequenceTensorTypes()) .Alias(0, 0), IdentityOp); } // namespace cuda diff --git a/onnxruntime/core/providers/cuda/tensor/identity_op.h b/onnxruntime/core/providers/cuda/tensor/identity_op.h index 4bac78d042..35f695c9d9 100644 --- a/onnxruntime/core/providers/cuda/tensor/identity_op.h +++ b/onnxruntime/core/providers/cuda/tensor/identity_op.h @@ -15,34 +15,75 @@ class IdentityOp final : public CudaKernel { } Status ComputeInternal(OpKernelContext* context) const override { - const Tensor* X = context->Input(0); - if (X == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "input count mismatch"); - const TensorShape& shape = X->Shape(); - Tensor* Y = context->Output(0, shape); - auto X_type = X->DataType(); - - const void* source = X->DataRaw(X_type); - void* target = Y->MutableDataRaw(X_type); - //If source and target pointers are not equal, we need to copy the data. - if (target != source) { - CUDA_RETURN_IF_ERROR(cudaMemcpyAsync(target, source, X->Shape().Size() * X->DataType()->Size(), cudaMemcpyDeviceToDevice, Stream())); - } - - if (is_dropout) { - Tensor* mask = context->Output(1, shape); - // a 'nullptr' returned would make it an unused optional output - if (mask != nullptr) { - // Opset 7 differs with Opset 10 in that the type of the 'mask' - // output is tied with the type of the input in Opset 7 whereas - // the type of 'mask' in Opset 10 is 'bool' always - // so we have a common solution - void* mask_data = mask->MutableDataRaw(); - // In 'test'/'inference' mode, there are no input values dropped out - // so fill the buffer with 0/false - CUDA_RETURN_IF_ERROR(cudaMemsetAsync(mask_data, 0, mask->SizeInBytes(), Stream())); + auto X_ml_type = context->InputType(0); + if (X_ml_type->IsTensorType()) { + const Tensor* X = context->Input(0); + if (nullptr == X) { + return Status(common::ONNXRUNTIME, common::FAIL, + "IdentityOp cuda: input count mismatch."); } - } + const TensorShape& shape = X->Shape(); + Tensor* Y = context->Output(0, shape); + if (nullptr == Y) { + return Status(common::ONNXRUNTIME, common::FAIL, + "IdentityOp cuda: failed to allocate output tensor."); + } + auto X_type = X->DataType(); + const void* source = X->DataRaw(X_type); + void* target = Y->MutableDataRaw(X_type); + //If source and target pointers are not equal, we need to copy the data. + if (target != source) { + CUDA_RETURN_IF_ERROR(cudaMemcpyAsync(target, source, X->Shape().Size() * X->DataType()->Size(), cudaMemcpyDeviceToDevice, Stream())); + } + + if (is_dropout) { + Tensor* mask = context->Output(1, shape); + // a 'nullptr' returned would make it an unused optional output + if (mask != nullptr) { + // Opset 7 differs with Opset 10 in that the type of the 'mask' + // output is tied with the type of the input in Opset 7 whereas + // the type of 'mask' in Opset 10 is 'bool' always + // so we have a common solution + void* mask_data = mask->MutableDataRaw(); + // In 'test'/'inference' mode, there are no input values dropped out + // so fill the buffer with 0/false + CUDA_RETURN_IF_ERROR(cudaMemsetAsync(mask_data, 0, mask->SizeInBytes(), Stream())); + } + } + } else if (X_ml_type->IsTensorSequenceType()) { + const TensorSeq* X = context->Input(0); + if (nullptr == X) { + return Status(common::ONNXRUNTIME, common::FAIL, + "IdentityOp cuda: input tensor is missing."); + } + TensorSeq* Y = context->Output(0); + if (nullptr == Y) { + return Status(common::ONNXRUNTIME, common::FAIL, + "IdentityOp cuda: failed to allocate output tensor sequence."); + } + auto X_type = X->DataType(); + Y->SetType(X_type); + AllocatorPtr alloc; + auto status = context->GetTempSpaceAllocator(&alloc); + if (!status.IsOK()) { + return Status(common::ONNXRUNTIME, common::FAIL, + "IdentityOp cuda: unable to get an allocator."); + } + auto X_size = X->Size(); + for (size_t i = 0; i < X_size; ++i) { + const Tensor& source_tensor = X->Get(i); + std::unique_ptr target_tensor = Tensor::Create(X_type, source_tensor.Shape(), alloc); + CUDA_RETURN_IF_ERROR(cudaMemcpyAsync(target_tensor->MutableDataRaw(), + source_tensor.DataRaw(), + source_tensor.SizeInBytes(), + cudaMemcpyDeviceToDevice, Stream())); + Y->Add(std::move(*target_tensor)); + } + } else { + return Status(common::ONNXRUNTIME, common::FAIL, + "IdentityOp cuda: unsupported input type."); + } return Status::OK(); } }; diff --git a/onnxruntime/core/providers/shared_library/provider_api.h b/onnxruntime/core/providers/shared_library/provider_api.h index aa3304718f..a2333972c2 100644 --- a/onnxruntime/core/providers/shared_library/provider_api.h +++ b/onnxruntime/core/providers/shared_library/provider_api.h @@ -169,6 +169,7 @@ struct OpKernelContext; struct OpKernelInfo; struct PrimitiveDataTypeBase; struct Tensor; +struct TensorSeq; class UnsqueezeBase; class SliceBase; diff --git a/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc b/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc index 058211379d..014e6a0dc0 100644 --- a/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc +++ b/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc @@ -98,6 +98,8 @@ AllocatorPtr AllocatorManager::GetAllocator(int id, OrtMemType mem_type) const { template <> MLDataType DataTypeImpl::GetType() { return Provider_GetHost()->DataTypeImpl__GetType_Tensor(); } template <> +MLDataType DataTypeImpl::GetType() { return Provider_GetHost()->DataTypeImpl__GetType_TensorSeq(); } +template <> MLDataType DataTypeImpl::GetType() { return Provider_GetHost()->DataTypeImpl__GetType_bool(); } template <> MLDataType DataTypeImpl::GetType() { return Provider_GetHost()->DataTypeImpl__GetType_int8(); } diff --git a/onnxruntime/core/providers/shared_library/provider_interfaces.h b/onnxruntime/core/providers/shared_library/provider_interfaces.h index 7261b5f1ea..d680a616c7 100644 --- a/onnxruntime/core/providers/shared_library/provider_interfaces.h +++ b/onnxruntime/core/providers/shared_library/provider_interfaces.h @@ -409,6 +409,7 @@ struct ProviderHost { // DataTypeImpl virtual MLDataType DataTypeImpl__GetType_Tensor() = 0; + virtual MLDataType DataTypeImpl__GetType_TensorSeq() = 0; virtual MLDataType DataTypeImpl__GetType_bool() = 0; virtual MLDataType DataTypeImpl__GetType_int8() = 0; virtual MLDataType DataTypeImpl__GetType_uint8() = 0; @@ -562,10 +563,13 @@ struct ProviderHost { // OpKernelContext virtual const Tensor* OpKernelContext__Input_Tensor(const OpKernelContext* p, int index) = 0; + virtual const TensorSeq* OpKernelContext__Input_TensorSeq(const OpKernelContext* p, int index) = 0; virtual const Tensor& OpKernelContext__RequiredInput_Tensor(const OpKernelContext* p, int index) = 0; virtual Tensor* OpKernelContext__Output_Tensor(OpKernelContext* p, int index) = 0; + virtual TensorSeq* OpKernelContext__Output_TensorSeq(OpKernelContext* p, int index) = 0; virtual Tensor* OpKernelContext__Output(OpKernelContext* p, int index, const TensorShape& shape) = 0; virtual Tensor& OpKernelContext__RequiredOutput(OpKernelContext* p, int index, const TensorShape& shape) = 0; + virtual MLDataType OpKernelContext__InputType(const OpKernelContext* p, int index) = 0; virtual int OpKernelContext__InputCount(const OpKernelContext* p) = 0; virtual int OpKernelContext__OutputCount(const OpKernelContext* p) = 0; virtual Status OpKernelContext__GetTempSpaceAllocator(const OpKernelContext* p, AllocatorPtr* output) = 0; @@ -660,6 +664,13 @@ struct ProviderHost { virtual int32_t Tensor__GetElementType(const Tensor* p) = 0; virtual MLDataType Tensor__DataType(const Tensor* p) = 0; + // TensorSeq + virtual MLDataType TensorSeq__DataType(const TensorSeq* p) noexcept = 0; + virtual void TensorSeq__SetType(TensorSeq* p, MLDataType data_type) = 0; + virtual size_t TensorSeq__Size(const TensorSeq* p) noexcept = 0; + virtual const Tensor& TensorSeq__Get(const TensorSeq* p, size_t i) = 0; + virtual void TensorSeq__Add(TensorSeq* p, Tensor&& tensor) = 0; + // AllocatorManager virtual void AllocatorManager__InsertAllocator(AllocatorManager* p, AllocatorPtr allocator) = 0; virtual AllocatorPtr AllocatorManager__GetAllocator(const AllocatorManager* p, int id, OrtMemType mem_type) = 0; @@ -1445,6 +1456,8 @@ struct OpKernelContext final { const T* Input(int index) const; int InputCount() const { return g_host->OpKernelContext__InputCount(this); } + MLDataType InputType(int index) const { return g_host->OpKernelContext__InputType(this, index); } + template T* Output(int index); @@ -1466,11 +1479,21 @@ inline const Tensor* OpKernelContext::Input(int index) const { return g_host->OpKernelContext__Input_Tensor(this, index); } +template <> +inline const TensorSeq* OpKernelContext::Input(int index) const { + return g_host->OpKernelContext__Input_TensorSeq(this, index); +} + template <> inline Tensor* OpKernelContext::Output(int index) { return g_host->OpKernelContext__Output_Tensor(this, index); } +template <> +inline TensorSeq* OpKernelContext::Output(int index) { + return g_host->OpKernelContext__Output_TensorSeq(this, index); +} + template <> inline const Tensor& OpKernelContext::RequiredInput(int index) const { return g_host->OpKernelContext__RequiredInput_Tensor(this, index); @@ -1663,6 +1686,15 @@ inline const BFloat16* Tensor::Data() const { return g_host->Tensor__D template <> inline const MLFloat16* Tensor::Data() const { return g_host->Tensor__Data_MLFloat16(this); } +//TensorSeq +struct TensorSeq final { + MLDataType DataType() const noexcept { return g_host->TensorSeq__DataType(this); } + void SetType(MLDataType elem_type) { g_host->TensorSeq__SetType(this, elem_type); } + size_t Size() const noexcept { return g_host->TensorSeq__Size(this); } + const Tensor& Get(size_t i) const { return g_host->TensorSeq__Get(this, i); } + void Add(Tensor&& tensor) { g_host->TensorSeq__Add(this, std::move(tensor)); } +}; + template <> inline gsl::span Tensor::DataAsSpan() const { return g_host->Tensor__DataAsSpan_int64(this); } From d8bcb3d6a4eaef5b01300a96a8746a8b6f0907f4 Mon Sep 17 00:00:00 2001 From: Chandru Ramakrishnan <41447659+chandru-r@users.noreply.github.com> Date: Sun, 30 May 2021 11:11:10 -0400 Subject: [PATCH 38/47] Added virtual destructor to adasum_interface.h (#7882) --- .../orttraining/core/framework/adasum/adasum_interface.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/orttraining/orttraining/core/framework/adasum/adasum_interface.h b/orttraining/orttraining/core/framework/adasum/adasum_interface.h index 0cd164b31f..b74e28fcaf 100644 --- a/orttraining/orttraining/core/framework/adasum/adasum_interface.h +++ b/orttraining/orttraining/core/framework/adasum/adasum_interface.h @@ -60,6 +60,8 @@ class AdasumInterface { virtual const Communicator_type* GetReductionComms() = 0; + virtual ~AdasumInterface() = default; + protected: // Communication primitives required for Adasum algorithm virtual void PointToPointSendRecv(void* input_data_buffer, From 3a72932c4a4afc2e72c6b3bfc17b3c0a248e6232 Mon Sep 17 00:00:00 2001 From: Hariharan Seshadri Date: Sun, 30 May 2021 21:12:32 -0700 Subject: [PATCH 39/47] Don't hold onto unnecessary numpy references while binding numpy objectas as inputs (#7881) --- onnxruntime/python/onnxruntime_inference_collection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onnxruntime/python/onnxruntime_inference_collection.py b/onnxruntime/python/onnxruntime_inference_collection.py index bb9d2cad6c..b2e98c23d5 100644 --- a/onnxruntime/python/onnxruntime_inference_collection.py +++ b/onnxruntime/python/onnxruntime_inference_collection.py @@ -355,7 +355,7 @@ class IOBinding: ''' def __init__(self, session): self._iobinding = C.SessionIOBinding(session._sess) - self._numpy_obj_references = [] + self._numpy_obj_references = {} def bind_cpu_input(self, name, arr_on_cpu): ''' @@ -366,7 +366,7 @@ class IOBinding: # Hold a reference to the numpy object as the bound OrtValue is backed # directly by the data buffer of the numpy object and so the numpy object # must be around until this IOBinding instance is around - self._numpy_obj_references.append(arr_on_cpu) + self._numpy_obj_references[name] = arr_on_cpu self._iobinding.bind_input(name, arr_on_cpu) def bind_input(self, name, device_type, device_id, element_type, shape, buffer_ptr): From 81ed6c55bf3fb7e0e5f85ccf2e71092eeefcf3d7 Mon Sep 17 00:00:00 2001 From: Tracy Sharpe <42477615+tracysh@users.noreply.github.com> Date: Tue, 1 Jun 2021 08:54:04 -0700 Subject: [PATCH 40/47] fix grouped pointwise convolution (#7885) --- .../core/providers/cpu/nn/qlinearconv.cc | 42 ++++++++++--------- .../providers/cpu/nn/qlinearconv_op_test.cc | 20 +++++++++ 2 files changed, 42 insertions(+), 20 deletions(-) diff --git a/onnxruntime/core/providers/cpu/nn/qlinearconv.cc b/onnxruntime/core/providers/cpu/nn/qlinearconv.cc index 882138d50b..ff20ff0f8c 100644 --- a/onnxruntime/core/providers/cpu/nn/qlinearconv.cc +++ b/onnxruntime/core/providers/cpu/nn/qlinearconv.cc @@ -510,14 +510,27 @@ Status QLinearConv::Compute(OpKernelContext* context) const { static_cast(kernel_size)); } else { for (int64_t group_id = 0; group_id < group_count; ++group_id) { + MLAS_GEMM_U8X8_DATA_PARAMS gemm_params; + gemm_params.ZeroPointA = X_zero_point_value; + if (packed_W_buffer_) { + gemm_params.B = static_cast(packed_W_buffer_.get()) + group_id * packed_W_size_, + gemm_params.BIsPacked = true; + } else { + gemm_params.B = reordered_W + group_id * group_output_channels, + gemm_params.ldb = static_cast(M); + } + gemm_params.ZeroPointB = &W_zero_point_value; + gemm_params.C = worker_gemm_output + group_id * group_output_channels; + gemm_params.ldc = static_cast(M); + // Prepare the im2col transformation or use the input buffer directly for // pointwise convolutions. - const uint8_t* worker_gemm_input; + const auto* group_input_data = input_data + group_id * group_input_channels; if (col_buffer) { auto* worker_col_buffer = static_cast(col_buffer.get()) + output_start * kernel_dim; if (kernel_rank == 2) { math::Im2col()( - input_data + group_id * group_input_channels, + group_input_data, group_input_channels, C, input_shape[0], @@ -537,7 +550,7 @@ Status QLinearConv::Compute(OpKernelContext* context) const { X_zero_point_value); } else if (kernel_rank == 1) { math::Im2col()( - input_data + group_id * group_input_channels, + group_input_data, group_input_channels, C, 1, @@ -559,9 +572,11 @@ Status QLinearConv::Compute(OpKernelContext* context) const { // Use the im2col buffer prepared outside the thread, indexed by group. worker_col_buffer += group_id * col_buffer_size; } - worker_gemm_input = worker_col_buffer; + gemm_params.A = worker_col_buffer; + gemm_params.lda = static_cast(kernel_dim); } else { - worker_gemm_input = input_data + output_start * kernel_dim; + gemm_params.A = group_input_data + output_start * C; + gemm_params.lda = static_cast(C); } MLAS_GEMM_U8X8_SHAPE_PARAMS gemm_shape; @@ -570,20 +585,6 @@ Status QLinearConv::Compute(OpKernelContext* context) const { gemm_shape.K = static_cast(kernel_dim); gemm_shape.BIsSigned = is_W_signed; - MLAS_GEMM_U8X8_DATA_PARAMS gemm_params; - gemm_params.A = worker_gemm_input; - gemm_params.lda = static_cast(kernel_dim); - gemm_params.ZeroPointA = X_zero_point_value; - if (packed_W_buffer_) { - gemm_params.B = static_cast(packed_W_buffer_.get()) + group_id * packed_W_size_, - gemm_params.BIsPacked = true; - } else { - gemm_params.B = reordered_W + group_id * group_output_channels, - gemm_params.ldb = static_cast(M); - } - gemm_params.ZeroPointB = &W_zero_point_value; - gemm_params.C = worker_gemm_output + group_id * group_output_channels; - gemm_params.ldc = static_cast(M); MlasGemm(gemm_shape, gemm_params, nullptr); } } @@ -597,7 +598,8 @@ Status QLinearConv::Compute(OpKernelContext* context) const { output_scales.data(), output_scales.size() > 1, Y_zero_point_value, - 0,0, + 0, + 0, static_cast(output_count), static_cast(M)); }; diff --git a/onnxruntime/test/providers/cpu/nn/qlinearconv_op_test.cc b/onnxruntime/test/providers/cpu/nn/qlinearconv_op_test.cc index ea1869aec8..0916669e86 100644 --- a/onnxruntime/test/providers/cpu/nn/qlinearconv_op_test.cc +++ b/onnxruntime/test/providers/cpu/nn/qlinearconv_op_test.cc @@ -736,6 +736,26 @@ TEST(QLinearConvTest, Conv2D_U8S8_Groups_PerChannel) { test.Run(); } +TEST(QLinearConvTest, Conv2D_U8S8_Groups_Pointwise) { + QLinearConvOpTester test; + test.GenerateRandomInput({1, 12, 17, 13}, .03f, 7); + test.GenerateRandomWeights({15, 4, 1, 1}, .10f, 0); + test.GenerateRandomBias(); + test.SetGroups(3); + test.SetOutputScaleAndZeroPoint(.26f, 88); + test.Run(); +} + +TEST(QLinearConvTest, Conv3D_U8S8_Groups_Pointwise) { + QLinearConvOpTester test; + test.GenerateRandomInput({2, 4, 13, 17, 13}, .03f, 7); + test.GenerateRandomWeights({6, 2, 1, 1, 1}, .10f, 0); + test.GenerateRandomBias(); + test.SetGroups(2); + test.SetOutputScaleAndZeroPoint(.26f, 88); + test.Run(); +} + TEST(QLinearConvTest, Conv1D_U8S8_Depthwise) { for (int64_t channels : std::initializer_list{7, 8, 9, 16, 25, 64}) { QLinearConvOpTester test; From e7e200ee59df07e3eb3aeb8220171ef72fcd1a4e Mon Sep 17 00:00:00 2001 From: Guoyu Wang <62914304+gwang-msft@users.noreply.github.com> Date: Tue, 1 Jun 2021 11:01:37 -0700 Subject: [PATCH 41/47] Add test for iOS package (#7816) * Add test for iOS package * Add readme * fix pep8 warning * Addressed CR comments, fixed CI failure * Address CR comments * Update readme.md * Update package name and readme, added comments to the podspec --- .../platform/ios/ios_package_test/.gitignore | 26 + .../platform/ios/ios_package_test/Podfile | 15 + .../platform/ios/ios_package_test/README.md | 47 ++ .../project.pbxproj | 474 ++++++++++++++++++ .../contents.xcworkspacedata | 7 + .../xcshareddata/IDEWorkspaceChecks.plist | 8 + .../ios_package_test/AppDelegate.h | 14 + .../ios_package_test/AppDelegate.m | 40 ++ .../Base.lproj/LaunchScreen.storyboard | 25 + .../Base.lproj/Main.storyboard | 24 + .../ios_package_test/Info.plist | 66 +++ .../ios_package_test/ios_package_test/main.m | 18 + .../ios_package_testTests/Info.plist | 22 + .../ios_package_test_c_api.m | 97 ++++ .../ios_package_test_cpp_api.mm | 68 +++ .../ios/ios_package_test/models/sigmoid.ort | Bin 0 -> 1208 bytes .../onnxruntime-mobile.podspec.template | 18 + tools/ci_build/build.py | 24 +- .../github/apple/test_ios_packages.py | 99 ++++ .../azure-pipelines/mac-ios-ci-pipeline.yml | 2 + 20 files changed, 1084 insertions(+), 10 deletions(-) create mode 100644 onnxruntime/test/platform/ios/ios_package_test/.gitignore create mode 100644 onnxruntime/test/platform/ios/ios_package_test/Podfile create mode 100644 onnxruntime/test/platform/ios/ios_package_test/README.md create mode 100644 onnxruntime/test/platform/ios/ios_package_test/ios_package_test.xcodeproj/project.pbxproj create mode 100644 onnxruntime/test/platform/ios/ios_package_test/ios_package_test.xcodeproj/project.xcworkspace/contents.xcworkspacedata create mode 100644 onnxruntime/test/platform/ios/ios_package_test/ios_package_test.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist create mode 100644 onnxruntime/test/platform/ios/ios_package_test/ios_package_test/AppDelegate.h create mode 100644 onnxruntime/test/platform/ios/ios_package_test/ios_package_test/AppDelegate.m create mode 100644 onnxruntime/test/platform/ios/ios_package_test/ios_package_test/Base.lproj/LaunchScreen.storyboard create mode 100644 onnxruntime/test/platform/ios/ios_package_test/ios_package_test/Base.lproj/Main.storyboard create mode 100644 onnxruntime/test/platform/ios/ios_package_test/ios_package_test/Info.plist create mode 100644 onnxruntime/test/platform/ios/ios_package_test/ios_package_test/main.m create mode 100644 onnxruntime/test/platform/ios/ios_package_test/ios_package_testTests/Info.plist create mode 100644 onnxruntime/test/platform/ios/ios_package_test/ios_package_testTests/ios_package_test_c_api.m create mode 100644 onnxruntime/test/platform/ios/ios_package_test/ios_package_testTests/ios_package_test_cpp_api.mm create mode 100644 onnxruntime/test/platform/ios/ios_package_test/models/sigmoid.ort create mode 100644 onnxruntime/test/platform/ios/ios_package_test/onnxruntime-mobile.podspec.template create mode 100644 tools/ci_build/github/apple/test_ios_packages.py diff --git a/onnxruntime/test/platform/ios/ios_package_test/.gitignore b/onnxruntime/test/platform/ios/ios_package_test/.gitignore new file mode 100644 index 0000000000..910554681a --- /dev/null +++ b/onnxruntime/test/platform/ios/ios_package_test/.gitignore @@ -0,0 +1,26 @@ +# Xcode +# +# gitignore contributors: remember to update Global/Xcode.gitignore, Objective-C.gitignore & Swift.gitignore + +## User settings +xcuserdata/ + +## compatibility with Xcode 8 and earlier (ignoring not required starting Xcode 9) +*.xcscmblueprint +*.xccheckout + +## compatibility with Xcode 3 and earlier (ignoring not required starting Xcode 4) +build/ +DerivedData/ +*.moved-aside +*.pbxuser +!default.pbxuser +*.mode1v3 +!default.mode1v3 +*.mode2v3 +!default.mode2v3 +*.perspectivev3 +!default.perspectivev3 + +## Gcc Patch +/*.gcno diff --git a/onnxruntime/test/platform/ios/ios_package_test/Podfile b/onnxruntime/test/platform/ios/ios_package_test/Podfile new file mode 100644 index 0000000000..78a2390f85 --- /dev/null +++ b/onnxruntime/test/platform/ios/ios_package_test/Podfile @@ -0,0 +1,15 @@ +platform :ios, '13.0' + +target 'ios_package_test' do + # Comment the next line if you don't want to use dynamic frameworks + use_frameworks! + + # Pods for ios_package_test + pod 'onnxruntime-mobile', :podspec => './onnxruntime-mobile.podspec' + + target 'ios_package_testTests' do + inherit! :search_paths + # Pods for testing + end + +end diff --git a/onnxruntime/test/platform/ios/ios_package_test/README.md b/onnxruntime/test/platform/ios/ios_package_test/README.md new file mode 100644 index 0000000000..980804b708 --- /dev/null +++ b/onnxruntime/test/platform/ios/ios_package_test/README.md @@ -0,0 +1,47 @@ +# iOS End-to-End Test App for ORT-Mobile + +This End-to-End test app for iOS will test ORT Mobile C/C++ API framework using XCode and CocoaPods + +## Requirements + +- [Prerequisites for building ORT-Mobile for iOS](http://www.onnxruntime.ai/docs/how-to/build/android-ios.html#prerequisites-1) +- [CocoaPods](https://cocoapods.org/) + +## iOS End-to-End Test App Overview + +The iOS End-to-End Test App will use CocoaPods to install the Onnx Runtime C/C++ framework, and run basic End-to-End tests of Onnx Runtime C and C++ API. + +### Model used +- [sigmoid ONNX model](https://github.com/onnx/onnx/blob/f9b0cc99344869c246b8f4011b8586a39841284c/onnx/backend/test/data/node/test_sigmoid/model.onnx) converted to ORT format + + Here's the [document](http://www.onnxruntime.ai/docs/how-to/deploy-on-mobile.html#1-create-ort-format-model-and-configuration-file-with-required-operators) about how you can convert an ONNX model into ORT format. + +### Tests +- [Tests for C API ](./ios_package_testTests/ios_package_test_c_api.m) +- [Tests for C++ API ](./ios_package_testTests/ios_package_test_cpp_api.mm) + +## Build and Test iOS Framework using [build.py](../../../../../tools/ci_build/build.py) + +Use the [build for iOS simulator](http://www.onnxruntime.ai/docs/how-to/build/android-ios.html#cross-build-for-ios-simulator) with `--build_apple_framework` + +## Run the iOS End-to-End Test App standalone + +### Requirements + +- A pre-built ORT Mobile iOS framework, which can be built using the [instruction](#build-and-test-ios-framework-using-buildpy) above. The framework can be found as `/iOS//-iphonesimulator/onnxruntime.framework` + +### Steps + +1. Go to this folder +2. Copy the [onnxruntime-mobile.podspec.template](./onnxruntime-mobile.podspec.template) to `onnxruntime-mobile.podspec` +3. Update the `onnxruntime-mobile.podspec`, replace `${ORT_BASE_FRAMEWORK_ARCHIVE}` with the path of a zip archive contains the pre-built ORT Mobile iOS framework +4. Run `pod install` to install the pre-built ORT Mobile iOS framework +5. Run the following command to perform the test + +``` + xcrun xcodebuild \ + -workspace ./ios_package_test.xcworkspace \ + -destination '' \ + -scheme ios_package_test \ + test +``` \ No newline at end of file diff --git a/onnxruntime/test/platform/ios/ios_package_test/ios_package_test.xcodeproj/project.pbxproj b/onnxruntime/test/platform/ios/ios_package_test/ios_package_test.xcodeproj/project.pbxproj new file mode 100644 index 0000000000..f76b6c01dd --- /dev/null +++ b/onnxruntime/test/platform/ios/ios_package_test/ios_package_test.xcodeproj/project.pbxproj @@ -0,0 +1,474 @@ +// !$*UTF8*$! +{ + archiveVersion = 1; + classes = { + }; + objectVersion = 50; + objects = { + +/* Begin PBXBuildFile section */ + 229E5921265869BF006E41AE /* AppDelegate.m in Sources */ = {isa = PBXBuildFile; fileRef = 229E5920265869BF006E41AE /* AppDelegate.m */; }; + 229E592A265869BF006E41AE /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 229E5928265869BF006E41AE /* Main.storyboard */; }; + 229E592F265869C2006E41AE /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 229E592D265869C2006E41AE /* LaunchScreen.storyboard */; }; + 229E5932265869C2006E41AE /* main.m in Sources */ = {isa = PBXBuildFile; fileRef = 229E5931265869C2006E41AE /* main.m */; }; + 229E593C265869C2006E41AE /* ios_package_test_cpp_api.mm in Sources */ = {isa = PBXBuildFile; fileRef = 229E593B265869C2006E41AE /* ios_package_test_cpp_api.mm */; }; + 229E595926586B4A006E41AE /* sigmoid.ort in Resources */ = {isa = PBXBuildFile; fileRef = 229E595826586B4A006E41AE /* sigmoid.ort */; }; + 229E595A26586B4A006E41AE /* sigmoid.ort in Resources */ = {isa = PBXBuildFile; fileRef = 229E595826586B4A006E41AE /* sigmoid.ort */; }; + 22DEADEE265905A7005CBD1C /* ios_package_test_c_api.m in Sources */ = {isa = PBXBuildFile; fileRef = 22DEADED265905A7005CBD1C /* ios_package_test_c_api.m */; }; +/* End PBXBuildFile section */ + +/* Begin PBXContainerItemProxy section */ + 229E5938265869C2006E41AE /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 229E5914265869BF006E41AE /* Project object */; + proxyType = 1; + remoteGlobalIDString = 229E591B265869BF006E41AE; + remoteInfo = ios_package_test; + }; +/* End PBXContainerItemProxy section */ + +/* Begin PBXFileReference section */ + 229E591C265869BF006E41AE /* ios_package_test.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = ios_package_test.app; sourceTree = BUILT_PRODUCTS_DIR; }; + 229E591F265869BF006E41AE /* AppDelegate.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = AppDelegate.h; sourceTree = ""; }; + 229E5920265869BF006E41AE /* AppDelegate.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = AppDelegate.m; sourceTree = ""; }; + 229E5929265869BF006E41AE /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/Main.storyboard; sourceTree = ""; }; + 229E592E265869C2006E41AE /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = ""; }; + 229E5930265869C2006E41AE /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; + 229E5931265869C2006E41AE /* main.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = main.m; sourceTree = ""; }; + 229E5937265869C2006E41AE /* ios_package_testTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = ios_package_testTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; + 229E593B265869C2006E41AE /* ios_package_test_cpp_api.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = ios_package_test_cpp_api.mm; sourceTree = ""; }; + 229E593D265869C2006E41AE /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; + 229E595826586B4A006E41AE /* sigmoid.ort */ = {isa = PBXFileReference; lastKnownFileType = file; path = sigmoid.ort; sourceTree = ""; }; + 22DEADED265905A7005CBD1C /* ios_package_test_c_api.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = ios_package_test_c_api.m; sourceTree = ""; }; +/* End PBXFileReference section */ + +/* Begin PBXFrameworksBuildPhase section */ + 229E5919265869BF006E41AE /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; + 229E5934265869C2006E41AE /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXFrameworksBuildPhase section */ + +/* Begin PBXGroup section */ + 229E5913265869BF006E41AE = { + isa = PBXGroup; + children = ( + 229E595426586A77006E41AE /* models */, + 229E591E265869BF006E41AE /* ios_package_test */, + 229E593A265869C2006E41AE /* ios_package_testTests */, + 229E591D265869BF006E41AE /* Products */, + ); + sourceTree = ""; + }; + 229E591D265869BF006E41AE /* Products */ = { + isa = PBXGroup; + children = ( + 229E591C265869BF006E41AE /* ios_package_test.app */, + 229E5937265869C2006E41AE /* ios_package_testTests.xctest */, + ); + name = Products; + sourceTree = ""; + }; + 229E591E265869BF006E41AE /* ios_package_test */ = { + isa = PBXGroup; + children = ( + 229E591F265869BF006E41AE /* AppDelegate.h */, + 229E5920265869BF006E41AE /* AppDelegate.m */, + 229E5928265869BF006E41AE /* Main.storyboard */, + 229E592D265869C2006E41AE /* LaunchScreen.storyboard */, + 229E5930265869C2006E41AE /* Info.plist */, + 229E5931265869C2006E41AE /* main.m */, + ); + path = ios_package_test; + sourceTree = ""; + }; + 229E593A265869C2006E41AE /* ios_package_testTests */ = { + isa = PBXGroup; + children = ( + 229E593B265869C2006E41AE /* ios_package_test_cpp_api.mm */, + 229E593D265869C2006E41AE /* Info.plist */, + 22DEADED265905A7005CBD1C /* ios_package_test_c_api.m */, + ); + path = ios_package_testTests; + sourceTree = ""; + }; + 229E595426586A77006E41AE /* models */ = { + isa = PBXGroup; + children = ( + 229E595826586B4A006E41AE /* sigmoid.ort */, + ); + path = models; + sourceTree = ""; + }; +/* End PBXGroup section */ + +/* Begin PBXNativeTarget section */ + 229E591B265869BF006E41AE /* ios_package_test */ = { + isa = PBXNativeTarget; + buildConfigurationList = 229E594B265869C2006E41AE /* Build configuration list for PBXNativeTarget "ios_package_test" */; + buildPhases = ( + 229E5918265869BF006E41AE /* Sources */, + 229E5919265869BF006E41AE /* Frameworks */, + 229E591A265869BF006E41AE /* Resources */, + ); + buildRules = ( + ); + dependencies = ( + ); + name = ios_package_test; + productName = ios_package_test; + productReference = 229E591C265869BF006E41AE /* ios_package_test.app */; + productType = "com.apple.product-type.application"; + }; + 229E5936265869C2006E41AE /* ios_package_testTests */ = { + isa = PBXNativeTarget; + buildConfigurationList = 229E594E265869C2006E41AE /* Build configuration list for PBXNativeTarget "ios_package_testTests" */; + buildPhases = ( + 229E5933265869C2006E41AE /* Sources */, + 229E5934265869C2006E41AE /* Frameworks */, + 229E5935265869C2006E41AE /* Resources */, + ); + buildRules = ( + ); + dependencies = ( + 229E5939265869C2006E41AE /* PBXTargetDependency */, + ); + name = ios_package_testTests; + productName = ios_package_testTests; + productReference = 229E5937265869C2006E41AE /* ios_package_testTests.xctest */; + productType = "com.apple.product-type.bundle.unit-test"; + }; +/* End PBXNativeTarget section */ + +/* Begin PBXProject section */ + 229E5914265869BF006E41AE /* Project object */ = { + isa = PBXProject; + attributes = { + LastUpgradeCheck = 1250; + TargetAttributes = { + 229E591B265869BF006E41AE = { + CreatedOnToolsVersion = 12.5; + }; + 229E5936265869C2006E41AE = { + CreatedOnToolsVersion = 12.5; + TestTargetID = 229E591B265869BF006E41AE; + }; + }; + }; + buildConfigurationList = 229E5917265869BF006E41AE /* Build configuration list for PBXProject "ios_package_test" */; + compatibilityVersion = "Xcode 9.3"; + developmentRegion = en; + hasScannedForEncodings = 0; + knownRegions = ( + en, + Base, + ); + mainGroup = 229E5913265869BF006E41AE; + productRefGroup = 229E591D265869BF006E41AE /* Products */; + projectDirPath = ""; + projectRoot = ""; + targets = ( + 229E591B265869BF006E41AE /* ios_package_test */, + 229E5936265869C2006E41AE /* ios_package_testTests */, + ); + }; +/* End PBXProject section */ + +/* Begin PBXResourcesBuildPhase section */ + 229E591A265869BF006E41AE /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 229E592F265869C2006E41AE /* LaunchScreen.storyboard in Resources */, + 229E595926586B4A006E41AE /* sigmoid.ort in Resources */, + 229E592A265869BF006E41AE /* Main.storyboard in Resources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; + 229E5935265869C2006E41AE /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 229E595A26586B4A006E41AE /* sigmoid.ort in Resources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXResourcesBuildPhase section */ + +/* Begin PBXSourcesBuildPhase section */ + 229E5918265869BF006E41AE /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 229E5921265869BF006E41AE /* AppDelegate.m in Sources */, + 229E5932265869C2006E41AE /* main.m in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; + 229E5933265869C2006E41AE /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 229E593C265869C2006E41AE /* ios_package_test_cpp_api.mm in Sources */, + 22DEADEE265905A7005CBD1C /* ios_package_test_c_api.m in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXSourcesBuildPhase section */ + +/* Begin PBXTargetDependency section */ + 229E5939265869C2006E41AE /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + target = 229E591B265869BF006E41AE /* ios_package_test */; + targetProxy = 229E5938265869C2006E41AE /* PBXContainerItemProxy */; + }; +/* End PBXTargetDependency section */ + +/* Begin PBXVariantGroup section */ + 229E5928265869BF006E41AE /* Main.storyboard */ = { + isa = PBXVariantGroup; + children = ( + 229E5929265869BF006E41AE /* Base */, + ); + name = Main.storyboard; + sourceTree = ""; + }; + 229E592D265869C2006E41AE /* LaunchScreen.storyboard */ = { + isa = PBXVariantGroup; + children = ( + 229E592E265869C2006E41AE /* Base */, + ); + name = LaunchScreen.storyboard; + sourceTree = ""; + }; +/* End PBXVariantGroup section */ + +/* Begin XCBuildConfiguration section */ + 229E5949265869C2006E41AE /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++14"; + CLANG_CXX_LIBRARY = "libc++"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = dwarf; + ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_TESTABILITY = YES; + GCC_C_LANGUAGE_STANDARD = gnu11; + GCC_DYNAMIC_NO_PIC = NO; + GCC_NO_COMMON_BLOCKS = YES; + GCC_OPTIMIZATION_LEVEL = 0; + GCC_PREPROCESSOR_DEFINITIONS = ( + "DEBUG=1", + "$(inherited)", + ); + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 13.0; + MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; + MTL_FAST_MATH = YES; + ONLY_ACTIVE_ARCH = YES; + SDKROOT = iphoneos; + }; + name = Debug; + }; + 229E594A265869C2006E41AE /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++14"; + CLANG_CXX_LIBRARY = "libc++"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + ENABLE_NS_ASSERTIONS = NO; + ENABLE_STRICT_OBJC_MSGSEND = YES; + GCC_C_LANGUAGE_STANDARD = gnu11; + GCC_NO_COMMON_BLOCKS = YES; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 13.0; + MTL_ENABLE_DEBUG_INFO = NO; + MTL_FAST_MATH = YES; + SDKROOT = iphoneos; + VALIDATE_PRODUCT = YES; + }; + name = Release; + }; + 229E594C265869C2006E41AE /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; + CODE_SIGN_STYLE = Automatic; + INFOPLIST_FILE = ios_package_test/Info.plist; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + ); + PRODUCT_BUNDLE_IDENTIFIER = "ai.onnxruntime.tests.ios-package-test"; + PRODUCT_NAME = "$(TARGET_NAME)"; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Debug; + }; + 229E594D265869C2006E41AE /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; + CODE_SIGN_STYLE = Automatic; + INFOPLIST_FILE = ios_package_test/Info.plist; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + ); + PRODUCT_BUNDLE_IDENTIFIER = "ai.onnxruntime.tests.ios-package-test"; + PRODUCT_NAME = "$(TARGET_NAME)"; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Release; + }; + 229E594F265869C2006E41AE /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + BUNDLE_LOADER = "$(TEST_HOST)"; + CODE_SIGN_STYLE = Automatic; + INFOPLIST_FILE = ios_package_testTests/Info.plist; + IPHONEOS_DEPLOYMENT_TARGET = 13.0; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + "@loader_path/Frameworks", + ); + PRODUCT_BUNDLE_IDENTIFIER = "ai.onnxruntime.tests.ios-package-testTests"; + PRODUCT_NAME = "$(TARGET_NAME)"; + TARGETED_DEVICE_FAMILY = "1,2"; + TEST_HOST = "$(BUILT_PRODUCTS_DIR)/ios_package_test.app/ios_package_test"; + }; + name = Debug; + }; + 229E5950265869C2006E41AE /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + BUNDLE_LOADER = "$(TEST_HOST)"; + CODE_SIGN_STYLE = Automatic; + INFOPLIST_FILE = ios_package_testTests/Info.plist; + IPHONEOS_DEPLOYMENT_TARGET = 13.0; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + "@loader_path/Frameworks", + ); + PRODUCT_BUNDLE_IDENTIFIER = "ai.onnxruntime.tests.ios-package-testTests"; + PRODUCT_NAME = "$(TARGET_NAME)"; + TARGETED_DEVICE_FAMILY = "1,2"; + TEST_HOST = "$(BUILT_PRODUCTS_DIR)/ios_package_test.app/ios_package_test"; + }; + name = Release; + }; +/* End XCBuildConfiguration section */ + +/* Begin XCConfigurationList section */ + 229E5917265869BF006E41AE /* Build configuration list for PBXProject "ios_package_test" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 229E5949265869C2006E41AE /* Debug */, + 229E594A265869C2006E41AE /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + 229E594B265869C2006E41AE /* Build configuration list for PBXNativeTarget "ios_package_test" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 229E594C265869C2006E41AE /* Debug */, + 229E594D265869C2006E41AE /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + 229E594E265869C2006E41AE /* Build configuration list for PBXNativeTarget "ios_package_testTests" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 229E594F265869C2006E41AE /* Debug */, + 229E5950265869C2006E41AE /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; +/* End XCConfigurationList section */ + }; + rootObject = 229E5914265869BF006E41AE /* Project object */; +} diff --git a/onnxruntime/test/platform/ios/ios_package_test/ios_package_test.xcodeproj/project.xcworkspace/contents.xcworkspacedata b/onnxruntime/test/platform/ios/ios_package_test/ios_package_test.xcodeproj/project.xcworkspace/contents.xcworkspacedata new file mode 100644 index 0000000000..919434a625 --- /dev/null +++ b/onnxruntime/test/platform/ios/ios_package_test/ios_package_test.xcodeproj/project.xcworkspace/contents.xcworkspacedata @@ -0,0 +1,7 @@ + + + + + diff --git a/onnxruntime/test/platform/ios/ios_package_test/ios_package_test.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist b/onnxruntime/test/platform/ios/ios_package_test/ios_package_test.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist new file mode 100644 index 0000000000..18d981003d --- /dev/null +++ b/onnxruntime/test/platform/ios/ios_package_test/ios_package_test.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist @@ -0,0 +1,8 @@ + + + + + IDEDidComputeMac32BitWarning + + + diff --git a/onnxruntime/test/platform/ios/ios_package_test/ios_package_test/AppDelegate.h b/onnxruntime/test/platform/ios/ios_package_test/ios_package_test/AppDelegate.h new file mode 100644 index 0000000000..63c992ff83 --- /dev/null +++ b/onnxruntime/test/platform/ios/ios_package_test/ios_package_test/AppDelegate.h @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +// +// AppDelegate.h +// ios_package_test +// + +#import + +@interface AppDelegate : UIResponder + + +@end + diff --git a/onnxruntime/test/platform/ios/ios_package_test/ios_package_test/AppDelegate.m b/onnxruntime/test/platform/ios/ios_package_test/ios_package_test/AppDelegate.m new file mode 100644 index 0000000000..c060260927 --- /dev/null +++ b/onnxruntime/test/platform/ios/ios_package_test/ios_package_test/AppDelegate.m @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +// +// AppDelegate.m +// ios_package_test +// + +#import "AppDelegate.h" + +@interface AppDelegate () + +@end + +@implementation AppDelegate + + +- (BOOL)application:(UIApplication *)application didFinishLaunchingWithOptions:(NSDictionary *)launchOptions { + // Override point for customization after application launch. + return YES; +} + + +#pragma mark - UISceneSession lifecycle + + +- (UISceneConfiguration *)application:(UIApplication *)application configurationForConnectingSceneSession:(UISceneSession *)connectingSceneSession options:(UISceneConnectionOptions *)options { + // Called when a new scene session is being created. + // Use this method to select a configuration to create the new scene with. + return [[UISceneConfiguration alloc] initWithName:@"Default Configuration" sessionRole:connectingSceneSession.role]; +} + + +- (void)application:(UIApplication *)application didDiscardSceneSessions:(NSSet *)sceneSessions { + // Called when the user discards a scene session. + // If any sessions were discarded while the application was not running, this will be called shortly after application:didFinishLaunchingWithOptions. + // Use this method to release any resources that were specific to the discarded scenes, as they will not return. +} + + +@end diff --git a/onnxruntime/test/platform/ios/ios_package_test/ios_package_test/Base.lproj/LaunchScreen.storyboard b/onnxruntime/test/platform/ios/ios_package_test/ios_package_test/Base.lproj/LaunchScreen.storyboard new file mode 100644 index 0000000000..865e9329f3 --- /dev/null +++ b/onnxruntime/test/platform/ios/ios_package_test/ios_package_test/Base.lproj/LaunchScreen.storyboard @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/onnxruntime/test/platform/ios/ios_package_test/ios_package_test/Base.lproj/Main.storyboard b/onnxruntime/test/platform/ios/ios_package_test/ios_package_test/Base.lproj/Main.storyboard new file mode 100644 index 0000000000..808a21ce77 --- /dev/null +++ b/onnxruntime/test/platform/ios/ios_package_test/ios_package_test/Base.lproj/Main.storyboard @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/onnxruntime/test/platform/ios/ios_package_test/ios_package_test/Info.plist b/onnxruntime/test/platform/ios/ios_package_test/ios_package_test/Info.plist new file mode 100644 index 0000000000..72bf2c4f59 --- /dev/null +++ b/onnxruntime/test/platform/ios/ios_package_test/ios_package_test/Info.plist @@ -0,0 +1,66 @@ + + + + + CFBundleDevelopmentRegion + $(DEVELOPMENT_LANGUAGE) + CFBundleExecutable + $(EXECUTABLE_NAME) + CFBundleIdentifier + $(PRODUCT_BUNDLE_IDENTIFIER) + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + $(PRODUCT_NAME) + CFBundlePackageType + $(PRODUCT_BUNDLE_PACKAGE_TYPE) + CFBundleShortVersionString + 1.0 + CFBundleVersion + 1 + LSRequiresIPhoneOS + + UIApplicationSceneManifest + + UIApplicationSupportsMultipleScenes + + UISceneConfigurations + + UIWindowSceneSessionRoleApplication + + + UISceneConfigurationName + Default Configuration + UISceneDelegateClassName + SceneDelegate + UISceneStoryboardFile + Main + + + + + UIApplicationSupportsIndirectInputEvents + + UILaunchStoryboardName + LaunchScreen + UIMainStoryboardFile + Main + UIRequiredDeviceCapabilities + + armv7 + + UISupportedInterfaceOrientations + + UIInterfaceOrientationPortrait + UIInterfaceOrientationLandscapeLeft + UIInterfaceOrientationLandscapeRight + + UISupportedInterfaceOrientations~ipad + + UIInterfaceOrientationPortrait + UIInterfaceOrientationPortraitUpsideDown + UIInterfaceOrientationLandscapeLeft + UIInterfaceOrientationLandscapeRight + + + diff --git a/onnxruntime/test/platform/ios/ios_package_test/ios_package_test/main.m b/onnxruntime/test/platform/ios/ios_package_test/ios_package_test/main.m new file mode 100644 index 0000000000..a4fe174a60 --- /dev/null +++ b/onnxruntime/test/platform/ios/ios_package_test/ios_package_test/main.m @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +// +// main.m +// ios_package_test +// + +#import +#import "AppDelegate.h" + +int main(int argc, char * argv[]) { + NSString * appDelegateClassName; + @autoreleasepool { + // Setup code that might create autoreleased objects goes here. + appDelegateClassName = NSStringFromClass([AppDelegate class]); + } + return UIApplicationMain(argc, argv, nil, appDelegateClassName); +} diff --git a/onnxruntime/test/platform/ios/ios_package_test/ios_package_testTests/Info.plist b/onnxruntime/test/platform/ios/ios_package_test/ios_package_testTests/Info.plist new file mode 100644 index 0000000000..64d65ca495 --- /dev/null +++ b/onnxruntime/test/platform/ios/ios_package_test/ios_package_testTests/Info.plist @@ -0,0 +1,22 @@ + + + + + CFBundleDevelopmentRegion + $(DEVELOPMENT_LANGUAGE) + CFBundleExecutable + $(EXECUTABLE_NAME) + CFBundleIdentifier + $(PRODUCT_BUNDLE_IDENTIFIER) + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + $(PRODUCT_NAME) + CFBundlePackageType + $(PRODUCT_BUNDLE_PACKAGE_TYPE) + CFBundleShortVersionString + 1.0 + CFBundleVersion + 1 + + diff --git a/onnxruntime/test/platform/ios/ios_package_test/ios_package_testTests/ios_package_test_c_api.m b/onnxruntime/test/platform/ios/ios_package_test/ios_package_testTests/ios_package_test_c_api.m new file mode 100644 index 0000000000..5210469f59 --- /dev/null +++ b/onnxruntime/test/platform/ios/ios_package_test/ios_package_testTests/ios_package_test_c_api.m @@ -0,0 +1,97 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +// +// ios_package_test_c_api.m +// ios_package_testTests +// +// This file hosts the tests of ORT C API, for tests of ORT C++ API, please see ios_package_test_cpp_api.mm +// + +#import +#include +#include + +#define ASSERT_ON_ERROR(expr) \ + do { \ + OrtStatus* status = (expr); \ + XCTAssertEqual(NULL, status, @"Failed with error message: %@", \ + @(ort_env_->GetErrorMessage(status))); \ + } while (0) + +@interface ios_package_test_c_api : XCTestCase { + const OrtApi* ort_env_; +} + +@end + +@implementation ios_package_test_c_api + +- (void)setUp { + // Put setup code here. This method is called before the invocation of each test method in the class. + ort_env_ = OrtGetApiBase()->GetApi(ORT_API_VERSION); +} + +- (void)tearDown { + // Put teardown code here. This method is called after the invocation of each test method in the class. +} + +- (void)testCAPI { + // This is an e2e test for ORT C API + OrtEnv* env = NULL; + ASSERT_ON_ERROR(ort_env_->CreateEnv(ORT_LOGGING_LEVEL_WARNING, "testCAPI", &env)); + + // initialize session options if needed + OrtSessionOptions* session_options; + ASSERT_ON_ERROR(ort_env_->CreateSessionOptions(&session_options)); + ASSERT_ON_ERROR(ort_env_->SetIntraOpNumThreads(session_options, 1)); + + OrtSession* session; + NSString* ns_model_path = [[NSBundle mainBundle] pathForResource:@"sigmoid" ofType:@"ort"]; + ASSERT_ON_ERROR(ort_env_->CreateSession(env, ns_model_path.UTF8String, session_options, &session)); + + size_t input_tensor_size = 3 * 4 * 5; + float input_tensor_values[input_tensor_size]; + float expected_output_values[input_tensor_size]; + const char* input_node_names[] = {"x"}; + const char* output_node_names[] = {"y"}; + const int64_t input_node_dims[] = {3, 4, 5}; + + for (size_t i = 0; i < input_tensor_size; i++) { + input_tensor_values[i] = (float)i - 30; + expected_output_values[i] = 1.0f / (1 + exp(-input_tensor_values[i])); + } + + OrtMemoryInfo* memory_info; + ASSERT_ON_ERROR(ort_env_->CreateCpuMemoryInfo(OrtArenaAllocator, OrtMemTypeDefault, &memory_info)); + OrtValue* input_tensor = NULL; + ASSERT_ON_ERROR(ort_env_->CreateTensorWithDataAsOrtValue( + memory_info, input_tensor_values, input_tensor_size * sizeof(float), + input_node_dims, 3, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, &input_tensor)); + int is_tensor; + ASSERT_ON_ERROR(ort_env_->IsTensor(input_tensor, &is_tensor)); + XCTAssertNotEqual(is_tensor, 0); + ort_env_->ReleaseMemoryInfo(memory_info); + + OrtValue* output_tensor = NULL; + ASSERT_ON_ERROR(ort_env_->Run(session, NULL, input_node_names, + (const OrtValue* const*)&input_tensor, 1, + output_node_names, 1, &output_tensor)); + ASSERT_ON_ERROR(ort_env_->IsTensor(output_tensor, &is_tensor)); + XCTAssertNotEqual(is_tensor, 0); + + // Get pointer to output tensor float values + float* output_values; + ASSERT_ON_ERROR(ort_env_->GetTensorMutableData(output_tensor, (void**)&output_values)); + + for (size_t i = 0; i < input_tensor_size; i++) { + XCTAssertEqualWithAccuracy(expected_output_values[i], output_values[i], 1e-6); + } + + ort_env_->ReleaseValue(output_tensor); + ort_env_->ReleaseValue(input_tensor); + ort_env_->ReleaseSession(session); + ort_env_->ReleaseSessionOptions(session_options); + ort_env_->ReleaseEnv(env); +} + +@end diff --git a/onnxruntime/test/platform/ios/ios_package_test/ios_package_testTests/ios_package_test_cpp_api.mm b/onnxruntime/test/platform/ios/ios_package_test/ios_package_testTests/ios_package_test_cpp_api.mm new file mode 100644 index 0000000000..2b0bc887ee --- /dev/null +++ b/onnxruntime/test/platform/ios/ios_package_test/ios_package_testTests/ios_package_test_cpp_api.mm @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +// +// ios_package_test_cpp_api.mm +// ios_package_testTests +// +// This file hosts the tests of ORT C++ API, for tests of ORT C API, please see ios_package_test_c_api.mm +// + +#import +#include +#include + +@interface ios_package_test_cpp_api : XCTestCase + +@end + +@implementation ios_package_test_cpp_api + +- (void)setUp { + // Put setup code here. This method is called before the invocation of each test method in the class. +} + +- (void)tearDown { + // Put teardown code here. This method is called after the invocation of each test method in the class. +} + +- (void)testCppAPI { + // This is an e2e test for ORT C++ API + Ort::Env env(ORT_LOGGING_LEVEL_WARNING, "testCppAPI"); + + // initialize session options if needed + Ort::SessionOptions session_options; + session_options.SetIntraOpNumThreads(1); + + NSString* ns_model_path = [[NSBundle mainBundle] pathForResource:@"sigmoid" ofType:@"ort"]; + Ort::Session session(env, ns_model_path.UTF8String, session_options); + + size_t input_tensor_size = 3 * 4 * 5; + float input_tensor_values[input_tensor_size]; + float expected_output_values[input_tensor_size]; + const char* input_node_names[] = {"x"}; + const char* output_node_names[] = {"y"}; + const int64_t input_node_dims[] = {3, 4, 5}; + + for (size_t i = 0; i < input_tensor_size; i++) { + input_tensor_values[i] = (float)i - 30; + expected_output_values[i] = 1.0f / (1 + exp(-input_tensor_values[i])); + } + + auto memory_info = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); + Ort::Value input_tensor = + Ort::Value::CreateTensor(memory_info, input_tensor_values, input_tensor_size, input_node_dims, 3); + XCTAssert(input_tensor.IsTensor()); + + auto output_tensors = session.Run(Ort::RunOptions{nullptr}, input_node_names, + &input_tensor, 1, output_node_names, 1); + XCTAssertEqual(output_tensors.size(), 1); + XCTAssert(output_tensors.front().IsTensor()); + + // Get pointer to output tensor float values + float* output_values = output_tensors.front().GetTensorMutableData(); + for (size_t i = 0; i < input_tensor_size; i++) { + XCTAssertEqualWithAccuracy(expected_output_values[i], output_values[i], 1e-6); + } +} + +@end diff --git a/onnxruntime/test/platform/ios/ios_package_test/models/sigmoid.ort b/onnxruntime/test/platform/ios/ios_package_test/models/sigmoid.ort new file mode 100644 index 0000000000000000000000000000000000000000..6336fed141a5ecbcf7b4c550f4cfbf24edcd06ef GIT binary patch literal 1208 zcmZWpJ#Q015S;@<4hYCftjGcsxp3hEjua^{HP3d$5wL_tAGg=p#c1LSz`b~o~wk)CH}cRt?iT+$SgM~{0Cp&%EfF4N*m z4VJ(XSq5I$L_Ys53i>chIfCD?z6aeocz^5p?>BF0QyS6&SWQrmQiCtD0JKn~4SYBw za+X(C8q!AAb!pm6@?g#?;?{rxK%YIRe*sH4qu&ImmjKRTxcaAaINs&8%!9`b>6HBh z9hq;~vgbT}@+8_;FGAl2y1;p06~t-x04MD4@C63#L;R5Iz3$WdBNc7uI!(G+x}*Cl zW8M_-L~q8a?$fOZMw3rKy*%Yid`Yt#T0PWWFN(svuXro4*^{%LoD>B$-}esC0_e{n zhPn<7<(ajqYng5KaLIBubBO0-lA5jM z_ zc)pvca}D6a=S%8!Zo+5h5`=-F 'MIT' } + spec.authors = { "ONNX Runtime" => "onnxruntime@microsoft.com" } + spec.platform = :ios, '13.0' + # if you are going to use a file as the spec.source, add 'file:' before your file path + spec.source = { :http => '${ORT_BASE_FRAMEWORK_ARCHIVE}' } + spec.vendored_frameworks = 'onnxruntime.framework' +end diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index ec19f79f51..decac8cb95 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -1257,16 +1257,20 @@ def run_android_tests(args, source_dir, build_dir, config, cwd): def run_ios_tests(args, source_dir, config, cwd): - cpr = run_subprocess(["xcodebuild", "test-without-building", "-project", "./onnxruntime.xcodeproj", - "-configuration", config, - "-scheme", "onnxruntime_test_all_xc", "-destination", - "platform=iOS Simulator,OS=latest,name=iPhone SE (2nd generation)"], cwd=cwd) - if cpr.returncode == 0: - cpr = run_subprocess(["xcodebuild", "test-without-building", "-project", "./onnxruntime.xcodeproj", - "-configuration", config, - "-scheme", "onnxruntime_shared_lib_test_xc", "-destination", - "platform=iOS Simulator,OS=latest,name=iPhone SE (2nd generation)"], cwd=cwd) - cpr.check_returncode() + run_subprocess(["xcodebuild", "test-without-building", "-project", "./onnxruntime.xcodeproj", + "-configuration", config, + "-scheme", "onnxruntime_test_all_xc", "-destination", + "platform=iOS Simulator,OS=latest,name=iPhone SE (2nd generation)"], cwd=cwd) + + run_subprocess(["xcodebuild", "test-without-building", "-project", "./onnxruntime.xcodeproj", + "-configuration", config, + "-scheme", "onnxruntime_shared_lib_test_xc", "-destination", + "platform=iOS Simulator,OS=latest,name=iPhone SE (2nd generation)"], cwd=cwd) + + if args.build_apple_framework: + package_test_py = os.path.join(source_dir, 'tools', 'ci_build', 'github', 'apple', 'test_ios_packages.py') + framework_dir = os.path.join(cwd, config + '-' + args.ios_sysroot) + run_subprocess([sys.executable, package_test_py, '--c_framework_dir', framework_dir], cwd=cwd) def run_orttraining_test_orttrainer_frontend_separately(cwd): diff --git a/tools/ci_build/github/apple/test_ios_packages.py b/tools/ci_build/github/apple/test_ios_packages.py new file mode 100644 index 0000000000..aab81364b5 --- /dev/null +++ b/tools/ci_build/github/apple/test_ios_packages.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import argparse +import os +import pathlib +import shutil +import subprocess + +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +REPO_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, "..", "..", "..", "..")) + + +def _test_ios_packages(args): + # check if CocoaPods is installed + if shutil.which('pod') is None: + if args.fail_if_cocoapods_missing: + raise ValueError('CocoaPods is required for this test') + else: + print('CocoaPods is not installed, ignore this test') + return + + # Now we need to create a zip file contains the framework and the podspec file, both of these 2 files + # should be under the c_framework_dir + c_framework_dir = args.c_framework_dir.resolve() + if not c_framework_dir.is_dir(): + raise FileNotFoundError('c_framework_dir {} is not a folder.'.format(c_framework_dir)) + + framework_path = os.path.join(c_framework_dir, 'onnxruntime.framework') + if not pathlib.Path(framework_path).exists(): + raise FileNotFoundError('{} does not have onnxruntime.framework'.format(c_framework_dir)) + + # create a temp folder + import tempfile + with tempfile.TemporaryDirectory() as temp_dir: + # create a zip file contains the framework + # TODO, move this into a util function + local_pods_dir = os.path.join(temp_dir, 'local_pods') + os.makedirs(local_pods_dir, exist_ok=True) + # shutil.make_archive require target file as full path without extension + zip_base_filename = os.path.join(local_pods_dir, 'onnxruntime-mobile') + zip_file_path = zip_base_filename + '.zip' + shutil.make_archive(zip_base_filename, 'zip', root_dir=c_framework_dir, base_dir='onnxruntime.framework') + + # copy the test project to the temp_dir + test_proj_path = os.path.join(REPO_DIR, 'onnxruntime', 'test', 'platform', 'ios', 'ios_package_test') + target_proj_path = os.path.join(temp_dir, 'ios_package_test') + shutil.copytree(test_proj_path, target_proj_path) + + # update the podspec to point to the local framework zip file + local_podspec_path = os.path.join(target_proj_path, 'onnxruntime-mobile.podspec') + local_podspec_template = os.path.join(target_proj_path, 'onnxruntime-mobile.podspec.template') + with open(local_podspec_template, 'r') as file: + file_data = file.read() + + # replace the target strings + file_data = file_data.replace('${ORT_BASE_FRAMEWORK_ARCHIVE}', 'file:' + zip_file_path) + with open(os.path.join(REPO_DIR, 'VERSION_NUMBER')) as version_file: + file_data = file_data.replace('${ORT_VERSION}', version_file.readline().strip()) + + # write the updated podspec + with open(local_podspec_path, 'w') as file: + file.write(file_data) + + # install pods first + subprocess.run(['pod', 'install'], shell=False, check=True, cwd=target_proj_path) + + # run the tests + subprocess.run(['xcrun', 'xcodebuild', 'test', + '-workspace', './ios_package_test.xcworkspace', + '-scheme', 'ios_package_test', + '-destination', 'platform=iOS Simulator,OS=latest,name=iPhone SE (2nd generation)'], + shell=False, check=True, cwd=target_proj_path) + + +def parse_args(): + parser = argparse.ArgumentParser( + os.path.basename(__file__), + description='Test iOS framework using CocoaPods package.' + ) + + parser.add_argument('--fail_if_cocoapods_missing', action='store_true', + help='This script will fail if CocoaPods is not installed, ' + 'will not throw error unless fail_if_cocoapod_missing is set.') + + parser.add_argument('--c_framework_dir', type=pathlib.Path, required=True, + help='Provide the parent directory for C/C++ framework') + + return parser.parse_args() + + +def main(): + args = parse_args() + _test_ios_packages(args) + + +if __name__ == '__main__': + main() diff --git a/tools/ci_build/github/azure-pipelines/mac-ios-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/mac-ios-ci-pipeline.yml index 750aa525ac..b31ab975fa 100644 --- a/tools/ci_build/github/azure-pipelines/mac-ios-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/mac-ios-ci-pipeline.yml @@ -13,6 +13,7 @@ jobs: --apple_deploy_target 12.1 \ --use_xcode \ --config RelWithDebInfo \ + --build_apple_framework \ --parallel displayName: (CPU EP) Build onnxruntime for iOS x86_64 and run tests using simulator - script: | @@ -25,5 +26,6 @@ jobs: --apple_deploy_target 12.1 \ --use_xcode \ --config RelWithDebInfo \ + --build_apple_framework \ --parallel displayName: (CoreML EP) Build onnxruntime for iOS x86_64 and run tests using simulator From 6d9062641c020c52f888a07b8d85fc6678cab66d Mon Sep 17 00:00:00 2001 From: Wei-Sheng Chin Date: Tue, 1 Jun 2021 11:22:45 -0700 Subject: [PATCH 42/47] Basic data parallel tests for ORTModule (#7812) * Test Pytorch DDP with ORTModule * Remove unused MP model * Update orttraining/orttraining/test/python/orttraining_test_ort_module_pytorch_ddp.py * Update orttraining/orttraining/test/python/orttraining_test_ort_module_pytorch_ddp.py * Change file name * Fix import * Skip a test * Address a comment * Add test back --- ...orttraining_ortmodule_distributed_tests.py | 10 ++ .../orttraining_test_ortmodule_pytorch_ddp.py | 159 ++++++++++++++++++ 2 files changed, 169 insertions(+) create mode 100644 orttraining/orttraining/test/python/orttraining_test_ortmodule_pytorch_ddp.py diff --git a/orttraining/orttraining/test/python/orttraining_ortmodule_distributed_tests.py b/orttraining/orttraining/test/python/orttraining_ortmodule_distributed_tests.py index 80e009c4f9..b03af9099a 100644 --- a/orttraining/orttraining/test/python/orttraining_ortmodule_distributed_tests.py +++ b/orttraining/orttraining/test/python/orttraining_ortmodule_distributed_tests.py @@ -33,6 +33,13 @@ def run_ortmodule_deepspeed_zero_stage_1_tests(cwd, log, data_dir): run_subprocess(command, cwd=cwd, log=log).check_returncode() +def run_pytorch_ddp_tests(cwd, log): + log.debug('Running: ORTModule Pytorch DDP tests') + + command = [sys.executable, 'orttraining_test_ortmodule_pytorch_ddp.py', '--use_ort_module'] + + run_subprocess(command, cwd=cwd, log=log).check_returncode() + def run_ortmodule_deepspeed_pipeline_parallel_tests(cwd, log): log.debug('Running: ORTModule deepspeed pipeline parallel tests') @@ -56,7 +63,10 @@ def main(): log.info("Running ortmodule tests pipeline") + run_pytorch_ddp_tests(cwd, log) + run_ortmodule_deepspeed_zero_stage_1_tests(cwd, log, args.mnist) + run_ortmodule_deepspeed_pipeline_parallel_tests(cwd, log) run_ortmodule_fairscale_sharded_optimizer_tests(cwd, log, args.mnist) return 0 diff --git a/orttraining/orttraining/test/python/orttraining_test_ortmodule_pytorch_ddp.py b/orttraining/orttraining/test/python/orttraining_test_ortmodule_pytorch_ddp.py new file mode 100644 index 0000000000..8eaa1e5704 --- /dev/null +++ b/orttraining/orttraining/test/python/orttraining_test_ortmodule_pytorch_ddp.py @@ -0,0 +1,159 @@ +# This test script is a modified version of Pytorch's tutorial. +# For details, see https://pytorch.org/tutorials/intermediate/ddp_tutorial.html. +import os +import sys +import tempfile +import torch +import argparse + +import torch.distributed as dist +import torch.nn as nn +import torch.optim as optim +import torch.multiprocessing as mp + +from torch.nn.parallel import DistributedDataParallel as DDP + +import onnxruntime +from onnxruntime.training.ortmodule import ORTModule + +def setup(rank, world_size): + os.environ['MASTER_ADDR'] = 'localhost' + os.environ['MASTER_PORT'] = '12355' + + # initialize the process group + dist.init_process_group("gloo", rank=rank, world_size=world_size) + +def cleanup(): + dist.destroy_process_group() + + +class ToyModel(nn.Module): + def __init__(self): + super(ToyModel, self).__init__() + self.net1 = nn.Linear(10, 10) + self.relu = nn.ReLU() + self.net2 = nn.Linear(10, 5) + + def forward(self, x): + return self.net2(self.relu(self.net1(x))) + + +def demo_basic(rank, world_size, use_ort_module): + torch.manual_seed(0) + print(f"Running basic DDP example on rank {rank}.") + setup(rank, world_size) + + # create model and move it to GPU with id rank + model = ToyModel().to(rank) + if use_ort_module: + model = ORTModule(model) + print(f" Rank {rank} uses ORTModule."); + else: + print(f" Rank {rank} uses Pytorch's nn.Module."); + + ddp_model = DDP(model, device_ids=[rank]) + + loss_fn = nn.MSELoss() + optimizer = optim.Adagrad(ddp_model.parameters(), lr=0.01) + + x = torch.randn(20, 10).to(rank) + y = torch.randn(20, 5).to(rank) + + loss_history = [] + + for i in range(5): + optimizer.zero_grad() + p = ddp_model(x) + loss = loss_fn(p, y) + with torch.no_grad(): + print(f" Rank {rank} at iteration {i} has loss {loss}.") + loss.backward() + optimizer.step() + with torch.no_grad(): + loss_history.append(torch.unsqueeze(loss, 0)) + + loss_history = torch.cat(loss_history).cpu() + expected_loss_history = torch.FloatTensor([1.4909229278564453, 1.432194471359253, 1.39592707157135, 1.367714762687683, 1.3445055484771729]) + + assert torch.allclose(expected_loss_history, loss_history) + + cleanup() + +def demo_checkpoint(rank, world_size, use_ort_module): + torch.manual_seed(rank) + print(f"Running DDP checkpoint example on rank {rank}.") + setup(rank, world_size) + + if use_ort_module: + print(f" Rank {rank} uses ORTModule."); + model = ToyModel().to(rank) + model = ORTModule(model) + else: + print(f" Rank {rank} uses Pytorch's nn.Module."); + model = ToyModel().to(rank) + + ddp_model = DDP(model, device_ids=[rank]) + + loss_fn = nn.MSELoss() + optimizer = optim.SGD(ddp_model.parameters(), lr=0.001) + + CHECKPOINT_PATH = os.path.join(tempfile.gettempdir(), "model.checkpoint") + if rank == 0: + # All processes should see same parameters as they all start from same + # random parameters and gradients are synchronized in backward passes. + # Therefore, saving it in one process is sufficient. + torch.save(ddp_model.state_dict(), CHECKPOINT_PATH) + + # Use a barrier() to make sure that process 1 loads the model after process + # 0 saves it. + dist.barrier() + # configure map_location properly + map_location = {'cuda:%d' % 0: 'cuda:%d' % rank} + ddp_model.load_state_dict( + torch.load(CHECKPOINT_PATH, map_location=map_location)) + + optimizer.zero_grad() + outputs = ddp_model(torch.randn(20, 10)) + labels = torch.randn(20, 5).to(rank) + loss_fn = nn.MSELoss() + loss = loss_fn(outputs, labels) + loss.backward() + optimizer.step() + + print(f"Rank {rank} sees loss {loss}") + + if rank == 0: + assert torch.allclose(loss.cpu(), torch.FloatTensor([1.4909229278564453])) + elif rank == 1: + assert torch.allclose(loss.cpu(), torch.FloatTensor([1.0177688598632812])) + elif rank == 2: + assert torch.allclose(loss.cpu(), torch.FloatTensor([1.290669322013855])) + elif rank == 3: + assert torch.allclose(loss.cpu(), torch.FloatTensor([0.825118362903595])) + else: + assert False + + # Not necessary to use a dist.barrier() to guard the file deletion below + # as the AllReduce ops in the backward pass of DDP already served as + # a synchronization. + + if rank == 0: + os.remove(CHECKPOINT_PATH) + + cleanup() + +def run_demo(demo_fn, world_size, use_ort_module): + mp.spawn(demo_fn, + args=(world_size, use_ort_module), + nprocs=world_size, + join=True) + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument('--use_ort_module', action='store_true') + return parser.parse_args() + +if __name__ == "__main__": + args = parse_args() + run_demo(demo_basic, 4, args.use_ort_module) + run_demo(demo_checkpoint, 4, args.use_ort_module) From f9587d6051213c143d271396230dd58601aed778 Mon Sep 17 00:00:00 2001 From: Yulong Wang Date: Tue, 1 Jun 2021 17:35:04 -0700 Subject: [PATCH 43/47] [js/web] update README.md (#7894) --- js/web/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/js/web/README.md b/js/web/README.md index 731f87cbb2..d2d3a0882c 100644 --- a/js/web/README.md +++ b/js/web/README.md @@ -10,9 +10,9 @@ The [Open Neural Network Exchange](http://onnx.ai/) (ONNX) is an open standard f ### Why ONNX Runtime Web -With ONNX Runtime Web, web developers can score pre-trained ONNX models directly on browsers with various benefits of reducing server-client communication and protecting user privacy, as well as offering install-free and cross-platform in-browser ML experience. +With ONNX Runtime Web, web developers can score models directly on browsers with various benefits including reducing server-client communication and protecting user privacy, as well as offering install-free and cross-platform in-browser ML experience. -ONNX Runtime Web can run on both CPU and GPU. For running on CPU, [WebAssembly](https://developer.mozilla.org/en-US/docs/WebAssembly) is adopted to execute the model at near-native speed. Furthermore, ONNX Runtime Web utilizes [Web Workers](https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Using_web_workers) to provide a "multi-threaded" environment to parallelize data processing. Empirical evaluation shows very promising performance gains on CPU by taking full advantage of WebAssembly and Web Workers. For running on GPUs, a popular standard for accessing GPU capabilities - WebGL is adopted. ONNX Runtime Web has further adopted several novel optimization techniques for reducing data transfer between CPU and GPU, as well as some techniques to reduce GPU processing cycles to further push the performance to the maximum. +ONNX Runtime Web can run on both CPU and GPU. On CPU side, [WebAssembly](https://developer.mozilla.org/en-US/docs/WebAssembly) is adopted to execute the model at near-native speed. ONNX Runtime Web complies the native ONNX Runtime CPU engine into WebAssembly backend by using Emscripten, so it supports most functionalities native ONNX Runtime offers, including full ONNX operator coverage, multi-threading, [ONNX Runtime Quantization](https://www.onnxruntime.ai/docs/how-to/quantization.html) as well as [ONNX Runtime Mobile](http://www.onnxruntime.ai/docs/how-to/deploy-on-mobile.html). For performance acceleration with GPUs, ONNX Runtime Web leverages WebGL, a popular standard for accessing GPU capabilities. We are keeping improving op coverage and optimizing performance in WebGL backend. See [Compatibility](#Compatibility) and [Operators Supported](#Operators) for a list of platforms and operators ONNX Runtime Web currently supports. From a9f7eef75476c7e3cf18ee83a75183e5f274c9f2 Mon Sep 17 00:00:00 2001 From: Ryan Hill <38674843+RyanUnderhill@users.noreply.github.com> Date: Tue, 1 Jun 2021 19:28:00 -0700 Subject: [PATCH 44/47] Add API_IMPL_* blocks around shared provider methods as they are C APIs (#7908) --- .../core/framework/provider_bridge_ort.cc | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/onnxruntime/core/framework/provider_bridge_ort.cc b/onnxruntime/core/framework/provider_bridge_ort.cc index 8dc938a055..a326845286 100644 --- a/onnxruntime/core/framework/provider_bridge_ort.cc +++ b/onnxruntime/core/framework/provider_bridge_ort.cc @@ -7,6 +7,7 @@ #include "core/framework/compute_capability.h" #include "core/framework/data_types.h" #include "core/framework/data_transfer_manager.h" +#include "core/framework/error_code_helper.h" #include "core/framework/execution_provider.h" #include "core/framework/kernel_registry.h" #include "core/framework/provider_bridge_ort.h" @@ -1096,6 +1097,7 @@ INcclService& INcclService::GetInstance() { } // namespace onnxruntime ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProvider_Dnnl, _In_ OrtSessionOptions* options, int use_arena) { + API_IMPL_BEGIN auto factory = onnxruntime::CreateExecutionProviderFactory_Dnnl(use_arena); if (!factory) { return OrtApis::CreateStatus(ORT_FAIL, "OrtSessionOptionsAppendExecutionProvider_Dnnl: Failed to load shared library"); @@ -1103,9 +1105,11 @@ ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProvider_Dnnl, _In_ OrtSessi options->provider_factories.push_back(factory); return nullptr; + API_IMPL_END } ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProvider_Tensorrt, _In_ OrtSessionOptions* options, int device_id) { + API_IMPL_BEGIN auto factory = onnxruntime::CreateExecutionProviderFactory_Tensorrt(device_id); if (!factory) { return OrtApis::CreateStatus(ORT_FAIL, "OrtSessionOptionsAppendExecutionProvider_Tensorrt: Failed to load shared library"); @@ -1113,9 +1117,11 @@ ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProvider_Tensorrt, _In_ OrtS options->provider_factories.push_back(factory); return nullptr; + API_IMPL_END } ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider_TensorRT, _In_ OrtSessionOptions* options, _In_ const OrtTensorRTProviderOptions* tensorrt_options) { + API_IMPL_BEGIN auto factory = onnxruntime::CreateExecutionProviderFactory_Tensorrt(tensorrt_options); if (!factory) { return OrtApis::CreateStatus(ORT_FAIL, "SessionOptionsAppendExecutionProvider_Tensorrt: Failed to load shared library"); @@ -1123,9 +1129,11 @@ ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider_TensorRT, _In options->provider_factories.push_back(factory); return nullptr; + API_IMPL_END } ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider_OpenVINO, _In_ OrtSessionOptions* options, _In_ const OrtOpenVINOProviderOptions* provider_options) { + API_IMPL_BEGIN auto factory = onnxruntime::CreateExecutionProviderFactory_OpenVINO(provider_options); if (!factory) { return OrtApis::CreateStatus(ORT_FAIL, "SessionOptionsAppendExecutionProvider_OpenVINO: Failed to load shared library"); @@ -1133,10 +1141,11 @@ ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider_OpenVINO, _In options->provider_factories.push_back(factory); return nullptr; + API_IMPL_END } ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProvider_OpenVINO, _In_ OrtSessionOptions* options, _In_ const char* device_type) { - OrtOpenVINOProviderOptions provider_options; + OrtOpenVINOProviderOptions provider_options{}; provider_options.device_type = device_type; return OrtApis::SessionOptionsAppendExecutionProvider_OpenVINO(options, &provider_options); } @@ -1149,18 +1158,23 @@ ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProvider_CUDA, _In_ OrtSessi } ORT_API_STATUS_IMPL(OrtApis::SetCurrentGpuDeviceId, _In_ int device_id) { + API_IMPL_BEGIN if (auto* info = onnxruntime::GetProviderInfo_CUDA()) return info->SetCurrentGpuDeviceId(device_id); return CreateStatus(ORT_FAIL, "CUDA execution provider is not enabled."); + API_IMPL_END } ORT_API_STATUS_IMPL(OrtApis::GetCurrentGpuDeviceId, _In_ int* device_id) { + API_IMPL_BEGIN if (auto* info = onnxruntime::GetProviderInfo_CUDA()) return info->GetCurrentGpuDeviceId(device_id); return CreateStatus(ORT_FAIL, "CUDA execution provider is not enabled."); + API_IMPL_END } ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider_CUDA, _In_ OrtSessionOptions* options, _In_ const OrtCUDAProviderOptions* cuda_options) { + API_IMPL_BEGIN auto factory = onnxruntime::CreateExecutionProviderFactory_Cuda(cuda_options); if (!factory) { return OrtApis::CreateStatus(ORT_FAIL, "OrtSessionOptionsAppendExecutionProvider_Cuda: Failed to load shared library"); @@ -1168,4 +1182,5 @@ ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider_CUDA, _In_ Or options->provider_factories.push_back(factory); return nullptr; + API_IMPL_END } From 38ca0f48398f10b6eae745988c7a1ebd276bbba5 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Tue, 1 Jun 2021 20:28:34 -0700 Subject: [PATCH 45/47] Change CMAKE_CUDA_STANDARD to C++17 for Windows GPU build (#7883) --- cmake/CMakeLists.txt | 7 +++++-- onnxruntime/core/providers/cpu/tensor/upsample.h | 4 ++-- onnxruntime/core/providers/cuda/cuda_execution_provider.h | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 7e8367ae3d..49a83933b8 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -1385,8 +1385,11 @@ if (onnxruntime_USE_CUDA) endif() enable_language(CUDA) message( STATUS "CMAKE_CUDA_COMPILER_VERSION: ${CMAKE_CUDA_COMPILER_VERSION}") - - set(CMAKE_CUDA_STANDARD 14) + if (WIN32) + set(CMAKE_CUDA_STANDARD 17) + else() + set(CMAKE_CUDA_STANDARD 14) + endif() file(TO_CMAKE_PATH ${onnxruntime_CUDNN_HOME} onnxruntime_CUDNN_HOME) set(ONNXRUNTIME_CUDA_LIBRARIES ${CUDA_LIBRARIES}) diff --git a/onnxruntime/core/providers/cpu/tensor/upsample.h b/onnxruntime/core/providers/cpu/tensor/upsample.h index c2aea9374b..0b48cd6fed 100644 --- a/onnxruntime/core/providers/cpu/tensor/upsample.h +++ b/onnxruntime/core/providers/cpu/tensor/upsample.h @@ -18,8 +18,8 @@ constexpr const char* UpsampleModeCubic = "cubic"; // is a 4x4 matrix const size_t CubicModeGridLength = 4; -using GetNearestPixelFunc = std::function; -using GetOriginalCoordinateFunc = std::function; +using GetNearestPixelFunc = int64_t(*)(float, bool); +using GetOriginalCoordinateFunc = float (*)(float, float, float, float, float, float); enum UpsampleMode { NN = 0, // nearest neighbour diff --git a/onnxruntime/core/providers/cuda/cuda_execution_provider.h b/onnxruntime/core/providers/cuda/cuda_execution_provider.h index c1c35284ee..e8848e06f2 100644 --- a/onnxruntime/core/providers/cuda/cuda_execution_provider.h +++ b/onnxruntime/core/providers/cuda/cuda_execution_provider.h @@ -178,7 +178,7 @@ class CUDAExecutionProvider : public IExecutionProvider { p.reset(); }); } - std::shared_ptr p{std::make_shared()}; + std::shared_ptr p = std::make_shared(); }; static const std::shared_ptr& PerThreadContextCache() { From 3d734a1cdcd3c45cf8f37302e70d58ca6e56f18a Mon Sep 17 00:00:00 2001 From: Ryan Hill <38674843+RyanUnderhill@users.noreply.github.com> Date: Tue, 1 Jun 2021 22:33:04 -0700 Subject: [PATCH 46/47] Missing logic for cuda nuget package (#7911) --- tools/nuget/generate_nuspec_for_native_nuget.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/nuget/generate_nuspec_for_native_nuget.py b/tools/nuget/generate_nuspec_for_native_nuget.py index 60e600d129..b8a5afc95f 100644 --- a/tools/nuget/generate_nuspec_for_native_nuget.py +++ b/tools/nuget/generate_nuspec_for_native_nuget.py @@ -25,7 +25,7 @@ def parse_arguments(): parser.add_argument("--is_release_build", required=False, default=None, type=str, help="Flag indicating if the build is a release build. Accepted values: true/false.") parser.add_argument("--execution_provider", required=False, default='None', type=str, - choices=['dnnl', 'openvino', 'tensorrt', 'None'], + choices=['cuda', 'dnnl', 'openvino', 'tensorrt', 'None'], help="The selected execution provider for this build.") return parser.parse_args() @@ -359,7 +359,7 @@ def generate_files(list, args): nuget_dependencies['openvino_ep_shared_lib']) + runtimes_target + args.target_architecture + '\\native" />') - if args.execution_provider == "cuda": + if args.execution_provider == "cuda" or is_cuda_gpu_package: files_list.append('') From 0fbec1b9c1d8f34a44ac5e7efc1fee0a0a08ac84 Mon Sep 17 00:00:00 2001 From: Scott McKay Date: Wed, 2 Jun 2021 17:47:40 +1000 Subject: [PATCH 47/47] Update the operator documentation generation (#7787) * Update the operator documentation generation - Make layout a little nicer - Update to latest supported operators including training - Fix some links that are broken when the docs content is copied to github-pages - Fix incorrect usage of 'onnx.ai.ml' as the default domain - ML ops are now separated from the real default domain of 'onnx.ai' - Include CPU, CUDA and training kernels - exclude DNNL as it's not an EP we own * There are separate paths for CUDA and CUDNN as they are not guaranteed to be in the same location on a Windows machine. Use the CUDNN path when looking for the CUDNN library. * Enable validation of both contrib ops and operator kernels in build Filter generation so it's deterministic Add ability for CI to publish the md files as build artifacts if they differ so a developer can download and add to their PR to resolve any diffs. Remove workarounds for github-pages as that will now link to the github docs which display correctly --- docs/ContribOperators.md | 347 +------- docs/OperatorKernels.md | 802 +++++++++--------- .../python/onnxruntime_pybind_state.cc | 15 +- tools/ci_build/build.py | 107 ++- .../azure-pipelines/win-ci-pipeline.yml | 4 +- .../azure-pipelines/win-gpu-ci-pipeline.yml | 23 +- tools/python/gen_contrib_doc.py | 42 +- tools/python/gen_opkernel_doc.py | 77 +- 8 files changed, 563 insertions(+), 854 deletions(-) diff --git a/docs/ContribOperators.md b/docs/ContribOperators.md index f8fe5d030d..ce59fa1337 100644 --- a/docs/ContribOperators.md +++ b/docs/ContribOperators.md @@ -1,7 +1,6 @@ ## Contrib Operator Schemas -*This file is automatically generated from the - [def files](/onnxruntime/core/graph/contrib_ops/contrib_defs.cc) via [this script](/tools/python/gen_contrib_doc.py). - Do not modify directly and instead edit operator definitions.* +*This file is automatically generated from the registered contrib operator schemas by [this script](https://github.com/microsoft/onnxruntime/blob/master/tools/python/gen_contrib_doc.py). +Do not modify directly.* * com.microsoft * com.microsoft.Attention @@ -58,15 +57,6 @@ * com.microsoft.Unique * com.microsoft.WordConvEmbedding * experimental com.microsoft.IsAllFinite -* com.microsoft.nchwc - * com.microsoft.nchwc.AveragePool - * com.microsoft.nchwc.Conv - * com.microsoft.nchwc.GlobalAveragePool - * com.microsoft.nchwc.GlobalMaxPool - * com.microsoft.nchwc.MaxPool - * com.microsoft.nchwc.ReorderInput - * com.microsoft.nchwc.ReorderOutput - * com.microsoft.nchwc.Upsample ## com.microsoft ### **com.microsoft.Attention** @@ -2762,336 +2752,3 @@ No versioning maintained for experimental ops. -## com.microsoft.nchwc -### **com.microsoft.nchwc.AveragePool** - - For internal use. - -#### Version - -This version of the operator has been available since version 1 of the 'com.microsoft.nchwc' operator set. - -#### Attributes - -
-
auto_pad : string
-
-
ceil_mode : int
-
-
count_include_pad : int
-
-
dilations : list of ints
-
-
kernel_shape : list of ints (required)
-
-
pads : list of ints
-
-
strides : list of ints
-
-
- -#### Inputs - -
-
X : T
-
-
- -#### Outputs - -
-
Y : T
-
-
- -#### Type Constraints - -
-
T : tensor(float)
-
Constrain input and output types to float tensors
-
- - -### **com.microsoft.nchwc.Conv** - - For internal use. - -#### Version - -This version of the operator has been available since version 1 of the 'com.microsoft.nchwc' operator set. - -#### Attributes - -
-
activation : string
-
-
activation_params : list of floats
-
-
auto_pad : string
-
-
dilations : list of ints
-
-
group : int
-
-
kernel_shape : list of ints
-
-
pads : list of ints
-
-
strides : list of ints
-
-
- -#### Inputs (2 - 4) - -
-
X : T
-
-
W : T
-
-
B (optional) : T
-
-
Sum (optional) : T
-
-
- -#### Outputs - -
-
Y : T
-
-
- -#### Type Constraints - -
-
T : tensor(float)
-
Constrain input and output types to float tensors
-
- - -### **com.microsoft.nchwc.GlobalAveragePool** - - For internal use. - -#### Version - -This version of the operator has been available since version 1 of the 'com.microsoft.nchwc' operator set. - -#### Inputs - -
-
X : T
-
-
- -#### Outputs - -
-
Y : T
-
-
- -#### Type Constraints - -
-
T : tensor(float)
-
Constrain input and output types to float tensors
-
- - -### **com.microsoft.nchwc.GlobalMaxPool** - - For internal use. - -#### Version - -This version of the operator has been available since version 1 of the 'com.microsoft.nchwc' operator set. - -#### Inputs - -
-
X : T
-
-
- -#### Outputs - -
-
Y : T
-
-
- -#### Type Constraints - -
-
T : tensor(float)
-
Constrain input and output types to float tensors
-
- - -### **com.microsoft.nchwc.MaxPool** - - For internal use. - -#### Version - -This version of the operator has been available since version 1 of the 'com.microsoft.nchwc' operator set. - -#### Attributes - -
-
auto_pad : string
-
-
ceil_mode : int
-
-
dilations : list of ints
-
-
kernel_shape : list of ints (required)
-
-
pads : list of ints
-
-
storage_order : int
-
-
strides : list of ints
-
-
- -#### Inputs - -
-
X : T
-
-
- -#### Outputs - -
-
Y : T
-
-
- -#### Type Constraints - -
-
T : tensor(float)
-
Constrain input and output types to float tensors
-
- - -### **com.microsoft.nchwc.ReorderInput** - - For internal use. - -#### Version - -This version of the operator has been available since version 1 of the 'com.microsoft.nchwc' operator set. - -#### Attributes - -
-
channels_last : int
-
-
- -#### Inputs - -
-
X : T
-
-
- -#### Outputs - -
-
Y : T
-
-
- -#### Type Constraints - -
-
T : tensor(float)
-
Constrain input and output types to float tensors
-
- - -### **com.microsoft.nchwc.ReorderOutput** - - For internal use. - -#### Version - -This version of the operator has been available since version 1 of the 'com.microsoft.nchwc' operator set. - -#### Attributes - -
-
channels : int
-
-
channels_last : int
-
-
- -#### Inputs - -
-
X : T
-
-
- -#### Outputs - -
-
Y : T
-
-
- -#### Type Constraints - -
-
T : tensor(float)
-
Constrain input and output types to float tensors
-
- - -### **com.microsoft.nchwc.Upsample** - - For internal use. - -#### Version - -This version of the operator has been available since version 1 of the 'com.microsoft.nchwc' operator set. - -#### Attributes - -
-
coordinate_transformation_mode : string
-
-
mode : string
-
-
scales : list of ints
-
-
- -#### Inputs - -
-
X : T
-
-
- -#### Outputs - -
-
Y : T
-
-
- -#### Type Constraints - -
-
T : tensor(float)
-
Constrain input and output types to float tensors
-
- - diff --git a/docs/OperatorKernels.md b/docs/OperatorKernels.md index 2b2fa79019..ac255c5b73 100644 --- a/docs/OperatorKernels.md +++ b/docs/OperatorKernels.md @@ -1,699 +1,715 @@ -## Supported Operators Data Types -*This file is automatically generated from the - [def files](/onnxruntime/core/providers/cpu/cpu_execution_provider.cc) via [this script](/tools/python/gen_opkernel_doc.py). - Do not modify directly and instead edit operator definitions.* +## Supported Operators and Data Types +*This file is automatically generated from the registered kernels by [this script](https://github.com/microsoft/onnxruntime/blob/master/tools/python/gen_opkernel_doc.py). +Do not modify directly.* +## Execution Providers +- [CPUExecutionProvider](#cpuexecutionprovider) +- [CUDAExecutionProvider](#cudaexecutionprovider) + +--------------- + + ## Operators implemented by CPUExecutionProvider | Op Name | Parameters | OpSet Version | Types Supported | |---------|------------|---------------|-----------------| -|**Operator Domain:** *ai.onnx.ml*|||| -|Abs|(*in* X:**T**, *out* Y:**T**)|13+|**T** = tensor(double), tensor(float), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|**Operator Domain:** *ai.onnx*|||| +|Abs|*in* X:**T**
*out* Y:**T**|13+|**T** = tensor(double), tensor(float), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[6, 12]|**T** = tensor(double), tensor(float), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| -|Acos|(*in* input:**T**, *out* output:**T**)|7+|**T** = tensor(float)| -|Acosh|(*in* input:**T**, *out* output:**T**)|9+|**T** = tensor(float)| -|Add|(*in* A:**T**, *in* B:**T**, *out* C:**T**)|14+|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)| +|Acos|*in* input:**T**
*out* output:**T**|7+|**T** = tensor(float)| +|Acosh|*in* input:**T**
*out* output:**T**|9+|**T** = tensor(float)| +|Add|*in* A:**T**
*in* B:**T**
*out* C:**T**|14+|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)| |||13|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)| |||[7, 12]|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)| -|Affine|(*in* X:**T**, *out* Y:**T**)|1+|**T** = tensor(float)| -|And|(*in* A:**T**, *in* B:**T**, *out* C:**T1**)|7+|**T** = tensor(bool)
**T1** = tensor(bool)| -|ArgMax|(*in* data:**T**, *out* reduced:**tensor(int64)**)|13+|**T** = tensor(double), tensor(float), tensor(int32)| +|Affine|*in* X:**T**
*out* Y:**T**|1+|**T** = tensor(float)| +|And|*in* A:**T**
*in* B:**T**
*out* C:**T1**|7+|**T** = tensor(bool)
**T1** = tensor(bool)| +|ArgMax|*in* data:**T**
*out* reduced:**tensor(int64)**|13+|**T** = tensor(double), tensor(float), tensor(int32)| |||[11, 12]|**T** = tensor(double), tensor(float), tensor(int32)| |||[1, 10]|**T** = tensor(float), tensor(int32)| -|ArgMin|(*in* data:**T**, *out* reduced:**tensor(int64)**)|13+|**T** = tensor(double), tensor(float), tensor(int32)| +|ArgMin|*in* data:**T**
*out* reduced:**tensor(int64)**|13+|**T** = tensor(double), tensor(float), tensor(int32)| |||[11, 12]|**T** = tensor(double), tensor(float), tensor(int32)| |||[1, 10]|**T** = tensor(float), tensor(int32)| -|ArrayFeatureExtractor|(*in* X:**T**, *in* Y:**tensor(int64)**, *out* Z:**T**)|1+|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64), tensor(string)| -|Asin|(*in* input:**T**, *out* output:**T**)|7+|**T** = tensor(float)| -|Asinh|(*in* input:**T**, *out* output:**T**)|9+|**T** = tensor(float)| -|Atan|(*in* input:**T**, *out* output:**T**)|7+|**T** = tensor(float)| -|Atanh|(*in* input:**T**, *out* output:**T**)|9+|**T** = tensor(float)| -|AveragePool|(*in* X:**T**, *out* Y:**T**)|11+|**T** = tensor(float)| +|Asin|*in* input:**T**
*out* output:**T**|7+|**T** = tensor(float)| +|Asinh|*in* input:**T**
*out* output:**T**|9+|**T** = tensor(float)| +|Atan|*in* input:**T**
*out* output:**T**|7+|**T** = tensor(float)| +|Atanh|*in* input:**T**
*out* output:**T**|9+|**T** = tensor(float)| +|AveragePool|*in* X:**T**
*out* Y:**T**|11+|**T** = tensor(float)| |||10|**T** = tensor(float)| |||[7, 9]|**T** = tensor(float)| -|BatchNormalization|(*in* X:**T**, *in* scale:**T**, *in* B:**T**, *in* input_mean:**U**, *in* input_var:**U**, *out* Y:**T**, *out* running_mean:**U**, *out* running_var:**U**) or (*in* X:**T**, *in* scale:**T**, *in* B:**T**, *in* mean:**T**, *in* var:**T**, *out* Y:**T**, *out* mean:**T**, *out* var:**T**, *out* saved_mean:**T**, *out* saved_var:**T**)|14+|**T** = tensor(double), tensor(float)| +|BatchNormalization|*in* X:**T**
*in* scale:**T**
*in* B:**T**
*in* input_mean:**U**
*in* input_var:**U**
*out* Y:**T**
*out* running_mean:**U**
*out* running_var:**U**

or

*in* X:**T**
*in* scale:**T**
*in* B:**T**
*in* mean:**T**
*in* var:**T**
*out* Y:**T**
*out* mean:**T**
*out* var:**T**
*out* saved_mean:**T**
*out* saved_var:**T**|14+|**T** = tensor(double), tensor(float)| |||[9, 13]|**T** = tensor(double), tensor(float)| |||[7, 8]|**T** = tensor(double), tensor(float)| -|Binarizer|(*in* X:**T**, *out* Y:**T**)|1+|**T** = tensor(float)| -|BitShift|(*in* X:**T**, *in* Y:**T**, *out* Z:**T**)|11+|**T** = tensor(uint32), tensor(uint64), tensor(uint8)| -|Cast|(*in* input:**T1**, *out* output:**T2**)|13+|**T1** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**T2** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|BitShift|*in* X:**T**
*in* Y:**T**
*out* Z:**T**|11+|**T** = tensor(uint32), tensor(uint64), tensor(uint8)| +|Cast|*in* input:**T1**
*out* output:**T2**|13+|**T1** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**T2** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[6, 12]|**T1** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**T2** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| -|CastMap|(*in* X:**T1**, *out* Y:**T2**)|1+|**T1** = map(int64,tensor(float)), map(int64,tensor(string))
**T2** = tensor(float), tensor(int64), tensor(string)| -|CategoryMapper|(*in* X:**T1**, *out* Y:**T2**)|1+|**T1** = tensor(int64), tensor(string)
**T2** = tensor(int64), tensor(string)| -|Ceil|(*in* X:**T**, *out* Y:**T**)|13+|**T** = tensor(float)| +|Ceil|*in* X:**T**
*out* Y:**T**|13+|**T** = tensor(float)| |||[6, 12]|**T** = tensor(float)| -|Celu|(*in* X:**T**, *out* Y:**T**)|12+|**T** = tensor(float)| -|Clip|(*in* input:**T**, *in* min:**T**, *in* max:**T**, *out* output:**T**) or (*in* input:**T**, *out* output:**T**)|13+|**T** = tensor(double), tensor(float), tensor(int64), tensor(int8), tensor(uint64), tensor(uint8)| +|Celu|*in* X:**T**
*out* Y:**T**|12+|**T** = tensor(float)| +|Clip|*in* input:**T**
*in* min:**T**
*in* max:**T**
*out* output:**T**

or

*in* input:**T**
*out* output:**T**|13+|**T** = tensor(double), tensor(float), tensor(int64), tensor(int8), tensor(uint64), tensor(uint8)| |||12|**T** = tensor(double), tensor(float), tensor(int64), tensor(int8), tensor(uint64), tensor(uint8)| |||11|**T** = tensor(float)| |||[6, 10]|**T** = tensor(float)| -|Compress|(*in* input:**T**, *in* condition:**T1**, *out* output:**T**)|11+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**T1** = tensor(bool)| +|Compress|*in* input:**T**
*in* condition:**T1**
*out* output:**T**|11+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**T1** = tensor(bool)| |||[9, 10]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**T1** = tensor(bool)| -|Concat|(*in* inputs:**T**, *out* concat_result:**T**)|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Concat|*in* inputs:**T**
*out* concat_result:**T**|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[11, 12]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[4, 10]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| -|ConcatFromSequence|(*in* input_sequence:**S**, *out* concat_result:**T**)|11+|**S** = seq(tensor(bfloat16)), seq(tensor(bool)), seq(tensor(double)), seq(tensor(float)), seq(tensor(float16)), seq(tensor(int16)), seq(tensor(int32)), seq(tensor(int64)), seq(tensor(int8)), seq(tensor(string)), seq(tensor(uint16)), seq(tensor(uint32)), seq(tensor(uint64)), seq(tensor(uint8))| -|ConstantOfShape|(*in* input:**T1**, *out* output:**T2**)|9+|**T1** = tensor(int64)
**T2** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| -|Conv|(*in* X:**T**, *in* W:**T**, *in* B:**T**, *out* Y:**T**)|11+|**T** = tensor(float)| +|ConcatFromSequence|*in* input_sequence:**S**
*out* concat_result:**T**|11+|**S** = seq(tensor(bfloat16)), seq(tensor(bool)), seq(tensor(double)), seq(tensor(float)), seq(tensor(float16)), seq(tensor(int16)), seq(tensor(int32)), seq(tensor(int64)), seq(tensor(int8)), seq(tensor(string)), seq(tensor(uint16)), seq(tensor(uint32)), seq(tensor(uint64)), seq(tensor(uint8))| +|ConstantOfShape|*in* input:**T1**
*out* output:**T2**|9+|**T1** = tensor(int64)
**T2** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Conv|*in* X:**T**
*in* W:**T**
*in* B:**T**
*out* Y:**T**|11+|**T** = tensor(float)| |||[1, 10]|**T** = tensor(float)| -|ConvInteger|(*in* x:**T1**, *in* w:**T2**, *in* x_zero_point:**T1**, *in* w_zero_point:**T2**, *out* y:**T3**)|10+|**T1** = tensor(uint8)
**T2** = tensor(uint8)
**T3** = tensor(int32)| -|ConvTranspose|(*in* X:**T**, *in* W:**T**, *in* B:**T**, *out* Y:**T**)|11+|**T** = tensor(float)| +|ConvInteger|*in* x:**T1**
*in* w:**T2**
*in* x_zero_point:**T1**
*in* w_zero_point:**T2**
*out* y:**T3**|10+|**T1** = tensor(uint8)
**T2** = tensor(uint8)
**T3** = tensor(int32)| +|ConvTranspose|*in* X:**T**
*in* W:**T**
*in* B:**T**
*out* Y:**T**|11+|**T** = tensor(float)| |||[1, 10]|**T** = tensor(float)| -|Cos|(*in* input:**T**, *out* output:**T**)|7+|**T** = tensor(float)| -|Cosh|(*in* input:**T**, *out* output:**T**)|9+|**T** = tensor(float)| -|Crop|(*in* input:**T**, *out* output:**T**)|1+|**T** = tensor(float)| -|CumSum|(*in* x:**T**, *in* axis:**T2**, *out* y:**T**)|14+|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)
**T2** = tensor(int32), tensor(int64)| +|Cos|*in* input:**T**
*out* output:**T**|7+|**T** = tensor(float)| +|Cosh|*in* input:**T**
*out* output:**T**|9+|**T** = tensor(float)| +|Crop|*in* input:**T**
*out* output:**T**|1+|**T** = tensor(float)| +|CumSum|*in* x:**T**
*in* axis:**T2**
*out* y:**T**|14+|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)
**T2** = tensor(int32), tensor(int64)| |||[11, 13]|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)
**T2** = tensor(int32), tensor(int64)| -|DepthToSpace|(*in* input:**T**, *out* output:**T**)|13+|**T** = tensor(float)| +|DepthToSpace|*in* input:**T**
*out* output:**T**|13+|**T** = tensor(float)| |||[11, 12]|**T** = tensor(float)| |||[1, 10]|**T** = tensor(float)| -|DequantizeLinear|(*in* x:**T**, *in* x_scale:**tensor(float)**, *in* x_zero_point:**T**, *out* y:**tensor(float)**)|13+|**T** = tensor(int32), tensor(int8), tensor(uint8)| +|DequantizeLinear|*in* x:**T**
*in* x_scale:**tensor(float)**
*in* x_zero_point:**T**
*out* y:**tensor(float)**|13+|**T** = tensor(int32), tensor(int8), tensor(uint8)| |||[10, 12]|**T** = tensor(int32), tensor(int8), tensor(uint8)| -|Det|(*in* X:**T**, *out* Y:**T**)|11+|**T** = tensor(float)| -|DictVectorizer|(*in* X:**T1**, *out* Y:**T2**)|1+|**T1** = map(int64,tensor(double)), map(int64,tensor(float)), map(int64,tensor(string)), map(string,tensor(double)), map(string,tensor(float)), map(string,tensor(int64))
**T2** = tensor(double), tensor(float), tensor(int64), tensor(string)| -|Div|(*in* A:**T**, *in* B:**T**, *out* C:**T**)|14+|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)| +|Det|*in* X:**T**
*out* Y:**T**|11+|**T** = tensor(float)| +|Div|*in* A:**T**
*in* B:**T**
*out* C:**T**|14+|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)| |||13|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)| |||[7, 12]|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)| -|Dropout|(*in* data:**T**, *in* ratio:**T1**, *in* training_mode:**T2**, *out* output:**T**, *out* mask:**T2**) or (*in* data:**T**, *out* output:**T**, *out* mask:**T**) or (*in* data:**T**, *out* output:**T**, *out* mask:**T1**)|13+|**T** = tensor(double), tensor(float)
**T1** = tensor(double), tensor(float)
**T2** = tensor(bool)| +|Dropout|*in* data:**T**
*in* ratio:**T1**
*in* training_mode:**T2**
*out* output:**T**
*out* mask:**T2**

or

*in* data:**T**
*out* output:**T**
*out* mask:**T**

or

*in* data:**T**
*out* output:**T**
*out* mask:**T1**|13+|**T** = tensor(double), tensor(float)
**T1** = tensor(double), tensor(float)
**T2** = tensor(bool)| |||12|**T** = tensor(double), tensor(float)
**T1** = tensor(double), tensor(float)
**T2** = tensor(bool)| |||[10, 11]|**T** = tensor(double), tensor(float), tensor(float16)
**T1** = tensor(bool)| |||[7, 9]|**T** = tensor(double), tensor(float), tensor(float16)
**T1** = tensor(bool)| -|DynamicQuantizeLinear|(*in* x:**T1**, *out* y:**T2**, *out* y_scale:**tensor(float)**, *out* y_zero_point:**T2**)|11+|**T2** = tensor(uint8)| -|DynamicSlice|(*in* data:**T**, *in* starts:**Tind**, *in* ends:**Tind**, *in* axes:**Tind**, *out* output:**T**)|1+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**Tind** = tensor(int32), tensor(int64)| -|Einsum|(*in* Inputs:**T**, *out* Output:**T**)|12+|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)| -|Elu|(*in* X:**T**, *out* Y:**T**)|6+|**T** = tensor(float)| -|Equal|(*in* A:**T**, *in* B:**T**, *out* C:**T1**)|13+|**T** = tensor(bool), tensor(double), tensor(float), tensor(int32), tensor(int64)
**T1** = tensor(bool)| +|DynamicQuantizeLinear|*in* x:**T1**
*out* y:**T2**
*out* y_scale:**tensor(float)**
*out* y_zero_point:**T2**|11+|**T2** = tensor(uint8)| +|DynamicSlice|*in* data:**T**
*in* starts:**Tind**
*in* ends:**Tind**
*in* axes:**Tind**
*out* output:**T**|1+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**Tind** = tensor(int32), tensor(int64)| +|Einsum|*in* Inputs:**T**
*out* Output:**T**|12+|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)| +|Elu|*in* X:**T**
*out* Y:**T**|6+|**T** = tensor(float)| +|Equal|*in* A:**T**
*in* B:**T**
*out* C:**T1**|13+|**T** = tensor(bool), tensor(double), tensor(float), tensor(int32), tensor(int64)
**T1** = tensor(bool)| |||[11, 12]|**T** = tensor(bool), tensor(double), tensor(float), tensor(int32), tensor(int64)
**T1** = tensor(bool)| |||[7, 10]|**T** = tensor(bool), tensor(double), tensor(float), tensor(int32), tensor(int64)
**T1** = tensor(bool)| -|Erf|(*in* input:**T**, *out* output:**T**)|13+|**T** = tensor(float)| +|Erf|*in* input:**T**
*out* output:**T**|13+|**T** = tensor(float)| |||[9, 12]|**T** = tensor(float)| -|Exp|(*in* input:**T**, *out* output:**T**)|13+|**T** = tensor(double), tensor(float)| +|Exp|*in* input:**T**
*out* output:**T**|13+|**T** = tensor(double), tensor(float)| |||[6, 12]|**T** = tensor(double), tensor(float)| -|Expand|(*in* input:**T**, *in* shape:**tensor(int64)**, *out* output:**T**)|13+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Expand|*in* input:**T**
*in* shape:**tensor(int64)**
*out* output:**T**|13+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[8, 12]|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| -|EyeLike|(*in* input:**T1**, *out* output:**T2**)|9+|**T1** = tensor(double), tensor(float), tensor(int32), tensor(int64), tensor(uint64)
**T2** = tensor(double), tensor(float), tensor(int32), tensor(int64), tensor(uint64)| -|FeatureVectorizer|(*in* X:**T1**, *out* Y:**tensor(float)**)|1+|**T1** = tensor(double), tensor(float), tensor(int32), tensor(int64)| -|Flatten|(*in* input:**T**, *out* output:**T**)|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|EyeLike|*in* input:**T1**
*out* output:**T2**|9+|**T1** = tensor(double), tensor(float), tensor(int32), tensor(int64), tensor(uint64)
**T2** = tensor(double), tensor(float), tensor(int32), tensor(int64), tensor(uint64)| +|Flatten|*in* input:**T**
*out* output:**T**|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[11, 12]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[9, 10]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[1, 8]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| -|Floor|(*in* X:**T**, *out* Y:**T**)|13+|**T** = tensor(float)| +|Floor|*in* X:**T**
*out* Y:**T**|13+|**T** = tensor(float)| |||[6, 12]|**T** = tensor(float)| -|GRU|(*in* X:**T**, *in* W:**T**, *in* R:**T**, *in* B:**T**, *in* sequence_lens:**T1**, *in* initial_h:**T**, *out* Y:**T**, *out* Y_h:**T**)|7+|**T** = tensor(double), tensor(float)
**T1** = tensor(int32)| -|Gather|(*in* data:**T**, *in* indices:**Tind**, *out* output:**T**)|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**Tind** = tensor(int32), tensor(int64)| +|GRU|*in* X:**T**
*in* W:**T**
*in* R:**T**
*in* B:**T**
*in* sequence_lens:**T1**
*in* initial_h:**T**
*out* Y:**T**
*out* Y_h:**T**|14+|**T** = tensor(double), tensor(float)
**T1** = tensor(int32)| +|||[7, 13]|**T** = tensor(double), tensor(float)
**T1** = tensor(int32)| +|Gather|*in* data:**T**
*in* indices:**Tind**
*out* output:**T**|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**Tind** = tensor(int32), tensor(int64)| |||[11, 12]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**Tind** = tensor(int32), tensor(int64)| |||[1, 10]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**Tind** = tensor(int32), tensor(int64)| -|GatherElements|(*in* data:**T**, *in* indices:**Tind**, *out* output:**T**)|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**Tind** = tensor(int32), tensor(int64)| +|GatherElements|*in* data:**T**
*in* indices:**Tind**
*out* output:**T**|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**Tind** = tensor(int32), tensor(int64)| |||[11, 12]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**Tind** = tensor(int32), tensor(int64)| -|GatherND|(*in* data:**T**, *in* indices:**tensor(int64)**, *out* output:**T**)|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**Tind** = tensor(int64)| +|GatherND|*in* data:**T**
*in* indices:**tensor(int64)**
*out* output:**T**|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**Tind** = tensor(int64)| |||12|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**Tind** = tensor(int64)| |||11|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**Tind** = tensor(int64)| -|Gemm|(*in* A:**T**, *in* B:**T**, *in* C:**T**, *out* Y:**T**)|13+|**T** = tensor(double), tensor(float)| +|Gemm|*in* A:**T**
*in* B:**T**
*in* C:**T**
*out* Y:**T**|13+|**T** = tensor(double), tensor(float)| |||[11, 12]|**T** = tensor(double), tensor(float)| |||[9, 10]|**T** = tensor(double), tensor(float)| |||[7, 8]|**T** = tensor(double), tensor(float)| -|GlobalAveragePool|(*in* X:**T**, *out* Y:**T**)|1+|**T** = tensor(float)| -|GlobalLpPool|(*in* X:**T**, *out* Y:**T**)|2+|**T** = tensor(float)| -|GlobalMaxPool|(*in* X:**T**, *out* Y:**T**)|1+|**T** = tensor(float)| -|Greater|(*in* A:**T**, *in* B:**T**, *out* C:**T1**)|13+|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)
**T1** = tensor(bool)| +|GlobalAveragePool|*in* X:**T**
*out* Y:**T**|1+|**T** = tensor(float)| +|GlobalLpPool|*in* X:**T**
*out* Y:**T**|2+|**T** = tensor(float)| +|GlobalMaxPool|*in* X:**T**
*out* Y:**T**|1+|**T** = tensor(float)| +|Greater|*in* A:**T**
*in* B:**T**
*out* C:**T1**|13+|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)
**T1** = tensor(bool)| |||[9, 12]|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)
**T1** = tensor(bool)| |||[7, 8]|**T** = tensor(double), tensor(float)
**T1** = tensor(bool)| -|GreaterOrEqual|(*in* A:**T**, *in* B:**T**, *out* C:**T1**)|12+|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)
**T1** = tensor(bool)| -|HardSigmoid|(*in* X:**T**, *out* Y:**T**)|6+|**T** = tensor(float)| -|Hardmax|(*in* input:**T**, *out* output:**T**)|13+|**T** = tensor(float)| +|GreaterOrEqual|*in* A:**T**
*in* B:**T**
*out* C:**T1**|12+|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)
**T1** = tensor(bool)| +|HardSigmoid|*in* X:**T**
*out* Y:**T**|6+|**T** = tensor(float)| +|Hardmax|*in* input:**T**
*out* output:**T**|13+|**T** = tensor(float)| |||[11, 12]|**T** = tensor(float)| |||[1, 10]|**T** = tensor(float)| -|Identity|(*in* input:**T**, *out* output:**T**) or (*in* input:**V**, *out* output:**V**)|14+|**V** = seq(tensor(bfloat16)), seq(tensor(bool)), seq(tensor(double)), seq(tensor(float)), seq(tensor(float16)), seq(tensor(int16)), seq(tensor(int32)), seq(tensor(int64)), seq(tensor(int8)), seq(tensor(string)), seq(tensor(uint16)), seq(tensor(uint32)), seq(tensor(uint64)), seq(tensor(uint8)), tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Identity|*in* input:**T**
*out* output:**T**

or

*in* input:**V**
*out* output:**V**|14+|**V** = seq(tensor(bfloat16)), seq(tensor(bool)), seq(tensor(double)), seq(tensor(float)), seq(tensor(float16)), seq(tensor(int16)), seq(tensor(int32)), seq(tensor(int64)), seq(tensor(int8)), seq(tensor(string)), seq(tensor(uint16)), seq(tensor(uint32)), seq(tensor(uint64)), seq(tensor(uint8)), tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||13|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[1, 12]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| -|If|(*in* cond:**B**, *out* outputs:**V**)|13+|**B** = tensor(bool)
**V** = seq(tensor(bfloat16)), seq(tensor(bool)), seq(tensor(double)), seq(tensor(float)), seq(tensor(float16)), seq(tensor(int16)), seq(tensor(int32)), seq(tensor(int64)), seq(tensor(int8)), seq(tensor(string)), seq(tensor(uint16)), seq(tensor(uint32)), seq(tensor(uint64)), seq(tensor(uint8)), tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|If|*in* cond:**B**
*out* outputs:**V**|13+|**B** = tensor(bool)
**V** = seq(tensor(bfloat16)), seq(tensor(bool)), seq(tensor(double)), seq(tensor(float)), seq(tensor(float16)), seq(tensor(int16)), seq(tensor(int32)), seq(tensor(int64)), seq(tensor(int8)), seq(tensor(string)), seq(tensor(uint16)), seq(tensor(uint32)), seq(tensor(uint64)), seq(tensor(uint8)), tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[11, 12]|**B** = tensor(bool)
**V** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[1, 10]|**B** = tensor(bool)
**V** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| -|ImageScaler|(*in* input:**T**, *out* output:**T**)|1+|**T** = tensor(float)| -|Imputer|(*in* X:**T**, *out* Y:**T**)|1+|**T** = tensor(float), tensor(int64)| -|InstanceNormalization|(*in* input:**T**, *in* scale:**T**, *in* B:**T**, *out* output:**T**)|6+|**T** = tensor(float)| -|IsInf|(*in* X:**T1**, *out* Y:**T2**)|10+|**T1** = tensor(double), tensor(float)
**T2** = tensor(bool)| -|IsNaN|(*in* X:**T1**, *out* Y:**T2**)|13+|**T1** = tensor(float), tensor(float16)
**T2** = tensor(bool)| +|ImageScaler|*in* input:**T**
*out* output:**T**|1+|**T** = tensor(float)| +|InstanceNormalization|*in* input:**T**
*in* scale:**T**
*in* B:**T**
*out* output:**T**|6+|**T** = tensor(float)| +|IsInf|*in* X:**T1**
*out* Y:**T2**|10+|**T1** = tensor(double), tensor(float)
**T2** = tensor(bool)| +|IsNaN|*in* X:**T1**
*out* Y:**T2**|13+|**T1** = tensor(float), tensor(float16)
**T2** = tensor(bool)| |||[9, 12]|**T1** = tensor(float), tensor(float16)
**T2** = tensor(bool)| -|LRN|(*in* X:**T**, *out* Y:**T**)|13+|**T** = tensor(float)| +|LRN|*in* X:**T**
*out* Y:**T**|13+|**T** = tensor(float)| |||[1, 12]|**T** = tensor(float)| -|LSTM|(*in* X:**T**, *in* W:**T**, *in* R:**T**, *in* B:**T**, *in* sequence_lens:**T1**, *in* initial_h:**T**, *in* initial_c:**T**, *in* P:**T**, *out* Y:**T**, *out* Y_h:**T**, *out* Y_c:**T**)|7+|**T** = tensor(double), tensor(float)
**T1** = tensor(int32)| -|LabelEncoder|(*in* X:**T1**, *out* Y:**T2**)|2+|**T1** = tensor(float), tensor(int64), tensor(string)
**T2** = tensor(float), tensor(int64), tensor(string)| -|||1|**T1** = tensor(int64), tensor(string)
**T2** = tensor(int64), tensor(string)| -|LayerNormalization|(*in* X:**T**, *in* Scale:**T**, *in* B:**T**, *out* Y:**T**, *out* Mean:**U**, *out* InvStdDev:**U**)|1+|**T** = tensor(double), tensor(float)| -|LeakyRelu|(*in* X:**T**, *out* Y:**T**)|6+|**T** = tensor(float)| -|Less|(*in* A:**T**, *in* B:**T**, *out* C:**T1**)|13+|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)
**T1** = tensor(bool)| +|LSTM|*in* X:**T**
*in* W:**T**
*in* R:**T**
*in* B:**T**
*in* sequence_lens:**T1**
*in* initial_h:**T**
*in* initial_c:**T**
*in* P:**T**
*out* Y:**T**
*out* Y_h:**T**
*out* Y_c:**T**|14+|**T** = tensor(double), tensor(float)
**T1** = tensor(int32)| +|||[7, 13]|**T** = tensor(double), tensor(float)
**T1** = tensor(int32)| +|LayerNormalization|*in* X:**T**
*in* Scale:**T**
*in* B:**T**
*out* Y:**T**
*out* Mean:**U**
*out* InvStdDev:**U**|1+|**T** = tensor(double), tensor(float)| +|LeakyRelu|*in* X:**T**
*out* Y:**T**|6+|**T** = tensor(float)| +|Less|*in* A:**T**
*in* B:**T**
*out* C:**T1**|13+|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)
**T1** = tensor(bool)| |||[9, 12]|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)
**T1** = tensor(bool)| |||[7, 8]|**T** = tensor(double), tensor(float)
**T1** = tensor(bool)| -|LessOrEqual|(*in* A:**T**, *in* B:**T**, *out* C:**T1**)|12+|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)
**T1** = tensor(bool)| -|LinearClassifier|(*in* X:**T1**, *out* Y:**T2**, *out* Z:**tensor(float)**)|1+|**T1** = tensor(double), tensor(float), tensor(int32), tensor(int64)
**T2** = tensor(int64), tensor(string)| -|LinearRegressor|(*in* X:**T**, *out* Y:**tensor(float)**)|1+|**T** = tensor(float)| -|Log|(*in* input:**T**, *out* output:**T**)|13+|**T** = tensor(double), tensor(float)| +|LessOrEqual|*in* A:**T**
*in* B:**T**
*out* C:**T1**|12+|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)
**T1** = tensor(bool)| +|Log|*in* input:**T**
*out* output:**T**|13+|**T** = tensor(double), tensor(float)| |||[6, 12]|**T** = tensor(double), tensor(float)| -|LogSoftmax|(*in* input:**T**, *out* output:**T**)|13+|**T** = tensor(double), tensor(float)| +|LogSoftmax|*in* input:**T**
*out* output:**T**|13+|**T** = tensor(double), tensor(float)| |||[11, 12]|**T** = tensor(double), tensor(float)| |||[1, 10]|**T** = tensor(double), tensor(float)| -|Loop|(*in* M:**I**, *in* cond:**B**, *in* v_initial:**V**, *out* v_final_and_scan_outputs:**V**)|13+|**B** = tensor(bool)
**I** = tensor(int64)
**V** = seq(tensor(bfloat16)), seq(tensor(bool)), seq(tensor(double)), seq(tensor(float)), seq(tensor(float16)), seq(tensor(int16)), seq(tensor(int32)), seq(tensor(int64)), seq(tensor(int8)), seq(tensor(string)), seq(tensor(uint16)), seq(tensor(uint32)), seq(tensor(uint64)), seq(tensor(uint8)), tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Loop|*in* M:**I**
*in* cond:**B**
*in* v_initial:**V**
*out* v_final_and_scan_outputs:**V**|13+|**B** = tensor(bool)
**I** = tensor(int64)
**V** = seq(tensor(bfloat16)), seq(tensor(bool)), seq(tensor(double)), seq(tensor(float)), seq(tensor(float16)), seq(tensor(int16)), seq(tensor(int32)), seq(tensor(int64)), seq(tensor(int8)), seq(tensor(string)), seq(tensor(uint16)), seq(tensor(uint32)), seq(tensor(uint64)), seq(tensor(uint8)), tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[11, 12]|**B** = tensor(bool)
**I** = tensor(int64)
**V** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[1, 10]|**B** = tensor(bool)
**I** = tensor(int64)
**V** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| -|LpNormalization|(*in* input:**T**, *out* output:**T**)|1+|**T** = tensor(double), tensor(float)| -|LpPool|(*in* X:**T**, *out* Y:**T**)|11+|**T** = tensor(float)| +|LpNormalization|*in* input:**T**
*out* output:**T**|1+|**T** = tensor(double), tensor(float)| +|LpPool|*in* X:**T**
*out* Y:**T**|11+|**T** = tensor(float)| |||[2, 10]|**T** = tensor(float)| -|MatMul|(*in* A:**T**, *in* B:**T**, *out* Y:**T**)|13+|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| +|MatMul|*in* A:**T**
*in* B:**T**
*out* Y:**T**|13+|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| |||[9, 12]|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| |||[1, 8]|**T** = tensor(double), tensor(float)| -|MatMulInteger|(*in* A:**T1**, *in* B:**T2**, *in* a_zero_point:**T1**, *in* b_zero_point:**T2**, *out* Y:**T3**)|10+|**T1** = tensor(uint8)
**T2** = tensor(int8), tensor(uint8)
**T3** = tensor(int32)| -|Max|(*in* data_0:**T**, *out* max:**T**)|13+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| +|MatMulInteger|*in* A:**T1**
*in* B:**T2**
*in* a_zero_point:**T1**
*in* b_zero_point:**T2**
*out* Y:**T3**|10+|**T1** = tensor(uint8)
**T2** = tensor(int8), tensor(uint8)
**T3** = tensor(int32)| +|Max|*in* data_0:**T**
*out* max:**T**|13+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| |||12|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| |||[8, 11]|**T** = tensor(double), tensor(float)| |||[6, 7]|**T** = tensor(float)| -|MaxPool|(*in* X:**T**, *out* Y:**T**) or (*in* X:**T**, *out* Y:**T**, *out* Indices:**I**)|12+|**I** = tensor(int64)
**T** = tensor(double), tensor(float), tensor(int8), tensor(uint8)| +|MaxPool|*in* X:**T**
*out* Y:**T**

or

*in* X:**T**
*out* Y:**T**
*out* Indices:**I**|12+|**I** = tensor(int64)
**T** = tensor(double), tensor(float), tensor(int8), tensor(uint8)| |||[8, 11]|**I** = tensor(int64)
**T** = tensor(double), tensor(float)| |||[1, 7]|**T** = tensor(float)| -|MaxRoiPool|(*in* X:**T**, *in* rois:**T**, *out* Y:**T**)|1+|**T** = tensor(float)| -|MaxUnpool|(*in* X:**T1**, *in* I:**T2**, *in* output_shape:**T2**, *out* output:**T1**)|11+|**T1** = tensor(float)
**T2** = tensor(int64)| +|MaxRoiPool|*in* X:**T**
*in* rois:**T**
*out* Y:**T**|1+|**T** = tensor(float)| +|MaxUnpool|*in* X:**T1**
*in* I:**T2**
*in* output_shape:**T2**
*out* output:**T1**|11+|**T1** = tensor(float)
**T2** = tensor(int64)| |||[9, 10]|**T1** = tensor(float)
**T2** = tensor(int64)| -|Mean|(*in* data_0:**T**, *out* mean:**T**)|13+|**T** = tensor(float)| +|Mean|*in* data_0:**T**
*out* mean:**T**|13+|**T** = tensor(float)| |||[8, 12]|**T** = tensor(float)| |||[6, 7]|**T** = tensor(float)| -|MeanVarianceNormalization|(*in* X:**T**, *out* Y:**T**) or (*in* input:**T**, *out* output:**T**)|13+|**T** = tensor(float)| +|MeanVarianceNormalization|*in* X:**T**
*out* Y:**T**

or

*in* input:**T**
*out* output:**T**|13+|**T** = tensor(float)| |||[9, 12]|**T** = tensor(float)| |||[1, 8]|**T** = tensor(float)| -|Min|(*in* data_0:**T**, *out* min:**T**)|13+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| +|Min|*in* data_0:**T**
*out* min:**T**|13+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| |||12|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| |||[8, 11]|**T** = tensor(double), tensor(float)| |||[6, 7]|**T** = tensor(float)| -|Mod|(*in* A:**T**, *in* B:**T**, *out* C:**T**)|13+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Mod|*in* A:**T**
*in* B:**T**
*out* C:**T**|13+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[10, 12]|**T** = tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| -|Mul|(*in* A:**T**, *in* B:**T**, *out* C:**T**)|14+|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)| +|Mul|*in* A:**T**
*in* B:**T**
*out* C:**T**|14+|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)| |||13|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)| |||[7, 12]|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)| -|Multinomial|(*in* input:**T1**, *out* output:**T2**)|7+|**T1** = tensor(float)
**T2** = tensor(int32), tensor(int64)| -|Neg|(*in* X:**T**, *out* Y:**T**)|13+|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64), tensor(int8)| +|Multinomial|*in* input:**T1**
*out* output:**T2**|7+|**T1** = tensor(float)
**T2** = tensor(int32), tensor(int64)| +|Neg|*in* X:**T**
*out* Y:**T**|13+|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64), tensor(int8)| |||[6, 12]|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64), tensor(int8)| -|NonZero|(*in* X:**T**, *out* Y:**tensor(int64)**)|13+|**T** = tensor(bool), tensor(float), tensor(int32), tensor(int64), tensor(uint8)| +|NonZero|*in* X:**T**
*out* Y:**tensor(int64)**|13+|**T** = tensor(bool), tensor(float), tensor(int32), tensor(int64), tensor(uint8)| |||[9, 12]|**T** = tensor(bool), tensor(float), tensor(int32), tensor(int64), tensor(uint8)| -|Normalizer|(*in* X:**T**, *out* Y:**tensor(float)**)|1+|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)| -|Not|(*in* X:**T**, *out* Y:**T**)|1+|**T** = tensor(bool)
**T1** = tensor(bool)| -|OneHot|(*in* indices:**T1**, *in* depth:**T2**, *in* values:**T3**, *out* output:**T3**)|11+|**T1** = tensor(float), tensor(int32), tensor(int64)
**T2** = tensor(float), tensor(int32), tensor(int64)
**T3** = tensor(float), tensor(int32), tensor(int64), tensor(string)| +|Not|*in* X:**T**
*out* Y:**T**|1+|**T** = tensor(bool)
**T1** = tensor(bool)| +|OneHot|*in* indices:**T1**
*in* depth:**T2**
*in* values:**T3**
*out* output:**T3**|11+|**T1** = tensor(float), tensor(int32), tensor(int64)
**T2** = tensor(float), tensor(int32), tensor(int64)
**T3** = tensor(float), tensor(int32), tensor(int64), tensor(string)| |||[9, 10]|**T1** = tensor(float), tensor(int32), tensor(int64)
**T2** = tensor(float), tensor(int32), tensor(int64)
**T3** = tensor(float), tensor(int32), tensor(int64), tensor(string)| -|OneHotEncoder|(*in* X:**T**, *out* Y:**tensor(float)**)|1+|**T** = tensor(double), tensor(float), tensor(int64), tensor(string)| -|Or|(*in* A:**T**, *in* B:**T**, *out* C:**T1**)|7+|**T** = tensor(bool)
**T1** = tensor(bool)| -|PRelu|(*in* X:**T**, *in* slope:**T**, *out* Y:**T**)|9+|**T** = tensor(float)| +|Or|*in* A:**T**
*in* B:**T**
*out* C:**T1**|7+|**T** = tensor(bool)
**T1** = tensor(bool)| +|PRelu|*in* X:**T**
*in* slope:**T**
*out* Y:**T**|9+|**T** = tensor(float)| |||[7, 8]|**T** = tensor(float)| -|Pad|(*in* data:**T**, *in* pads:**tensor(int64)**, *in* constant_value:**T**, *out* output:**T**) or (*in* data:**T**, *out* output:**T**)|13+|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64), tensor(int8), tensor(uint32), tensor(uint64), tensor(uint8)| +|Pad|*in* data:**T**
*in* pads:**tensor(int64)**
*in* constant_value:**T**
*out* output:**T**

or

*in* data:**T**
*out* output:**T**|13+|**T** = tensor(bool), tensor(double), tensor(float), tensor(int32), tensor(int64), tensor(int8), tensor(uint32), tensor(uint64), tensor(uint8)| |||[11, 12]|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64), tensor(int8), tensor(uint32), tensor(uint64), tensor(uint8)| |||[2, 10]|**T** = tensor(double), tensor(float)| -|ParametricSoftplus|(*in* X:**T**, *out* Y:**T**)|1+|**T** = tensor(float)| -|Pow|(*in* X:**T**, *in* Y:**T**, *out* Z:**T**) or (*in* X:**T**, *in* Y:**T1**, *out* Z:**T**)|13+|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)
**T1** = tensor(double), tensor(float), tensor(int32), tensor(int64)| +|ParametricSoftplus|*in* X:**T**
*out* Y:**T**|1+|**T** = tensor(float)| +|Pow|*in* X:**T**
*in* Y:**T**
*out* Z:**T**

or

*in* X:**T**
*in* Y:**T1**
*out* Z:**T**|13+|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)
**T1** = tensor(double), tensor(float), tensor(int32), tensor(int64)| |||12|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)
**T1** = tensor(double), tensor(float), tensor(int32), tensor(int64)| |||[7, 11]|**T** = tensor(double), tensor(float)| -|QLinearConv|(*in* x:**T1**, *in* x_scale:**tensor(float)**, *in* x_zero_point:**T1**, *in* w:**T2**, *in* w_scale:**tensor(float)**, *in* w_zero_point:**T2**, *in* y_scale:**tensor(float)**, *in* y_zero_point:**T3**, *in* B:**T4**, *out* y:**T3**)|10+|**T1** = tensor(uint8)
**T2** = tensor(int8), tensor(uint8)
**T3** = tensor(uint8)
**T4** = tensor(int32)| -|QLinearMatMul|(*in* a:**T1**, *in* a_scale:**tensor(float)**, *in* a_zero_point:**T1**, *in* b:**T2**, *in* b_scale:**tensor(float)**, *in* b_zero_point:**T2**, *in* y_scale:**tensor(float)**, *in* y_zero_point:**T3**, *out* y:**T3**)|10+|**T1** = tensor(uint8)
**T2** = tensor(int8), tensor(uint8)
**T3** = tensor(uint8)| -|QuantizeLinear|(*in* x:**T1**, *in* y_scale:**tensor(float)**, *in* y_zero_point:**T2**, *out* y:**T2**)|13+|**T1** = tensor(float)
**T2** = tensor(int8), tensor(uint8)| +|QLinearConv|*in* x:**T1**
*in* x_scale:**tensor(float)**
*in* x_zero_point:**T1**
*in* w:**T2**
*in* w_scale:**tensor(float)**
*in* w_zero_point:**T2**
*in* y_scale:**tensor(float)**
*in* y_zero_point:**T3**
*in* B:**T4**
*out* y:**T3**|10+|**T1** = tensor(uint8)
**T2** = tensor(int8), tensor(uint8)
**T3** = tensor(uint8)
**T4** = tensor(int32)| +|QLinearMatMul|*in* a:**T1**
*in* a_scale:**tensor(float)**
*in* a_zero_point:**T1**
*in* b:**T2**
*in* b_scale:**tensor(float)**
*in* b_zero_point:**T2**
*in* y_scale:**tensor(float)**
*in* y_zero_point:**T3**
*out* y:**T3**|10+|**T1** = tensor(uint8)
**T2** = tensor(int8), tensor(uint8)
**T3** = tensor(uint8)| +|QuantizeLinear|*in* x:**T1**
*in* y_scale:**tensor(float)**
*in* y_zero_point:**T2**
*out* y:**T2**|13+|**T1** = tensor(float)
**T2** = tensor(int8), tensor(uint8)| |||[10, 12]|**T1** = tensor(float)
**T2** = tensor(int8), tensor(uint8)| -|RNN|(*in* X:**T**, *in* W:**T**, *in* R:**T**, *in* B:**T**, *in* sequence_lens:**T1**, *in* initial_h:**T**, *out* Y:**T**, *out* Y_h:**T**)|7+|**T** = tensor(float)
**T1** = tensor(int32)| -|RandomNormal|(*out* output:**T**)|1+|**T** = tensor(double), tensor(float)| -|RandomNormalLike|(*in* input:**T1**, *out* output:**T2**)|1+|**T1** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**T2** = tensor(double), tensor(float)| -|RandomUniform|(*out* output:**T**)|1+|**T** = tensor(double), tensor(float)| -|RandomUniformLike|(*in* input:**T1**, *out* output:**T2**)|1+|**T1** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**T2** = tensor(double), tensor(float)| -|Range|(*in* start:**T**, *in* limit:**T**, *in* delta:**T**, *out* output:**T**)|11+|**T** = tensor(double), tensor(float), tensor(int16), tensor(int32), tensor(int64)| -|Reciprocal|(*in* X:**T**, *out* Y:**T**)|13+|**T** = tensor(double), tensor(float)| +|RNN|*in* X:**T**
*in* W:**T**
*in* R:**T**
*in* B:**T**
*in* sequence_lens:**T1**
*in* initial_h:**T**
*out* Y:**T**
*out* Y_h:**T**|14+|**T** = tensor(float)
**T1** = tensor(int32)| +|||[7, 13]|**T** = tensor(float)
**T1** = tensor(int32)| +|RandomNormal|*out* output:**T**|1+|**T** = tensor(double), tensor(float)| +|RandomNormalLike|*in* input:**T1**
*out* output:**T2**|1+|**T1** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**T2** = tensor(double), tensor(float)| +|RandomUniform|*out* output:**T**|1+|**T** = tensor(double), tensor(float)| +|RandomUniformLike|*in* input:**T1**
*out* output:**T2**|1+|**T1** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**T2** = tensor(double), tensor(float)| +|Range|*in* start:**T**
*in* limit:**T**
*in* delta:**T**
*out* output:**T**|11+|**T** = tensor(double), tensor(float), tensor(int16), tensor(int32), tensor(int64)| +|Reciprocal|*in* X:**T**
*out* Y:**T**|13+|**T** = tensor(double), tensor(float)| |||[6, 12]|**T** = tensor(double), tensor(float)| -|ReduceL1|(*in* data:**T**, *out* reduced:**T**)|13+|**T** = tensor(float), tensor(int32)| +|ReduceL1|*in* data:**T**
*out* reduced:**T**|13+|**T** = tensor(float), tensor(int32)| |||[11, 12]|**T** = tensor(float), tensor(int32)| |||[1, 10]|**T** = tensor(float), tensor(int32)| -|ReduceL2|(*in* data:**T**, *out* reduced:**T**)|13+|**T** = tensor(float), tensor(int32)| +|ReduceL2|*in* data:**T**
*out* reduced:**T**|13+|**T** = tensor(float), tensor(int32)| |||[11, 12]|**T** = tensor(float), tensor(int32)| |||[1, 10]|**T** = tensor(float), tensor(int32)| -|ReduceLogSum|(*in* data:**T**, *out* reduced:**T**)|13+|**T** = tensor(float), tensor(int32)| +|ReduceLogSum|*in* data:**T**
*out* reduced:**T**|13+|**T** = tensor(float), tensor(int32)| |||[11, 12]|**T** = tensor(float), tensor(int32)| |||[1, 10]|**T** = tensor(float), tensor(int32)| -|ReduceLogSumExp|(*in* data:**T**, *out* reduced:**T**)|13+|**T** = tensor(double), tensor(float), tensor(int32)| +|ReduceLogSumExp|*in* data:**T**
*out* reduced:**T**|13+|**T** = tensor(double), tensor(float), tensor(int32)| |||[11, 12]|**T** = tensor(double), tensor(float), tensor(int32)| |||[1, 10]|**T** = tensor(double), tensor(float), tensor(int32)| -|ReduceMax|(*in* data:**T**, *out* reduced:**T**)|13+|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64), tensor(int8), tensor(uint8)| +|ReduceMax|*in* data:**T**
*out* reduced:**T**|13+|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64), tensor(int8), tensor(uint8)| |||12|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64), tensor(int8), tensor(uint8)| |||11|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)| |||[1, 10]|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)| -|ReduceMean|(*in* data:**T**, *out* reduced:**T**)|13+|**T** = tensor(double), tensor(float), tensor(int32)| +|ReduceMean|*in* data:**T**
*out* reduced:**T**|13+|**T** = tensor(double), tensor(float), tensor(int32)| |||[11, 12]|**T** = tensor(double), tensor(float), tensor(int32)| |||[1, 10]|**T** = tensor(double), tensor(float), tensor(int32)| -|ReduceMin|(*in* data:**T**, *out* reduced:**T**)|13+|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64), tensor(int8), tensor(uint8)| +|ReduceMin|*in* data:**T**
*out* reduced:**T**|13+|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64), tensor(int8), tensor(uint8)| |||12|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64), tensor(int8), tensor(uint8)| |||11|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)| |||[1, 10]|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)| -|ReduceProd|(*in* data:**T**, *out* reduced:**T**)|13+|**T** = tensor(float), tensor(int32), tensor(int64)| +|ReduceProd|*in* data:**T**
*out* reduced:**T**|13+|**T** = tensor(float), tensor(int32), tensor(int64)| |||[11, 12]|**T** = tensor(float), tensor(int32), tensor(int64)| |||[1, 10]|**T** = tensor(float), tensor(int32), tensor(int64)| -|ReduceSum|(*in* data:**T**, *in* axes:**tensor(int64)**, *out* reduced:**T**) or (*in* data:**T**, *out* reduced:**T**)|13+|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)| +|ReduceSum|*in* data:**T**
*in* axes:**tensor(int64)**
*out* reduced:**T**

or

*in* data:**T**
*out* reduced:**T**|13+|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)| |||[11, 12]|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)| |||[1, 10]|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)| -|ReduceSumSquare|(*in* data:**T**, *out* reduced:**T**)|13+|**T** = tensor(double), tensor(float), tensor(int32)| +|ReduceSumSquare|*in* data:**T**
*out* reduced:**T**|13+|**T** = tensor(double), tensor(float), tensor(int32)| |||[11, 12]|**T** = tensor(double), tensor(float), tensor(int32)| |||[1, 10]|**T** = tensor(double), tensor(float), tensor(int32)| -|Relu|(*in* X:**T**, *out* Y:**T**)|14+|**T** = tensor(double), tensor(float)| +|Relu|*in* X:**T**
*out* Y:**T**|14+|**T** = tensor(double), tensor(float), tensor(int32), tensor(int8)| |||13|**T** = tensor(double), tensor(float)| |||[6, 12]|**T** = tensor(double), tensor(float)| -|Reshape|(*in* data:**T**, *in* shape:**tensor(int64)**, *out* reshaped:**T**) or (*in* data:**T**, *out* reshaped:**T**)|14+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**shape** = tensor(int64)| +|Reshape|*in* data:**T**
*in* shape:**tensor(int64)**
*out* reshaped:**T**

or

*in* data:**T**
*out* reshaped:**T**|14+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**shape** = tensor(int64)| |||13|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**shape** = tensor(int64)| |||[5, 12]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**shape** = tensor(int64)| |||[1, 4]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| -|Resize|(*in* X:**T**, *in* scales:**tensor(float)**, *out* Y:**T**) or (*in* X:**T1**, *in* roi:**T2**, *in* scales:**tensor(float)**, *in* sizes:**tensor(int64)**, *out* Y:**T1**)|13+|**T1** = tensor(float), tensor(int32), tensor(uint8)| +|Resize|*in* X:**T**
*in* scales:**tensor(float)**
*out* Y:**T**

or

*in* X:**T1**
*in* roi:**T2**
*in* scales:**tensor(float)**
*in* sizes:**tensor(int64)**
*out* Y:**T1**|13+|**T1** = tensor(float), tensor(int32), tensor(uint8)| |||[11, 12]|**T1** = tensor(float), tensor(int32), tensor(uint8)| |||10|**T** = tensor(float), tensor(int32), tensor(uint8)| -|ReverseSequence|(*in* input:**T**, *in* sequence_lens:**tensor(int64)**, *out* Y:**T**)|10+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| -|RoiAlign|(*in* X:**T1**, *in* rois:**T1**, *in* batch_indices:**T2**, *out* Y:**T1**)|10+|**T** = tensor(double), tensor(float)
**T2** = tensor(int64)| -|Round|(*in* X:**T**, *out* Y:**T**)|11+|**T** = tensor(double), tensor(float), tensor(float16)| -|SVMClassifier|(*in* X:**T1**, *out* Y:**T2**, *out* Z:**tensor(float)**)|1+|**T1** = tensor(double), tensor(float), tensor(int32), tensor(int64)
**T2** = tensor(int64), tensor(string)| -|SVMRegressor|(*in* X:**T**, *out* Y:**tensor(float)**)|1+|**T** = tensor(float)| -|Scale|(*in* input:**T**, *out* output:**T**)|1+|**T** = tensor(float)| -|ScaledTanh|(*in* input:**T**, *out* output:**T**)|1+|**T** = tensor(float)| -|Scaler|(*in* X:**T**, *out* Y:**tensor(float)**)|1+|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)| -|Scan|(*in* initial_state_and_scan_inputs:**V**, *out* final_state_and_scan_outputs:**V**) or (*in* sequence_lens:**I**, *in* initial_state_and_scan_inputs:**V**, *out* final_state_and_scan_outputs:**V**)|11+|**I** = tensor(int64)
**V** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|ReverseSequence|*in* input:**T**
*in* sequence_lens:**tensor(int64)**
*out* Y:**T**|10+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|RoiAlign|*in* X:**T1**
*in* rois:**T1**
*in* batch_indices:**T2**
*out* Y:**T1**|10+|**T** = tensor(double), tensor(float)
**T2** = tensor(int64)| +|Round|*in* X:**T**
*out* Y:**T**|11+|**T** = tensor(double), tensor(float), tensor(float16)| +|Scale|*in* input:**T**
*out* output:**T**|1+|**T** = tensor(float)| +|ScaledTanh|*in* input:**T**
*out* output:**T**|1+|**T** = tensor(float)| +|Scan|*in* initial_state_and_scan_inputs:**V**
*out* final_state_and_scan_outputs:**V**

or

*in* sequence_lens:**I**
*in* initial_state_and_scan_inputs:**V**
*out* final_state_and_scan_outputs:**V**|11+|**I** = tensor(int64)
**V** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[9, 10]|**I** = tensor(int64)
**V** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||8|**I** = tensor(int64)
**V** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| -|Scatter|(*in* data:**T**, *in* indices:**Tind**, *in* updates:**T**, *out* output:**T**)|[9, 10]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**Tind** = tensor(int32), tensor(int64)| -|ScatterElements|(*in* data:**T**, *in* indices:**Tind**, *in* updates:**T**, *out* output:**T**)|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**Tind** = tensor(int32), tensor(int64)| +|Scatter|*in* data:**T**
*in* indices:**Tind**
*in* updates:**T**
*out* output:**T**|[9, 10]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**Tind** = tensor(int32), tensor(int64)| +|ScatterElements|*in* data:**T**
*in* indices:**Tind**
*in* updates:**T**
*out* output:**T**|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**Tind** = tensor(int32), tensor(int64)| |||[11, 12]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**Tind** = tensor(int32), tensor(int64)| -|ScatterND|(*in* data:**T**, *in* indices:**tensor(int64)**, *in* updates:**T**, *out* output:**T**)|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|ScatterND|*in* data:**T**
*in* indices:**tensor(int64)**
*in* updates:**T**
*out* output:**T**|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[11, 12]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| -|Selu|(*in* X:**T**, *out* Y:**T**)|6+|**T** = tensor(float)| -|SequenceAt|(*in* input_sequence:**S**, *in* position:**I**, *out* tensor:**T**)|11+|**I** = tensor(int32), tensor(int64)
**S** = seq(tensor(bfloat16)), seq(tensor(bool)), seq(tensor(double)), seq(tensor(float)), seq(tensor(float16)), seq(tensor(int16)), seq(tensor(int32)), seq(tensor(int64)), seq(tensor(int8)), seq(tensor(string)), seq(tensor(uint16)), seq(tensor(uint32)), seq(tensor(uint64)), seq(tensor(uint8))
**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| -|SequenceConstruct|(*in* inputs:**T**, *out* output_sequence:**S**)|11+|**S** = seq(tensor(bfloat16)), seq(tensor(bool)), seq(tensor(double)), seq(tensor(float)), seq(tensor(float16)), seq(tensor(int16)), seq(tensor(int32)), seq(tensor(int64)), seq(tensor(int8)), seq(tensor(string)), seq(tensor(uint16)), seq(tensor(uint32)), seq(tensor(uint64)), seq(tensor(uint8))
**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| -|SequenceEmpty|(*out* output:**S**)|11+|**S** = seq(tensor(bfloat16)), seq(tensor(bool)), seq(tensor(double)), seq(tensor(float)), seq(tensor(float16)), seq(tensor(int16)), seq(tensor(int32)), seq(tensor(int64)), seq(tensor(int8)), seq(tensor(string)), seq(tensor(uint16)), seq(tensor(uint32)), seq(tensor(uint64)), seq(tensor(uint8))| -|SequenceErase|(*in* input_sequence:**S**, *in* position:**I**, *out* output_sequence:**S**)|11+|**I** = tensor(int32), tensor(int64)
**S** = seq(tensor(bfloat16)), seq(tensor(bool)), seq(tensor(double)), seq(tensor(float)), seq(tensor(float16)), seq(tensor(int16)), seq(tensor(int32)), seq(tensor(int64)), seq(tensor(int8)), seq(tensor(string)), seq(tensor(uint16)), seq(tensor(uint32)), seq(tensor(uint64)), seq(tensor(uint8))| -|SequenceInsert|(*in* input_sequence:**S**, *in* tensor:**T**, *in* position:**I**, *out* output_sequence:**S**)|11+|**I** = tensor(int32), tensor(int64)
**S** = seq(tensor(bfloat16)), seq(tensor(bool)), seq(tensor(double)), seq(tensor(float)), seq(tensor(float16)), seq(tensor(int16)), seq(tensor(int32)), seq(tensor(int64)), seq(tensor(int8)), seq(tensor(string)), seq(tensor(uint16)), seq(tensor(uint32)), seq(tensor(uint64)), seq(tensor(uint8))| -|SequenceLength|(*in* input_sequence:**S**, *out* length:**I**)|11+|**I** = tensor(int64)
**S** = seq(tensor(bfloat16)), seq(tensor(bool)), seq(tensor(double)), seq(tensor(float)), seq(tensor(float16)), seq(tensor(int16)), seq(tensor(int32)), seq(tensor(int64)), seq(tensor(int8)), seq(tensor(string)), seq(tensor(uint16)), seq(tensor(uint32)), seq(tensor(uint64)), seq(tensor(uint8))| -|Shape|(*in* data:**T**, *out* shape:**T1**)|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**T1** = tensor(int64)| +|Selu|*in* X:**T**
*out* Y:**T**|6+|**T** = tensor(float)| +|SequenceAt|*in* input_sequence:**S**
*in* position:**I**
*out* tensor:**T**|11+|**I** = tensor(int32), tensor(int64)
**S** = seq(tensor(bfloat16)), seq(tensor(bool)), seq(tensor(double)), seq(tensor(float)), seq(tensor(float16)), seq(tensor(int16)), seq(tensor(int32)), seq(tensor(int64)), seq(tensor(int8)), seq(tensor(string)), seq(tensor(uint16)), seq(tensor(uint32)), seq(tensor(uint64)), seq(tensor(uint8))
**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|SequenceConstruct|*in* inputs:**T**
*out* output_sequence:**S**|11+|**S** = seq(tensor(bfloat16)), seq(tensor(bool)), seq(tensor(double)), seq(tensor(float)), seq(tensor(float16)), seq(tensor(int16)), seq(tensor(int32)), seq(tensor(int64)), seq(tensor(int8)), seq(tensor(string)), seq(tensor(uint16)), seq(tensor(uint32)), seq(tensor(uint64)), seq(tensor(uint8))
**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|SequenceEmpty|*out* output:**S**|11+|**S** = seq(tensor(bfloat16)), seq(tensor(bool)), seq(tensor(double)), seq(tensor(float)), seq(tensor(float16)), seq(tensor(int16)), seq(tensor(int32)), seq(tensor(int64)), seq(tensor(int8)), seq(tensor(string)), seq(tensor(uint16)), seq(tensor(uint32)), seq(tensor(uint64)), seq(tensor(uint8))| +|SequenceErase|*in* input_sequence:**S**
*in* position:**I**
*out* output_sequence:**S**|11+|**I** = tensor(int32), tensor(int64)
**S** = seq(tensor(bfloat16)), seq(tensor(bool)), seq(tensor(double)), seq(tensor(float)), seq(tensor(float16)), seq(tensor(int16)), seq(tensor(int32)), seq(tensor(int64)), seq(tensor(int8)), seq(tensor(string)), seq(tensor(uint16)), seq(tensor(uint32)), seq(tensor(uint64)), seq(tensor(uint8))| +|SequenceInsert|*in* input_sequence:**S**
*in* tensor:**T**
*in* position:**I**
*out* output_sequence:**S**|11+|**I** = tensor(int32), tensor(int64)
**S** = seq(tensor(bfloat16)), seq(tensor(bool)), seq(tensor(double)), seq(tensor(float)), seq(tensor(float16)), seq(tensor(int16)), seq(tensor(int32)), seq(tensor(int64)), seq(tensor(int8)), seq(tensor(string)), seq(tensor(uint16)), seq(tensor(uint32)), seq(tensor(uint64)), seq(tensor(uint8))| +|SequenceLength|*in* input_sequence:**S**
*out* length:**I**|11+|**I** = tensor(int64)
**S** = seq(tensor(bfloat16)), seq(tensor(bool)), seq(tensor(double)), seq(tensor(float)), seq(tensor(float16)), seq(tensor(int16)), seq(tensor(int32)), seq(tensor(int64)), seq(tensor(int8)), seq(tensor(string)), seq(tensor(uint16)), seq(tensor(uint32)), seq(tensor(uint64)), seq(tensor(uint8))| +|Shape|*in* data:**T**
*out* shape:**T1**|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**T1** = tensor(int64)| |||[1, 12]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**T1** = tensor(int64)| -|Shrink|(*in* input:**T**, *out* output:**T**)|9+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| -|Sigmoid|(*in* X:**T**, *out* Y:**T**)|13+|**T** = tensor(double), tensor(float)| +|Shrink|*in* input:**T**
*out* output:**T**|9+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Sigmoid|*in* X:**T**
*out* Y:**T**|13+|**T** = tensor(double), tensor(float)| |||[6, 12]|**T** = tensor(double), tensor(float)| -|Sign|(*in* input:**T**, *out* output:**T**)|13+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Sign|*in* input:**T**
*out* output:**T**|13+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[9, 12]|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| -|SimplifiedLayerNormalization|(*in* X:**T**, *in* scale:**T**, *out* Y:**T**, *out* inv_std_var:**U**)|1+|**T** = tensor(double), tensor(float)| -|Sin|(*in* input:**T**, *out* output:**T**)|7+|**T** = tensor(double), tensor(float)| -|Sinh|(*in* input:**T**, *out* output:**T**)|9+|**T** = tensor(float)| -|Size|(*in* data:**T**, *out* size:**T1**)|13+|**T** = tensor(bool), tensor(double), tensor(float), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**T1** = tensor(int64)| +|SimplifiedLayerNormalization|*in* X:**T**
*in* scale:**T**
*out* Y:**T**
*out* inv_std_var:**U**|1+|**T** = tensor(double), tensor(float)| +|Sin|*in* input:**T**
*out* output:**T**|7+|**T** = tensor(double), tensor(float)| +|Sinh|*in* input:**T**
*out* output:**T**|9+|**T** = tensor(float)| +|Size|*in* data:**T**
*out* size:**T1**|13+|**T** = tensor(bool), tensor(double), tensor(float), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**T1** = tensor(int64)| |||[1, 12]|**T** = tensor(bool), tensor(double), tensor(float), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**T1** = tensor(int64)| -|Slice|(*in* data:**T**, *in* starts:**Tind**, *in* ends:**Tind**, *in* axes:**Tind**, *in* steps:**Tind**, *out* output:**T**) or (*in* data:**T**, *out* output:**T**)|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**Tind** = tensor(int32), tensor(int64)| +|Slice|*in* data:**T**
*in* starts:**Tind**
*in* ends:**Tind**
*in* axes:**Tind**
*in* steps:**Tind**
*out* output:**T**

or

*in* data:**T**
*out* output:**T**|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**Tind** = tensor(int32), tensor(int64)| |||[11, 12]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**Tind** = tensor(int32), tensor(int64)| |||10|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**Tind** = tensor(int32), tensor(int64)| |||[1, 9]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| -|Softmax|(*in* input:**T**, *out* output:**T**)|13+|**T** = tensor(double), tensor(float)| +|Softmax|*in* input:**T**
*out* output:**T**|13+|**T** = tensor(double), tensor(float)| |||[11, 12]|**T** = tensor(double), tensor(float)| |||[1, 10]|**T** = tensor(double), tensor(float)| -|Softplus|(*in* X:**T**, *out* Y:**T**)|1+|**T** = tensor(float)| -|Softsign|(*in* input:**T**, *out* output:**T**)|1+|**T** = tensor(float)| -|SpaceToDepth|(*in* input:**T**, *out* output:**T**)|13+|**T** = tensor(float)| +|Softplus|*in* X:**T**
*out* Y:**T**|1+|**T** = tensor(float)| +|Softsign|*in* input:**T**
*out* output:**T**|1+|**T** = tensor(float)| +|SpaceToDepth|*in* input:**T**
*out* output:**T**|13+|**T** = tensor(float)| |||[1, 12]|**T** = tensor(float)| -|Split|(*in* input:**T**, *in* split:**T**, *out* outputs...:**T**) or (*in* input:**T**, *in* split:**tensor(int64)**, *out* outputs:**T**) or (*in* input:**T**, *out* outputs:**T**)|13+|**T** = tensor(float), tensor(int32), tensor(int64), tensor(string), tensor(uint8)| +|Split|*in* input:**T**
*in* split:**T**
*out* outputs...:**T**

or

*in* input:**T**
*in* split:**tensor(int64)**
*out* outputs:**T**

or

*in* input:**T**
*out* outputs:**T**|13+|**T** = tensor(float), tensor(int32), tensor(int64), tensor(string), tensor(uint8)| |||[11, 12]|**T** = tensor(float), tensor(int32), tensor(int64), tensor(string), tensor(uint8)| |||[2, 10]|**T** = tensor(float), tensor(int32), tensor(int64), tensor(string), tensor(uint8)| -|SplitToSequence|(*in* input:**T**, *in* split:**I**, *out* output_sequence:**S**)|11+|**I** = tensor(int32), tensor(int64)
**S** = seq(tensor(bfloat16)), seq(tensor(bool)), seq(tensor(double)), seq(tensor(float)), seq(tensor(float16)), seq(tensor(int16)), seq(tensor(int32)), seq(tensor(int64)), seq(tensor(int8)), seq(tensor(string)), seq(tensor(uint16)), seq(tensor(uint32)), seq(tensor(uint64)), seq(tensor(uint8))
**T** = tensor(double), tensor(float), tensor(int32), tensor(int64), tensor(string)| -|Sqrt|(*in* X:**T**, *out* Y:**T**)|13+|**T** = tensor(double), tensor(float)| +|SplitToSequence|*in* input:**T**
*in* split:**I**
*out* output_sequence:**S**|11+|**I** = tensor(int32), tensor(int64)
**S** = seq(tensor(bfloat16)), seq(tensor(bool)), seq(tensor(double)), seq(tensor(float)), seq(tensor(float16)), seq(tensor(int16)), seq(tensor(int32)), seq(tensor(int64)), seq(tensor(int8)), seq(tensor(string)), seq(tensor(uint16)), seq(tensor(uint32)), seq(tensor(uint64)), seq(tensor(uint8))
**T** = tensor(double), tensor(float), tensor(int32), tensor(int64), tensor(string)| +|Sqrt|*in* X:**T**
*out* Y:**T**|13+|**T** = tensor(double), tensor(float)| |||[6, 12]|**T** = tensor(double), tensor(float)| -|Squeeze|(*in* data:**T**, *in* axes:**tensor(int64)**, *out* squeezed:**T**) or (*in* data:**T**, *out* squeezed:**T**)|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Squeeze|*in* data:**T**
*in* axes:**tensor(int64)**
*out* squeezed:**T**

or

*in* data:**T**
*out* squeezed:**T**|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[11, 12]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[1, 10]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| -|StringNormalizer|(*in* X:**tensor(string)**, *out* Y:**tensor(string)**)|10+|**T** = tensor(string)| -|Sub|(*in* A:**T**, *in* B:**T**, *out* C:**T**)|14+|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)| +|StringNormalizer|*in* X:**tensor(string)**
*out* Y:**tensor(string)**|10+|**T** = tensor(string)| +|Sub|*in* A:**T**
*in* B:**T**
*out* C:**T**|14+|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)| |||13|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)| |||[7, 12]|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)| -|Sum|(*in* data_0:**T**, *out* sum:**T**)|13+|**T** = tensor(double), tensor(float)| +|Sum|*in* data_0:**T**
*out* sum:**T**|13+|**T** = tensor(double), tensor(float)| |||[8, 12]|**T** = tensor(double), tensor(float)| |||[6, 7]|**T** = tensor(double), tensor(float)| -|Tan|(*in* input:**T**, *out* output:**T**)|7+|**T** = tensor(float)| -|Tanh|(*in* input:**T**, *out* output:**T**)|13+|**T** = tensor(double), tensor(float)| +|Tan|*in* input:**T**
*out* output:**T**|7+|**T** = tensor(float)| +|Tanh|*in* input:**T**
*out* output:**T**|13+|**T** = tensor(double), tensor(float)| |||[6, 12]|**T** = tensor(double), tensor(float)| -|TfIdfVectorizer|(*in* X:**T**, *out* Y:**T1**)|9+|**T** = tensor(int32), tensor(int64), tensor(string)
**T1** = tensor(float)| -|ThresholdedRelu|(*in* X:**T**, *out* Y:**T**)|10+|**T** = tensor(float)| +|TfIdfVectorizer|*in* X:**T**
*out* Y:**T1**|9+|**T** = tensor(int32), tensor(int64), tensor(string)
**T1** = tensor(float)| +|ThresholdedRelu|*in* X:**T**
*out* Y:**T**|10+|**T** = tensor(float)| |||[1, 9]|**T** = tensor(float)| -|Tile|(*in* input:**T**, *in* repeats:**T1**, *out* output:**T**) or (*in* input:**T**, *in* tiles:**T**, *in* axis:**T**, *out* output:**T**)|13+|**T** = tensor(bool), tensor(double), tensor(float), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**T1** = tensor(int64)| +|Tile|*in* input:**T**
*in* repeats:**T1**
*out* output:**T**

or

*in* input:**T**
*in* tiles:**T**
*in* axis:**T**
*out* output:**T**|13+|**T** = tensor(bool), tensor(double), tensor(float), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**T1** = tensor(int64)| |||[6, 12]|**T** = tensor(bool), tensor(double), tensor(float), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**T1** = tensor(int64)| -|TopK|(*in* X:**T**, *in* K:**tensor(int64)**, *out* Values:**T**, *out* Indices:**I**) or (*in* X:**T**, *out* Values:**T**, *out* Indices:**I**)|11+|**I** = tensor(int64)
**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)| +|TopK|*in* X:**T**
*in* K:**tensor(int64)**
*out* Values:**T**
*out* Indices:**I**

or

*in* X:**T**
*out* Values:**T**
*out* Indices:**I**|11+|**I** = tensor(int64)
**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)| |||10|**I** = tensor(int64)
**T** = tensor(double), tensor(float)| |||[1, 9]|**I** = tensor(int64)
**T** = tensor(double), tensor(float)| -|Transpose|(*in* data:**T**, *out* transposed:**T**)|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Transpose|*in* data:**T**
*out* transposed:**T**|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[1, 12]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| -|TreeEnsembleClassifier|(*in* X:**T1**, *out* Y:**T2**, *out* Z:**tensor(float)**)|1+|**T1** = tensor(double), tensor(float), tensor(int32), tensor(int64)
**T2** = tensor(int64), tensor(string)| -|TreeEnsembleRegressor|(*in* X:**T**, *out* Y:**tensor(float)**)|1+|**T** = tensor(double), tensor(float)| -|Trilu|(*in* input:**T**, *in* k:**tensor(int64)**, *out* output:**T**)|14+|**T** = tensor(double), tensor(float), tensor(int64)| -|Unique|(*in* X:**T**, *out* Y:**T**, *out* indices:**tensor(int64)**, *out* inverse_indices:**tensor(int64)**, *out* counts:**tensor(int64)**)|11+|**T** = tensor(float), tensor(int64), tensor(int8), tensor(string)| -|Unsqueeze|(*in* data:**T**, *in* axes:**tensor(int64)**, *out* expanded:**T**) or (*in* data:**T**, *out* expanded:**T**)|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Trilu|*in* input:**T**
*in* k:**tensor(int64)**
*out* output:**T**|14+|**T** = tensor(double), tensor(float), tensor(int64)| +|Unique|*in* X:**T**
*out* Y:**T**
*out* indices:**tensor(int64)**
*out* inverse_indices:**tensor(int64)**
*out* counts:**tensor(int64)**|11+|**T** = tensor(float), tensor(int64), tensor(int8), tensor(string)| +|Unsqueeze|*in* data:**T**
*in* axes:**tensor(int64)**
*out* expanded:**T**

or

*in* data:**T**
*out* expanded:**T**|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[11, 12]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[1, 10]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| -|Upsample|(*in* X:**T**, *in* scales:**tensor(float)**, *out* Y:**T**) or (*in* X:**T**, *out* Y:**T**)|9|**T** = tensor(float), tensor(int32), tensor(uint8)| +|Upsample|*in* X:**T**
*in* scales:**tensor(float)**
*out* Y:**T**

or

*in* X:**T**
*out* Y:**T**|9|**T** = tensor(float), tensor(int32), tensor(uint8)| |||[7, 8]|**T** = tensor(float), tensor(int32), tensor(uint8)| -|Where|(*in* condition:**B**, *in* X:**T**, *in* Y:**T**, *out* output:**T**)|9+|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64), tensor(string), tensor(uint8)| -|Xor|(*in* A:**T**, *in* B:**T**, *out* C:**T1**)|7+|**T** = tensor(bool)
**T1** = tensor(bool)| -|ZipMap|(*in* X:**tensor(float)**, *out* Z:**T**)|1+|**T** = seq(map(int64,tensor(float))), seq(map(string,tensor(float)))| +|Where|*in* condition:**B**
*in* X:**T**
*in* Y:**T**
*out* output:**T**|9+|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64), tensor(string), tensor(uint8)| +|Xor|*in* A:**T**
*in* B:**T**
*out* C:**T1**|7+|**T** = tensor(bool)
**T1** = tensor(bool)| +| | +| | +|**Operator Domain:** *ai.onnx.ml*|||| +|ArrayFeatureExtractor|*in* X:**T**
*in* Y:**tensor(int64)**
*out* Z:**T**|1+|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64), tensor(string)| +|Binarizer|*in* X:**T**
*out* Y:**T**|1+|**T** = tensor(float)| +|CastMap|*in* X:**T1**
*out* Y:**T2**|1+|**T1** = map(int64,tensor(float)), map(int64,tensor(string))
**T2** = tensor(float), tensor(int64), tensor(string)| +|CategoryMapper|*in* X:**T1**
*out* Y:**T2**|1+|**T1** = tensor(int64), tensor(string)
**T2** = tensor(int64), tensor(string)| +|DictVectorizer|*in* X:**T1**
*out* Y:**T2**|1+|**T1** = map(int64,tensor(double)), map(int64,tensor(float)), map(int64,tensor(string)), map(string,tensor(double)), map(string,tensor(float)), map(string,tensor(int64))
**T2** = tensor(double), tensor(float), tensor(int64), tensor(string)| +|FeatureVectorizer|*in* X:**T1**
*out* Y:**tensor(float)**|1+|**T1** = tensor(double), tensor(float), tensor(int32), tensor(int64)| +|Imputer|*in* X:**T**
*out* Y:**T**|1+|**T** = tensor(float), tensor(int64)| +|LabelEncoder|*in* X:**T1**
*out* Y:**T2**|2+|**T1** = tensor(float), tensor(int64), tensor(string)
**T2** = tensor(float), tensor(int64), tensor(string)| +|||1|**T1** = tensor(int64), tensor(string)
**T2** = tensor(int64), tensor(string)| +|LinearClassifier|*in* X:**T1**
*out* Y:**T2**
*out* Z:**tensor(float)**|1+|**T1** = tensor(double), tensor(float), tensor(int32), tensor(int64)
**T2** = tensor(int64), tensor(string)| +|LinearRegressor|*in* X:**T**
*out* Y:**tensor(float)**|1+|**T** = tensor(float)| +|Normalizer|*in* X:**T**
*out* Y:**tensor(float)**|1+|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)| +|OneHotEncoder|*in* X:**T**
*out* Y:**tensor(float)**|1+|**T** = tensor(double), tensor(float), tensor(int64), tensor(string)| +|SVMClassifier|*in* X:**T1**
*out* Y:**T2**
*out* Z:**tensor(float)**|1+|**T1** = tensor(double), tensor(float), tensor(int32), tensor(int64)
**T2** = tensor(int64), tensor(string)| +|SVMRegressor|*in* X:**T**
*out* Y:**tensor(float)**|1+|**T** = tensor(float)| +|Scaler|*in* X:**T**
*out* Y:**tensor(float)**|1+|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64)| +|TreeEnsembleClassifier|*in* X:**T1**
*out* Y:**T2**
*out* Z:**tensor(float)**|1+|**T1** = tensor(double), tensor(float), tensor(int32), tensor(int64)
**T2** = tensor(int64), tensor(string)| +|TreeEnsembleRegressor|*in* X:**T**
*out* Y:**tensor(float)**|1+|**T** = tensor(double), tensor(float)| +|ZipMap|*in* X:**tensor(float)**
*out* Z:**T**|1+|**T** = seq(map(int64,tensor(float))), seq(map(string,tensor(float)))| | | | | |**Operator Domain:** *com.microsoft*|||| -|Attention|(*in* input:**T**, *in* weight:**T**, *in* bias:**T**, *in* mask_index:**M**, *in* past:**T**, *out* output:**T**, *out* present:**T**)|1+|**T** = tensor(float)| -|AttnLSTM|(*in* X:**T**, *in* W:**T**, *in* R:**T**, *in* B:**T**, *in* sequence_lens:**T1**, *in* initial_h:**T**, *in* initial_c:**T**, *in* P:**T**, *in* QW:**T**, *in* MW:**T**, *in* V:**T**, *in* M:**T**, *in* memory_seq_lens:**T1**, *in* AW:**T**, *out* Y:**T**, *out* Y_h:**T**, *out* Y_c:**T**)|1+|**T** = tensor(double), tensor(float)
**T1** = tensor(int32)| -|BiasGelu|(*in* A:**T**, *in* B:**T**, *out* C:**T**)|1+|**T** = tensor(float)| -|CDist|(*in* A:**T**, *in* B:**T**, *out* C:**T**)|1+|**T** = tensor(double), tensor(float)| -|ConvTransposeWithDynamicPads|(*in* X:**T**, *in* W:**T**, *in* Pads:**tensor(int64)**, *in* B:**T**, *out* Y:**T**)|1+|**T** = tensor(float)| -|CropAndResize|(*in* X:**T1**, *in* rois:**T1**, *in* batch_indices:**T2**, *in* crop_size:**T2**, *out* Y:**T1**)|1+|**T** = tensor(float)
**T2** = tensor(int32)| -|DequantizeLinear|(*in* x:**T1**, *in* x_scale:**T2**, *in* x_zero_point:**T1**, *out* y:**T2**)|1+|**T1** = tensor(int8), tensor(uint8)
**T2** = tensor(float)| -|DynamicQuantizeLSTM|(*in* X:**T**, *in* W:**T2**, *in* R:**T2**, *in* B:**T**, *in* sequence_lens:**T1**, *in* initial_h:**T**, *in* initial_c:**T**, *in* P:**T**, *in* W_scale:**T**, *in* W_zero_point:**T2**, *in* R_scale:**T**, *in* R_zero_point:**T2**, *out* Y:**T**, *out* Y_h:**T**, *out* Y_c:**T**)|1+|**T** = tensor(float)
**T1** = tensor(int32)
**T2** = tensor(int8), tensor(uint8)| -|DynamicQuantizeMatMul|(*in* A:**T1**, *in* B:**T2**, *in* b_scale:**T1**, *in* b_zero_point:**T2**, *in* bias:**T1**, *out* Y:**T1**)|1+|**T1** = tensor(float)
**T2** = tensor(int8), tensor(uint8)| -|EmbedLayerNormalization|(*in* input_ids:**T1**, *in* segment_ids:**T1**, *in* word_embedding:**T**, *in* position_embedding:**T**, *in* segment_embedding:**T**, *in* gamma:**T**, *in* beta:**T**, *in* mask:**T1**, *out* output:**T**, *out* mask_index:**T1**)|1+|**T** = tensor(float)| -|ExpandDims|(*in* X:**T**, *in* axis:**tensor(int32)**, *out* Y:**T**)|1+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**axis** = tensor(int32)| -|FastGelu|(*in* X:**T**, *in* bias:**T**, *out* Y:**T**)|1+|**T** = tensor(float)| -|FusedConv|(*in* X:**T**, *in* W:**T**, *in* B:**T**, *in* Z:**T**, *out* Y:**T**)|1+|**T** = tensor(float)| -|FusedGemm|(*in* A:**T**, *in* B:**T**, *in* C:**T**, *out* Y:**T**)|1+|**T** = tensor(float)| -|FusedMatMul|(*in* A:**T**, *in* B:**T**, *out* Y:**T**)|1+|**T** = tensor(float)| -|GatherND|(*in* data:**T**, *in* indices:**Tind**, *out* output:**T**)|1+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**Tind** = tensor(int32), tensor(int64)| -|Gelu|(*in* X:**T**, *out* Y:**T**)|1+|**T** = tensor(float)| -|Inverse|(*in* X:**T**, *out* Y:**T**)|1+|**T** = tensor(double), tensor(float), tensor(float16)| -|MatMulInteger16|(*in* A:**T1**, *in* B:**T2**, *out* Y:**T3**)|1+|**T1** = tensor(int16)
**T2** = tensor(int16)
**T3** = tensor(int32)| -|MatMulIntegerToFloat|(*in* A:**T1**, *in* B:**T2**, *in* a_scale:**T3**, *in* b_scale:**T3**, *in* a_zero_point:**T1**, *in* b_zero_point:**T2**, *in* bias:**T3**, *out* Y:**T3**)|1+|**T1** = tensor(uint8)
**T2** = tensor(int8), tensor(uint8)
**T3** = tensor(float)| -|MaxpoolWithMask|(*in* X:**T**, *in* M:**tensor(int32)**, *out* Y:**T**)|1+|**X** = tensor(float)| -|MurmurHash3|(*in* X:**T1**, *out* Y:**T2**)|1+|**T1** = tensor(double), tensor(float), tensor(int32), tensor(int64), tensor(string), tensor(uint32), tensor(uint64)
**T2** = tensor(int32), tensor(uint32)| -|NhwcMaxPool|(*in* x:**T**, *out* y:**T**)|1+|**T** = tensor(uint8)| -|Pad|(*in* data:**T**, *in* pads:**tensor(int64)**, *in* value:**T**, *out* output:**T**)|1+|**T** = tensor(float)| -|QAttention|(*in* input:**T1**, *in* weight:**T2**, *in* bias:**T3**, *in* input_scale:**T3**, *in* weight_scale:**T3**, *in* mask_index:**T4**, *in* input_zero_point:**T1**, *in* weight_zero_point:**T2**, *in* past:**T3**, *out* output:**T3**, *out* present:**T3**)|1+|**T1** = tensor(uint8)
**T2** = tensor(int8), tensor(uint8)
**T3** = tensor(float)
**T4** = tensor(int32)| -|QLinearAdd|(*in* A:**T**, *in* A_scale:**tensor(float)**, *in* A_zero_point:**T**, *in* B:**T**, *in* B_scale:**tensor(float)**, *in* B_zero_point:**T**, *in* C_scale:**tensor(float)**, *in* C_zero_point:**T**, *out* C:**T**)|1+|**T** = tensor(int8), tensor(uint8)| -|QLinearConv|(*in* x:**T1**, *in* x_scale:**tensor(float)**, *in* x_zero_point:**T1**, *in* w:**T2**, *in* w_scale:**tensor(float)**, *in* w_zero_point:**T2**, *in* y_scale:**tensor(float)**, *in* y_zero_point:**T3**, *in* B:**T4**, *out* y:**T3**)|1+|**T1** = tensor(uint8)
**T2** = tensor(int8), tensor(uint8)
**T3** = tensor(uint8)
**T4** = tensor(int32)| -|QLinearLeakyRelu|(*in* X:**T**, *in* X_scale:**tensor(float)**, *in* X_zero_point:**T**, *in* Y_scale:**tensor(float)**, *in* Y_zero_point:**T**, *out* Y:**T**)|1+|**T** = tensor(int8), tensor(uint8)| -|QLinearMul|(*in* A:**T**, *in* A_scale:**tensor(float)**, *in* A_zero_point:**T**, *in* B:**T**, *in* B_scale:**tensor(float)**, *in* B_zero_point:**T**, *in* C_scale:**tensor(float)**, *in* C_zero_point:**T**, *out* C:**T**)|1+|**T** = tensor(int8), tensor(uint8)| -|QLinearSigmoid|(*in* X:**T**, *in* X_scale:**tensor(float)**, *in* X_zero_point:**T**, *in* Y_scale:**tensor(float)**, *in* Y_zero_point:**T**, *out* Y:**T**)|1+|**T** = tensor(int8), tensor(uint8)| -|QuantizeLinear|(*in* x:**T1**, *in* y_scale:**T1**, *in* y_zero_point:**T2**, *out* y:**T2**)|1+|**T1** = tensor(float)
**T2** = tensor(int8), tensor(uint8)| -|Range|(*in* start:**T**, *in* limit:**T**, *in* delta:**T**, *out* Y:**T**)|1+|**T** = tensor(double), tensor(float), tensor(int16), tensor(int32), tensor(int64)| -|SampleOp|(*in* X:**T**, *out* Y:**T**)|1+|**T** = tensor(float)| -|SkipLayerNormalization|(*in* input:**T**, *in* skip:**T**, *in* gamma:**T**, *in* beta:**T**, *in* bias:**T**, *out* output:**T**, *out* mean:**U**, *out* inv_std_var:**U**)|1+|**T** = tensor(double), tensor(float)| -|Tokenizer|(*in* X:**T**, *out* Y:**T**)|1+|**T** = tensor(string)| -|TransposeMatMul|(*in* A:**T**, *in* B:**T**, *out* Y:**T**)|1+|**T** = tensor(float)| -|Trilu|(*in* X:**T**, *in* k:**tensor(int64)**, *out* Y:**T**)|1+|**T** = tensor(double), tensor(float), tensor(int64)| -|Unique|(*in* x:**T**, *out* y:**T**, *out* idx:**tensor(int64)**, *out* counts:**tensor(int64)**)|1+|**T** = tensor(float)| -|WordConvEmbedding|(*in* Sequence:**T**, *in* W:**T1**, *in* B:**T1**, *in* C:**T1**, *out* Y:**T1**)|1+|**T** = tensor(int32)
**T1** = tensor(float)| +|Attention|*in* input:**T**
*in* weight:**T**
*in* bias:**T**
*in* mask_index:**M**
*in* past:**T**
*out* output:**T**
*out* present:**T**|1+|**T** = tensor(float)| +|AttnLSTM|*in* X:**T**
*in* W:**T**
*in* R:**T**
*in* B:**T**
*in* sequence_lens:**T1**
*in* initial_h:**T**
*in* initial_c:**T**
*in* P:**T**
*in* QW:**T**
*in* MW:**T**
*in* V:**T**
*in* M:**T**
*in* memory_seq_lens:**T1**
*in* AW:**T**
*out* Y:**T**
*out* Y_h:**T**
*out* Y_c:**T**|1+|**T** = tensor(double), tensor(float)
**T1** = tensor(int32)| +|BiasGelu|*in* A:**T**
*in* B:**T**
*out* C:**T**|1+|**T** = tensor(float)| +|CDist|*in* A:**T**
*in* B:**T**
*out* C:**T**|1+|**T** = tensor(double), tensor(float)| +|ConvTransposeWithDynamicPads|*in* X:**T**
*in* W:**T**
*in* Pads:**tensor(int64)**
*in* B:**T**
*out* Y:**T**|1+|**T** = tensor(float)| +|CropAndResize|*in* X:**T1**
*in* rois:**T1**
*in* batch_indices:**T2**
*in* crop_size:**T2**
*out* Y:**T1**|1+|**T** = tensor(float)
**T2** = tensor(int32)| +|DequantizeLinear|*in* x:**T1**
*in* x_scale:**T2**
*in* x_zero_point:**T1**
*out* y:**T2**|1+|**T1** = tensor(int8), tensor(uint8)
**T2** = tensor(float)| +|DynamicQuantizeLSTM|*in* X:**T**
*in* W:**T2**
*in* R:**T2**
*in* B:**T**
*in* sequence_lens:**T1**
*in* initial_h:**T**
*in* initial_c:**T**
*in* P:**T**
*in* W_scale:**T**
*in* W_zero_point:**T2**
*in* R_scale:**T**
*in* R_zero_point:**T2**
*out* Y:**T**
*out* Y_h:**T**
*out* Y_c:**T**|1+|**T** = tensor(float)
**T1** = tensor(int32)
**T2** = tensor(int8), tensor(uint8)| +|DynamicQuantizeMatMul|*in* A:**T1**
*in* B:**T2**
*in* b_scale:**T1**
*in* b_zero_point:**T2**
*in* bias:**T1**
*out* Y:**T1**|1+|**T1** = tensor(float)
**T2** = tensor(int8), tensor(uint8)| +|EmbedLayerNormalization|*in* input_ids:**T1**
*in* segment_ids:**T1**
*in* word_embedding:**T**
*in* position_embedding:**T**
*in* segment_embedding:**T**
*in* gamma:**T**
*in* beta:**T**
*in* mask:**T1**
*out* output:**T**
*out* mask_index:**T1**|1+|**T** = tensor(float)| +|ExpandDims|*in* X:**T**
*in* axis:**tensor(int32)**
*out* Y:**T**|1+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**axis** = tensor(int32)| +|FastGelu|*in* X:**T**
*in* bias:**T**
*out* Y:**T**|1+|**T** = tensor(float)| +|FusedConv|*in* X:**T**
*in* W:**T**
*in* B:**T**
*in* Z:**T**
*out* Y:**T**|1+|**T** = tensor(float)| +|FusedGemm|*in* A:**T**
*in* B:**T**
*in* C:**T**
*out* Y:**T**|1+|**T** = tensor(float)| +|FusedMatMul|*in* A:**T**
*in* B:**T**
*out* Y:**T**|1+|**T** = tensor(float)| +|GatherND|*in* data:**T**
*in* indices:**Tind**
*out* output:**T**|1+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**Tind** = tensor(int32), tensor(int64)| +|Gelu|*in* X:**T**
*out* Y:**T**|1+|**T** = tensor(float)| +|Inverse|*in* X:**T**
*out* Y:**T**|1+|**T** = tensor(double), tensor(float), tensor(float16)| +|MatMulInteger16|*in* A:**T1**
*in* B:**T2**
*out* Y:**T3**|1+|**T1** = tensor(int16)
**T2** = tensor(int16)
**T3** = tensor(int32)| +|MatMulIntegerToFloat|*in* A:**T1**
*in* B:**T2**
*in* a_scale:**T3**
*in* b_scale:**T3**
*in* a_zero_point:**T1**
*in* b_zero_point:**T2**
*in* bias:**T3**
*out* Y:**T3**|1+|**T1** = tensor(uint8)
**T2** = tensor(int8), tensor(uint8)
**T3** = tensor(float)| +|MaxpoolWithMask|*in* X:**T**
*in* M:**tensor(int32)**
*out* Y:**T**|1+|**X** = tensor(float)| +|MurmurHash3|*in* X:**T1**
*out* Y:**T2**|1+|**T1** = tensor(double), tensor(float), tensor(int32), tensor(int64), tensor(string), tensor(uint32), tensor(uint64)
**T2** = tensor(int32), tensor(uint32)| +|NhwcMaxPool|*in* x:**T**
*out* y:**T**|1+|**T** = tensor(uint8)| +|Pad|*in* data:**T**
*in* pads:**tensor(int64)**
*in* value:**T**
*out* output:**T**|1+|**T** = tensor(float)| +|QAttention|*in* input:**T1**
*in* weight:**T2**
*in* bias:**T3**
*in* input_scale:**T3**
*in* weight_scale:**T3**
*in* mask_index:**T4**
*in* input_zero_point:**T1**
*in* weight_zero_point:**T2**
*in* past:**T3**
*out* output:**T3**
*out* present:**T3**|1+|**T1** = tensor(uint8)
**T2** = tensor(int8), tensor(uint8)
**T3** = tensor(float)
**T4** = tensor(int32)| +|QLinearAdd|*in* A:**T**
*in* A_scale:**tensor(float)**
*in* A_zero_point:**T**
*in* B:**T**
*in* B_scale:**tensor(float)**
*in* B_zero_point:**T**
*in* C_scale:**tensor(float)**
*in* C_zero_point:**T**
*out* C:**T**|1+|**T** = tensor(int8), tensor(uint8)| +|QLinearConv|*in* x:**T1**
*in* x_scale:**tensor(float)**
*in* x_zero_point:**T1**
*in* w:**T2**
*in* w_scale:**tensor(float)**
*in* w_zero_point:**T2**
*in* y_scale:**tensor(float)**
*in* y_zero_point:**T3**
*in* B:**T4**
*out* y:**T3**|1+|**T1** = tensor(uint8)
**T2** = tensor(int8), tensor(uint8)
**T3** = tensor(uint8)
**T4** = tensor(int32)| +|QLinearLeakyRelu|*in* X:**T**
*in* X_scale:**tensor(float)**
*in* X_zero_point:**T**
*in* Y_scale:**tensor(float)**
*in* Y_zero_point:**T**
*out* Y:**T**|1+|**T** = tensor(int8), tensor(uint8)| +|QLinearMul|*in* A:**T**
*in* A_scale:**tensor(float)**
*in* A_zero_point:**T**
*in* B:**T**
*in* B_scale:**tensor(float)**
*in* B_zero_point:**T**
*in* C_scale:**tensor(float)**
*in* C_zero_point:**T**
*out* C:**T**|1+|**T** = tensor(int8), tensor(uint8)| +|QLinearSigmoid|*in* X:**T**
*in* X_scale:**tensor(float)**
*in* X_zero_point:**T**
*in* Y_scale:**tensor(float)**
*in* Y_zero_point:**T**
*out* Y:**T**|1+|**T** = tensor(int8), tensor(uint8)| +|QuantizeLinear|*in* x:**T1**
*in* y_scale:**T1**
*in* y_zero_point:**T2**
*out* y:**T2**|1+|**T1** = tensor(float)
**T2** = tensor(int8), tensor(uint8)| +|Range|*in* start:**T**
*in* limit:**T**
*in* delta:**T**
*out* Y:**T**|1+|**T** = tensor(double), tensor(float), tensor(int16), tensor(int32), tensor(int64)| +|SampleOp|*in* X:**T**
*out* Y:**T**|1+|**T** = tensor(float)| +|SkipLayerNormalization|*in* input:**T**
*in* skip:**T**
*in* gamma:**T**
*in* beta:**T**
*in* bias:**T**
*out* output:**T**
*out* mean:**U**
*out* inv_std_var:**U**|1+|**T** = tensor(double), tensor(float)| +|Tokenizer|*in* X:**T**
*out* Y:**T**|1+|**T** = tensor(string)| +|TransposeMatMul|*in* A:**T**
*in* B:**T**
*out* Y:**T**|1+|**T** = tensor(float)| +|Trilu|*in* X:**T**
*in* k:**tensor(int64)**
*out* Y:**T**|1+|**T** = tensor(double), tensor(float), tensor(int64)| +|Unique|*in* x:**T**
*out* y:**T**
*out* idx:**tensor(int64)**
*out* counts:**tensor(int64)**|1+|**T** = tensor(float)| +|WordConvEmbedding|*in* Sequence:**T**
*in* W:**T1**
*in* B:**T1**
*in* C:**T1**
*out* Y:**T1**|1+|**T** = tensor(int32)
**T1** = tensor(float)| | | | | |**Operator Domain:** *com.microsoft.nchwc*|||| -|AveragePool|(*in* X:**T**, *out* Y:**T**)|1+|**T** = tensor(float)| -|Conv|(*in* X:**T**, *in* W:**T**, *in* B:**T**, *in* Sum:**T**, *out* Y:**T**)|1+|**T** = tensor(float)| -|GlobalAveragePool|(*in* X:**T**, *out* Y:**T**)|1+|**T** = tensor(float)| -|GlobalMaxPool|(*in* X:**T**, *out* Y:**T**)|1+|**T** = tensor(float)| -|MaxPool|(*in* X:**T**, *out* Y:**T**)|1+|**T** = tensor(float)| -|ReorderInput|(*in* X:**T**, *out* Y:**T**)|1+|**T** = tensor(float)| -|ReorderOutput|(*in* X:**T**, *out* Y:**T**)|1+|**T** = tensor(float)| -|Upsample|(*in* X:**T**, *out* Y:**T**)|1+|**T** = tensor(float)| +|AveragePool|*in* X:**T**
*out* Y:**T**|1+|**T** = tensor(float)| +|Conv|*in* X:**T**
*in* W:**T**
*in* B:**T**
*in* Sum:**T**
*out* Y:**T**|1+|**T** = tensor(float)| +|GlobalAveragePool|*in* X:**T**
*out* Y:**T**|1+|**T** = tensor(float)| +|GlobalMaxPool|*in* X:**T**
*out* Y:**T**|1+|**T** = tensor(float)| +|MaxPool|*in* X:**T**
*out* Y:**T**|1+|**T** = tensor(float)| +|ReorderInput|*in* X:**T**
*out* Y:**T**|1+|**T** = tensor(float)| +|ReorderOutput|*in* X:**T**
*out* Y:**T**|1+|**T** = tensor(float)| +|Upsample|*in* X:**T**
*out* Y:**T**|1+|**T** = tensor(float)| | | | | + + ## Operators implemented by CUDAExecutionProvider | Op Name | Parameters | OpSet Version | Types Supported | |---------|------------|---------------|-----------------| -|**Operator Domain:** *ai.onnx.ml*|||| -|Abs|(*in* X:**T**, *out* Y:**T**)|13+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|**Operator Domain:** *ai.onnx*|||| +|Abs|*in* X:**T**
*out* Y:**T**|13+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[6, 12]|**T** = tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| -|Add|(*in* A:**T**, *in* B:**T**, *out* C:**T**)|13+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| +|Add|*in* A:**T**
*in* B:**T**
*out* C:**T**|14+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| +|||13|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| |||[7, 12]|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| -|Affine|(*in* X:**T**, *out* Y:**T**)|1+|**T** = tensor(double), tensor(float), tensor(float16)| -|And|(*in* A:**T**, *in* B:**T**, *out* C:**T1**)|7+|**T** = tensor(bool)
**T1** = tensor(bool)| -|ArgMax|(*in* data:**T**, *out* reduced:**tensor(int64)**)|11+|**T** = tensor(double), tensor(float), tensor(float16)| +|Affine|*in* X:**T**
*out* Y:**T**|1+|**T** = tensor(double), tensor(float), tensor(float16)| +|And|*in* A:**T**
*in* B:**T**
*out* C:**T1**|7+|**T** = tensor(bool)
**T1** = tensor(bool)| +|ArgMax|*in* data:**T**
*out* reduced:**tensor(int64)**|11+|**T** = tensor(double), tensor(float), tensor(float16)| |||[1, 10]|**T** = tensor(double), tensor(float), tensor(float16)| -|ArgMin|(*in* data:**T**, *out* reduced:**tensor(int64)**)|11+|**T** = tensor(double), tensor(float), tensor(float16)| +|ArgMin|*in* data:**T**
*out* reduced:**tensor(int64)**|11+|**T** = tensor(double), tensor(float), tensor(float16)| |||[1, 10]|**T** = tensor(double), tensor(float), tensor(float16)| -|AveragePool|(*in* X:**T**, *out* Y:**T**)|11+|**T** = tensor(double), tensor(float), tensor(float16)| +|AveragePool|*in* X:**T**
*out* Y:**T**|11+|**T** = tensor(double), tensor(float), tensor(float16)| |||10|**I** = tensor(int64)
**T** = tensor(double), tensor(float), tensor(float16)| |||[7, 9]|**I** = tensor(int64)
**T** = tensor(double), tensor(float), tensor(float16)| -|BatchNormalization|(*in* X:**T**, *in* scale:**T**, *in* B:**T**, *in* input_mean:**U**, *in* input_var:**U**, *out* Y:**T**, *out* running_mean:**U**, *out* running_var:**U**) or (*in* X:**T**, *in* scale:**T**, *in* B:**T**, *in* mean:**T**, *in* var:**T**, *out* Y:**T**, *out* mean:**T**, *out* var:**T**, *out* saved_mean:**T**, *out* saved_var:**T**)|9+|**T** = tensor(double), tensor(float), tensor(float16)| +|BatchNormalization|*in* X:**T**
*in* scale:**T**
*in* B:**T**
*in* input_mean:**U**
*in* input_var:**U**
*out* Y:**T**
*out* running_mean:**U**
*out* running_var:**U**

or

*in* X:**T**
*in* scale:**T**
*in* B:**T**
*in* mean:**T**
*in* var:**T**
*out* Y:**T**
*out* mean:**T**
*out* var:**T**
*out* saved_mean:**T**
*out* saved_var:**T**|14+|**T** = tensor(double), tensor(float), tensor(float16)| +|||[9, 13]|**T** = tensor(double), tensor(float), tensor(float16)| |||[7, 8]|**T** = tensor(double), tensor(float), tensor(float16)| -|Cast|(*in* input:**T1**, *out* output:**T2**)|13+|**T1** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**T2** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Cast|*in* input:**T1**
*out* output:**T2**|13+|**T1** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**T2** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[9, 12]|**T1** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**T2** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[6, 8]|**T1** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**T2** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| -|Ceil|(*in* X:**T**, *out* Y:**T**)|13+|**T** = tensor(double), tensor(float), tensor(float16)| +|Ceil|*in* X:**T**
*out* Y:**T**|13+|**T** = tensor(double), tensor(float), tensor(float16)| |||[6, 12]|**T** = tensor(double), tensor(float), tensor(float16)| -|Clip|(*in* input:**T**, *in* min:**T**, *in* max:**T**, *out* output:**T**) or (*in* input:**T**, *out* output:**T**)|13+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int64), tensor(int8), tensor(uint64), tensor(uint8)| +|Clip|*in* input:**T**
*in* min:**T**
*in* max:**T**
*out* output:**T**

or

*in* input:**T**
*out* output:**T**|13+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int64), tensor(int8), tensor(uint64), tensor(uint8)| |||12|**T** = tensor(double), tensor(float), tensor(float16), tensor(int64), tensor(int8), tensor(uint64), tensor(uint8)| |||11|**T** = tensor(float)| |||[6, 10]|**T** = tensor(float)| -|Compress|(*in* input:**T**, *in* condition:**T1**, *out* output:**T**)|11+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**T1** = tensor(bool)| +|Compress|*in* input:**T**
*in* condition:**T1**
*out* output:**T**|11+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**T1** = tensor(bool)| |||[9, 10]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**T1** = tensor(bool)| -|Concat|(*in* inputs:**T**, *out* concat_result:**T**)|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Concat|*in* inputs:**T**
*out* concat_result:**T**|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[11, 12]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[4, 10]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| -|ConstantOfShape|(*in* input:**T1**, *out* output:**T2**)|9+|**T1** = tensor(int64)
**T2** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| -|Conv|(*in* X:**T**, *in* W:**T**, *in* B:**T**, *out* Y:**T**)|11+|**T** = tensor(double), tensor(float), tensor(float16)| +|ConstantOfShape|*in* input:**T1**
*out* output:**T2**|9+|**T1** = tensor(int64)
**T2** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Conv|*in* X:**T**
*in* W:**T**
*in* B:**T**
*out* Y:**T**|11+|**T** = tensor(double), tensor(float), tensor(float16)| |||[1, 10]|**T** = tensor(double), tensor(float), tensor(float16)| -|ConvTranspose|(*in* X:**T**, *in* W:**T**, *in* B:**T**, *out* Y:**T**)|11+|**T** = tensor(double), tensor(float), tensor(float16)| +|ConvTranspose|*in* X:**T**
*in* W:**T**
*in* B:**T**
*out* Y:**T**|11+|**T** = tensor(double), tensor(float), tensor(float16)| |||[1, 10]|**T** = tensor(double), tensor(float), tensor(float16)| -|Cos|(*in* input:**T**, *out* output:**T**)|7+|**T** = tensor(double), tensor(float), tensor(float16)| -|Crop|(*in* input:**T**, *out* output:**T**)|1+|**T** = tensor(double), tensor(float), tensor(float16)| -|CumSum|(*in* x:**T**, *in* axis:**T2**, *out* y:**T**)|14+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)
**T2** = tensor(int32), tensor(int64)| +|Cos|*in* input:**T**
*out* output:**T**|7+|**T** = tensor(double), tensor(float), tensor(float16)| +|Crop|*in* input:**T**
*out* output:**T**|1+|**T** = tensor(double), tensor(float), tensor(float16)| +|CumSum|*in* x:**T**
*in* axis:**T2**
*out* y:**T**|14+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)
**T2** = tensor(int32), tensor(int64)| |||[11, 13]|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)
**T2** = tensor(int32), tensor(int64)| -|DequantizeLinear|(*in* x:**T**, *in* x_scale:**tensor(float)**, *in* x_zero_point:**T**, *out* y:**tensor(float)**)|10+|**T** = tensor(int8), tensor(uint8)| -|Div|(*in* A:**T**, *in* B:**T**, *out* C:**T**)|13+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| +|DequantizeLinear|*in* x:**T**
*in* x_scale:**tensor(float)**
*in* x_zero_point:**T**
*out* y:**tensor(float)**|10+|**T** = tensor(int8), tensor(uint8)| +|Div|*in* A:**T**
*in* B:**T**
*out* C:**T**|14+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| +|||13|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| |||[7, 12]|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| -|Dropout|(*in* data:**T**, *in* ratio:**T1**, *in* training_mode:**T2**, *out* output:**T**, *out* mask:**T2**) or (*in* data:**T**, *out* output:**T**, *out* mask:**T**) or (*in* data:**T**, *out* output:**T**, *out* mask:**T1**)|13+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16)
**T1** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16)
**T2** = tensor(bool)| +|Dropout|*in* data:**T**
*in* ratio:**T1**
*in* training_mode:**T2**
*out* output:**T**
*out* mask:**T2**

or

*in* data:**T**
*out* output:**T**
*out* mask:**T**

or

*in* data:**T**
*out* output:**T**
*out* mask:**T1**|13+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16)
**T1** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16)
**T2** = tensor(bool)| |||12|**T** = tensor(double), tensor(float), tensor(float16)
**T1** = tensor(double), tensor(float), tensor(float16)
**T2** = tensor(bool)| |||[10, 11]|**T** = tensor(double), tensor(float), tensor(float16)
**T1** = tensor(bool)| |||[7, 9]|**T** = tensor(double), tensor(float), tensor(float16)| -|DynamicSlice|(*in* data:**T**, *in* starts:**Tind**, *in* ends:**Tind**, *in* axes:**Tind**, *out* output:**T**)|1+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**Tind** = tensor(int32), tensor(int64)| -|Einsum|(*in* Inputs:**T**, *out* Output:**T**)|12+|**T** = tensor(double), tensor(float), tensor(float16)| -|Elu|(*in* X:**T**, *out* Y:**T**)|6+|**T** = tensor(double), tensor(float), tensor(float16)| -|Equal|(*in* A:**T**, *in* B:**T**, *out* C:**T1**)|13+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)
**T1** = tensor(bool)| +|DynamicSlice|*in* data:**T**
*in* starts:**Tind**
*in* ends:**Tind**
*in* axes:**Tind**
*out* output:**T**|1+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**Tind** = tensor(int32), tensor(int64)| +|Einsum|*in* Inputs:**T**
*out* Output:**T**|12+|**T** = tensor(double), tensor(float), tensor(float16)| +|Elu|*in* X:**T**
*out* Y:**T**|6+|**T** = tensor(double), tensor(float), tensor(float16)| +|Equal|*in* A:**T**
*in* B:**T**
*out* C:**T1**|13+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)
**T1** = tensor(bool)| |||[11, 12]|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| |||[7, 10]|**T** = tensor(bool), tensor(int32), tensor(int64)| -|Erf|(*in* input:**T**, *out* output:**T**)|13+|**T** = tensor(double), tensor(float), tensor(float16)| +|Erf|*in* input:**T**
*out* output:**T**|13+|**T** = tensor(double), tensor(float), tensor(float16)| |||[9, 12]|**T** = tensor(double), tensor(float), tensor(float16)| -|Exp|(*in* input:**T**, *out* output:**T**)|13+|**T** = tensor(double), tensor(float), tensor(float16)| +|Exp|*in* input:**T**
*out* output:**T**|13+|**T** = tensor(double), tensor(float), tensor(float16)| |||[6, 12]|**T** = tensor(double), tensor(float), tensor(float16)| -|Expand|(*in* input:**T**, *in* shape:**tensor(int64)**, *out* output:**T**)|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Expand|*in* input:**T**
*in* shape:**tensor(int64)**
*out* output:**T**|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[8, 12]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| -|EyeLike|(*in* input:**T1**, *out* output:**T2**)|9+|**T1** = tensor(double), tensor(float), tensor(int32), tensor(int64), tensor(uint64)
**T2** = tensor(double), tensor(float), tensor(int32), tensor(int64), tensor(uint64)| -|Flatten|(*in* input:**T**, *out* output:**T**)|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|EyeLike|*in* input:**T1**
*out* output:**T2**|9+|**T1** = tensor(double), tensor(float), tensor(int32), tensor(int64), tensor(uint64)
**T2** = tensor(double), tensor(float), tensor(int32), tensor(int64), tensor(uint64)| +|Flatten|*in* input:**T**
*out* output:**T**|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[11, 12]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[9, 10]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[1, 8]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| -|Floor|(*in* X:**T**, *out* Y:**T**)|13+|**T** = tensor(double), tensor(float), tensor(float16)| +|Floor|*in* X:**T**
*out* Y:**T**|13+|**T** = tensor(double), tensor(float), tensor(float16)| |||[6, 12]|**T** = tensor(double), tensor(float), tensor(float16)| -|GRU|(*in* X:**T**, *in* W:**T**, *in* R:**T**, *in* B:**T**, *in* sequence_lens:**T1**, *in* initial_h:**T**, *out* Y:**T**, *out* Y_h:**T**)|7+|**T** = tensor(double), tensor(float), tensor(float16)
**T1** = tensor(int32)| -|Gather|(*in* data:**T**, *in* indices:**Tind**, *out* output:**T**)|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**Tind** = tensor(int32), tensor(int64)| +|GRU|*in* X:**T**
*in* W:**T**
*in* R:**T**
*in* B:**T**
*in* sequence_lens:**T1**
*in* initial_h:**T**
*out* Y:**T**
*out* Y_h:**T**|14+|**T** = tensor(double), tensor(float), tensor(float16)
**T1** = tensor(int32)| +|||[7, 13]|**T** = tensor(double), tensor(float), tensor(float16)
**T1** = tensor(int32)| +|Gather|*in* data:**T**
*in* indices:**Tind**
*out* output:**T**|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**Tind** = tensor(int32), tensor(int64)| |||[11, 12]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**Tind** = tensor(int32), tensor(int64)| |||[1, 10]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**Tind** = tensor(int32), tensor(int64)| -|GatherElements|(*in* data:**T**, *in* indices:**Tind**, *out* output:**T**)|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**Tind** = tensor(int32), tensor(int64)| +|GatherElements|*in* data:**T**
*in* indices:**Tind**
*out* output:**T**|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**Tind** = tensor(int32), tensor(int64)| |||[11, 12]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**Tind** = tensor(int32), tensor(int64)| -|GatherND|(*in* data:**T**, *in* indices:**tensor(int64)**, *out* output:**T**)|13+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16), tensor(int64)
**Tind** = tensor(int64)| +|GatherND|*in* data:**T**
*in* indices:**tensor(int64)**
*out* output:**T**|13+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16), tensor(int64)
**Tind** = tensor(int64)| |||12|**T** = tensor(double), tensor(float), tensor(float16), tensor(int64)
**Tind** = tensor(int64)| -|Gemm|(*in* A:**T**, *in* B:**T**, *in* C:**T**, *out* Y:**T**)|13+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16)| +|Gemm|*in* A:**T**
*in* B:**T**
*in* C:**T**
*out* Y:**T**|13+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16)| |||[11, 12]|**T** = tensor(double), tensor(float), tensor(float16)| |||[9, 10]|**T** = tensor(double), tensor(float), tensor(float16)| |||[7, 8]|**T** = tensor(double), tensor(float), tensor(float16)| -|GlobalAveragePool|(*in* X:**T**, *out* Y:**T**)|1+|**T** = tensor(double), tensor(float), tensor(float16)| -|GlobalMaxPool|(*in* X:**T**, *out* Y:**T**)|1+|**T** = tensor(double), tensor(float), tensor(float16)| -|Greater|(*in* A:**T**, *in* B:**T**, *out* C:**T1**)|13+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)
**T1** = tensor(bool)| +|GlobalAveragePool|*in* X:**T**
*out* Y:**T**|1+|**T** = tensor(double), tensor(float), tensor(float16)| +|GlobalMaxPool|*in* X:**T**
*out* Y:**T**|1+|**T** = tensor(double), tensor(float), tensor(float16)| +|Greater|*in* A:**T**
*in* B:**T**
*out* C:**T1**|13+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)
**T1** = tensor(bool)| |||[9, 12]|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| |||[7, 8]|**T** = tensor(double), tensor(float), tensor(float16)| -|GreaterOrEqual|(*in* A:**T**, *in* B:**T**, *out* C:**T1**)|12+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)
**T1** = tensor(bool)| -|HardSigmoid|(*in* X:**T**, *out* Y:**T**)|6+|**T** = tensor(double), tensor(float), tensor(float16)| -|Identity|(*in* input:**T**, *out* output:**T**) or (*in* input:**V**, *out* output:**V**)|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|GreaterOrEqual|*in* A:**T**
*in* B:**T**
*out* C:**T1**|12+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)
**T1** = tensor(bool)| +|HardSigmoid|*in* X:**T**
*out* Y:**T**|6+|**T** = tensor(double), tensor(float), tensor(float16)| +|Identity|*in* input:**T**
*out* output:**T**

or

*in* input:**V**
*out* output:**V**|14+|**V** = seq(tensor(bfloat16)), seq(tensor(bool)), seq(tensor(double)), seq(tensor(float)), seq(tensor(float16)), seq(tensor(int16)), seq(tensor(int32)), seq(tensor(int64)), seq(tensor(int8)), seq(tensor(string)), seq(tensor(uint16)), seq(tensor(uint32)), seq(tensor(uint64)), seq(tensor(uint8)), tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||13|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[1, 12]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| -|If|(*in* cond:**B**, *out* outputs:**V**)|13+|**B** = tensor(bool)
**V** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|If|*in* cond:**B**
*out* outputs:**V**|13+|**B** = tensor(bool)
**V** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[11, 12]|**B** = tensor(bool)
**V** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[1, 10]|**B** = tensor(bool)
**V** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| -|ImageScaler|(*in* input:**T**, *out* output:**T**)|1+|**T** = tensor(double), tensor(float), tensor(float16)| -|InstanceNormalization|(*in* input:**T**, *in* scale:**T**, *in* B:**T**, *out* output:**T**)|6+|**T** = tensor(double), tensor(float), tensor(float16)| -|LRN|(*in* X:**T**, *out* Y:**T**)|13+|**T** = tensor(double), tensor(float), tensor(float16)| +|ImageScaler|*in* input:**T**
*out* output:**T**|1+|**T** = tensor(double), tensor(float), tensor(float16)| +|InstanceNormalization|*in* input:**T**
*in* scale:**T**
*in* B:**T**
*out* output:**T**|6+|**T** = tensor(double), tensor(float), tensor(float16)| +|LRN|*in* X:**T**
*out* Y:**T**|13+|**T** = tensor(double), tensor(float), tensor(float16)| |||[1, 12]|**T** = tensor(double), tensor(float), tensor(float16)| -|LSTM|(*in* X:**T**, *in* W:**T**, *in* R:**T**, *in* B:**T**, *in* sequence_lens:**T1**, *in* initial_h:**T**, *in* initial_c:**T**, *in* P:**T**, *out* Y:**T**, *out* Y_h:**T**, *out* Y_c:**T**)|7+|**T** = tensor(double), tensor(float), tensor(float16)
**T1** = tensor(int32)| -|LayerNormalization|(*in* X:**T**, *in* Scale:**T**, *in* B:**T**, *out* Y:**T**, *out* Mean:**U**, *out* InvStdDev:**U**)|1+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16)
**U** = tensor(double), tensor(float)| -|LeakyRelu|(*in* X:**T**, *out* Y:**T**)|6+|**T** = tensor(double), tensor(float), tensor(float16)| -|Less|(*in* A:**T**, *in* B:**T**, *out* C:**T1**)|13+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)
**T1** = tensor(bool)| +|LSTM|*in* X:**T**
*in* W:**T**
*in* R:**T**
*in* B:**T**
*in* sequence_lens:**T1**
*in* initial_h:**T**
*in* initial_c:**T**
*in* P:**T**
*out* Y:**T**
*out* Y_h:**T**
*out* Y_c:**T**|14+|**T** = tensor(double), tensor(float), tensor(float16)
**T1** = tensor(int32)| +|||[7, 13]|**T** = tensor(double), tensor(float), tensor(float16)
**T1** = tensor(int32)| +|LayerNormalization|*in* X:**T**
*in* Scale:**T**
*in* B:**T**
*out* Y:**T**
*out* Mean:**U**
*out* InvStdDev:**U**|1+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16)
**U** = tensor(double), tensor(float)| +|LeakyRelu|*in* X:**T**
*out* Y:**T**|6+|**T** = tensor(double), tensor(float), tensor(float16)| +|Less|*in* A:**T**
*in* B:**T**
*out* C:**T1**|13+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)
**T1** = tensor(bool)| |||[9, 12]|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| |||[7, 8]|**T** = tensor(double), tensor(float), tensor(float16)| -|LessOrEqual|(*in* A:**T**, *in* B:**T**, *out* C:**T1**)|12+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)
**T1** = tensor(bool)| -|Log|(*in* input:**T**, *out* output:**T**)|13+|**T** = tensor(double), tensor(float), tensor(float16)| +|LessOrEqual|*in* A:**T**
*in* B:**T**
*out* C:**T1**|12+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)
**T1** = tensor(bool)| +|Log|*in* input:**T**
*out* output:**T**|13+|**T** = tensor(double), tensor(float), tensor(float16)| |||[6, 12]|**T** = tensor(double), tensor(float), tensor(float16)| -|LogSoftmax|(*in* input:**T**, *out* output:**T**)|13+|**T** = tensor(double), tensor(float), tensor(float16)| +|LogSoftmax|*in* input:**T**
*out* output:**T**|13+|**T** = tensor(double), tensor(float), tensor(float16)| |||[11, 12]|**T** = tensor(double), tensor(float), tensor(float16)| |||[1, 10]|**T** = tensor(double), tensor(float), tensor(float16)| -|Loop|(*in* M:**I**, *in* cond:**B**, *in* v_initial:**V**, *out* v_final_and_scan_outputs:**V**)|13+|**B** = tensor(bool)
**I** = tensor(int64)
**V** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Loop|*in* M:**I**
*in* cond:**B**
*in* v_initial:**V**
*out* v_final_and_scan_outputs:**V**|13+|**B** = tensor(bool)
**I** = tensor(int64)
**V** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[11, 12]|**B** = tensor(bool)
**I** = tensor(int64)
**V** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[1, 10]|**B** = tensor(bool)
**I** = tensor(int64)
**V** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| -|MatMul|(*in* A:**T**, *in* B:**T**, *out* Y:**T**)|13+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16)| +|MatMul|*in* A:**T**
*in* B:**T**
*out* Y:**T**|13+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16)| |||[9, 12]|**T** = tensor(double), tensor(float), tensor(float16)| |||[1, 8]|**T** = tensor(double), tensor(float), tensor(float16)| -|MatMulInteger|(*in* A:**T1**, *in* B:**T2**, *in* a_zero_point:**T1**, *in* b_zero_point:**T2**, *out* Y:**T3**)|10+|**T1** = tensor(int8)
**T2** = tensor(int8)
**T3** = tensor(int32)| -|Max|(*in* data_0:**T**, *out* max:**T**)|13+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| +|MatMulInteger|*in* A:**T1**
*in* B:**T2**
*in* a_zero_point:**T1**
*in* b_zero_point:**T2**
*out* Y:**T3**|10+|**T1** = tensor(int8)
**T2** = tensor(int8)
**T3** = tensor(int32)| +|Max|*in* data_0:**T**
*out* max:**T**|13+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| |||12|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| |||[6, 11]|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16)| -|MaxPool|(*in* X:**T**, *out* Y:**T**) or (*in* X:**T**, *out* Y:**T**, *out* Indices:**I**)|12+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int8), tensor(uint8)| +|MaxPool|*in* X:**T**
*out* Y:**T**

or

*in* X:**T**
*out* Y:**T**
*out* Indices:**I**|12+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int8), tensor(uint8)| |||11|**I** = tensor(int64)
**T** = tensor(double), tensor(float), tensor(float16)| |||10|**I** = tensor(int64)
**T** = tensor(double), tensor(float), tensor(float16)| |||[8, 9]|**I** = tensor(int64)
**T** = tensor(double), tensor(float), tensor(float16)| |||[1, 7]|**I** = tensor(int64)
**T** = tensor(double), tensor(float), tensor(float16)| -|MemcpyFromHost|(*in* X:**T**, *out* Y:**T**)|1+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| -|MemcpyToHost|(*in* X:**T**, *out* Y:**T**)|1+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| -|Min|(*in* data_0:**T**, *out* min:**T**)|13+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| +|MemcpyFromHost|*in* X:**T**
*out* Y:**T**|1+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|MemcpyToHost|*in* X:**T**
*out* Y:**T**|1+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Min|*in* data_0:**T**
*out* min:**T**|13+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| |||12|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| |||[6, 11]|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16)| -|Mul|(*in* A:**T**, *in* B:**T**, *out* C:**T**)|13+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| +|Mul|*in* A:**T**
*in* B:**T**
*out* C:**T**|14+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| +|||13|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| |||[7, 12]|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| -|Neg|(*in* X:**T**, *out* Y:**T**)|13+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8)| +|Neg|*in* X:**T**
*out* Y:**T**|13+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8)| |||[6, 12]|**T** = tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8)| -|NonZero|(*in* X:**T**, *out* Y:**tensor(int64)**)|13+|**T** = tensor(bool), tensor(float), tensor(int32), tensor(int64), tensor(uint8)| +|NonZero|*in* X:**T**
*out* Y:**tensor(int64)**|13+|**T** = tensor(bool), tensor(float), tensor(int32), tensor(int64), tensor(uint8)| |||[9, 12]|**T** = tensor(bool), tensor(float), tensor(int32), tensor(int64), tensor(uint8)| -|Not|(*in* X:**T**, *out* Y:**T**)|1+|**T** = tensor(bool)
**T1** = tensor(bool)| -|OneHot|(*in* indices:**T1**, *in* depth:**T2**, *in* values:**T3**, *out* output:**T3**)|11+|**T1** = tensor(int32), tensor(int64)
**T2** = tensor(int32), tensor(int64)
**T3** = tensor(float), tensor(float16), tensor(int64)| -|Or|(*in* A:**T**, *in* B:**T**, *out* C:**T1**)|7+|**T** = tensor(bool)
**T1** = tensor(bool)| -|PRelu|(*in* X:**T**, *in* slope:**T**, *out* Y:**T**)|9+|**T** = tensor(double), tensor(float), tensor(float16)| +|Not|*in* X:**T**
*out* Y:**T**|1+|**T** = tensor(bool)
**T1** = tensor(bool)| +|OneHot|*in* indices:**T1**
*in* depth:**T2**
*in* values:**T3**
*out* output:**T3**|11+|**T1** = tensor(int32), tensor(int64)
**T2** = tensor(int32), tensor(int64)
**T3** = tensor(float), tensor(float16), tensor(int64)| +|Or|*in* A:**T**
*in* B:**T**
*out* C:**T1**|7+|**T** = tensor(bool)
**T1** = tensor(bool)| +|PRelu|*in* X:**T**
*in* slope:**T**
*out* Y:**T**|9+|**T** = tensor(double), tensor(float), tensor(float16)| |||[7, 8]|**T** = tensor(double), tensor(float), tensor(float16)| -|Pad|(*in* data:**T**, *in* pads:**tensor(int64)**, *in* constant_value:**T**, *out* output:**T**) or (*in* data:**T**, *out* output:**T**)|11+|**T** = tensor(double), tensor(float), tensor(float16)| +|Pad|*in* data:**T**
*in* pads:**tensor(int64)**
*in* constant_value:**T**
*out* output:**T**

or

*in* data:**T**
*out* output:**T**|13+|**T** = tensor(double), tensor(float), tensor(float16)| +|||[11, 12]|**T** = tensor(double), tensor(float), tensor(float16)| |||[2, 10]|**T** = tensor(double), tensor(float), tensor(float16)| -|ParametricSoftplus|(*in* X:**T**, *out* Y:**T**)|1+|**T** = tensor(double), tensor(float), tensor(float16)| -|Pow|(*in* X:**T**, *in* Y:**T**, *out* Z:**T**) or (*in* X:**T**, *in* Y:**T1**, *out* Z:**T**)|13+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64)
**T1** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64)| +|ParametricSoftplus|*in* X:**T**
*out* Y:**T**|1+|**T** = tensor(double), tensor(float), tensor(float16)| +|Pow|*in* X:**T**
*in* Y:**T**
*out* Z:**T**

or

*in* X:**T**
*in* Y:**T1**
*out* Z:**T**|13+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64)
**T1** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64)| |||12|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64)
**T1** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64)| |||[7, 11]|**T** = tensor(double), tensor(float), tensor(float16)| -|QuantizeLinear|(*in* x:**T1**, *in* y_scale:**tensor(float)**, *in* y_zero_point:**T2**, *out* y:**T2**)|10+|**T1** = tensor(float)
**T2** = tensor(int8), tensor(uint8)| -|RNN|(*in* X:**T**, *in* W:**T**, *in* R:**T**, *in* B:**T**, *in* sequence_lens:**T1**, *in* initial_h:**T**, *out* Y:**T**, *out* Y_h:**T**)|7+|**T** = tensor(double), tensor(float), tensor(float16)
**T1** = tensor(int32)| -|Range|(*in* start:**T**, *in* limit:**T**, *in* delta:**T**, *out* output:**T**)|11+|**T** = tensor(double), tensor(float), tensor(int16), tensor(int32), tensor(int64)| -|Reciprocal|(*in* X:**T**, *out* Y:**T**)|13+|**T** = tensor(double), tensor(float), tensor(float16)| +|QuantizeLinear|*in* x:**T1**
*in* y_scale:**tensor(float)**
*in* y_zero_point:**T2**
*out* y:**T2**|10+|**T1** = tensor(float)
**T2** = tensor(int8), tensor(uint8)| +|RNN|*in* X:**T**
*in* W:**T**
*in* R:**T**
*in* B:**T**
*in* sequence_lens:**T1**
*in* initial_h:**T**
*out* Y:**T**
*out* Y_h:**T**|14+|**T** = tensor(double), tensor(float), tensor(float16)
**T1** = tensor(int32)| +|||[7, 13]|**T** = tensor(double), tensor(float), tensor(float16)
**T1** = tensor(int32)| +|Range|*in* start:**T**
*in* limit:**T**
*in* delta:**T**
*out* output:**T**|11+|**T** = tensor(double), tensor(float), tensor(int16), tensor(int32), tensor(int64)| +|Reciprocal|*in* X:**T**
*out* Y:**T**|13+|**T** = tensor(double), tensor(float), tensor(float16)| |||[6, 12]|**T** = tensor(double), tensor(float), tensor(float16)| -|ReduceL1|(*in* data:**T**, *out* reduced:**T**)|13+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32)| +|ReduceL1|*in* data:**T**
*out* reduced:**T**|13+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32)| |||[11, 12]|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32)| |||[1, 10]|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32)| -|ReduceL2|(*in* data:**T**, *out* reduced:**T**)|13+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32)| +|ReduceL2|*in* data:**T**
*out* reduced:**T**|13+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32)| |||[11, 12]|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32)| |||[1, 10]|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32)| -|ReduceLogSum|(*in* data:**T**, *out* reduced:**T**)|13+|**T** = tensor(double), tensor(float), tensor(float16)| +|ReduceLogSum|*in* data:**T**
*out* reduced:**T**|13+|**T** = tensor(double), tensor(float), tensor(float16)| |||[11, 12]|**T** = tensor(double), tensor(float), tensor(float16)| |||[1, 10]|**T** = tensor(double), tensor(float), tensor(float16)| -|ReduceLogSumExp|(*in* data:**T**, *out* reduced:**T**)|13+|**T** = tensor(double), tensor(float), tensor(float16)| +|ReduceLogSumExp|*in* data:**T**
*out* reduced:**T**|13+|**T** = tensor(double), tensor(float), tensor(float16)| |||[11, 12]|**T** = tensor(double), tensor(float), tensor(float16)| |||[1, 10]|**T** = tensor(double), tensor(float), tensor(float16)| -|ReduceMax|(*in* data:**T**, *out* reduced:**T**)|13+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(int8), tensor(uint8)| +|ReduceMax|*in* data:**T**
*out* reduced:**T**|13+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(int8), tensor(uint8)| |||12|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(int8), tensor(uint8)| |||11|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64)| |||[1, 10]|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64)| -|ReduceMean|(*in* data:**T**, *out* reduced:**T**)|13+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32)| +|ReduceMean|*in* data:**T**
*out* reduced:**T**|13+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32)| |||[11, 12]|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32)| |||[1, 10]|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32)| -|ReduceMin|(*in* data:**T**, *out* reduced:**T**)|13+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int8), tensor(uint8)| +|ReduceMin|*in* data:**T**
*out* reduced:**T**|13+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int8), tensor(uint8)| |||12|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int8), tensor(uint8)| |||11|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32)| |||[1, 10]|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32)| -|ReduceProd|(*in* data:**T**, *out* reduced:**T**)|13+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32)| +|ReduceProd|*in* data:**T**
*out* reduced:**T**|13+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32)| |||[11, 12]|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32)| |||[1, 10]|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32)| -|ReduceSum|(*in* data:**T**, *in* axes:**tensor(int64)**, *out* reduced:**T**) or (*in* data:**T**, *out* reduced:**T**)|13+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64)| +|ReduceSum|*in* data:**T**
*in* axes:**tensor(int64)**
*out* reduced:**T**

or

*in* data:**T**
*out* reduced:**T**|13+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64)| |||[11, 12]|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64)| |||[1, 10]|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64)| -|ReduceSumSquare|(*in* data:**T**, *out* reduced:**T**)|13+|**T** = tensor(double), tensor(float), tensor(float16)| +|ReduceSumSquare|*in* data:**T**
*out* reduced:**T**|13+|**T** = tensor(double), tensor(float), tensor(float16)| |||[11, 12]|**T** = tensor(double), tensor(float), tensor(float16)| |||[1, 10]|**T** = tensor(double), tensor(float), tensor(float16)| -|Relu|(*in* X:**T**, *out* Y:**T**)|13+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16)| +|Relu|*in* X:**T**
*out* Y:**T**|14+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16)| +|||13|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16)| |||[6, 12]|**T** = tensor(double), tensor(float), tensor(float16)| -|Reshape|(*in* data:**T**, *in* shape:**tensor(int64)**, *out* reshaped:**T**) or (*in* data:**T**, *out* reshaped:**T**)|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**shape** = tensor(int64)| +|Reshape|*in* data:**T**
*in* shape:**tensor(int64)**
*out* reshaped:**T**

or

*in* data:**T**
*out* reshaped:**T**|14+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**shape** = tensor(int64)| +|||13|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**shape** = tensor(int64)| |||[5, 12]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**shape** = tensor(int64)| |||[1, 4]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| -|Resize|(*in* X:**T**, *in* scales:**tensor(float)**, *out* Y:**T**) or (*in* X:**T1**, *in* roi:**T2**, *in* scales:**tensor(float)**, *in* sizes:**tensor(int64)**, *out* Y:**T1**)|13+|**T1** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(uint8)| +|Resize|*in* X:**T**
*in* scales:**tensor(float)**
*out* Y:**T**

or

*in* X:**T1**
*in* roi:**T2**
*in* scales:**tensor(float)**
*in* sizes:**tensor(int64)**
*out* Y:**T1**|13+|**T1** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(uint8)| |||[11, 12]|**T1** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(uint8)| |||10|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(uint8)| -|ReverseSequence|(*in* input:**T**, *in* sequence_lens:**tensor(int64)**, *out* Y:**T**)|10+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| -|RoiAlign|(*in* X:**T1**, *in* rois:**T1**, *in* batch_indices:**T2**, *out* Y:**T1**)|10+|**T** = tensor(double), tensor(float)
**T2** = tensor(int64)| -|Round|(*in* X:**T**, *out* Y:**T**)|11+|**T** = tensor(double), tensor(float), tensor(float16)| -|ScaledTanh|(*in* input:**T**, *out* output:**T**)|1+|**T** = tensor(double), tensor(float), tensor(float16)| -|Scan|(*in* initial_state_and_scan_inputs:**V**, *out* final_state_and_scan_outputs:**V**) or (*in* sequence_lens:**I**, *in* initial_state_and_scan_inputs:**V**, *out* final_state_and_scan_outputs:**V**)|11+|**I** = tensor(int64)
**V** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|ReverseSequence|*in* input:**T**
*in* sequence_lens:**tensor(int64)**
*out* Y:**T**|10+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|RoiAlign|*in* X:**T1**
*in* rois:**T1**
*in* batch_indices:**T2**
*out* Y:**T1**|10+|**T** = tensor(double), tensor(float)
**T2** = tensor(int64)| +|Round|*in* X:**T**
*out* Y:**T**|11+|**T** = tensor(double), tensor(float), tensor(float16)| +|ScaledTanh|*in* input:**T**
*out* output:**T**|1+|**T** = tensor(double), tensor(float), tensor(float16)| +|Scan|*in* initial_state_and_scan_inputs:**V**
*out* final_state_and_scan_outputs:**V**

or

*in* sequence_lens:**I**
*in* initial_state_and_scan_inputs:**V**
*out* final_state_and_scan_outputs:**V**|11+|**I** = tensor(int64)
**V** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[9, 10]|**I** = tensor(int64)
**V** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||8|**I** = tensor(int64)
**V** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| -|Scatter|(*in* data:**T**, *in* indices:**Tind**, *in* updates:**T**, *out* output:**T**)|[9, 10]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**Tind** = tensor(int32), tensor(int64)| -|ScatterElements|(*in* data:**T**, *in* indices:**Tind**, *in* updates:**T**, *out* output:**T**)|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**Tind** = tensor(int32), tensor(int64)| +|Scatter|*in* data:**T**
*in* indices:**Tind**
*in* updates:**T**
*out* output:**T**|[9, 10]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**Tind** = tensor(int32), tensor(int64)| +|ScatterElements|*in* data:**T**
*in* indices:**Tind**
*in* updates:**T**
*out* output:**T**|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**Tind** = tensor(int32), tensor(int64)| |||[11, 12]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**Tind** = tensor(int32), tensor(int64)| -|ScatterND|(*in* data:**T**, *in* indices:**tensor(int64)**, *in* updates:**T**, *out* output:**T**)|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|ScatterND|*in* data:**T**
*in* indices:**tensor(int64)**
*in* updates:**T**
*out* output:**T**|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[11, 12]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| -|Selu|(*in* X:**T**, *out* Y:**T**)|6+|**T** = tensor(double), tensor(float), tensor(float16)| -|Shape|(*in* data:**T**, *out* shape:**T1**)|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**T1** = tensor(int64)| +|Selu|*in* X:**T**
*out* Y:**T**|6+|**T** = tensor(double), tensor(float), tensor(float16)| +|Shape|*in* data:**T**
*out* shape:**T1**|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**T1** = tensor(int64)| |||[1, 12]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**T1** = tensor(int64)| -|Shrink|(*in* input:**T**, *out* output:**T**)|9+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| -|Sigmoid|(*in* X:**T**, *out* Y:**T**)|13+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16)| +|Shrink|*in* input:**T**
*out* output:**T**|9+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Sigmoid|*in* X:**T**
*out* Y:**T**|13+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16)| |||[6, 12]|**T** = tensor(double), tensor(float), tensor(float16)| -|SimplifiedLayerNormalization|(*in* X:**T**, *in* scale:**T**, *out* Y:**T**, *out* inv_std_var:**U**)|1+|**T** = tensor(double), tensor(float), tensor(float16)
**U** = tensor(double), tensor(float)| -|Sin|(*in* input:**T**, *out* output:**T**)|7+|**T** = tensor(double), tensor(float), tensor(float16)| -|Size|(*in* data:**T**, *out* size:**T1**)|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**T1** = tensor(int64)| +|SimplifiedLayerNormalization|*in* X:**T**
*in* scale:**T**
*out* Y:**T**
*out* inv_std_var:**U**|1+|**T** = tensor(double), tensor(float), tensor(float16)
**U** = tensor(double), tensor(float)| +|Sin|*in* input:**T**
*out* output:**T**|7+|**T** = tensor(double), tensor(float), tensor(float16)| +|Size|*in* data:**T**
*out* size:**T1**|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**T1** = tensor(int64)| |||[1, 12]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**T1** = tensor(int64)| -|Slice|(*in* data:**T**, *in* starts:**Tind**, *in* ends:**Tind**, *in* axes:**Tind**, *in* steps:**Tind**, *out* output:**T**) or (*in* data:**T**, *out* output:**T**)|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**Tind** = tensor(float), tensor(int32), tensor(int64)| +|Slice|*in* data:**T**
*in* starts:**Tind**
*in* ends:**Tind**
*in* axes:**Tind**
*in* steps:**Tind**
*out* output:**T**

or

*in* data:**T**
*out* output:**T**|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**Tind** = tensor(float), tensor(int32), tensor(int64)| |||[11, 12]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**Tind** = tensor(float), tensor(int32), tensor(int64)| |||10|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**Tind** = tensor(float), tensor(int32), tensor(int64)| |||[1, 9]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**Tind** = tensor(float), tensor(int32), tensor(int64)| -|Softmax|(*in* input:**T**, *out* output:**T**)|13+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16)| +|Softmax|*in* input:**T**
*out* output:**T**|13+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16)| |||[11, 12]|**T** = tensor(double), tensor(float), tensor(float16)| |||[1, 10]|**T** = tensor(double), tensor(float), tensor(float16)| -|Softplus|(*in* X:**T**, *out* Y:**T**)|1+|**T** = tensor(double), tensor(float), tensor(float16)| -|Softsign|(*in* input:**T**, *out* output:**T**)|1+|**T** = tensor(double), tensor(float), tensor(float16)| -|Split|(*in* input:**T**, *in* split:**T**, *out* outputs...:**T**) or (*in* input:**T**, *in* split:**tensor(int64)**, *out* outputs:**T**) or (*in* input:**T**, *out* outputs:**T**)|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Softplus|*in* X:**T**
*out* Y:**T**|1+|**T** = tensor(double), tensor(float), tensor(float16)| +|Softsign|*in* input:**T**
*out* output:**T**|1+|**T** = tensor(double), tensor(float), tensor(float16)| +|Split|*in* input:**T**
*in* split:**T**
*out* outputs...:**T**

or

*in* input:**T**
*in* split:**tensor(int64)**
*out* outputs:**T**

or

*in* input:**T**
*out* outputs:**T**|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[11, 12]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[2, 10]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| -|Sqrt|(*in* X:**T**, *out* Y:**T**)|13+|**T** = tensor(double), tensor(float), tensor(float16)| +|Sqrt|*in* X:**T**
*out* Y:**T**|13+|**T** = tensor(double), tensor(float), tensor(float16)| |||[6, 12]|**T** = tensor(double), tensor(float), tensor(float16)| -|Squeeze|(*in* data:**T**, *in* axes:**tensor(int64)**, *out* squeezed:**T**) or (*in* data:**T**, *out* squeezed:**T**)|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Squeeze|*in* data:**T**
*in* axes:**tensor(int64)**
*out* squeezed:**T**

or

*in* data:**T**
*out* squeezed:**T**|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[11, 12]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[1, 10]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| -|Sub|(*in* A:**T**, *in* B:**T**, *out* C:**T**)|13+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| +|Sub|*in* A:**T**
*in* B:**T**
*out* C:**T**|14+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| +|||13|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| |||[7, 12]|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| -|Sum|(*in* data_0:**T**, *out* sum:**T**)|13+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16)| +|Sum|*in* data_0:**T**
*out* sum:**T**|13+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16)| |||[8, 12]|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16)| |||[6, 7]|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16)| -|Tanh|(*in* input:**T**, *out* output:**T**)|13+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16)| +|Tanh|*in* input:**T**
*out* output:**T**|13+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16)| |||[6, 12]|**T** = tensor(double), tensor(float), tensor(float16)| -|ThresholdedRelu|(*in* X:**T**, *out* Y:**T**)|10+|**T** = tensor(double), tensor(float), tensor(float16)| +|ThresholdedRelu|*in* X:**T**
*out* Y:**T**|10+|**T** = tensor(double), tensor(float), tensor(float16)| |||1+|**T** = tensor(double), tensor(float), tensor(float16)| -|Tile|(*in* input:**T**, *in* repeats:**T1**, *out* output:**T**) or (*in* input:**T**, *in* tiles:**T**, *in* axis:**T**, *out* output:**T**)|13+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64)
**T1** = tensor(int64)| +|Tile|*in* input:**T**
*in* repeats:**T1**
*out* output:**T**

or

*in* input:**T**
*in* tiles:**T**
*in* axis:**T**
*out* output:**T**|13+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64)
**T1** = tensor(int64)| |||[6, 12]|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64)
**T1** = tensor(int64)| -|TopK|(*in* X:**T**, *in* K:**tensor(int64)**, *out* Values:**T**, *out* Indices:**I**) or (*in* X:**T**, *out* Values:**T**, *out* Indices:**I**)|11+|**I** = tensor(int64)
**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|TopK|*in* X:**T**
*in* K:**tensor(int64)**
*out* Values:**T**
*out* Indices:**I**

or

*in* X:**T**
*out* Values:**T**
*out* Indices:**I**|11+|**I** = tensor(int64)
**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||10|**I** = tensor(int64)
**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[1, 9]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| -|Transpose|(*in* data:**T**, *out* transposed:**T**)|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Transpose|*in* data:**T**
*out* transposed:**T**|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[1, 12]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| -|Unsqueeze|(*in* data:**T**, *in* axes:**tensor(int64)**, *out* expanded:**T**) or (*in* data:**T**, *out* expanded:**T**)|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Unsqueeze|*in* data:**T**
*in* axes:**tensor(int64)**
*out* expanded:**T**

or

*in* data:**T**
*out* expanded:**T**|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[11, 12]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[1, 10]|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| -|Upsample|(*in* X:**T**, *in* scales:**tensor(float)**, *out* Y:**T**) or (*in* X:**T**, *out* Y:**T**)|9|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(uint8)| +|Upsample|*in* X:**T**
*in* scales:**tensor(float)**
*out* Y:**T**

or

*in* X:**T**
*out* Y:**T**|9|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(uint8)| |||[7, 8]|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(uint8)| -|Where|(*in* condition:**B**, *in* X:**T**, *in* Y:**T**, *out* output:**T**)|9+|**B** = tensor(bool)
**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint8)| -|Xor|(*in* A:**T**, *in* B:**T**, *out* C:**T1**)|7+|**T** = tensor(bool)
**T1** = tensor(bool)| +|Where|*in* condition:**B**
*in* X:**T**
*in* Y:**T**
*out* output:**T**|9+|**B** = tensor(bool)
**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint8)| +|Xor|*in* A:**T**
*in* B:**T**
*out* C:**T1**|7+|**T** = tensor(bool)
**T1** = tensor(bool)| | | | | |**Operator Domain:** *com.microsoft*|||| -|Attention|(*in* input:**T**, *in* weight:**T**, *in* bias:**T**, *in* mask_index:**M**, *in* past:**T**, *out* output:**T**, *out* present:**T**)|1+|**T** = tensor(float), tensor(float16)| -|BiasDropout|(*in* data:**T**, *in* bias:**T**, *in* residual:**T**, *in* ratio:**T1**, *in* training_mode:**T2**, *out* output:**T**, *out* mask:**T2**)|1+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16)
**T1** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16)
**T2** = tensor(bool)| -|BiasGelu|(*in* A:**T**, *in* B:**T**, *out* C:**T**)|1+|**T** = tensor(double), tensor(float), tensor(float16)| -|BiasSoftmax|(*in* data:**T**, *in* bias:**T**, *out* output:**T**)|1+|**T** = tensor(double), tensor(float), tensor(float16)| -|ComplexMul|(*in* A:**T**, *in* B:**T**, *out* C:**T**)|1+|**T** = tensor(float), tensor(float16)| -|ComplexMulConj|(*in* A:**T**, *in* B:**T**, *out* C:**T**)|1+|**T** = tensor(float), tensor(float16)| -|ConvTransposeWithDynamicPads|(*in* X:**T**, *in* W:**T**, *in* Pads:**tensor(int64)**, *in* B:**T**, *out* Y:**T**)|1+|**T** = tensor(float)| -|DequantizeLinear|(*in* x:**T1**, *in* x_scale:**T2**, *in* x_zero_point:**T1**, *out* y:**T2**)|1+|**T1** = tensor(int8), tensor(uint8)
**T2** = tensor(float16)| -|EmbedLayerNormalization|(*in* input_ids:**T1**, *in* segment_ids:**T1**, *in* word_embedding:**T**, *in* position_embedding:**T**, *in* segment_embedding:**T**, *in* gamma:**T**, *in* beta:**T**, *in* mask:**T1**, *out* output:**T**, *out* mask_index:**T1**)|1+|**T** = tensor(float), tensor(float16)| -|FastGelu|(*in* X:**T**, *in* bias:**T**, *out* Y:**T**)|1+|**T** = tensor(bfloat16), tensor(float), tensor(float16)| -|FusedConv|(*in* X:**T**, *in* W:**T**, *in* B:**T**, *in* Z:**T**, *out* Y:**T**)|1+|**T** = tensor(float)| -|FusedMatMul|(*in* A:**T**, *in* B:**T**, *out* Y:**T**)|1+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16)| -|Gelu|(*in* X:**T**, *out* Y:**T**)|1+|**T** = tensor(double), tensor(float), tensor(float16)| -|Inverse|(*in* X:**T**, *out* Y:**T**)|1+|**T** = tensor(double), tensor(float), tensor(float16)| -|Irfft|(*in* X:**T**, *out* Y:**T**)|1+|**T** = tensor(double), tensor(float), tensor(float16)| -|LongformerAttention|(*in* input:**T**, *in* weight:**T**, *in* bias:**T**, *in* mask:**T**, *in* global_weight:**T**, *in* global_bias:**T**, *in* global:**G**, *out* output:**T**)|1+|**T** = tensor(float), tensor(float16)| -|QAttention|(*in* input:**T1**, *in* weight:**T2**, *in* bias:**T3**, *in* input_scale:**T3**, *in* weight_scale:**T3**, *in* mask_index:**T4**, *in* input_zero_point:**T1**, *in* weight_zero_point:**T2**, *in* past:**T3**, *out* output:**T3**, *out* present:**T3**)|1+|**T1** = tensor(int8)
**T2** = tensor(int8)
**T3** = tensor(float), tensor(float16)
**T4** = tensor(int32)| -|QuantizeLinear|(*in* x:**T1**, *in* y_scale:**T1**, *in* y_zero_point:**T2**, *out* y:**T2**)|1+|**T1** = tensor(float16)
**T2** = tensor(int8), tensor(uint8)| -|Rfft|(*in* X:**T**, *out* Y:**T**)|1+|**T** = tensor(double), tensor(float), tensor(float16)| -|SkipLayerNormalization|(*in* input:**T**, *in* skip:**T**, *in* gamma:**T**, *in* beta:**T**, *in* bias:**T**, *out* output:**T**, *out* mean:**U**, *out* inv_std_var:**U**)|1+|**T** = tensor(float), tensor(float16)| -|TransposeMatMul|(*in* A:**T**, *in* B:**T**, *out* Y:**T**)|1+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16)| -| | -| | - - -## Operators implemented by DnnlExecutionProvider - -| Op Name | Parameters | OpSet Version | Types Supported | -|---------|------------|---------------|-----------------| -|**Operator Domain:** *ai.onnx.ml*|||| -|Gemm|(*in* A:**T**, *in* B:**T**, *in* C:**T**, *out* Y:**T**)|7+|**T** = tensor(float)| +|Attention|*in* input:**T**
*in* weight:**T**
*in* bias:**T**
*in* mask_index:**M**
*in* past:**T**
*out* output:**T**
*out* present:**T**|1+|**T** = tensor(float), tensor(float16)| +|BiasDropout|*in* data:**T**
*in* bias:**T**
*in* residual:**T**
*in* ratio:**T1**
*in* training_mode:**T2**
*out* output:**T**
*out* mask:**T2**|1+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16)
**T1** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16)
**T2** = tensor(bool)| +|BiasGelu|*in* A:**T**
*in* B:**T**
*out* C:**T**|1+|**T** = tensor(double), tensor(float), tensor(float16)| +|BiasSoftmax|*in* data:**T**
*in* bias:**T**
*out* output:**T**|1+|**T** = tensor(double), tensor(float), tensor(float16)| +|ComplexMul|*in* A:**T**
*in* B:**T**
*out* C:**T**|1+|**T** = tensor(float), tensor(float16)| +|ComplexMulConj|*in* A:**T**
*in* B:**T**
*out* C:**T**|1+|**T** = tensor(float), tensor(float16)| +|ConvTransposeWithDynamicPads|*in* X:**T**
*in* W:**T**
*in* Pads:**tensor(int64)**
*in* B:**T**
*out* Y:**T**|1+|**T** = tensor(float)| +|DequantizeLinear|*in* x:**T1**
*in* x_scale:**T2**
*in* x_zero_point:**T1**
*out* y:**T2**|1+|**T1** = tensor(int8), tensor(uint8)
**T2** = tensor(float16)| +|EmbedLayerNormalization|*in* input_ids:**T1**
*in* segment_ids:**T1**
*in* word_embedding:**T**
*in* position_embedding:**T**
*in* segment_embedding:**T**
*in* gamma:**T**
*in* beta:**T**
*in* mask:**T1**
*out* output:**T**
*out* mask_index:**T1**|1+|**T** = tensor(float), tensor(float16)| +|FastGelu|*in* X:**T**
*in* bias:**T**
*out* Y:**T**|1+|**T** = tensor(bfloat16), tensor(float), tensor(float16)| +|FusedConv|*in* X:**T**
*in* W:**T**
*in* B:**T**
*in* Z:**T**
*out* Y:**T**|1+|**T** = tensor(float)| +|FusedMatMul|*in* A:**T**
*in* B:**T**
*out* Y:**T**|1+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16)| +|Gelu|*in* X:**T**
*out* Y:**T**|1+|**T** = tensor(double), tensor(float), tensor(float16)| +|Inverse|*in* X:**T**
*out* Y:**T**|1+|**T** = tensor(double), tensor(float), tensor(float16)| +|Irfft|*in* X:**T**
*out* Y:**T**|1+|**T** = tensor(double), tensor(float), tensor(float16)| +|LongformerAttention|*in* input:**T**
*in* weight:**T**
*in* bias:**T**
*in* mask:**T**
*in* global_weight:**T**
*in* global_bias:**T**
*in* global:**G**
*out* output:**T**|1+|**T** = tensor(float), tensor(float16)| +|QAttention|*in* input:**T1**
*in* weight:**T2**
*in* bias:**T3**
*in* input_scale:**T3**
*in* weight_scale:**T3**
*in* mask_index:**T4**
*in* input_zero_point:**T1**
*in* weight_zero_point:**T2**
*in* past:**T3**
*out* output:**T3**
*out* present:**T3**|1+|**T1** = tensor(int8)
**T2** = tensor(int8)
**T3** = tensor(float), tensor(float16)
**T4** = tensor(int32)| +|QuantizeLinear|*in* x:**T1**
*in* y_scale:**T1**
*in* y_zero_point:**T2**
*out* y:**T2**|1+|**T1** = tensor(float16)
**T2** = tensor(int8), tensor(uint8)| +|Rfft|*in* X:**T**
*out* Y:**T**|1+|**T** = tensor(double), tensor(float), tensor(float16)| +|SkipLayerNormalization|*in* input:**T**
*in* skip:**T**
*in* gamma:**T**
*in* beta:**T**
*in* bias:**T**
*out* output:**T**
*out* mean:**U**
*out* inv_std_var:**U**|1+|**T** = tensor(float), tensor(float16)| +|TransposeMatMul|*in* A:**T**
*in* B:**T**
*out* Y:**T**|1+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16)| | | | | diff --git a/onnxruntime/python/onnxruntime_pybind_state.cc b/onnxruntime/python/onnxruntime_pybind_state.cc index e352093593..c78b12aa7a 100644 --- a/onnxruntime/python/onnxruntime_pybind_state.cc +++ b/onnxruntime/python/onnxruntime_pybind_state.cc @@ -828,17 +828,10 @@ void addGlobalMethods(py::module& m, Environment& env) { std::vector> factories = { onnxruntime::CreateExecutionProviderFactory_CPU(0), #ifdef USE_CUDA - onnxruntime::CreateExecutionProviderFactory_CUDA( - [&]() { - CUDAExecutionProviderInfo info{}; - info.device_id = cuda_device_id; - info.gpu_mem_limit = gpu_mem_limit; - info.arena_extend_strategy = arena_extend_strategy; - info.cudnn_conv_algo_search = cudnn_conv_algo_search; - info.do_copy_in_default_stream = do_copy_in_default_stream; - info.external_allocator_info = external_allocator_info; - return info; - }()), + []() { + OrtCUDAProviderOptions provider_options{}; + return CreateExecutionProviderFactory_Cuda(&provider_options); + }(), #endif #ifdef USE_ROCM onnxruntime::CreateExecutionProviderFactory_ROCM( diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index decac8cb95..329ae821d2 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -149,8 +149,7 @@ def parse_arguments(): help="Use parallel build. The optional value specifies the maximum number of parallel jobs. " "If the optional value is 0 or unspecified, it is interpreted as the number of CPUs.") parser.add_argument("--test", action='store_true', help="Run unit tests.") - parser.add_argument( - "--skip_tests", action='store_true', help="Skip all tests.") + parser.add_argument("--skip_tests", action='store_true', help="Skip all tests.") # Training options parser.add_argument( @@ -184,9 +183,10 @@ def parse_arguments(): available test data directories.""") # generate documentation - parser.add_argument( - "--gen_doc", action='store_true', - help="Generate documentation on contrib ops") + parser.add_argument("--gen_doc", nargs='?', const='yes', type=str, + help="Generate documentation listing standard ONNX operators and types implemented by " + "various execution providers and contrib operator schemas. " + "Use `--gen_doc validate` to validate these match the current contents in /docs.") parser.add_argument( "--gen-api-doc", action='store_true', @@ -412,15 +412,11 @@ def parse_arguments(): "--use_full_protobuf", action='store_true', help="Use the full protobuf library") - parser.add_argument( - "--skip_onnx_tests", action='store_true', help="Explicitly disable " - "all onnx related tests. Note: Use --skip_tests to skip all tests.") - parser.add_argument( - "--skip_winml_tests", action='store_true', - help="Explicitly disable all WinML related tests") - parser.add_argument( - "--skip_nodejs_tests", action='store_true', - help="Explicitly disable all Node.js binding tests") + parser.add_argument("--skip_onnx_tests", action='store_true', + help="Explicitly disable all onnx related tests. Note: Use --skip_tests to skip all tests.") + parser.add_argument("--skip_winml_tests", action='store_true', help="Explicitly disable all WinML related tests") + parser.add_argument("--skip_nodejs_tests", action='store_true', help="Explicitly disable all Node.js binding tests") + parser.add_argument( "--enable_msvc_static_runtime", action='store_true', help="Enable static linking of MSVC runtimes.") @@ -1770,52 +1766,49 @@ def build_protoc_for_host(cmake_path, source_dir, build_dir, args): return expected_protoc_path -def generate_documentation(source_dir, build_dir, configs): +def generate_documentation(source_dir, build_dir, configs, validate): # Randomly choose one build config config = next(iter(configs)) cwd = get_config_build_dir(build_dir, config) if is_windows(): cwd = os.path.join(cwd, config) - operator_doc_path = os.path.join(source_dir, 'docs', 'ContribOperators.md') - opkernel_doc_path = os.path.join(source_dir, 'docs', 'OperatorKernels.md') - shutil.copy( - os.path.join(source_dir, 'tools', 'python', 'gen_contrib_doc.py'), cwd) - shutil.copy( - os.path.join(source_dir, 'tools', 'python', 'gen_opkernel_doc.py'), - cwd) - run_subprocess( - [sys.executable, - 'gen_contrib_doc.py', - '--output_path', operator_doc_path], cwd=cwd) - run_subprocess( - [sys.executable, - 'gen_opkernel_doc.py', - '--output_path', opkernel_doc_path], cwd=cwd) - docdiff = '' - try: - docdiff = subprocess.check_output(['git', 'diff', opkernel_doc_path], cwd=source_dir) - except subprocess.CalledProcessError: - print('git diff returned non-zero error code') - if len(docdiff) > 0: - # Show warning instead of throwing exception, because it is - # dependent on build configuration for including - # execution propviders - log.warning( - 'The updated opkernel document file ' + str(opkernel_doc_path) + - ' is different from the checked in version. Consider ' - 'regenerating the file with CPU, DNNL and CUDA providers enabled.') - log.debug('diff:\n' + str(docdiff)) - docdiff = '' - try: - docdiff = subprocess.check_output(['git', 'diff', operator_doc_path], cwd=source_dir) - except subprocess.CalledProcessError: - print('git diff returned non-zero error code') - if len(docdiff) > 0: - raise BuildError( - 'The updated operator document file ' + - str(operator_doc_path) + ' must be checked in.\n diff:\n' + - str(docdiff)) + contrib_op_doc_path = os.path.join(source_dir, 'docs', 'ContribOperators.md') + opkernel_doc_path = os.path.join(source_dir, 'docs', 'OperatorKernels.md') + shutil.copy(os.path.join(source_dir, 'tools', 'python', 'gen_contrib_doc.py'), cwd) + shutil.copy(os.path.join(source_dir, 'tools', 'python', 'gen_opkernel_doc.py'), cwd) + # limit to just com.microsoft (excludes purely internal stuff like com.microsoft.nchwc). + run_subprocess([sys.executable, 'gen_contrib_doc.py', '--output_path', contrib_op_doc_path, + '--domains', 'com.microsoft'], cwd=cwd) + # we currently limit the documentation created by a build to the CPU and CUDA EPs. + # Run get_opkernel_doc.py directly if you need/want documentation from other EPs that are enabled in the build. + run_subprocess([sys.executable, 'gen_opkernel_doc.py', '--output_path', opkernel_doc_path, + '--providers', 'CPU', 'CUDA'], cwd=cwd) + + if validate: + try: + have_diff = False + + def diff_file(path, regenerate_qualifiers=''): + diff = subprocess.check_output(['git', 'diff', path], cwd=source_dir) + if diff: + nonlocal have_diff + have_diff = True + log.warning('The updated document {} is different from the checked in version. ' + 'Please regenerate the file{}, or copy the updated version from the ' + 'CI build\'s published artifacts if applicable.'.format(path, regenerate_qualifiers)) + log.debug('diff:\n' + str(diff)) + + diff_file(opkernel_doc_path, ' with CPU and CUDA execution providers enabled') + diff_file(contrib_op_doc_path) + + if have_diff: + # Output for the CI to publish the updated md files as an artifact + print('##vso[task.setvariable variable=DocUpdateNeeded]true') + raise BuildError('Generated documents have diffs. Check build output for details.') + + except subprocess.CalledProcessError: + raise BuildError('git diff returned non-zero error code') def main(): @@ -2052,10 +2045,13 @@ def main(): "currently through this script") if not is_docker() and not args.use_acl and not args.use_armnn: install_python_deps() + if args.enable_pybind and is_windows(): install_python_deps(args.numpy_version) + if args.enable_onnx_tests: setup_test_data(build_dir, configs) + if args.use_cuda and args.cuda_version is None: if is_windows(): # cuda_version is used while generating version_info.py on Windows. @@ -2064,6 +2060,7 @@ def main(): args.cuda_version = "" if args.use_rocm and args.rocm_version is None: args.rocm_version = "" + generate_build_tree( cmake_path, source_dir, build_dir, cuda_home, cudnn_home, rocm_home, mpi_home, nccl_home, tensorrt_home, migraphx_home, acl_home, acl_libs, armnn_home, armnn_libs, @@ -2142,7 +2139,7 @@ def main(): args.use_dnnl) if args.gen_doc and (args.build or args.test): - generate_documentation(source_dir, build_dir, configs) + generate_documentation(source_dir, build_dir, configs, args.gen_doc == 'validate') if args.gen_api_doc and (args.build or args.test): print('Generating Python doc for ORTModule...') diff --git a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml index 99be609ebd..be47768715 100644 --- a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml @@ -14,7 +14,7 @@ jobs: UseOmp: '--use_openmp' EnvSetupScript: setup_env.bat buildArch: x64 - additionalBuildFlags: --use_dnnl --build_java --build_nodejs --gen_doc + additionalBuildFlags: --use_dnnl --build_java --build_nodejs msbuildPlatform: x64 isX86: false x64_release: @@ -22,7 +22,7 @@ jobs: UseOmp: '' EnvSetupScript: setup_env.bat buildArch: x64 - additionalBuildFlags: --use_dnnl --build_java --build_nodejs --gen_doc + additionalBuildFlags: --use_dnnl --build_java --build_nodejs msbuildPlatform: x64 isX86: false x86_release: diff --git a/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml index 2150ac121c..27dd47a650 100644 --- a/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml @@ -21,6 +21,7 @@ jobs: msbuildPlatform: x64 isX86: false ALLOW_RELEASED_ONNX_OPSET_ONLY: '0' + DocUpdateNeeded: false timeoutInMinutes: 180 workspace: clean: all @@ -94,7 +95,7 @@ jobs: displayName: 'Generate cmake config' inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) $(UseOmp) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_wheel --disable_rtti --use_winml --build_shared_lib --enable_onnx_tests --enable_wcos $(additionalBuildFlags)' + arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) $(UseOmp) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_wheel --disable_rtti --use_winml --build_shared_lib --enable_onnx_tests --enable_wcos --gen_doc $(additionalBuildFlags)' workingDirectory: '$(Build.BinariesDirectory)' - task: VSBuild@1 @@ -147,12 +148,28 @@ jobs: - powershell: | Get-ChildItem -Path dist/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname} - python $(Build.SourcesDirectory)\tools\ci_build\build.py --config $(BuildConfig) --build_dir $(Build.BinariesDirectory) $(UseOmp) --skip_submodule_sync --build_shared_lib --test --cmake_generator "Visual Studio 16 2019" --build_wheel --disable_rtti --use_winml --build_shared_lib --enable_onnx_tests --enable_wcos $(additionalBuildFlags) - + python $(Build.SourcesDirectory)\tools\ci_build\build.py --config $(BuildConfig) --build_dir $(Build.BinariesDirectory) $(UseOmp) --test --enable_onnx_tests --gen_doc validate $(additionalBuildFlags) workingDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)' condition: and(succeeded(), eq('${{ parameters.RunOnnxRuntimeTests}}', true)) displayName: 'Run tests' + # if the validation from --gen_doc failed it sets a variable so we can publish the latest version of the docs + # as an artifact, allowing a developer to download this and replace the current version instead of having to build + # and generate the docs locally themselves. handle each of the two md files separately - simpler than copying + # them to another location and publishing from there in a single task. + - task: PublishBuildArtifacts@1 + condition: and(failed(), eq(variables['DocUpdateNeeded'], 'true')) + inputs: + pathtoPublish: '$(Build.SourcesDirectory)/docs/OperatorKernels.md' + artifactName: 'OperatorKernels.md' + + - task: PublishBuildArtifacts@1 + condition: and(failed(), eq(variables['DocUpdateNeeded'], 'true')) + inputs: + pathtoPublish: '$(Build.SourcesDirectory)/docs/ContribOperators.md' + artifactName: 'ContribOperators.md' + + - task: PythonScript@0 displayName: 'Regenerate cmake config with STATIC_ANALYSIS=ON ' condition: and(succeeded(), eq(variables['BuildConfig'], 'RelWithDebInfo')) diff --git a/tools/python/gen_contrib_doc.py b/tools/python/gen_contrib_doc.py index f4e3f0cfea..ec59f1e216 100644 --- a/tools/python/gen_contrib_doc.py +++ b/tools/python/gen_contrib_doc.py @@ -10,6 +10,7 @@ from __future__ import unicode_literals from collections import defaultdict import io import os +import pathlib import sys import argparse @@ -18,7 +19,7 @@ import numpy as np # type: ignore import onnxruntime.capi.onnxruntime_pybind11_state as rtpy from onnxruntime.capi.onnxruntime_pybind11_state import schemadef # noqa: F401 from onnxruntime.capi.onnxruntime_pybind11_state.schemadef import OpSchema # noqa: F401 -from typing import Any, Text, Sequence, Dict, List, Type, Set, Tuple +from typing import Any, Text, Sequence, Dict, List, Set, Tuple from onnx import AttributeProto, FunctionProto ONNX_ML = not bool(os.getenv('ONNX_ML') == '0') @@ -37,9 +38,13 @@ def display_number(v): # type: (int) -> Text return Text(v) -def should_render_domain(domain): # type: (Text) -> bool +def should_render_domain(domain, domain_filter): # type: (Text) -> bool if domain == ONNX_DOMAIN or domain == '' or domain == ONNX_ML_DOMAIN or domain == 'ai.onnx.ml': return False + + if domain_filter and domain not in domain_filter: + return False + return True @@ -308,19 +313,18 @@ def support_level_str(level): # type: (OpSchema.SupportType) -> Text # "experimental " if status == OperatorStatus.Value('EXPERIMENTAL') else "" # type: ignore -def main(args): # type: (Type[Args]) -> None +def main(output_path: str, domain_filter: [str]): - with io.open(args.output, 'w', newline='', encoding="utf-8") as fout: + with io.open(output_path, 'w', newline='', encoding="utf-8") as fout: fout.write('## Contrib Operator Schemas\n') fout.write( - "*This file is automatically generated from the\n" - " [def files](/onnxruntime/core/graph/contrib_ops/contrib_defs.cc) via " - "[this script](/tools/python/gen_contrib_doc.py).\n" - " Do not modify directly and instead edit operator definitions.*\n") + "*This file is automatically generated from the registered contrib operator schemas by " + "[this script](https://github.com/microsoft/onnxruntime/blob/master/tools/python/gen_contrib_doc.py).\n" + "Do not modify directly.*\n") # domain -> support level -> name -> [schema] - index = defaultdict(lambda: defaultdict(lambda: defaultdict(list)) # type: Dict[Text, Dict[int, Dict[Text, List[OpSchema]]]] # noqa: E501 - ) + index = defaultdict(lambda: defaultdict(lambda: defaultdict(list))) # type: Dict[Text, Dict[int, Dict[Text, List[OpSchema]]]] # noqa: E501 + for schema in rtpy.get_all_operator_schema(): index[schema.domain][int(schema.support_level)][schema.name].append(schema) @@ -331,7 +335,7 @@ def main(args): # type: (Type[Args]) -> None operator_schemas = list() # type: List[Tuple[Text, List[Tuple[int, List[Tuple[Text, OpSchema, List[OpSchema]]]]]]] # noqa: E501 exsting_ops = set() # type: Set[Text] for domain, _supportmap in sorted(index.items()): - if not should_render_domain(domain): + if not should_render_domain(domain, domain_filter): continue processed_supportmap = list() @@ -384,13 +388,13 @@ def main(args): # type: (Type[Args]) -> None if __name__ == '__main__': - parser = argparse.ArgumentParser(description='ONNX Runtime Operator Documentation Generator') - parser.add_argument('--output_path', help='output markdown file path', - default=os.path.join(os.path.dirname(os.path.realpath(__file__)), 'ContribOperators.md') - ) + parser = argparse.ArgumentParser(description='ONNX Runtime Contrib Operator Documentation Generator') + parser.add_argument('--domains', nargs='+', + help="Filter to specified domains. " + "e.g. `--domains com.microsoft com.microsoft.nchwc`") + parser.add_argument('--output_path', help='output markdown file path', type=pathlib.Path, required=True, + default=os.path.join(os.path.dirname(os.path.realpath(__file__)), 'ContribOperators.md')) args = parser.parse_args() + output_path = args.output_path.resolve() - class Args(object): - output = args.output_path - - main(Args) + main(output_path, args.domains) diff --git a/tools/python/gen_opkernel_doc.py b/tools/python/gen_opkernel_doc.py index 9481846b96..992115eebd 100644 --- a/tools/python/gen_opkernel_doc.py +++ b/tools/python/gen_opkernel_doc.py @@ -2,13 +2,12 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. - -from collections import defaultdict +import argparse import io import os -import argparse +import pathlib +from collections import defaultdict -from typing import Type import onnxruntime.capi.onnxruntime_pybind11_state as rtpy @@ -44,36 +43,49 @@ def format_param_strings(params): if firstparam: firstparam = False else: - s += ' or ' + s += '

or

' s += param return s -def main(args): # type: (Type[Args]) -> None +def expand_providers(provider_filter: [str]): + providers = set() + if provider_filter: + for provider in provider_filter: + p = provider.lower() + if not p.endswith('executionprovider'): + p += 'executionprovider' + providers.add(p) - with io.open(args.output, 'w', newline='', encoding="utf-8") as fout: - fout.write('## Supported Operators Data Types\n') + return providers + + +def main(output_path: pathlib.Path, provider_filter: [str]): + + providers = expand_providers(provider_filter) + + with io.open(output_path, 'w', newline='', encoding="utf-8") as fout: + fout.write('## Supported Operators and Data Types\n') fout.write( - "*This file is automatically generated from the\n" - " [def files](/onnxruntime/core/providers/cpu/cpu_execution_provider.cc) via " - "[this script](/tools/python/gen_opkernel_doc.py).\n" - " Do not modify directly and instead edit operator definitions.*\n") + "*This file is automatically generated from the registered kernels by " + "[this script](https://github.com/microsoft/onnxruntime/blob/master/tools/python/gen_opkernel_doc.py).\n" + "Do not modify directly.*\n\n") opdef = rtpy.get_all_operator_schema() paramdict = {} for schema in opdef: inputs = schema.inputs domain = schema.domain if (domain == ''): - domain = 'ai.onnx.ml' + domain = 'ai.onnx' fullname = domain+'.'+schema.name - paramstr = '(' + paramstr = '' firstinput = True if inputs: for inp in inputs: if firstinput: firstinput = False else: - paramstr += ', ' + paramstr += '
' paramstr += '*in* {}:**{}**'.format(inp.name, inp.typeStr) outputs = schema.outputs @@ -82,10 +94,10 @@ def main(args): # type: (Type[Args]) -> None if firstinput: firstinput = False else: - paramstr += ', ' + paramstr += '
' paramstr += '*out* {}:**{}**'.format(outp.name, outp.typeStr) - paramstr += ')' + paramstr += '' paramset = paramdict.get(fullname, None) if paramset is None: paramdict[fullname] = set() @@ -96,12 +108,23 @@ def main(args): # type: (Type[Args]) -> None for op in rtpy.get_all_opkernel_def(): domain = op.domain if (domain == ''): - domain = 'ai.onnx.ml' + domain = 'ai.onnx' index[op.provider][domain][op.op_name].append(op) - fout.write('\n') + # TOC + fout.write('## Execution Providers\n\n') + for provider in sorted(index.keys()): + if providers and provider.lower() not in providers: + continue + fout.write('- [{}](#{})\n'.format(provider, provider.lower())) + fout.write('\n---------------') + for provider, domainmap in sorted(index.items()): - fout.write('\n\n## Operators implemented by '+provider+'\n\n') + if providers and provider.lower() not in providers: + continue + + fout.write('\n\n\n\n'.format(provider.lower())) + fout.write('## Operators implemented by {}\n\n'.format(provider)) fout.write('| Op Name | Parameters | OpSet Version | Types Supported |\n') fout.write('|---------|------------|---------------|-----------------|\n') for domain, namemap in sorted(domainmap.items()): @@ -138,11 +161,13 @@ def main(args): # type: (Type[Args]) -> None if __name__ == '__main__': parser = argparse.ArgumentParser(description='ONNX Runtime Operator Kernel Documentation Generator') - parser.add_argument('--output_path', help='output markdown file path', - default=os.path.join(os.path.dirname(os.path.realpath(__file__)), 'OperatorKernels.md') - ) + parser.add_argument('--providers', nargs='+', + help="Filter to specified execution providers. Case-insensitive. " + "Matches provider names from /include/onnxruntime/core/graph/constants.h'. " + "'ExecutionProvider' is automatically appended as needed. " + "e.g. `--providers cpu cuda` will match CPUExecutionProvider and CUDAExecutionProvider.") + parser.add_argument('--output_path', help='output markdown file path', type=pathlib.Path, required=True, + default=os.path.join(os.path.dirname(os.path.realpath(__file__)), 'OperatorKernels.md')) args = parser.parse_args() - class Args(object): - output = args.output_path - main(Args) + main(args.output_path, args.providers)