diff --git a/VERSION_NUMBER b/VERSION_NUMBER index 4cda8f19ed..dc1e644a10 100644 --- a/VERSION_NUMBER +++ b/VERSION_NUMBER @@ -1 +1 @@ -1.5.2 +1.6.0 diff --git a/docs/Versioning.md b/docs/Versioning.md index 6edaaea577..4aa736977b 100644 --- a/docs/Versioning.md +++ b/docs/Versioning.md @@ -26,11 +26,13 @@ For more details on ONNX Release versions, see [this page](https://github.com/on | ONNX Runtime release version | ONNX release version | ONNX opset version | ONNX ML opset version | Supported ONNX IR version | [Windows ML Availability](https://docs.microsoft.com/en-us/windows/ai/windows-ml/release-notes/)| |------------------------------|--------------------|--------------------|----------------------|------------------|------------------| -| 1.5.2 | **1.7** down to 1.2 | 12 | 2 | 6 | Windows AI 1.5+ | -| 1.5.1 | **1.7** down to 1.2 | 12 | 2 | 6 | Windows AI 1.5+ | -| 1.4.0 | **1.7** down to 1.2 | 12 | 2 | 6 | Windows AI 1.4+ | -| 1.3.1 | **1.7** down to 1.2 | 12 | 2 | 6 | Windows AI 1.4+ | -| 1.3.0 | **1.7** down to 1.2 | 12 | 2 | 6 | Windows AI 1.3+ | +| 1.6.0 | **1.8** down to 1.2 | 13 | 2 | 7 | Windows AI 1.6+ | +| 1.5.3 | **1.7** down to 1.2 | 12 | 2 | 7 | Windows AI 1.5+ | +| 1.5.2 | **1.7** down to 1.2 | 12 | 2 | 7 | Windows AI 1.5+ | +| 1.5.1 | **1.7** down to 1.2 | 12 | 2 | 7 | Windows AI 1.5+ | +| 1.4.0 | **1.7** down to 1.2 | 12 | 2 | 7 | Windows AI 1.4+ | +| 1.3.1 | **1.7** down to 1.2 | 12 | 2 | 7 | Windows AI 1.4+ | +| 1.3.0 | **1.7** down to 1.2 | 12 | 2 | 7 | Windows AI 1.3+ | | 1.2.0
1.1.2
1.1.1
1.1.0 | **1.6** down to 1.2 | 11 | 2 | 6 | Windows AI 1.3+ | | 1.0.0 | **1.6** down to 1.2 | 11 | 2 | 6 | Windows AI 1.3+ | | 0.5.0 | **1.5** down to 1.2 | 10 | 1 | 5 | Windows AI 1.3+ | diff --git a/docs/python/README.rst b/docs/python/README.rst index 59582d456d..5bc52fe4ff 100644 --- a/docs/python/README.rst +++ b/docs/python/README.rst @@ -8,6 +8,16 @@ For more information on ONNX Runtime, please see `aka.ms/onnxruntime // This value is used in structures passed to ORT so that a newer version of ORT will still work with them -#define ORT_API_VERSION 5 +#define ORT_API_VERSION 6 #ifdef __cplusplus extern "C" { diff --git a/nodejs/package-lock.json b/nodejs/package-lock.json index dbc45af31a..d2986b83d4 100644 --- a/nodejs/package-lock.json +++ b/nodejs/package-lock.json @@ -1,6 +1,6 @@ { "name": "onnxruntime", - "version": "1.5.2", + "version": "1.6.0", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -1779,9 +1779,9 @@ "dev": true }, "highlight.js": { - "version": "10.2.1", - "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-10.2.1.tgz", - "integrity": "sha1-CXhP4ulWEqu+/VEJSJRdT+b6lmg=", + "version": "10.4.1", + "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-10.4.1.tgz", + "integrity": "sha512-yR5lWvNz7c85OhVAEAeFhVCc/GV4C30Fjzc/rCP0aCWzc1UUOPUk55dK/qdwTZHBvMZo+eZ2jpk62ndX/xMFlg==", "dev": true }, "hosted-git-info": { @@ -4258,4 +4258,4 @@ } } } -} +} \ No newline at end of file diff --git a/nodejs/package.json b/nodejs/package.json index 86cf9104bf..a1129a8577 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -1,7 +1,7 @@ { "name": "onnxruntime", "description": "Node.js binding of ONNXRuntime", - "version": "1.5.2", + "version": "1.6.0", "main": "./lib/index.js", "types": "./types/lib/index.d.ts", "scripts": { @@ -69,4 +69,4 @@ "dependencies": { "prebuild-install": "^5.3.5" } -} +} \ No newline at end of file diff --git a/onnxruntime/__init__.py b/onnxruntime/__init__.py index c46534ade7..a0dfd33df8 100644 --- a/onnxruntime/__init__.py +++ b/onnxruntime/__init__.py @@ -7,7 +7,7 @@ ONNX Runtime is a performance-focused scoring engine for Open Neural Network Exc For more information on ONNX Runtime, please see `aka.ms/onnxruntime `_ or the `Github project `_. """ -__version__ = "1.5.2" +__version__ = "1.6.0" __author__ = "Microsoft" from onnxruntime.capi._pybind_state import get_all_providers, get_available_providers, get_device, set_seed, \ diff --git a/onnxruntime/core/providers/cpu/ml/tree_ensemble_classifier.cc b/onnxruntime/core/providers/cpu/ml/tree_ensemble_classifier.cc index f4512e5df0..216bacd607 100644 --- a/onnxruntime/core/providers/cpu/ml/tree_ensemble_classifier.cc +++ b/onnxruntime/core/providers/cpu/ml/tree_ensemble_classifier.cc @@ -139,7 +139,7 @@ template TreeEnsembleClassifier::TreeEnsembleClassifier(const OpKernelInfo& info) : OpKernel(info), tree_ensemble_( - 100, + 80, 50, info.GetAttrOrDefault("aggregate_function", "SUM"), info.GetAttrsOrDefault("base_values"), diff --git a/onnxruntime/core/providers/cpu/ml/tree_ensemble_common.h b/onnxruntime/core/providers/cpu/ml/tree_ensemble_common.h index 0dbdf544bc..4a06f5a7a2 100644 --- a/onnxruntime/core/providers/cpu/ml/tree_ensemble_common.h +++ b/onnxruntime/core/providers/cpu/ml/tree_ensemble_common.h @@ -262,126 +262,180 @@ void TreeEnsembleCommon::ComputeAgg(concurrency::ThreadPool* ttp, const ITYPE* x_data = X->template Data(); OTYPE* z_data = Z->template MutableData(); int64_t* label_data = label == nullptr ? nullptr : label->template MutableData(); + auto max_num_threads = concurrency::ThreadPool::DegreeOfParallelism(ttp); if (n_targets_or_classes_ == 1) { if (N == 1) { ScoreValue score = {0, 0}; - if (n_trees_ <= parallel_tree_) { + if (n_trees_ <= parallel_tree_) { /* section A: 1 output, 1 row and not enough trees to parallelize */ for (int64_t j = 0; j < n_trees_; ++j) { agg.ProcessTreeNodePrediction1(score, *ProcessTreeNodeLeave(roots_[j], x_data)); } - } else { - std::vector> scores_t(n_trees_, {0, 0}); + } else { /* section B: 1 output, 1 row and enough trees to parallelize */ + std::vector> scores(n_trees_, {0, 0}); concurrency::ThreadPool::TryBatchParallelFor( ttp, SafeInt(n_trees_), - [this, &scores_t, &agg, x_data](ptrdiff_t j) { - agg.ProcessTreeNodePrediction1(scores_t[j], *ProcessTreeNodeLeave(roots_[j], x_data)); + [this, &scores, &agg, x_data](ptrdiff_t j) { + agg.ProcessTreeNodePrediction1(scores[j], *ProcessTreeNodeLeave(roots_[j], x_data)); }, 0); - for (auto it = scores_t.cbegin(); it != scores_t.cend(); ++it) { + for (auto it = scores.cbegin(); it != scores.cend(); ++it) { agg.MergePrediction1(score, *it); } } - agg.FinalizeScores1(z_data, score, label_data); - } else { - if (N <= parallel_N_) { - ScoreValue score; - size_t j; + } else if (N <= parallel_N_) { /* section C: 1 output, 2+ rows but not enough rows to parallelize */ + ScoreValue score; + size_t j; - for (int64_t i = 0; i < N; ++i) { - score = {0, 0}; - for (j = 0; j < static_cast(n_trees_); ++j) { - agg.ProcessTreeNodePrediction1(score, *ProcessTreeNodeLeave(roots_[j], x_data + i * stride)); - } - - agg.FinalizeScores1(z_data + i * n_targets_or_classes_, score, - label_data == nullptr ? nullptr : (label_data + i)); + for (int64_t i = 0; i < N; ++i) { + score = {0, 0}; + for (j = 0; j < static_cast(n_trees_); ++j) { + agg.ProcessTreeNodePrediction1(score, *ProcessTreeNodeLeave(roots_[j], x_data + i * stride)); } - } else { - concurrency::ThreadPool::TryBatchParallelFor( - ttp, - SafeInt(N), - [this, &agg, x_data, z_data, stride, label_data](ptrdiff_t i) { - ScoreValue score = {0, 0}; - for (size_t j = 0; j < static_cast(n_trees_); ++j) { - agg.ProcessTreeNodePrediction1(score, *ProcessTreeNodeLeave(roots_[j], x_data + i * stride)); - } - agg.FinalizeScores1(z_data + i * n_targets_or_classes_, score, - label_data == nullptr ? nullptr : (label_data + i)); - }, - 0); + agg.FinalizeScores1(z_data + i, score, + label_data == nullptr ? nullptr : (label_data + i)); } + } else if (n_trees_ > max_num_threads) { /* section D: 1 output, 2+ rows and enough trees to parallelize */ + auto num_threads = std::min(max_num_threads, SafeInt(n_trees_)); + std::vector> scores(num_threads * N); + concurrency::ThreadPool::TrySimpleParallelFor( + ttp, + num_threads, + [this, &agg, &scores, num_threads, x_data, N, stride](ptrdiff_t batch_num) { + auto work = concurrency::ThreadPool::PartitionWork(batch_num, num_threads, this->n_trees_); + for (int64_t i = 0; i < N; ++i) { + scores[batch_num * N + i] = {0, 0}; + } + for (auto j = work.start; j < work.end; ++j) { + for (int64_t i = 0; i < N; ++i) { + agg.ProcessTreeNodePrediction1(scores[batch_num * N + i], *ProcessTreeNodeLeave(roots_[j], x_data + i * stride)); + } + } + }); + + concurrency::ThreadPool::TrySimpleParallelFor( + ttp, + num_threads, + [&agg, &scores, num_threads, label_data, z_data, N](ptrdiff_t batch_num) { + auto work = concurrency::ThreadPool::PartitionWork(batch_num, num_threads, N); + for (auto i = work.start; i < work.end; ++i) { + for (int64_t j = 1; j < num_threads; ++j) { + agg.MergePrediction1(scores[i], scores[j * N + i]); + } + agg.FinalizeScores1(z_data + i, scores[i], + label_data == nullptr ? nullptr : (label_data + i)); + } + }); + } else { /* section E: 1 output, 2+ rows, parallelization by rows */ + concurrency::ThreadPool::TryBatchParallelFor( + ttp, + SafeInt(N), + [this, &agg, x_data, z_data, stride, label_data](ptrdiff_t i) { + ScoreValue score = {0, 0}; + for (size_t j = 0; j < static_cast(n_trees_); ++j) { + agg.ProcessTreeNodePrediction1(score, *ProcessTreeNodeLeave(roots_[j], x_data + i * stride)); + } + + agg.FinalizeScores1(z_data + i, score, + label_data == nullptr ? nullptr : (label_data + i)); + }, + 0); } } else { - if (N == 1) { - std::vector> scores(n_targets_or_classes_, {0, 0}); - if (n_trees_ <= parallel_tree_) { + if (N == 1) { /* section A2: 2+ outputs, 1 row, not enough trees to parallelize */ + if (n_trees_ <= parallel_tree_) { /* section A2 */ + std::vector> scores(n_targets_or_classes_, {0, 0}); for (int64_t j = 0; j < n_trees_; ++j) { agg.ProcessTreeNodePrediction(scores, *ProcessTreeNodeLeave(roots_[j], x_data)); } - } else { - // split the work into one block per thread so we can re-use the 'private_scores' vector as much as possible - // TODO: Refine the number of threads used - auto num_threads = std::min(concurrency::ThreadPool::DegreeOfParallelism(ttp), SafeInt(n_trees_)); - OrtMutex merge_mutex; + agg.FinalizeScores(scores, z_data, -1, label_data); + } else { /* section B2: 2+ outputs, 1 row, enough trees to parallelize */ + auto num_threads = std::min(max_num_threads, SafeInt(n_trees_)); + std::vector>> scores(num_threads); concurrency::ThreadPool::TrySimpleParallelFor( ttp, num_threads, - [this, &agg, &scores, &merge_mutex, num_threads, x_data](ptrdiff_t batch_num) { - std::vector> private_scores(n_targets_or_classes_, {0, 0}); + [this, &agg, &scores, num_threads, x_data](ptrdiff_t batch_num) { + scores[batch_num].resize(n_targets_or_classes_, {0, 0}); auto work = concurrency::ThreadPool::PartitionWork(batch_num, num_threads, n_trees_); for (auto j = work.start; j < work.end; ++j) { - agg.ProcessTreeNodePrediction(private_scores, *ProcessTreeNodeLeave(roots_[j], x_data)); + agg.ProcessTreeNodePrediction(scores[batch_num], *ProcessTreeNodeLeave(roots_[j], x_data)); } - - std::lock_guard lock(merge_mutex); - agg.MergePrediction(scores, private_scores); }); - } - - agg.FinalizeScores(scores, z_data, -1, label_data); - } else { - if (N <= parallel_N_) { - std::vector> scores(n_targets_or_classes_); - size_t j; - - for (int64_t i = 0; i < N; ++i) { - std::fill(scores.begin(), scores.end(), ScoreValue({0, 0})); - for (j = 0; j < roots_.size(); ++j) { - agg.ProcessTreeNodePrediction(scores, *ProcessTreeNodeLeave(roots_[j], x_data + i * stride)); - } - - agg.FinalizeScores(scores, z_data + i * n_targets_or_classes_, -1, - label_data == nullptr ? nullptr : (label_data + i)); + for (size_t i = 1; i < scores.size(); ++i) { + agg.MergePrediction(scores[0], scores[i]); } - } else { - // split the work into one block per thread so we can re-use the 'scores' vector as much as possible - // TODO: Refine the number of threads used. - auto num_threads = std::min(concurrency::ThreadPool::DegreeOfParallelism(ttp), SafeInt(N)); - concurrency::ThreadPool::TrySimpleParallelFor( - ttp, - num_threads, - [this, &agg, num_threads, x_data, z_data, label_data, N, stride](ptrdiff_t batch_num) { - size_t j; - std::vector> scores(n_targets_or_classes_); - auto work = concurrency::ThreadPool::PartitionWork(batch_num, num_threads, N); - - for (auto i = work.start; i < work.end; ++i) { - std::fill(scores.begin(), scores.end(), ScoreValue({0, 0})); - for (j = 0; j < roots_.size(); ++j) { - agg.ProcessTreeNodePrediction(scores, *ProcessTreeNodeLeave(roots_[j], x_data + i * stride)); - } - - agg.FinalizeScores(scores, - z_data + i * n_targets_or_classes_, -1, - label_data == nullptr ? nullptr : (label_data + i)); - } - }); + agg.FinalizeScores(scores[0], z_data, -1, label_data); } + } else if (N <= parallel_N_) { /* section C2: 2+ outputs, 2+ rows, not enough rows to parallelize */ + std::vector> scores(n_targets_or_classes_); + size_t j; + + for (int64_t i = 0; i < N; ++i) { + std::fill(scores.begin(), scores.end(), ScoreValue({0, 0})); + for (j = 0; j < roots_.size(); ++j) { + agg.ProcessTreeNodePrediction(scores, *ProcessTreeNodeLeave(roots_[j], x_data + i * stride)); + } + + agg.FinalizeScores(scores, z_data + i * n_targets_or_classes_, -1, + label_data == nullptr ? nullptr : (label_data + i)); + } + } else if (n_trees_ >= max_num_threads) { /* section: D2: 2+ outputs, 2+ rows, enough trees to parallelize*/ + auto num_threads = std::min(max_num_threads, SafeInt(n_trees_)); + std::vector>> scores(num_threads * N); + concurrency::ThreadPool::TrySimpleParallelFor( + ttp, + num_threads, + [this, &agg, &scores, num_threads, x_data, N, stride](ptrdiff_t batch_num) { + auto work = concurrency::ThreadPool::PartitionWork(batch_num, num_threads, this->n_trees_); + for (int64_t i = 0; i < N; ++i) { + scores[batch_num * N + i].resize(n_targets_or_classes_, {0, 0}); + } + for (auto j = work.start; j < work.end; ++j) { + for (int64_t i = 0; i < N; ++i) { + agg.ProcessTreeNodePrediction(scores[batch_num * N + i], *ProcessTreeNodeLeave(roots_[j], x_data + i * stride)); + } + } + }); + + concurrency::ThreadPool::TrySimpleParallelFor( + ttp, + num_threads, + [this, &agg, &scores, num_threads, label_data, z_data, N](ptrdiff_t batch_num) { + auto work = concurrency::ThreadPool::PartitionWork(batch_num, num_threads, N); + for (auto i = work.start; i < work.end; ++i) { + for (int64_t j = 1; j < num_threads; ++j) { + agg.MergePrediction(scores[i], scores[j * N + i]); + } + agg.FinalizeScores(scores[i], z_data + i * this->n_targets_or_classes_, -1, + label_data == nullptr ? nullptr : (label_data + i)); + } + }); + } else { /* section E2: 2+ outputs, 2+ rows, parallelization by rows */ + auto num_threads = std::min(max_num_threads, SafeInt(N)); + concurrency::ThreadPool::TrySimpleParallelFor( + ttp, + num_threads, + [this, &agg, num_threads, x_data, z_data, label_data, N, stride](ptrdiff_t batch_num) { + size_t j; + std::vector> scores(n_targets_or_classes_); + auto work = concurrency::ThreadPool::PartitionWork(batch_num, num_threads, N); + + for (auto i = work.start; i < work.end; ++i) { + std::fill(scores.begin(), scores.end(), ScoreValue({0, 0})); + for (j = 0; j < roots_.size(); ++j) { + agg.ProcessTreeNodePrediction(scores, *ProcessTreeNodeLeave(roots_[j], x_data + i * stride)); + } + + agg.FinalizeScores(scores, + z_data + i * n_targets_or_classes_, -1, + label_data == nullptr ? nullptr : (label_data + i)); + } + }); } } } // namespace detail diff --git a/onnxruntime/core/providers/cpu/ml/treeregressor.cc b/onnxruntime/core/providers/cpu/ml/treeregressor.cc index cfee2ccae8..960e4fcf97 100644 --- a/onnxruntime/core/providers/cpu/ml/treeregressor.cc +++ b/onnxruntime/core/providers/cpu/ml/treeregressor.cc @@ -24,7 +24,7 @@ template TreeEnsembleRegressor::TreeEnsembleRegressor(const OpKernelInfo& info) : OpKernel(info), tree_ensemble_( - 100, + 80, 50, info.GetAttrOrDefault("aggregate_function", "SUM"), info.GetAttrsOrDefault("base_values"), diff --git a/onnxruntime/core/providers/cpu/reduction/reduction_ops.cc b/onnxruntime/core/providers/cpu/reduction/reduction_ops.cc index 14dc9151e0..fd3a0d485d 100644 --- a/onnxruntime/core/providers/cpu/reduction/reduction_ops.cc +++ b/onnxruntime/core/providers/cpu/reduction/reduction_ops.cc @@ -74,10 +74,10 @@ namespace onnxruntime { x); #define REGISTER_UNARY_ELEMENTWISE_VERSIONED_KERNEL_INT8_ONLY(x, startVer, endVer) \ - ONNX_CPU_OPERATOR_VERSIONED_TYPED_KERNEL( \ - x, \ - startVer, \ - endVer, \ + ONNX_CPU_OPERATOR_VERSIONED_TYPED_KERNEL( \ + x, \ + startVer, \ + endVer, \ int8_t, \ KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), \ x); @@ -138,7 +138,6 @@ REGISTER_UNARY_ELEMENTWISE_KERNEL_INT64_ONLY(ReduceMax, 13); REGISTER_UNARY_ELEMENTWISE_KERNEL_INT8_ONLY(ReduceMax, 13); REGISTER_UNARY_ELEMENTWISE_KERNEL_UINT8_ONLY(ReduceMax, 13); - REGISTER_UNARY_ELEMENTWISE_VERSIONED_KERNEL(ReduceMean, 1, 10); REGISTER_UNARY_ELEMENTWISE_VERSIONED_KERNEL(ReduceMean, 11, 12); REGISTER_UNARY_ELEMENTWISE_KERNEL(ReduceMean, 13); @@ -361,6 +360,9 @@ void NoTransposeReduce(Tensor* output, const TensorShape& new_input_shape, const if (last_results.last_loop_red_size == 0 || last_results.last_loop_size == 0) return; } + ORT_ENFORCE(last_results.last_loop_red_size > 0); + ORT_ENFORCE(last_results.last_loop_size > 0); + ORT_ENFORCE(last_results.projected_index.size() > 0); int64_t denominator = last_results.last_loop_red_size * last_results.projected_index.size(); if (AGG::two_loops()) { diff --git a/onnxruntime/core/providers/cpu/reduction/reduction_ops.h b/onnxruntime/core/providers/cpu/reduction/reduction_ops.h index 9b0c30b880..092e6d0abe 100644 --- a/onnxruntime/core/providers/cpu/reduction/reduction_ops.h +++ b/onnxruntime/core/providers/cpu/reduction/reduction_ops.h @@ -24,6 +24,14 @@ class ResultsNoTransposePrepareForReduce { std::vector unprojected_index; int64_t last_loop_size; int64_t last_loop_inc; + + ResultsNoTransposePrepareForReduce() : input_shape(), reduced_axes(), projected_index(), unprojected_index() { + last_loop_red_size = 0; + last_loop_red_inc = 0; + last_loop_size = 0; + last_loop_inc = 0; + } + bool equal(const std::vector& local_input_shape, const std::vector& local_reduced_axes) { if (input_shape.size() != local_input_shape.size()) return false; diff --git a/onnxruntime/core/providers/cpu/tensor/transpose.cc b/onnxruntime/core/providers/cpu/tensor/transpose.cc index 7503ab946b..06777cd48a 100644 --- a/onnxruntime/core/providers/cpu/tensor/transpose.cc +++ b/onnxruntime/core/providers/cpu/tensor/transpose.cc @@ -15,23 +15,85 @@ namespace onnxruntime { etc. */ -// ComputeOffset: compute offset into a tensor. This is essentially the dot-product of -// index and stride, restricted to the specified number of axes. -static inline size_t ComputeOffset(const std::vector& index, const std::vector& stride, int64_t num_axes) { - size_t offset = 0; - for (int64_t j = 0; j < num_axes; ++j) { - offset += index[j] * stride[j]; +struct MultiIndex { + size_t n_axes; + std::vector index; + std::vector upper_bound; + std::vector stride; + + /* There is one MultiIndex instance per axis in the tensor. + * The array keeps track of the position of a pointer walking through the data. + * Any function using it creates an array of MultiIndex + * then calls function IncrementIndexAndComputeOffsetSetup + * to initialize the array. This constructor does not initialize + * anything because it would be overwritten by function + * IncrementIndexAndComputeOffsetSetup. This one calls method Init. + * Function IncrementIndexAndComputeOffset is called to increment + * the array of MultiIndex to move to the next data in the tensor. + */ + MultiIndex() : index(), upper_bound(), stride() { n_axes = 0; } + + void Init(size_t num_axes) { + index.resize(num_axes); + upper_bound.resize(num_axes); + stride.resize(num_axes); + n_axes = num_axes; } - return offset; + + void InitAxis(size_t n_axis, size_t i, size_t n, int64_t s) { + index[n_axis] = i; + upper_bound[n_axis] = n; + stride[n_axis] = s; + } +}; + +/* This function initializes an array of MultiIndex of size num_axes (one instance per axis). +* target_dims is the shape of the transposed tensor, stride is linked to the tensor to +* be transposed, if source_dims is the shape, stride[i] = source_dims[i+1] * source_dims[i+2] * ... * 1. +* element_size is the size of the tensor element (sizeof(float), sizeof(double)). +*/ +static void IncrementIndexAndComputeOffsetSetup(MultiIndex& mindex, size_t num_axes, const std::vector& target_dims, + const std::vector& stride, size_t element_size) { + mindex.Init(num_axes); + size_t naxes = 0; + for (size_t i = 0; i < num_axes; ++i) { + if (target_dims[i] == 1) + continue; + mindex.InitAxis(naxes, 0, static_cast(target_dims[i]), stride[i] * element_size); + ++naxes; + } + ORT_ENFORCE(naxes > 0, "Method IncrementIndexAndComputeOffset assumes this value is strictly positive."); + mindex.n_axes = naxes; } -// IncrementIndex: Increment an index into a tensor (in lexicographic ordering), wrapping -// around the specified upper_bound. -static inline void IncrementIndex(std::vector& index, const std::vector& upper_bound, int64_t num_axes) { - for (int64_t k = num_axes - 1; k >= 0; --k) { - index[k]++; - if (index[k] < upper_bound[k]) break; - index[k] = 0; +/* This function increments an array of MultiIndex initialized by function IncrementIndexAndComputeOffsetSetup. +* It increments the last dimension, checks if it stays within boundary. If it stays in, it returns, +* otherwise, it reset the dimension to zero and increments the previous one. +* While doing that, every modification brought to the array of indices is applied on the +* pointer local_source. It avoids computing again local_source from the source tensor. +* At every time, the following condition is verified: +* local_source = source + (sum_i mindex[i].index * mindex[i].stride +*/ +template +static inline void IncrementIndexAndComputeOffset(MultiIndex& mindex, const T*& local_source) { + // Increment the last dimension. + int pos = static_cast(mindex.n_axes) - 1; + local_source += mindex.stride[pos]; + // Checks it stays within boundaries. + if (++mindex.index[pos] < mindex.upper_bound[pos]) + return; + // If not, loops on other indices. + // The first test is outside the loop to be faster. + // As it is the most common case. + local_source -= mindex.stride[pos] * mindex.index[pos]; + mindex.index[pos] = 0; + --pos; + for (; pos >= 0; --pos) { + local_source += mindex.stride[pos]; + if (++mindex.index[pos] < mindex.upper_bound[pos]) + break; + local_source -= mindex.stride[pos] * mindex.index[pos]; + mindex.index[pos] = 0; } } @@ -55,17 +117,14 @@ static void DoTransposeImpl(int64_t num_axes, const std::vector& target size_t num_blocks, size_t num_elts_in_block, const std::vector& stride, const uint8_t* source, uint8_t* target, size_t element_size) { size_t blocksize = num_elts_in_block * element_size; - // index used to iterate over target iteration-space - std::vector target_index(num_axes, 0); + MultiIndex mindex; + IncrementIndexAndComputeOffsetSetup(mindex, num_axes, target_dims, stride, element_size); + + const uint8_t* local_source = source; for (size_t i = 0; i < num_blocks; ++i) { - // convert target_index into an offset in source data - size_t source_offset = ComputeOffset(target_index, stride, num_axes); - - // copy - memcpy(target, source + source_offset * element_size, blocksize); - - // increment target_index: - IncrementIndex(target_index, target_dims, num_axes); + ORT_ENFORCE((local_source >= source) && (local_source < source + num_blocks * blocksize)); + memcpy(target, local_source, blocksize); + IncrementIndexAndComputeOffset(mindex, local_source); target += blocksize; } } @@ -73,17 +132,15 @@ static void DoTransposeImpl(int64_t num_axes, const std::vector& target static void DoTransposeImpl(int64_t num_axes, const std::vector& target_dims, size_t num_blocks, size_t num_elts_in_block, const std::vector& stride, const std::string* source, std::string* target) { - // index used to iterate over target iteration-space - std::vector target_index(num_axes, 0); + ORT_ENFORCE(num_axes > 0, "Transpose not implemented for empty tensors."); + MultiIndex mindex; + IncrementIndexAndComputeOffsetSetup(mindex, num_axes, target_dims, stride, 1); + + const std::string* local_source = source; for (size_t i = 0; i < num_blocks; ++i) { - // convert target_index into an offset in source data - size_t source_offset = ComputeOffset(target_index, stride, num_axes); - - // copy - DoTransposeSingleBlock(num_elts_in_block, source + source_offset, target); - - // increment target_index: - IncrementIndex(target_index, target_dims, num_axes); + ORT_ENFORCE((local_source >= source) && (local_source < source + num_blocks * num_elts_in_block)); + DoTransposeSingleBlock(num_elts_in_block, local_source, target); + IncrementIndexAndComputeOffset(mindex, local_source); target += num_elts_in_block; } } @@ -93,67 +150,40 @@ inline void CopyPrim(uint8_t* target, const uint8_t* source) { *reinterpret_cast(target) = *reinterpret_cast(source); } +// The function does not check num_axes > 0 but this is expected. +template +static void TypedDoTransposeEltWise(int64_t num_axes, const std::vector& target_dims, size_t num_blocks, + const std::vector& stride, const uint8_t* source, uint8_t* target) { + MultiIndex mindex; + IncrementIndexAndComputeOffsetSetup(mindex, num_axes, target_dims, stride, sizeof(T)); + + const uint8_t* local_source = source; + uint8_t* target_end = target + sizeof(T) * num_blocks; + for (; target != target_end; target += sizeof(T)) { + ORT_ENFORCE((local_source >= source) && (local_source < source + sizeof(T) * num_blocks)); + CopyPrim(target, local_source); + IncrementIndexAndComputeOffset(mindex, local_source); + } +} + // DoTransposeEltWise: specialization of DoTranspose for the num_elts_in_block=1 case. // copies source tensor to target, transposing elements. // The stride vector indicates the transposition. -static void DoTransposeEltWise(int64_t num_axes, const std::vector& target_dims, size_t num_blocks, - const std::vector& stride, const uint8_t* source, uint8_t* target, - size_t element_size) { - // index used to iterate over target iteration-space - std::vector target_index(num_axes, 0); - +void DoTransposeEltWise(int64_t num_axes, const std::vector& target_dims, size_t num_blocks, + const std::vector& stride, const uint8_t* source, uint8_t* target, + size_t element_size) { switch (element_size) { case sizeof(uint64_t): - for (size_t i = 0; i < num_blocks; ++i) { - // convert target_index into an offset in source data - size_t source_offset = ComputeOffset(target_index, stride, num_axes); - - // copy - CopyPrim(target, source + (source_offset * element_size)); - - // increment target_index: - IncrementIndex(target_index, target_dims, num_axes); - target += element_size; - } + TypedDoTransposeEltWise(num_axes, target_dims, num_blocks, stride, source, target); break; case sizeof(uint32_t): - for (size_t i = 0; i < num_blocks; ++i) { - // convert target_index into an offset in source data - size_t source_offset = ComputeOffset(target_index, stride, num_axes); - - // copy - CopyPrim(target, source + (source_offset * element_size)); - - // increment target_index: - IncrementIndex(target_index, target_dims, num_axes); - target += element_size; - } + TypedDoTransposeEltWise(num_axes, target_dims, num_blocks, stride, source, target); break; case sizeof(uint16_t): - for (size_t i = 0; i < num_blocks; ++i) { - // convert target_index into an offset in source data - size_t source_offset = ComputeOffset(target_index, stride, num_axes); - - // copy - CopyPrim(target, source + (source_offset * element_size)); - - // increment target_index: - IncrementIndex(target_index, target_dims, num_axes); - target += element_size; - } + TypedDoTransposeEltWise(num_axes, target_dims, num_blocks, stride, source, target); break; case sizeof(uint8_t): - for (size_t i = 0; i < num_blocks; ++i) { - // convert target_index into an offset in source data - size_t source_offset = ComputeOffset(target_index, stride, num_axes); - - // copy - *target = *(source + (source_offset * element_size)); - - // increment target_index: - IncrementIndex(target_index, target_dims, num_axes); - target += element_size; - } + TypedDoTransposeEltWise(num_axes, target_dims, num_blocks, stride, source, target); break; default: assert(false); @@ -162,17 +192,16 @@ static void DoTransposeEltWise(int64_t num_axes, const std::vector& tar static void DoTransposeEltWise(int64_t num_axes, const std::vector& target_dims, size_t num_blocks, const std::vector& stride, const std::string* source, std::string* target) { + ORT_ENFORCE(num_axes > 0, "Transpose not implemented for empty tensors."); + MultiIndex mindex; + IncrementIndexAndComputeOffsetSetup(mindex, num_axes, target_dims, stride, 1); + // index used to iterate over target iteration-space - std::vector target_index(num_axes, 0); + const std::string* local_source = source; for (size_t i = 0; i < num_blocks; ++i) { - // convert target_index into an offset in source data - size_t source_offset = ComputeOffset(target_index, stride, num_axes); - - // copy - *target = *(source + source_offset); - - // increment target_index: - IncrementIndex(target_index, target_dims, num_axes); + ORT_ENFORCE((local_source >= source) && (local_source < source + num_blocks)); + *target = *local_source; + IncrementIndexAndComputeOffset(mindex, local_source); target++; } } @@ -274,13 +303,15 @@ template static void SimpleTransposeSingleAxisOutwards(const T* input_data, T* output_data, int64_t num_loops, int64_t num_writers, int64_t writes_per_loop, int64_t writes_per_writer_per_loop) { + const T* end; for (int64_t l = 0; l < num_loops; ++l) { T* output_for_first_writer = output_data; for (auto wwpl = 0; wwpl < writes_per_writer_per_loop; ++wwpl) { T* output_for_current_writer = output_for_first_writer; - for (int64_t w = 0; w < num_writers; ++w) { + end = input_data + num_writers; + for (; input_data != end;) { *output_for_current_writer = *input_data++; // skip to output position for next writer @@ -379,13 +410,15 @@ template static void SimpleTransposeSingleAxisInwards(const T* input_data, T* output_data, int64_t num_loops, int64_t num_readers, int64_t reads_per_loop, int64_t reads_per_reader_per_loop) { + T* end; for (int64_t l = 0; l < num_loops; ++l) { const T* input_for_first_reader = input_data; for (auto rrpl = 0; rrpl < reads_per_reader_per_loop; ++rrpl) { const T* input_for_current_reader = input_for_first_reader; - for (int64_t r = 0; r < num_readers; ++r) { + end = output_data + num_readers; + for (; output_data != end;) { *output_data++ = *input_for_current_reader; // skip to input position for next reader input_for_current_reader += reads_per_reader_per_loop; @@ -560,6 +593,20 @@ static bool IsMovingSingleAxis(const std::vector& permutations, size_t& return single_axis_moved; } +bool IsTransposeReshape(const std::vector& perm, const std::vector& input_dims) { + // As long as the dims with values > 1 stay in the same order, it's a reshape. + // Example: Shape=(1,1,1024,4096) -> perm=(2,0,3,1). + size_t last_permuted_axis = 0; + for (size_t i = 0; i < perm.size(); ++i) { + if (input_dims[perm[i]] == 1) + continue; + if (perm[i] < last_permuted_axis) + return false; + last_permuted_axis = perm[i]; + } + return true; +} + //`input_shape_override` overrides the shape of `input` for compute purposes. Status TransposeBase::DoTranspose(const std::vector& permutations, const Tensor& input, Tensor& output, const TensorShape* input_shape_override) { @@ -572,6 +619,14 @@ Status TransposeBase::DoTranspose(const std::vector& permutations, const status = ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Mismatched data types between input and output Tensors. ", input_type, " != ", output_type); } else { + TensorShape shape = input_shape_override ? *input_shape_override : input.Shape(); + if (IsTransposeReshape(permutations, shape.GetDims())) { + // As long as the dims with values > 1 stay in the same order, it's a reshape. + // Example: Shape=(1,1,1024,4096) -> perm=(2,0,3,1). + CopyCpuTensor(&input, &output); + return Status::OK(); + } + size_t from = 0, to = 0; bool moving_single_axis = IsMovingSingleAxis(permutations, from, to); @@ -607,6 +662,13 @@ Status Transpose::Compute(OpKernelContext* ctx) const { if (output_shape.Size() == 0) return Status::OK(); + if (IsTransposeReshape(*p_perm, input_dims)) { + // As long as the dims with values > 1 stay in the same order, it's a reshape. + // Example: Shape=(1,1,1024,4096) -> perm=(2,0,3,1). + CopyCpuTensor(&X, &Y); + return Status::OK(); + } + size_t from = 0, to = 0; bool moving_single_axis = IsMovingSingleAxis(*p_perm, from, to); diff --git a/onnxruntime/core/providers/cpu/tensor/transpose.h b/onnxruntime/core/providers/cpu/tensor/transpose.h index 3cb56f6ddc..341975d475 100644 --- a/onnxruntime/core/providers/cpu/tensor/transpose.h +++ b/onnxruntime/core/providers/cpu/tensor/transpose.h @@ -10,6 +10,16 @@ namespace onnxruntime { +/** Tells if the transpose is equivalent to a reshape: + empty dimensions can change place, not empty dimensions must be in + the same order in the permuted tenosr. +*/ +bool IsTransposeReshape(const std::vector& perm, const std::vector& input_dims); + +void DoTransposeEltWise(int64_t num_axes, const std::vector& target_dims, size_t num_blocks, + const std::vector& stride, const uint8_t* source, uint8_t* target, + size_t element_size); + class TransposeBase { public: /** diff --git a/onnxruntime/core/session/onnxruntime_c_api.cc b/onnxruntime/core/session/onnxruntime_c_api.cc index 13fa90f263..db097983d8 100644 --- a/onnxruntime/core/session/onnxruntime_c_api.cc +++ b/onnxruntime/core/session/onnxruntime_c_api.cc @@ -2063,7 +2063,6 @@ static constexpr OrtApi ort_api_1_to_6 = { &OrtApis::SetGlobalSpinControl, // End of Version 5 - DO NOT MODIFY ABOVE (see above text for more information) - // Version 6 - In development, feel free to add/remove/rearrange here &OrtApis::AddInitializer, &OrtApis::CreateEnvWithCustomLoggerAndGlobalThreadPools, &OrtApis::SessionOptionsAppendExecutionProvider_CUDA, @@ -2071,6 +2070,9 @@ static constexpr OrtApi ort_api_1_to_6 = { &OrtApis::SetGlobalDenormalAsZero, &OrtApis::CreateArenaCfg, &OrtApis::ReleaseArenaCfg, + // End of Version 6 - DO NOT MODIFY ABOVE (see above text for more information) + + // Version 7 - In development, feel free to add/remove/rearrange here }; // Assert to do a limited check to ensure Version 1 of OrtApi never changes (will detect an addition or deletion but not if they cancel out each other) diff --git a/onnxruntime/python/tools/quantization/calibrate.py b/onnxruntime/python/tools/quantization/calibrate.py index 1a1bafea4f..eeccdc6868 100644 --- a/onnxruntime/python/tools/quantization/calibrate.py +++ b/onnxruntime/python/tools/quantization/calibrate.py @@ -58,6 +58,7 @@ class ONNXCalibrater: model = onnx.shape_inference.infer_shapes(model) value_infos = {vi.name: vi for vi in model.graph.value_info} value_infos.update({ot.name: ot for ot in model.graph.output}) + value_infos.update({it.name: it for it in model.graph.input}) added_nodes = [] added_outputs = [] @@ -264,4 +265,4 @@ def calibrate(model_path, quantization_params_dict = calibrater.calculate_quantization_params(dict_for_quantization) print("Calibrated,quantized parameters calculated and returned.") - return quantization_params_dict \ No newline at end of file + return quantization_params_dict diff --git a/onnxruntime/python/tools/quantization/onnx_quantizer.py b/onnxruntime/python/tools/quantization/onnx_quantizer.py index abe10af477..661621cc6b 100644 --- a/onnxruntime/python/tools/quantization/onnx_quantizer.py +++ b/onnxruntime/python/tools/quantization/onnx_quantizer.py @@ -806,7 +806,7 @@ class ONNXQuantizer: # Quantize the input initializer = find_by_name(node_input, self.model.initializer()) if initializer is not None: - weight = self._get_quantized_weight(initializer, self.weight_qType) + weight = self._get_quantized_weight(initializer, self.weight_qType if initializer_use_weight_qType else self.input_qType) # Update graph self._update_weight(weight) diff --git a/onnxruntime/test/providers/cpu/ml/treeregressor_test.cc b/onnxruntime/test/providers/cpu/ml/treeregressor_test.cc index 87a3ecde9d..e6effd1427 100644 --- a/onnxruntime/test/providers/cpu/ml/treeregressor_test.cc +++ b/onnxruntime/test/providers/cpu/ml/treeregressor_test.cc @@ -8,7 +8,31 @@ namespace onnxruntime { namespace test { template -void GenTreeAndRunTest(const std::vector& X, const std::vector& base_values, const std::vector& results, const std::string& aggFunction, bool one_obs = false) { +void _multiply_update_array(std::vector& data, int n, T inc = 0) { + std::vector copy = data; + data.resize(copy.size() * n); + T cst = 0; + for (int i = 0; i < n; ++i) { + for (size_t j = 0; j < copy.size(); ++j) { + data[j + i * copy.size()] = copy[j] + cst; + } + cst += inc; + } +} + +void _multiply_update_array_string(std::vector& data, int n) { + std::vector copy = data; + data.resize(copy.size() * n); + for (int i = 0; i < n; ++i) { + for (size_t j = 0; j < copy.size(); ++j) { + data[j + i * copy.size()] = copy[j]; + } + } +} + +template +void GenTreeAndRunTest(const std::vector& X, const std::vector& base_values, const std::vector& results, const std::string& aggFunction, + bool one_obs = false, int64_t n_obs = 8, int n_trees = 1) { OpTester test("TreeEnsembleRegressor", 1, onnxruntime::kMLDomain); //tree @@ -26,6 +50,21 @@ void GenTreeAndRunTest(const std::vector& X, const std::vector& base_v std::vector target_weights = {1.5f, 27.5f, 2.25f, 20.75f, 2.f, 23.f, 3.f, 14.f, 0.f, 41.f, 1.83333333f, 24.5f, 0.f, 41.f, 2.75f, 16.25f, 2.f, 23.f, 3.f, 14.f, 2.66666667f, 17.f, 2.f, 23.f, 3.f, 14.f}; std::vector classes = {0, 1}; + if (n_trees > 1) { + // Multiplies the number of trees to test the parallelization by trees. + _multiply_update_array(lefts, n_trees); + _multiply_update_array(rights, n_trees); + _multiply_update_array(treeids, n_trees, (int64_t)3); + _multiply_update_array(nodeids, n_trees); + _multiply_update_array(featureids, n_trees); + _multiply_update_array(thresholds, n_trees); + _multiply_update_array_string(modes, n_trees); + _multiply_update_array(target_treeids, n_trees, (int64_t)3); + _multiply_update_array(target_nodeids, n_trees); + _multiply_update_array(target_classids, n_trees); + _multiply_update_array(target_weights, n_trees); + } + //add attributes test.AddAttribute("nodes_truenodeids", lefts); test.AddAttribute("nodes_falsenodeids", rights); @@ -51,6 +90,8 @@ void GenTreeAndRunTest(const std::vector& X, const std::vector& base_v } // default function is SUM //fill input data + std::vector xn; + std::vector yn; if (one_obs) { auto X1 = X; auto results1 = results; @@ -58,13 +99,69 @@ void GenTreeAndRunTest(const std::vector& X, const std::vector& base_v results1.resize(2); test.AddInput("X", {1, 3}, X1); test.AddOutput("Y", {1, 2}, results1); - } else { + } else if (n_obs == 8) { test.AddInput("X", {8, 3}, X); test.AddOutput("Y", {8, 2}, results); + } else { + int64_t i; + size_t k; + ASSERT_TRUE(n_obs % 8 == 0); + xn.resize(n_obs * 3); + yn.resize(n_obs * 2); + for (i = 0; i < n_obs; i += 8) { + for (k = 0; k < 24; ++k) { + xn[i * 3 + k] = X[k]; + } + for (k = 0; k < 16; ++k) { + yn[i * 2 + k] = results[k]; + } + } + ASSERT_TRUE(i == n_obs); + test.AddInput("X", {n_obs, 3}, xn); + test.AddOutput("Y", {n_obs, 2}, yn); } + test.Run(); } // namespace test +TEST(MLOpTest, TreeRegressorMultiTargetBatchTreeA2) { + // TreeEnsemble implements different paths depending on n_trees or N. + // This test and the next ones go through all sections for multi-targets. + std::vector X = {1.f, 0.0f, 0.4f, 3.0f, 44.0f, -3.f, 12.0f, 12.9f, -312.f, 23.0f, 11.3f, -222.f, 23.0f, 11.3f, -222.f, 23.0f, 3311.3f, -222.f, 23.0f, 11.3f, -222.f, 43.0f, 413.3f, -114.f}; + std::vector results = {1.33333333f, 29.f, 3.f, 14.f, 2.f, 23.f, 2.f, 23.f, 2.f, 23.f, 2.66666667f, 17.f, 2.f, 23.f, 3.f, 14.f}; + std::vector base_values{0.f, 0.f}; + GenTreeAndRunTest(X, base_values, results, "AVERAGE", true, 8, 1); // section A2 +} + +TEST(MLOpTest, TreeRegressorMultiTargetBatchTreeB2) { + std::vector X = {1.f, 0.0f, 0.4f, 3.0f, 44.0f, -3.f, 12.0f, 12.9f, -312.f, 23.0f, 11.3f, -222.f, 23.0f, 11.3f, -222.f, 23.0f, 3311.3f, -222.f, 23.0f, 11.3f, -222.f, 43.0f, 413.3f, -114.f}; + std::vector results = {1.33333333f, 29.f, 3.f, 14.f, 2.f, 23.f, 2.f, 23.f, 2.f, 23.f, 2.66666667f, 17.f, 2.f, 23.f, 3.f, 14.f}; + std::vector base_values{0.f, 0.f}; + GenTreeAndRunTest(X, base_values, results, "AVERAGE", true, 8, 130); // section B2 +} + +TEST(MLOpTest, TreeRegressorMultiTargetBatchTreeC2) { + std::vector X = {1.f, 0.0f, 0.4f, 3.0f, 44.0f, -3.f, 12.0f, 12.9f, -312.f, 23.0f, 11.3f, -222.f, 23.0f, 11.3f, -222.f, 23.0f, 3311.3f, -222.f, 23.0f, 11.3f, -222.f, 43.0f, 413.3f, -114.f}; + std::vector results = {1.33333333f, 29.f, 3.f, 14.f, 2.f, 23.f, 2.f, 23.f, 2.f, 23.f, 2.66666667f, 17.f, 2.f, 23.f, 3.f, 14.f}; + std::vector base_values{0.f, 0.f}; + GenTreeAndRunTest(X, base_values, results, "AVERAGE", false, 200, 130); // section C2 +} + +TEST(MLOpTest, TreeRegressorMultiTargetBatchTreeD2) { + std::vector X = {1.f, 0.0f, 0.4f, 3.0f, 44.0f, -3.f, 12.0f, 12.9f, -312.f, 23.0f, 11.3f, -222.f, 23.0f, 11.3f, -222.f, 23.0f, 3311.3f, -222.f, 23.0f, 11.3f, -222.f, 43.0f, 413.3f, -114.f}; + std::vector results = {1.33333333f, 29.f, 3.f, 14.f, 2.f, 23.f, 2.f, 23.f, 2.f, 23.f, 2.66666667f, 17.f, 2.f, 23.f, 3.f, 14.f}; + std::vector base_values{0.f, 0.f}; + GenTreeAndRunTest(X, base_values, results, "AVERAGE", true, 8, 30); // section D2 + GenTreeAndRunTest(X, base_values, results, "AVERAGE", false, 200, 30); // section D2 +} + +TEST(MLOpTest, TreeRegressorMultiTargetBatchTreeE2) { + std::vector X = {1.f, 0.0f, 0.4f, 3.0f, 44.0f, -3.f, 12.0f, 12.9f, -312.f, 23.0f, 11.3f, -222.f, 23.0f, 11.3f, -222.f, 23.0f, 3311.3f, -222.f, 23.0f, 11.3f, -222.f, 43.0f, 413.3f, -114.f}; + std::vector results = {1.33333333f, 29.f, 3.f, 14.f, 2.f, 23.f, 2.f, 23.f, 2.f, 23.f, 2.66666667f, 17.f, 2.f, 23.f, 3.f, 14.f}; + std::vector base_values{0.f, 0.f}; + GenTreeAndRunTest(X, base_values, results, "AVERAGE", false, 200, 1); // section E2 +} + TEST(MLOpTest, TreeRegressorMultiTargetAverage) { std::vector X = {1.f, 0.0f, 0.4f, 3.0f, 44.0f, -3.f, 12.0f, 12.9f, -312.f, 23.0f, 11.3f, -222.f, 23.0f, 11.3f, -222.f, 23.0f, 3311.3f, -222.f, 23.0f, 11.3f, -222.f, 43.0f, 413.3f, -114.f}; std::vector results = {1.33333333f, 29.f, 3.f, 14.f, 2.f, 23.f, 2.f, 23.f, 2.f, 23.f, 2.66666667f, 17.f, 2.f, 23.f, 3.f, 14.f}; @@ -97,7 +194,7 @@ TEST(MLOpTest, TreeRegressorMultiTargetMaxDouble) { GenTreeAndRunTest(X, base_values, results, "MAX", true); } -void GenTreeAndRunTest1(const std::string& aggFunction, bool one_obs) { +void GenTreeAndRunTest1(const std::string& aggFunction, bool one_obs, int64_t n_obs = 3, int n_trees = 1) { OpTester test("TreeEnsembleRegressor", 1, onnxruntime::kMLDomain); //tree @@ -115,6 +212,21 @@ void GenTreeAndRunTest1(const std::string& aggFunction, bool one_obs) { std::vector target_weights = {33.33333f, 16.66666f, 33.33333f, -3.33333f, 16.66666f, -3.333333f}; std::vector classes = {0, 1}; + if (n_trees > 1) { + // Multiplies the number of trees to test the parallelization by trees. + _multiply_update_array(lefts, n_trees); + _multiply_update_array(rights, n_trees); + _multiply_update_array(treeids, n_trees, (int64_t)3); + _multiply_update_array(nodeids, n_trees); + _multiply_update_array(featureids, n_trees); + _multiply_update_array(thresholds, n_trees); + _multiply_update_array_string(modes, n_trees); + _multiply_update_array(target_treeids, n_trees, (int64_t)3); + _multiply_update_array(target_nodeids, n_trees); + _multiply_update_array(target_classids, n_trees); + _multiply_update_array(target_weights, n_trees); + } + std::vector results; if (aggFunction == "AVERAGE") { test.AddAttribute("aggregate_function", "AVERAGE"); @@ -149,16 +261,32 @@ void GenTreeAndRunTest1(const std::string& aggFunction, bool one_obs) { // SUM aggregation by default -- no need to add explicitly //fill input data + std::vector xn, yn; if (one_obs) { + ASSERT_TRUE(n_obs == 3); auto X1 = X; auto results1 = results; X1.resize(2); results1.resize(1); test.AddInput("X", {1, 2}, X1); test.AddOutput("Y", {1, 1}, results1); - } else { + } else if (n_obs == 3) { test.AddInput("X", {3, 2}, X); test.AddOutput("Y", {3, 1}, results); + } else { + ASSERT_TRUE(n_obs % 3 == 0); + xn.resize(n_obs * 2); + yn.resize(n_obs); + for (int64_t i = 0; i < n_obs; i += 3) { + for (size_t k = 0; k < 6; ++k) { + xn[i * 2 + k] = X[k]; + } + for (size_t k = 0; k < 3; ++k) { + yn[i + k] = results[k]; + } + } + test.AddInput("X", {n_obs, 2}, xn); + test.AddOutput("Y", {n_obs, 1}, yn); } test.Run(); } @@ -168,6 +296,34 @@ TEST(MLOpTest, TreeRegressorSingleTargetSum) { GenTreeAndRunTest1("SUM", true); } +TEST(MLOpTest, TreeRegressorSingleTargetSumBatch) { + GenTreeAndRunTest1("SUM", false, 201); + GenTreeAndRunTest1("SUM", false, 40002); +} + +TEST(MLOpTest, TreeRegressorSingleTargetBatchTreeA) { + // TreeEnsemble implements different paths depending on n_trees or N. + // This test and the next ones goe through all sections for one target. + GenTreeAndRunTest1("SUM", true, 3, 1); // section A +} + +TEST(MLOpTest, TreeRegressorSingleTargetBatchTreeB) { + GenTreeAndRunTest1("AVERAGE", true, 3, 30); // section B +} + +TEST(MLOpTest, TreeRegressorSingleTargetBatchTreeC) { + GenTreeAndRunTest1("AVERAGE", false, 3, 1); // section C +} + +TEST(MLOpTest, TreeRegressorSingleTargetBatchTreeD) { + GenTreeAndRunTest1("AVERAGE", false, 201, 30); // section D + GenTreeAndRunTest1("AVERAGE", false, 201, 130); // section D +} + +TEST(MLOpTest, TreeRegressorSingleTargetBatchTreeE) { + GenTreeAndRunTest1("AVERAGE", false, 201, 1); // section E +} + TEST(MLOpTest, TreeRegressorSingleTargetAverage) { GenTreeAndRunTest1("AVERAGE", false); GenTreeAndRunTest1("AVERAGE", true); diff --git a/onnxruntime/test/providers/cpu/tensor/transpose_test.cc b/onnxruntime/test/providers/cpu/tensor/transpose_test.cc index a40c805ca8..6317f068c7 100644 --- a/onnxruntime/test/providers/cpu/tensor/transpose_test.cc +++ b/onnxruntime/test/providers/cpu/tensor/transpose_test.cc @@ -4,10 +4,23 @@ #include "gtest/gtest.h" #include "test/providers/provider_test_utils.h" #include "test/providers/compare_provider_test_utils.h" +#include "core/providers/cpu/tensor/transpose.h" namespace onnxruntime { namespace test { +TEST(TransposeOpTest, IsTransposeReshapeTest) { + std::vector input_dims{1, 2, 3, 4, 1}; + std::vector perm{0, 1, 2, 3, 4}; + ASSERT_TRUE(IsTransposeReshape(perm, input_dims)); + perm = std::vector{1, 2, 3, 0, 4}; + ASSERT_TRUE(IsTransposeReshape(perm, input_dims)); + perm = std::vector{4, 1, 0, 2, 3}; + ASSERT_TRUE(IsTransposeReshape(perm, input_dims)); + perm = std::vector{4, 1, 0, 3, 2}; + ASSERT_FALSE(IsTransposeReshape(perm, input_dims)); +} + // Some of the tests can't run on TensorrtExecutionProvider because of errors. // Those tests will fallback to other EPs. @@ -124,11 +137,11 @@ TEST(TransposeOpTest, TwoDim_int16) { 2, 5, 3, 6}; - #if defined(OPENVINO_CONFIG_MYRIAD) || defined(OPENVINO_CONFIG_VAD_M) - TransposeTest(input_shape, input_vals, &perm, expected_shape, expected_vals, true, false); - #else - TransposeTest(input_shape, input_vals, &perm, expected_shape, expected_vals); - #endif +#if defined(OPENVINO_CONFIG_MYRIAD) || defined(OPENVINO_CONFIG_VAD_M) + TransposeTest(input_shape, input_vals, &perm, expected_shape, expected_vals, true, false); +#else + TransposeTest(input_shape, input_vals, &perm, expected_shape, expected_vals); +#endif } TEST(TransposeOpTest, TwoDim_mlfloat16) { @@ -246,6 +259,39 @@ TEST(TransposeOpTest, ThreeDimSuffix) { TransposeTest(input_shape, input_vals, &perm, expected_shape, expected_vals, false); //TensorRT: illegal error } +TEST(TransposeOpTest, TransposeReshape) { + std::vector input_shape({1, 4, 2, 1, 3}); + std::vector input_vals = { + 1.0f, 2.0f, 3.0f, + 4.0f, 5.0f, 6.0f, + + 1.1f, 2.1f, 3.1f, + 4.1f, 5.1f, 6.1f, + + 1.2f, 2.2f, 3.2f, + 4.2f, 5.2f, 6.2f, + + 1.3f, 2.3f, 3.3f, + 4.3f, 5.3f, 6.3f}; + + std::vector perm = {1, 3, 2, 4, 0}; + std::vector expected_shape({4, 1, 2, 3, 1}); + auto expected_vals = { + 1.0f, 2.0f, 3.0f, + 4.0f, 5.0f, 6.0f, + + 1.1f, 2.1f, 3.1f, + 4.1f, 5.1f, 6.1f, + + 1.2f, 2.2f, 3.2f, + 4.2f, 5.2f, 6.2f, + + 1.3f, 2.3f, 3.3f, + 4.3f, 5.3f, 6.3f}; + + TransposeTest(input_shape, input_vals, &perm, expected_shape, expected_vals, false); //TensorRT: illegal error +} + TEST(TransposeOpTest, ThreeDimStr) { std::vector input_shape({4, 2, 3}); std::vector input_vals = { @@ -419,10 +465,112 @@ TEST(TransposeOpTest, SingleAxisMovingInwardsBlockCopy) { TransposeTest(input_shape, input_vals, &perm, expected_shape, expected_vals, false); } +TEST(TransposeOpTest, NDim) { + std::vector input_shape({2, 2, 2, 2}); + std::vector input_vals = {1.0f, 2.0f, 3.0f, 4.0f, + 5.0f, 6.0f, 7.0f, 8.0f, + 9.0f, 10.0f, 11.0f, 12.0f, + 13.0f, 14.0f, 15.0f, 16.0f}; + + std::vector perm = {1, 0, 2, 3}; + auto expected_vals = {1.0f, 2.0f, 3.0f, 4.0f, + 9.0f, 10.0f, 11.0f, 12.0f, + 5.0f, 6.0f, 7.0f, 8.0f, + 13.0f, 14.0f, 15.0f, 16.0f}; + TransposeTest(input_shape, input_vals, &perm, input_shape, expected_vals); + + perm = {1, 0, 3, 2}; + auto expected_vals2 = {1.0f, 3.0f, 2.0f, 4.0f, + 9.0f, 11.0f, 10.0f, 12.0f, + 5.0f, 7.0f, 6.0f, 8.0f, + 13.0f, 15.0f, 14.0f, 16.0f}; + TransposeTest(input_shape, input_vals, &perm, input_shape, expected_vals2); +} + +TEST(TransposeOpTest, DoTransposeImpl) { + std::vector input_shape({5, 2, 1, 3}); + std::vector input_vals(30); + for (auto it = input_vals.begin(); it != input_vals.end(); ++it) { + *it = static_cast(std::distance(input_vals.begin(), it)); + } + std::vector perm = {2, 1, 0, 3}; + std::vector expected_shape({1, 2, 5, 3}); + auto expected_vals = {0.0f, 1.0f, 2.0f, 6.0f, 7.0f, 8.0f, + 12.0f, 13.0f, 14.0f, 18.0f, 19.0f, 20.0f, + 24.0f, 25.0f, 26.0f, 3.0f, 4.0f, 5.0f, + 9.0f, 10.0f, 11.0f, 15.0f, 16.0f, 17.0f, + 21.0f, 22.0f, 23.0f, 27.0f, 28.0f, 29.0f}; + TransposeTest(input_shape, input_vals, &perm, expected_shape, expected_vals); +} + +TEST(TransposeOpTest, DoTransposeImplString) { + std::vector input_shape({5, 2, 1, 3}); + std::vector input_vals(30); + for (auto it = input_vals.begin(); it != input_vals.end(); ++it) { + *it = std::string("n") + std::to_string(static_cast(std::distance(input_vals.begin(), it))); + } + std::vector perm = {2, 1, 0, 3}; + std::vector expected_shape({1, 2, 5, 3}); + std::initializer_list expected_vals = {"n0", "n1", "n2", "n6", "n7", "n8", + "n12", "n13", "n14", "n18", "n19", "n20", + "n24", "n25", "n26", "n3", "n4", "n5", + "n9", "n10", "n11", "n15", "n16", "n17", + "n21", "n22", "n23", "n27", "n28", "n29"}; + TransposeTest(input_shape, input_vals, &perm, expected_shape, expected_vals); +} + +TEST(TransposeOpTest, DoTransposeEltWise) { + // Configuration where DoTransposeEltWise is called. + std::vector input_shape({2, 2, 2, 2}); + std::vector input_vals = {1.0f, 2.0f, 3.0f, 4.0f, + 5.0f, 6.0f, 7.0f, 8.0f, + 9.0f, 10.0f, 11.0f, 12.0f, + 13.0f, 14.0f, 15.0f, 16.0f}; + + std::vector perm = {1, 0, 3, 2}; + auto expected_vals2 = {1.0f, 3.0f, 2.0f, 4.0f, + 9.0f, 11.0f, 10.0f, 12.0f, + 5.0f, 7.0f, 6.0f, 8.0f, + 13.0f, 15.0f, 14.0f, 16.0f}; + TransposeTest(input_shape, input_vals, &perm, input_shape, expected_vals2); + + // Specific test which tests that function DoTransposeEltWise does not + // copy values outside the target buffer. + TensorShape tensor_shape(input_shape); + std::vector stride(input_shape.size()); + for (size_t i = 0; i < input_shape.size(); i++) { + size_t inpdim = perm[i]; + if (inpdim + 1 < input_shape.size()) + stride[i] = tensor_shape.SizeFromDimension(inpdim + 1); + else + stride[i] = 1; + } + + std::vector input_vals_end = {1.0f, 2.0f, 3.0f, 4.0f, + 5.0f, 6.0f, 7.0f, 8.0f, + 9.0f, 10.0f, 11.0f, 12.0f, + 13.0f, 14.0f, 15.0f, 16.0f, + -1.0f, -1.0f}; + std::vector target(input_vals_end.size(), 17.0f); + + std::vector expected_vals3 = {1.0f, 3.0f, 2.0f, 4.0f, + 9.0f, 11.0f, 10.0f, 12.0f, + 5.0f, 7.0f, 6.0f, 8.0f, + 13.0f, 15.0f, 14.0f, 16.0f, + 17.0f, 17.0f}; + + DoTransposeEltWise(input_shape.size(), input_shape, 16, + stride, (uint8_t*)input_vals_end.data(), (uint8_t*)target.data(), + sizeof(float)); + for (size_t i = 0; i < input_vals_end.size(); ++i) { + ASSERT_TRUE(target[i] == expected_vals3[i]); + } +} + #if USE_CUDA - constexpr const char* kGpuExecutionProvider = kCudaExecutionProvider; +constexpr const char* kGpuExecutionProvider = kCudaExecutionProvider; #elif USE_ROCM - constexpr const char* kGpuExecutionProvider = kRocmExecutionProvider; +constexpr const char* kGpuExecutionProvider = kRocmExecutionProvider; #endif #if defined(USE_CUDA) || defined(USE_ROCM) diff --git a/package/rpm/onnxruntime.spec b/package/rpm/onnxruntime.spec index de01bbb00a..164809dccd 100644 --- a/package/rpm/onnxruntime.spec +++ b/package/rpm/onnxruntime.spec @@ -1,5 +1,5 @@ Name: onnxruntime -Version: 1.5.2 +Version: 1.6.0 Release: 1%{?dist} Summary: onnxruntime