From bca8daf76247c649674ef0afc29cb6cc6e055922 Mon Sep 17 00:00:00 2001 From: Scott McKay Date: Thu, 24 Jan 2019 08:10:39 +1000 Subject: [PATCH] Update ONNX. Implement Scan 9 changes (#366) * Update ONNX version to pickup Scan spec change that adds scan_output_axes. Add logic to transpose an output - write to temporary buffer when executing subgraph - transpose temporary buffer into Scan output when execution completes Add unit tests * Update to ONNX dbf3581835e3a05716e10587511d7ab3b2cdc386 to pickup inferencing bugfix. Update test to match. * Disable some tests for opset 9 operators that haven't been implemented yet. --- cmake/external/onnx | 2 +- .../core/providers/cpu/controlflow/scan.h | 3 +- .../core/providers/cpu/controlflow/scan_9.cc | 133 +++++++++++------- .../providers/cpu/controlflow/scan_utils.cc | 69 +++++++-- .../providers/cpu/controlflow/scan_utils.h | 39 ++++- onnxruntime/test/onnx/main.cc | 18 ++- .../providers/cpu/controlflow/scan_test.cc | 126 +++++++++++++---- .../linux/docker/scripts/install_deps.sh | 4 +- 8 files changed, 299 insertions(+), 95 deletions(-) diff --git a/cmake/external/onnx b/cmake/external/onnx index c4cf11269c..dbf3581835 160000 --- a/cmake/external/onnx +++ b/cmake/external/onnx @@ -1 +1 @@ -Subproject commit c4cf11269c1ef9bf1f459bb5b1b68a5f66840321 +Subproject commit dbf3581835e3a05716e10587511d7ab3b2cdc386 diff --git a/onnxruntime/core/providers/cpu/controlflow/scan.h b/onnxruntime/core/providers/cpu/controlflow/scan.h index 5e2849d31b..75dc35c5cd 100644 --- a/onnxruntime/core/providers/cpu/controlflow/scan.h +++ b/onnxruntime/core/providers/cpu/controlflow/scan.h @@ -20,6 +20,7 @@ class Scan final : public OpKernel { int64_t num_scan_inputs_; std::vector input_directions_; std::vector output_directions_; - std::vector axes_; + std::vector input_axes_; + std::vector output_axes_; }; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/controlflow/scan_9.cc b/onnxruntime/core/providers/cpu/controlflow/scan_9.cc index 6a16d97038..7dffa6cef4 100644 --- a/onnxruntime/core/providers/cpu/controlflow/scan_9.cc +++ b/onnxruntime/core/providers/cpu/controlflow/scan_9.cc @@ -103,7 +103,8 @@ class ScanImpl { int64_t num_scan_inputs, const std::vector& input_directions, const std::vector& output_directions, - const std::vector& axes); + const std::vector& input_axes, + const std::vector& output_axes); // Initialize by validating all the inputs, and allocating the output tensors Status Initialize(); @@ -124,6 +125,7 @@ class ScanImpl { Status AllocateOutputTensors(); Status CreateLoopStateVariables(std::vector& loop_state_variables); + Status TransposeOutput(); using ConstTensorSlicerIterators = std::vector::Iterator>; using MutableTensorSlicerIterators = std::vector::Iterator>; @@ -132,17 +134,19 @@ class ScanImpl { const SessionState& session_state_; const GraphViewer& subgraph_; - int num_loop_state_variables_; - int num_scan_inputs_; int num_variadic_inputs_; int num_variadic_outputs_; + int num_loop_state_variables_; + int num_scan_inputs_; + int num_scan_outputs_; int64_t sequence_len_ = -1; const std::vector& input_directions_; const std::vector& output_directions_; - const std::vector& axes_from_attribute_; - std::vector axes_; + const std::vector& input_axes_from_attribute_; + const std::vector& output_axes_from_attribute_; + std::vector input_axes_; // inputs for graph. either original input value or transposed input if an axis other than 0 was specified std::vector inputs_; @@ -171,11 +175,19 @@ Scan<9>::Scan(const OpKernelInfo& info) : OpKernel(info) { ReadDirections(info, "scan_input_directions", input_directions_, num_scan_inputs_); ReadDirections(info, "scan_output_directions", output_directions_, num_scan_outputs); - if (info.GetAttrs("axes", axes_).IsOK()) { - ORT_ENFORCE(gsl::narrow_cast(axes_.size()) == num_scan_inputs_, - "Number of entries in 'axes' was ", axes_.size(), " but expected ", num_scan_inputs_); + if (info.GetAttrs("scan_input_axes", input_axes_).IsOK()) { + ORT_ENFORCE(gsl::narrow_cast(input_axes_.size()) == num_scan_inputs_, + "Number of entries in 'scan_input_axes' was ", input_axes_.size(), " but expected ", num_scan_inputs_); } else { - axes_ = std::vector(num_scan_inputs_, 0); + input_axes_ = std::vector(num_scan_inputs_, 0); + } + + if (info.GetAttrs("scan_output_axes", output_axes_).IsOK()) { + ORT_ENFORCE(gsl::narrow_cast(output_axes_.size()) == num_scan_outputs, + "Number of entries in 'scan_output_axes' was ", output_axes_.size(), " but expected ", + num_scan_outputs); + } else { + output_axes_ = std::vector(num_scan_outputs, 0); } } @@ -185,7 +197,8 @@ Status Scan<9>::Compute(OpKernelContext* ctx) const { auto* session_state = ctx_internal->SubgraphSessionState("body"); ORT_ENFORCE(session_state, "Subgraph SessionState was not found for 'body' attribute."); - ScanImpl scan_impl{*ctx_internal, *session_state, num_scan_inputs_, input_directions_, output_directions_, axes_}; + ScanImpl scan_impl{*ctx_internal, *session_state, num_scan_inputs_, input_directions_, output_directions_, + input_axes_, output_axes_}; auto status = scan_impl.Initialize(); ORT_RETURN_IF_ERROR(status); @@ -200,47 +213,24 @@ ScanImpl::ScanImpl(OpKernelContextInternal& context, int64_t num_scan_inputs, const std::vector& input_directions, const std::vector& output_directions, - const std::vector& axes) + const std::vector& input_axes, + const std::vector& output_axes) : context_{context}, session_state_{session_state}, subgraph_{*session_state.GetGraphViewer()}, num_scan_inputs_{gsl::narrow_cast(num_scan_inputs)}, input_directions_{input_directions}, output_directions_{output_directions}, - axes_from_attribute_{axes}, + input_axes_from_attribute_{input_axes}, + output_axes_from_attribute_{output_axes}, implicit_inputs_{context_.GetImplicitInputs()} { num_variadic_inputs_ = context_.NumVariadicInputs(0); num_variadic_outputs_ = context_.OutputCount(); num_loop_state_variables_ = num_variadic_inputs_ - num_scan_inputs_; + num_scan_outputs_ = num_variadic_outputs_ - num_loop_state_variables_; inputs_.reserve(num_scan_inputs_); - axes_.reserve(num_scan_inputs_); -} - -/** -Calculate the transpose permutations and output shape by shifting the chosen axis to the first dimension. -The other dimension indexes or values are pushed in order after the chosen axis. - -e.g. if shape is {2, 3, 4} and axis 1 is chosen the permutations will be {1, 0, 2} and output shape will be {3, 2, 4} - if axis 2 is chosen the permutations will be {2, 0, 1} and the output shape will be {4, 2, 3} -*/ -static void CalculateTransposedShape(const TensorShape& input_shape, int64_t axis, - std::vector& permutations, std::vector& output_shape) { - int64_t rank = input_shape.NumDimensions(); - const auto& dims = input_shape.GetDims(); - - permutations.reserve(rank); - permutations.push_back(axis); - - output_shape.reserve(rank); - output_shape.push_back(dims[axis]); - - for (int64_t i = 0; i < rank; ++i) { - if (i != axis) { - permutations.push_back(i); - output_shape.push_back(dims[i]); - } - } + input_axes_.reserve(num_scan_inputs_); } Status ScanImpl::Initialize() { @@ -279,7 +269,7 @@ Status ScanImpl::ValidateSubgraphInput(int start_input, int end_input, " Expected ", min_dims_required, " dimensions or more but input had shape of ", input_shape); - auto seq_len_dim = axes_[i - num_loop_state_variables_]; + auto seq_len_dim = input_axes_[i - num_loop_state_variables_]; auto this_seq_len = input_shape[seq_len_dim]; if (sequence_len_ < 0) { @@ -306,10 +296,10 @@ Status ScanImpl::ValidateInput() { " inputs but Scan was only given ", num_variadic_inputs_); } - // validate/calculate the axes values and populate axes_. - // we already checked that axes_from_attribute_.size() == num_scan_inputs_. + // validate/calculate the input axes values and populate input_axes_. + // we already checked that input_axes_from_attribute_.size() == num_scan_inputs_ for (int i = 0; i < num_scan_inputs_; ++i) { - auto axis = axes_from_attribute_[i]; + auto axis = input_axes_from_attribute_[i]; // zero is always valid, so only do extra checks for non-zero values if (axis != 0) { @@ -318,14 +308,17 @@ Status ScanImpl::ValidateInput() { if (axis >= -input_rank && axis < input_rank) axis = HandleNegativeAxis(axis, input_rank); else - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Invalid value in axes for input ", i, + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Invalid value in scan_input_axes for input ", i, " of ", axis, ". Input tensor rank was ", input_rank); } - axes_.push_back(axis); + input_axes_.push_back(axis); } - // no validation for loop state variables + // we're not guaranteed to have complete output shapes, so delay checking output_axes_from_attribute_ + // values until after execution. + + // no validation for loop state variables. // validate the scan inputs auto status = ValidateSubgraphInput(num_loop_state_variables_, num_variadic_inputs_, graph_inputs); @@ -339,7 +332,7 @@ Status ScanImpl::SetupInputs() { AllocatorPtr alloc; for (int i = 0; i < num_scan_inputs_; ++i) { - auto sequence_dim = axes_[i]; + auto sequence_dim = input_axes_[i]; if (sequence_dim == 0) { // no transpose required @@ -393,8 +386,12 @@ Status ScanImpl::AllocateOutputTensors() { direction = static_cast(output_directions_[scan_output_index]); } - status = AllocateOutput(context_, subgraph_, i, false, -1, sequence_len_, output_iter, direction); + // if we need to transpose later, we need to use a temporary output buffer when executing the subgraph + bool temporary = output_axes_from_attribute_[scan_output_index] != 0; + + status = AllocateOutput(context_, subgraph_, i, false, -1, sequence_len_, output_iter, direction, temporary); ORT_RETURN_IF_ERROR(status); + output_iterators_.push_back(std::move(output_iter)); } @@ -448,6 +445,45 @@ Status ScanImpl::Execute() { num_variadic_inputs_, num_variadic_outputs_, implicit_inputs_, subgraph_output_names_, output_iterators_); + ORT_RETURN_IF_ERROR(status); + + status = TransposeOutput(); + + return status; +} + +Status ScanImpl::TransposeOutput() { + auto status = Status::OK(); + + for (int i = 0; i < num_scan_outputs_; ++i) { + auto axis = output_axes_from_attribute_[i]; + + if (axis != 0) { + auto output_index = i + num_loop_state_variables_; + const MLValue& temporary_output_mlvalue = output_iterators_[output_index]->GetOutput(); + const Tensor& temporary_output_tensor = temporary_output_mlvalue.Get(); + + int64_t output_rank = temporary_output_tensor.Shape().NumDimensions(); + + // check axis is valid for input_rank and also handle any negative axis value + if (axis >= -output_rank && axis < output_rank) + axis = HandleNegativeAxis(axis, output_rank); + else + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Invalid value in scan_output_axes for output ", i, + " of ", axis, ". Output tensor rank was ", output_rank); + + std::vector permutations; + std::vector new_shape; + CalculateTransposedShape(temporary_output_tensor.Shape(), axis, permutations, new_shape); + + Tensor* output = context_.Output(output_index, new_shape); + ORT_ENFORCE(output, "Outputs from Scan are not optional and should never be null."); + + status = TransposeBase::DoTranspose(permutations, temporary_output_tensor, *output); + ORT_RETURN_IF_ERROR(status); + } + } + return status; } @@ -457,4 +493,5 @@ ONNX_CPU_OPERATOR_KERNEL(Scan, .TypeConstraint("I", DataTypeImpl::GetTensorType()) .TypeConstraint("V", DataTypeImpl::AllTensorTypes()), Scan<9>); + } // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/controlflow/scan_utils.cc b/onnxruntime/core/providers/cpu/controlflow/scan_utils.cc index f4039cfda2..a8caacc724 100644 --- a/onnxruntime/core/providers/cpu/controlflow/scan_utils.cc +++ b/onnxruntime/core/providers/cpu/controlflow/scan_utils.cc @@ -13,6 +13,7 @@ #include "gsl/gsl_algorithm" +#include "core/framework/mldata_type_utils.h" #include "core/framework/op_kernel_context_internal.h" #include "core/framework/sequential_executor.h" #include "core/framework/tensorprotoutils.h" @@ -48,7 +49,8 @@ void ReadDirections(const OpKernelInfo& info, const std::string& attr_name, Status AllocateOutput(OpKernelContextInternal& context, const GraphViewer& subgraph, int output_index, bool is_loop_state_var, int64_t batch_size, int64_t sequence_len, - std::unique_ptr& output_iterator, ScanDirection direction) { + std::unique_ptr& output_iterator, ScanDirection direction, + bool temporary) { // use the shape from the subgraph output. we require this to be specified in the model or inferable. auto& graph_outputs = subgraph.GetOutputs(); auto* graph_output = graph_outputs.at(output_index); @@ -78,8 +80,18 @@ Status AllocateOutput(OpKernelContextInternal& context, const GraphViewer& subgr scan_output_dims.insert(scan_output_dims.cend(), graph_output_dims.cbegin(), graph_output_dims.cend()); - OutputIterator::Create(context, output_index, is_loop_state_var, is_v8, TensorShape(scan_output_dims), - output_iterator, direction); + if (!temporary) { + OutputIterator::Create(context, output_index, is_loop_state_var, is_v8, TensorShape(scan_output_dims), + output_iterator, direction); + } else { + auto mltype = utils::GetMLDataType(*graph_output); + + // the outputs from Scan are constrained to tensors, so we can safely cast to TensorTypeBase + auto ml_data_type = static_cast(mltype)->GetElementType(); + + OutputIterator::Create(context, output_index, is_loop_state_var, is_v8, TensorShape(scan_output_dims), + output_iterator, direction, temporary, ml_data_type); + } return Status::OK(); } @@ -206,6 +218,25 @@ MLValue AllocateTensorInMLValue(const MLDataType data_type, const TensorShape& s DataTypeImpl::GetType()->GetDeleteFunc()}; }; +void CalculateTransposedShape(const TensorShape& input_shape, int64_t axis, + std::vector& permutations, std::vector& output_shape) { + int64_t rank = input_shape.NumDimensions(); + const auto& dims = input_shape.GetDims(); + + permutations.reserve(rank); + permutations.push_back(axis); + + output_shape.reserve(rank); + output_shape.push_back(dims[axis]); + + for (int64_t i = 0; i < rank; ++i) { + if (i != axis) { + permutations.push_back(i); + output_shape.push_back(dims[i]); + } + } +} + LoopStateVariable::LoopStateVariable(const MLValue& original_value, MLValue& final_value, const int64_t sequence_len, @@ -280,14 +311,18 @@ OutputIterator::OutputIterator(OpKernelContextInternal& context, bool is_loop_state_var, bool is_v8, TensorShape final_shape, - ScanDirection direction) + ScanDirection direction, + bool temporary, + MLDataType data_type) : context_{context}, is_v8_{is_v8}, output_index_{output_index}, final_shape_{final_shape}, is_loop_state_var_{is_loop_state_var}, direction_{direction}, - cur_iteration_{0} { + cur_iteration_{0}, + temporary_{temporary}, + data_type_{data_type} { is_concrete_shape_ = final_shape_.Size() >= 0; if (is_v8) { @@ -330,13 +365,25 @@ Status OutputIterator::Initialize() { Status OutputIterator::AllocateFinalBuffer() { // make sure a single buffer for the full output is created upfront. // we slice this into per-iteration pieces in Execute using MLValueTensorSlicer. - auto* tensor = context_.Output(output_index_, final_shape_); - - if (!tensor) - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Failed to create output tensor for output #", output_index_); - // get the output tensor we just created as an MLValue - final_output_mlvalue_ = context_.GetOutputMLValue(output_index_); + if (!temporary_) { + // we can write directly to the Scan output + auto* tensor = context_.Output(output_index_, final_shape_); + + if (!tensor) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Failed to create output tensor for output #", output_index_); + } + + final_output_mlvalue_ = context_.GetOutputMLValue(output_index_); + } else { + // we need to do a transpose at the end so need to write to a temporary buffer when executing the subgraph. + AllocatorPtr alloc; + auto status = context_.GetTempSpaceAllocator(&alloc); + ORT_RETURN_IF_ERROR(status); + + temporary_final_output_mlvalue_ = AllocateTensorInMLValue(data_type_, final_shape_, alloc); + final_output_mlvalue_ = &temporary_final_output_mlvalue_; + } // if it's v8 there's always a batch size dimension so we need a slicer to hide that from each iteration if (is_v8_) { diff --git a/onnxruntime/core/providers/cpu/controlflow/scan_utils.h b/onnxruntime/core/providers/cpu/controlflow/scan_utils.h index 6785ede045..b3ce70e280 100644 --- a/onnxruntime/core/providers/cpu/controlflow/scan_utils.h +++ b/onnxruntime/core/providers/cpu/controlflow/scan_utils.h @@ -66,6 +66,9 @@ Class that co-ordinates writing to slices of the overall Scan output buffer retu If the subgraph has a symbolic dimension in an output it will use a temporary MLValue for the first execution in order to discover the output shape. Once the shape is known, it will switch to using the overall output buffer to avoid copies. +If 'temporary' is true it will use a temporary MLValue for the overall output as well. Set this to true if the output +needs to be transposed before being returned by the Scan operator. The data_type also needs to be provided if +'temporary' is true to do the allocation. */ class OutputIterator { public: @@ -75,8 +78,11 @@ class OutputIterator { bool is_v8, TensorShape final_shape, std::unique_ptr& iterator, - ScanDirection direction = ScanDirection::kForward) { - iterator.reset(new OutputIterator(context, output_index, is_loop_state_var, is_v8, final_shape, direction)); + ScanDirection direction = ScanDirection::kForward, + bool temporary = false, + MLDataType data_type = nullptr) { + iterator.reset(new OutputIterator(context, output_index, is_loop_state_var, is_v8, final_shape, + direction, temporary, data_type)); return iterator->Initialize(); } @@ -89,13 +95,20 @@ class OutputIterator { memset(tensor->MutableDataRaw(), 0, tensor->Size()); } + const MLValue& GetOutput() const { + ORT_ENFORCE(final_output_mlvalue_, "Attempt to retrieve final output before it was set."); + return *final_output_mlvalue_; + } + private: OutputIterator(OpKernelContextInternal& context, int output_index, bool is_loop_state_var, bool is_v8, TensorShape final_shape, - ScanDirection direction); + ScanDirection direction, + bool temporary, + MLDataType data_type); Status Initialize(); Status AllocateFinalBuffer(); @@ -122,6 +135,13 @@ class OutputIterator { // we can allocate final_output_mlvalue_ and use the slicers. MLValue first_output_; + // if true allocate temporary_final_output_mlvalue_ with data_type_ using the temporary allocator + // and point final_output_value_ at that. + // if false, final_output_value_ is an output from the Scan operator and allocated using the context_. + bool temporary_; + MLDataType data_type_; + MLValue temporary_final_output_mlvalue_; + MLValue* final_output_mlvalue_; }; @@ -131,7 +151,8 @@ void ReadDirections(const OpKernelInfo& info, const std::string& attr_name, Status AllocateOutput(OpKernelContextInternal& context, const GraphViewer& subgraph, int output_index, bool is_loop_state_var, int64_t batch_size, int64_t sequence_len, std::unique_ptr& output_iterator, - ScanDirection direction = ScanDirection::kForward); + ScanDirection direction = ScanDirection::kForward, + bool temporary = false); Status IterateSequence(OpKernelContextInternal& context, const SessionState& session_state, @@ -148,6 +169,16 @@ Status IterateSequence(OpKernelContextInternal& context, MLValue AllocateTensorInMLValue(const MLDataType data_type, const TensorShape& shape, AllocatorPtr& allocator); +/** +Calculate the transpose permutations and output shape by shifting the chosen axis to the first dimension. +The other dimension indexes or values are pushed in order after the chosen axis. + +e.g. if shape is {2, 3, 4} and axis 1 is chosen the permutations will be {1, 0, 2} and output shape will be {3, 2, 4} + if axis 2 is chosen the permutations will be {2, 0, 1} and the output shape will be {4, 2, 3} +*/ +void CalculateTransposedShape(const TensorShape& input_shape, int64_t axis, + std::vector& permutations, std::vector& output_shape); + } // namespace detail } // namespace scan } // namespace onnxruntime diff --git a/onnxruntime/test/onnx/main.cc b/onnxruntime/test/onnx/main.cc index 4b3a8043dc..631e56eecb 100644 --- a/onnxruntime/test/onnx/main.cc +++ b/onnxruntime/test/onnx/main.cc @@ -335,7 +335,23 @@ int real_main(int argc, char* argv[]) { {"where_example", "opset 9 not supported yet"}, {"constantofshape_float_ones", "opset 9 not supported yet"}, {"batchnorm_example", "opset 9 not supported yet"}, - {"batchnorm_epsilon", "opset 9 not supported yet"}}; + {"batchnorm_epsilon", "opset 9 not supported yet"}, + {"cast_DOUBLE_to_FLOAT16", "Cast opset 9 not supported yet"}, + {"cast_DOUBLE_to_FLOAT", "Cast opset 9 not supported yet"}, + {"cast_FLOAT_to_DOUBLE", "Cast opset 9 not supported yet"}, + {"cast_STRING_to_FLOAT", "Cast opset 9 not supported yet"}, + {"cast_FLOAT16_to_FLOAT", "Cast opset 9 not supported yet"}, + {"cast_FLOAT_to_STRING", "Cast opset 9 not supported yet"}, + {"cast_FLOAT_to_FLOAT16", "Cast opset 9 not supported yet"}, + {"cast_FLOAT16_to_DOUBLE", "Cast opset 9 not supported yet"}, + {"nonzero_example", "NonZero opset 9 not supported yet"}, + {"tfidfvectorizer_tf_uniandbigrams_skip5", "TfIdfVectorizer opset 9 not supported yet"}, + {"tfidfvectorizer_tf_batch_onlybigrams_skip0", "TfIdfVectorizer opset 9 not supported yet"}, + {"tfidfvectorizer_tf_onlybigrams_skip5", "TfIdfVectorizer opset 9 not supported yet"}, + {"tfidfvectorizer_tf_only_bigrams_skip0", "TfIdfVectorizer opset 9 not supported yet"}, + {"tfidfvectorizer_tf_onlybigrams_levelempty", "TfIdfVectorizer opset 9 not supported yet"}, + {"tfidfvectorizer_tf_batch_uniandbigrams_skip5", "TfIdfVectorizer opset 9 not supported yet"}, + {"tfidfvectorizer_tf_batch_onlybigrams_skip5", "TfIdfVectorizer opset 9 not supported yet"}}; #ifdef USE_CUDA broken_tests["maxpool_2d_default"] = "cudnn pooling only support input dimension >= 3"; diff --git a/onnxruntime/test/providers/cpu/controlflow/scan_test.cc b/onnxruntime/test/providers/cpu/controlflow/scan_test.cc index f017a1e282..d140a3f560 100644 --- a/onnxruntime/test/providers/cpu/controlflow/scan_test.cc +++ b/onnxruntime/test/providers/cpu/controlflow/scan_test.cc @@ -5,7 +5,8 @@ #include "gmock/gmock.h" #include "core/framework/session_state.h" #include "core/session/inference_session.h" - +#include "core/providers/common.h" +#include "core/providers/cpu/controlflow/scan_utils.h" #include "test/providers/provider_test_utils.h" #include "test/util/include/default_providers.h" @@ -314,8 +315,9 @@ static void RunTest_v8(const std::string test_name, int64_t batch_size, int64_t static void RunTest_v9(const std::string test_name, int64_t sequence_len, int64_t input_size, std::vector* input_directions, - std::vector* axes, std::vector* output_directions, + std::vector* input_axes, + std::vector* output_axes, std::vector& loop_state_in_0, std::vector input_0, std::vector input_1, @@ -342,14 +344,18 @@ static void RunTest_v9(const std::string test_name, int64_t sequence_len, int64_ test.AddAttribute>("scan_input_directions", *input_directions); } - if (axes != nullptr) { - test.AddAttribute>("axes", *axes); - } - if (output_directions != nullptr) { test.AddAttribute>("scan_output_directions", *output_directions); } + if (input_axes != nullptr) { + test.AddAttribute>("scan_input_axes", *input_axes); + } + + if (output_axes != nullptr) { + test.AddAttribute>("scan_output_axes", *output_axes); + } + test.AddShapeToTensorData(options.include_dim_values_in_main_graph); std::vector loop_state_shape; @@ -366,10 +372,29 @@ static void RunTest_v9(const std::string test_name, int64_t sequence_len, int64_ test.AddOutput("scan_loop_state_out_0", loop_state_shape, loop_state_out_0); std::vector output_shape{sequence_len, 1}; - test.AddOutput("scan_output_0", output_shape, output_0); - test.AddOutput("scan_output_1", output_shape, output_1); - test.AddOutput("scan_output_2", output_shape, output_2); - test.AddOutput("scan_output_3", output_shape, output_3); + + auto calculate_output_shape = [&](size_t output_index) { + if (output_axes && output_axes->size() > output_index) { + auto axis = output_axes->at(output_index); + auto rank = gsl::narrow_cast(output_shape.size()); + + // skip if this is an invalid input test and axis is out of the valid range + if (axis >= -rank && axis < rank) { + std::vector permutations; + std::vector new_shape; + scan::detail::CalculateTransposedShape(output_shape, HandleNegativeAxis(axis, output_shape.size()), + permutations, new_shape); + return new_shape; + } + } + + return output_shape; + }; + + test.AddOutput("scan_output_0", calculate_output_shape(0), output_0); + test.AddOutput("scan_output_1", calculate_output_shape(1), output_1); + test.AddOutput("scan_output_2", calculate_output_shape(2), output_2); + test.AddOutput("scan_output_3", calculate_output_shape(3), output_3); if (options.mixed_execution_providers) { // we want the CUDA provider to be first, and the CPU provider second. all except the Scannode should run on @@ -418,7 +443,7 @@ static void ShortSequenceOneInBatchOneLoopStateVar(const RunOptions& options, co expected_error); } else { RunTest_v9("ShortSequenceOneInBatchOneLoopStateVar", input_size, sequence_len, - nullptr, nullptr, nullptr, + nullptr, nullptr, nullptr, nullptr, iteration_count_in, input_0, input_1, iteration_count_out, output_0, output_1, output_2, output_3, options, @@ -725,7 +750,7 @@ TEST(Scan9, ReversedInput) { std::vector output_3{14.f, 12.f}; RunTest_v9("ReversedInput", sequence_len, input_size, - &input_directions, nullptr, nullptr, + &input_directions, nullptr, nullptr, nullptr, iteration_count_in, input_0, input_1, iteration_count_out, output_0, output_1, output_2, output_3); } @@ -754,7 +779,7 @@ TEST(Scan9, ReversedOutput) { std::vector output_3{12.f, 14.f}; RunTest_v9("ReversedOutput", sequence_len, input_size, - nullptr, nullptr, &output_directions, + nullptr, &output_directions, nullptr, nullptr, iteration_count_in, input_0, input_1, iteration_count_out, output_0, output_1, output_2, output_3); } @@ -766,7 +791,7 @@ TEST(Scan9, TransposeInput) { std::vector iteration_count_in{0.f}; // transpose should also support negative axis - std::vector axes{1, -1}; // transpose both inputs on axis 1 + std::vector input_axes{1, -1}; // transpose both inputs on axis 1 // inputs are {input_size, sequence_len}, but will be transposed to {sequence_len, input_size} by the axes values std::vector input_0{1.f, 3.f, @@ -785,7 +810,36 @@ TEST(Scan9, TransposeInput) { std::vector output_3{12.f, 14.f}; RunTest_v9("TransposeInput", sequence_len, input_size, - nullptr, &axes, nullptr, + nullptr, nullptr, &input_axes, nullptr, + iteration_count_in, input_0, input_1, + iteration_count_out, output_0, output_1, output_2, output_3); +} + +TEST(Scan9, TransposeOutput) { + const int64_t sequence_len = 2; + const int64_t input_size = 2; + + std::vector iteration_count_in{0.f}; + + // transpose also supports negative axis + std::vector output_axes{1, -1, 0, 0}; // transpose two outputs on axis 1, and leave 2 as is by using axis 0 + + std::vector input_0{1.f, 2.f, + 3.f, 4.f}; + std::vector input_1{11.f, 12.f, + 13.f, 14.f}; + + std::vector iteration_count_out{2.f}; // iteration_count_in + 1 for each item in sequence + + // whilst we transpose that only changes the shape from 2, 1 to 1, 2 so the data is the same. the expected + // shape is validated by RunTest_v9. + std::vector output_0{1.f, 3.f}; + std::vector output_1{2.f, 4.f}; + std::vector output_2{11.f, 13.f}; + std::vector output_3{12.f, 14.f}; + + RunTest_v9("TransposeOutput", sequence_len, input_size, + nullptr, nullptr, nullptr, &output_axes, iteration_count_in, input_0, input_1, iteration_count_out, output_0, output_1, output_2, output_3); } @@ -822,7 +876,7 @@ static void InvalidInput(bool is_v8) { "Invalid values in 'directions'."); } else { RunTest_v9("InvalidInputDirectionsValue", sequence_len, input_size, - &directions, nullptr, nullptr, + &directions, nullptr, nullptr, nullptr, iteration_count_in, input_0, input_1, iteration_count_out, output_0, output_1, output_2, output_3, {}, @@ -832,7 +886,7 @@ static void InvalidInput(bool is_v8) { std::vector output_directions = {0, 2, 1, 0}; RunTest_v9("InvalidOutputDirectionsValue", sequence_len, input_size, - nullptr, nullptr, &output_directions, + nullptr, &output_directions, nullptr, nullptr, iteration_count_in, input_0, input_1, iteration_count_out, output_0, output_1, output_2, output_3, {}, @@ -853,7 +907,7 @@ static void InvalidInput(bool is_v8) { "Number of entries in 'directions' was 3 but expected 2"); } else { RunTest_v9("InvalidNumEntriesInInputDirections", sequence_len, input_size, - &directions, nullptr, nullptr, + &directions, nullptr, nullptr, nullptr, iteration_count_in, input_0, input_1, iteration_count_out, output_0, output_1, output_2, output_3, {}, @@ -861,7 +915,7 @@ static void InvalidInput(bool is_v8) { "Number of entries in 'scan_input_directions' was 3 but expected 2"); RunTest_v9("InvalidNumEntriesInOutputDirections", sequence_len, input_size, - nullptr, nullptr, &directions, + nullptr, &directions, nullptr, nullptr, iteration_count_in, input_0, input_1, iteration_count_out, output_0, output_1, output_2, output_3, {}, @@ -870,23 +924,41 @@ static void InvalidInput(bool is_v8) { } if (!is_v8) { - std::vector axes = {2, -1}; // only 2 dims in input so 2 is invalid - RunTest_v9("InvalidEntryInAxes", sequence_len, input_size, - nullptr, &axes, nullptr, + std::vector input_axes = {2, -1}; // only 2 dims in input so 2 is invalid + RunTest_v9("InvalidEntryInInputAxes", sequence_len, input_size, + nullptr, nullptr, &input_axes, nullptr, iteration_count_in, input_0, input_1, iteration_count_out, output_0, output_1, output_2, output_3, {}, OpTester::ExpectResult::kExpectFailure, - "Invalid value in axes for input 0 of 2. Input tensor rank was 2"); + "Invalid value in scan_input_axes for input 0 of 2. Input tensor rank was 2"); - axes = {0, 1, 2}; - RunTest_v9("InvalidNumEntriesInAxes", sequence_len, input_size, - nullptr, &axes, nullptr, + input_axes = {0, 1, 2}; + RunTest_v9("InvalidNumEntriesInInputAxes", sequence_len, input_size, + nullptr, nullptr, &input_axes, nullptr, iteration_count_in, input_0, input_1, iteration_count_out, output_0, output_1, output_2, output_3, {}, OpTester::ExpectResult::kExpectFailure, - "[ShapeInferenceError] Number of axes specified (3) is not equal to number of scan inputs (2)."); + "[ShapeInferenceError] Number of scan input axes specified (3) is not equal to number of scan inputs (2)."); + + std::vector output_axes = {3, -1, 0, 0}; // 2 dims in output so 3 is invalid + RunTest_v9("InvalidEntryInOutputAxes", sequence_len, input_size, + nullptr, nullptr, nullptr, &output_axes, + iteration_count_in, input_0, input_1, + iteration_count_out, output_0, output_1, output_2, output_3, + {}, + OpTester::ExpectResult::kExpectFailure, + "[ShapeInferenceError] scan_output_axes axis value 3 is invalid for a tensor of rank 2"); + + output_axes = {0, 1, 2}; + RunTest_v9("InvalidNumEntriesInOutputAxes", sequence_len, input_size, + nullptr, nullptr, nullptr, &output_axes, + iteration_count_in, input_0, input_1, + iteration_count_out, output_0, output_1, output_2, output_3, + {}, + OpTester::ExpectResult::kExpectFailure, + "[ShapeInferenceError] Number of scan output axes specified (3) is not equal to number of scan outputs (4)."); } } diff --git a/tools/ci_build/github/linux/docker/scripts/install_deps.sh b/tools/ci_build/github/linux/docker/scripts/install_deps.sh index 87f0d5c5d7..7ca73605c0 100755 --- a/tools/ci_build/github/linux/docker/scripts/install_deps.sh +++ b/tools/ci_build/github/linux/docker/scripts/install_deps.sh @@ -34,8 +34,8 @@ else #Install ONNX #5af210ca8a1c73aa6bae8754c9346ec54d0a756e is v1.2.3 #bae6333e149a59a3faa9c4d9c44974373dcf5256 is v1.3.0 - #c4cf11269c1ef9bf1f459bb5b1b68a5f66840321 is v1.3.0 latest - for onnx_version in "5af210ca8a1c73aa6bae8754c9346ec54d0a756e" "bae6333e149a59a3faa9c4d9c44974373dcf5256" "c4cf11269c1ef9bf1f459bb5b1b68a5f66840321"; do + #dbf3581835e3a05716e10587511d7ab3b2cdc386 is v1.3.0 latest + for onnx_version in "5af210ca8a1c73aa6bae8754c9346ec54d0a756e" "bae6333e149a59a3faa9c4d9c44974373dcf5256" "dbf3581835e3a05716e10587511d7ab3b2cdc386"; do if [ -z ${lastest_onnx_version+x} ]; then echo "first pass"; else