From 9f633c5bd9ecccdb0d02e0088d516d67a9dab7cb Mon Sep 17 00:00:00 2001 From: Scott McKay Date: Thu, 3 Oct 2019 07:58:42 +1000 Subject: [PATCH] Update Cast op to use precision of 8 when casting floating point numbers to strings (#1210) * Update Cast op to use precision of 8 when casting floating point numbers to strings. This matches numpy precision. Update unit tests to include non-trivial floats in the input. Update onnx test infrastructure to document why the test cases are disabled --- .../core/providers/cpu/tensor/cast_op.cc | 27 +++++++++------ onnxruntime/test/onnx/main.cc | 7 ++-- .../providers/cpu/tensor/tensor_op_test.cc | 21 ++++++++---- .../test/python/onnx_backend_test_series.py | 33 +++++++++++-------- 4 files changed, 56 insertions(+), 32 deletions(-) diff --git a/onnxruntime/core/providers/cpu/tensor/cast_op.cc b/onnxruntime/core/providers/cpu/tensor/cast_op.cc index 341060df1a..928db73c47 100644 --- a/onnxruntime/core/providers/cpu/tensor/cast_op.cc +++ b/onnxruntime/core/providers/cpu/tensor/cast_op.cc @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. - +#include #include #include "core/common/common.h" #include "core/framework/op_kernel.h" @@ -72,19 +72,27 @@ template inline void CastToStringData(const Tensor* in, Tensor* out, const TensorShape& shape) { const int64_t len = shape.Size(); ORT_ENFORCE(len > 0); + auto input_data = in->DataAsSpan(); + auto output_data = out->MutableDataAsSpan(); + for (int i = 0; i < len; ++i) { - if (std::is_floating_point::value && std::isnan(in->Data()[i])) { - out->MutableData()[i] = "NaN"; - } else if (std::is_floating_point::value && std::isinf(in->Data()[i])) { - if (in->Data()[i] < std::numeric_limits::lowest()) { - out->MutableData()[i] = "-INF"; + if (std::is_floating_point::value && std::isnan(input_data[i])) { + output_data[i] = "NaN"; + } else if (std::is_floating_point::value && std::isinf(input_data[i])) { + if (input_data[i] < std::numeric_limits::lowest()) { + output_data[i] = "-INF"; } else { - out->MutableData()[i] = "INF"; + output_data[i] = "INF"; } } else { std::ostringstream convert; - convert << in->Data()[i]; - out->MutableData()[i] = convert.str(); + if (std::is_floating_point::value) { + // match numpy default behavior + convert << std::setprecision(8); + } + + convert << input_data[i]; + output_data[i] = convert.str(); } } } @@ -196,7 +204,6 @@ class Cast final : public OpKernel { ONNX_NAMESPACE::TensorProto_DataType to_; }; - const std::vector castOpTypeConstraints{ DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType(), diff --git a/onnxruntime/test/onnx/main.cc b/onnxruntime/test/onnx/main.cc index 8ee52e6847..989be674ab 100644 --- a/onnxruntime/test/onnx/main.cc +++ b/onnxruntime/test/onnx/main.cc @@ -374,8 +374,11 @@ int real_main(int argc, char* argv[], Ort::Env& env) { {"constantofshape_int_zeros", "test data bug", {"onnx141","onnx150"}}, {"convtranspose_1d", "1d convtranspose not supported yet"}, {"convtranspose_3d", "3d convtranspose not supported yet"}, - {"cast_STRING_to_FLOAT", "result differs"}, - {"cast_FLOAT_to_STRING", "result differs"}, + {"cast_STRING_to_FLOAT", "Linux CI has old ONNX python package with bad test data", {"onnx141"}}, + // Numpy float to string has unexpected rounding for some results given numpy default precision is meant to be 8. + // "e.g. 0.296140194 -> '0.2961402' not '0.29614019'. ORT produces the latter with precision set to 8, + // which doesn't match the expected output that was generated with numpy. + {"cast_FLOAT_to_STRING", "Numpy float to string has unexpected rounding for some results."}, {"tf_nasnet_large", "disable temporarily"}, {"tf_nasnet_mobile", "disable temporarily"}, {"tf_pnasnet_large", "disable temporarily"}, diff --git a/onnxruntime/test/providers/cpu/tensor/tensor_op_test.cc b/onnxruntime/test/providers/cpu/tensor/tensor_op_test.cc index 9bb0a844f0..19079e79d7 100644 --- a/onnxruntime/test/providers/cpu/tensor/tensor_op_test.cc +++ b/onnxruntime/test/providers/cpu/tensor/tensor_op_test.cc @@ -46,7 +46,7 @@ TEST(TensorOpTest, ShapeTest2D) { test.AddInput("data", {2, 3}, std::vector(6, 1.0f)); test.AddOutput("shape", {2}, {2, 3}); - test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});//TensorRT: volume of dimensions is not consistent with weights size + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: volume of dimensions is not consistent with weights size } TEST(TensorOpTest, ShapeTest3D) { @@ -54,7 +54,7 @@ TEST(TensorOpTest, ShapeTest3D) { test.AddInput("data", {2, 3, 4}, std::vector(24, 1.0f)); test.AddOutput("shape", {3}, {2, 3, 4}); - test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});//TensorRT: volume of dimensions is not consistent with weights size + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: volume of dimensions is not consistent with weights size } template shape{2, 2, 2}; - std::initializer_list string_data = {"-inf", "+INF", "2.0f", "3.0f", "4.0f", "5.0f", "NaN", "nan"}; - const std::initializer_list float_output = {-(std::numeric_limits::infinity()), std::numeric_limits::infinity(), 2.0f, 3.0f, 4.0f, 5.0f, NAN, NAN}; + std::initializer_list string_data = {"-inf", "+INF", "0.9767611f", "0.28280696f", + "-0.12019656f", "5.0f", "NaN", "nan"}; + const std::initializer_list float_output = {-(std::numeric_limits::infinity()), std::numeric_limits::infinity(), + 0.9767611f, 0.28280696f, + -0.12019656f, 5.0f, NAN, NAN}; TestCastOp(string_data, float_output, shape, TensorProto::FLOAT); std::initializer_list int_16_string_data = {"0", "1", "2", "3", "4", "5", "-32768", "32767"}; @@ -279,8 +282,13 @@ TEST(TensorOpTest, CastFromString) { TEST(TensorOpTest, CastToString) { const std::vector shape{2, 2, 2}; - const std::initializer_list float_input = {NAN, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, -std::numeric_limits::infinity(), std::numeric_limits::infinity()}; - std::initializer_list string_output = {"NaN", "1", "2", "3", "4", "5", "-INF", "INF"}; + const std::initializer_list float_input = {NAN, -1.f, 0.0391877927f, 0.296140194f, -0.120196559f, 5.0f, + -std::numeric_limits::infinity(), + std::numeric_limits::infinity()}; + + // float output precision is 8, so the expected output differs slightly from the input due to that + std::initializer_list string_output = {"NaN", "-1", "0.039187793", "0.29614019", + "-0.12019656", "5", "-INF", "INF"}; TestCastOp(float_input, string_output, shape, TensorProto::STRING); std::initializer_list int_string_data = {"0", "1", "2", "3", "4", "5", "6", "7"}; @@ -375,7 +383,6 @@ void MeanVarianceNormalizationFunctionAcrossChannels(std::vector axes) } TEST(TensorOpTest, MeanVarianceNormalizationCPUTest) { - // axes: {0, 1, 2, 3} for across_channels MeanVarianceNormalizationFunctionAcrossChannels({0, 1, 2, 3}); diff --git a/onnxruntime/test/python/onnx_backend_test_series.py b/onnxruntime/test/python/onnx_backend_test_series.py index 4c1db26639..61903645b4 100644 --- a/onnxruntime/test/python/onnx_backend_test_series.py +++ b/onnxruntime/test/python/onnx_backend_test_series.py @@ -36,7 +36,7 @@ class OrtBackendTest(onnx.backend.test.BackendTest): # ORT first supported opset 7, so models with nodes that require versions prior to opset 7 are not supported def tests_with_pre_opset7_dependencies_filters(): - filters = ('^test_AvgPool1d_cpu.*', + filters = ['^test_AvgPool1d_cpu.*', '^test_AvgPool1d_stride_cpu.*', '^test_AvgPool2d_cpu.*', '^test_AvgPool2d_stride_cpu.*', @@ -69,17 +69,24 @@ def tests_with_pre_opset7_dependencies_filters(): '^test_operator_mm_cpu.*', '^test_operator_non_float_params_cpu.*', '^test_operator_params_cpu.*', - '^test_operator_pow_cpu.*') + '^test_operator_pow_cpu.*'] return filters def unsupported_usages_filters(): - filters = ('^test_convtranspose_1d_cpu.*', # ConvTransponse supports 4-D only - '^test_convtranspose_3d_cpu.*') + filters = ['^test_convtranspose_1d_cpu.*', # ConvTransponse supports 4-D only + '^test_convtranspose_3d_cpu.*'] return filters +def other_tests_failing_permanently_filters(): + # Numpy float to string has unexpected rounding for some results given numpy default precision is meant to be 8. + # e.g. 0.296140194 -> '0.2961402' not '0.29614019'. ORT produces the latter with precision set to 8, which + # doesn't match the expected output that was generated with numpy. + filters = ['^test_cast_FLOAT_to_STRING_cpu.*'] + + return filters def create_backend_test(testname=None): backend_test = OrtBackendTest(c2, __name__) @@ -91,8 +98,7 @@ def create_backend_test(testname=None): backend_test.include(testname + '.*') else: # Tests that are failing temporarily and should be fixed - current_failing_tests = ('^test_cast_STRING_to_FLOAT_cpu.*', - '^test_cast_FLOAT_to_STRING_cpu.*', + current_failing_tests = [#'^test_cast_STRING_to_FLOAT_cpu.*', # old test data that is bad on Linux CI builds '^test_qlinearconv_cpu.*', '^test_gru_seq_length_cpu.*', '^test_bitshift_right_uint16_cpu.*', @@ -147,8 +153,8 @@ def create_backend_test(testname=None): '^test_onehot_*', '^test_constant_pad_cpu.*', '^test_edge_pad_cpu.*', - '^test_reflect_pad_cpu.*' - ) + '^test_reflect_pad_cpu.*' + ] # Example of how to disable tests for a specific provider. # if c2.supports_device('NGRAPH'): @@ -156,20 +162,21 @@ def create_backend_test(testname=None): if c2.supports_device('NGRAPH'): current_failing_tests = current_failing_tests + ('|^test_clip*',) current_failing_tests = current_failing_tests + ('|^test_depthtospace_crd*',) - current_failing_tests = current_failing_tests + ('|^test_argmax_negative_axis*',) + current_failing_tests = current_failing_tests + ('|^test_argmax_negative_axis*',) current_failing_tests = current_failing_tests + ('|^test_argmin_negative_axis*',) - current_failing_tests = current_failing_tests + ('|^test_hadmax_negative_axis*',) - current_failing_tests = current_failing_tests + ('|^test_gemm_default_no_bias_cpu.*',) + current_failing_tests = current_failing_tests + ('|^test_hadmax_negative_axis*',) + current_failing_tests = current_failing_tests + ('|^test_gemm_default_no_bias_cpu.*',) if c2.supports_device('OPENVINO_GPU_FP32') or c2.supports_device('OPENVINO_GPU_FP16'): current_failing_tests = current_failing_tests + ('^test_div_cpu*',) filters = current_failing_tests + \ tests_with_pre_opset7_dependencies_filters() + \ - unsupported_usages_filters() + unsupported_usages_filters() + \ + other_tests_failing_permanently_filters() backend_test.exclude('(' + '|'.join(filters) + ')') - print ('excluded tests:', filters) + print('excluded tests:', filters) # import all test cases at global scope to make # them visible to python.unittest.