From e8b0d2407157216a898d0575cf6fe03df6ece389 Mon Sep 17 00:00:00 2001 From: Gary Miguel Date: Tue, 14 Jun 2022 15:12:23 -0700 Subject: [PATCH] Support per-test tolerances for ONNX tests (#11775) Prior to this every test shared the same tolerances. This meant that if an ONNX test failed due to a small but acceptable difference in output, the only alternative was to disable the test entirely. In op set 17, the DFT operator is being added. Without this change, the tests for that operator fail because the output is off by about 5e-5. It's better to keep test coverage for this new op rather than disable the test entirely. Also prior to this change, the global tolerances were not shared between C++, JavaScript, and Python tests. Now they are. Also fix various minor issues raised by linters. Unblocks https://github.com/microsoft/onnxruntime/issues/11640. --- cmake/CMakeLists.txt | 1 + cmake/external/json | 2 +- cmake/onnxruntime_unittests.cmake | 2 +- docs/How_To_Update_ONNX_Dev_Notes.md | 28 ++-- js/node/test/test-runner.ts | 4 +- js/node/test/test-utils.ts | 36 ++++-- onnxruntime/test/onnx/TestCase.cc | 53 ++++++-- onnxruntime/test/onnx/TestCase.h | 28 +++- onnxruntime/test/onnx/main.cc | 49 +++++-- .../test/python/onnx_backend_test_series.py | 121 ++++++++++++------ .../onnx_backend_test_series_filters.jsonc | 16 ++- .../onnx_backend_test_series_overrides.jsonc | 13 ++ winml/test/model/model_tests.cpp | 4 +- 13 files changed, 252 insertions(+), 105 deletions(-) create mode 100644 onnxruntime/test/testdata/onnx_backend_test_series_overrides.jsonc diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 4d4a5a3b64..a3f9faeedb 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -434,6 +434,7 @@ if (onnxruntime_DISABLE_EXCEPTIONS) add_compile_definitions("ORT_NO_EXCEPTIONS") add_compile_definitions("MLAS_NO_EXCEPTION") add_compile_definitions("ONNX_NO_EXCEPTIONS") + add_compile_definitions("JSON_NOEXCEPTION") # https://json.nlohmann.me/api/macros/json_noexception/ if (MSVC) string(REGEX REPLACE "/EHsc" "/EHs-c-" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") diff --git a/cmake/external/json b/cmake/external/json index db78ac1d77..b5364faf9d 160000 --- a/cmake/external/json +++ b/cmake/external/json @@ -1 +1 @@ -Subproject commit db78ac1d7716f56fc9f1b030b715f872f93964e4 +Subproject commit b5364faf9d732052506cefc933d3f4e4f04513a5 diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index baf81e48aa..071baa968e 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -861,7 +861,7 @@ if (onnxruntime_BUILD_WEBASSEMBLY) endif() endif() -target_link_libraries(onnx_test_runner PRIVATE onnx_test_runner_common ${GETOPT_LIB_WIDE} ${onnx_test_libs}) +target_link_libraries(onnx_test_runner PRIVATE onnx_test_runner_common ${GETOPT_LIB_WIDE} ${onnx_test_libs} nlohmann_json::nlohmann_json) target_include_directories(onnx_test_runner PRIVATE ${ONNXRUNTIME_ROOT}) if (onnxruntime_USE_ROCM) target_include_directories(onnx_test_runner PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/amdgpu/onnxruntime ${CMAKE_CURRENT_BINARY_DIR}/amdgpu/orttraining) diff --git a/docs/How_To_Update_ONNX_Dev_Notes.md b/docs/How_To_Update_ONNX_Dev_Notes.md index c1d1f634b4..3c13e50862 100644 --- a/docs/How_To_Update_ONNX_Dev_Notes.md +++ b/docs/How_To_Update_ONNX_Dev_Notes.md @@ -1,9 +1,11 @@ -This is a note only for ONNX Runtime developers. +# How to update ONNX -It's very often, you need to update the ONNX submodule to a newer version in the upstream. Please follow the steps below, don't miss any! +This note is only for ONNX Runtime developers. -1. Update the ONNX subfolder -``` +If you need to update the ONNX submodule to a different version, follow the steps below. + +1. Update the ONNX submodule +```sh cd cmake/external/onnx git remote update git reset --hard @@ -15,22 +17,28 @@ git add onnx 1. Update [cgmanifests/generated/cgmanifest.json](/cgmanifests/generated/cgmanifest.json). This file should be generated. See [cgmanifests/README](/cgmanifests/README.md) for instructions. -1. Update [tools/ci_build/github/linux/docker/scripts/requirements.txt](/tools/ci_build/github/linux/docker/scripts/requirements.txt) and [tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt](/tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt). -Update the commit hash for `git+http://github.com/onnx/onnx.git@targetonnxcommithash#egg=onnx`. +1. Update [tools/ci_build/github/linux/docker/scripts/requirements.txt](/tools/ci_build/github/linux/docker/scripts/requirements.txt) + and [tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt](/tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt). + Update the commit hash for `git+http://github.com/onnx/onnx.git@targetonnxcommithash#egg=onnx`. -1. If there is any change to `cmake/external/onnx/onnx/*.in.proto`, you need to regenerate OnnxMl.cs. [Building onnxruntime with Nuget](https://onnxruntime.ai/docs/build/inferencing.html#build-nuget-packages) will do this. +1. If there is any change to `cmake/external/onnx/onnx/*.in.proto`, you need to regenerate OnnxMl.cs. + [Building onnxruntime with Nuget](https://onnxruntime.ai/docs/build/inferencing.html#build-nuget-packages) will do + this. -1. If you are updating ONNX from a released tag to a new commit, please tell Changming deploying the new test data along with other test models to our CI build machines. This is to ensure that our tests cover every ONNX opset. +1. If you are updating ONNX from a released tag to a new commit, please ask Changming (@snnn) to deploy the new test + data along with other test models to our CI build machines. This is to ensure that our tests cover every ONNX opset. 1. Send you PR, and **manually** queue a build for every packaging pipeline for your branch. -1. If there is a build failure in stage "Check out of dated documents" in WebAssembly CI pipeline, update ONNX Runtime Web WebGL operator support document: +1. If there is a build failure in stage "Check out of dated documents" in WebAssembly CI pipeline, update ONNX Runtime + Web WebGL operator support document: - Make sure Node.js is installed (see [Prerequisites](../js/README.md#Prerequisites) for instructions). - Follow step 1 in [js/Build](../js/README.md#Build-2) to install dependencies). - Follow instructions in [Generate document](../js/README.md#Generating-Document) to update document. Commit changes applied to file `docs/operators.md`. -1. Usually there would be some unitest failures, because you introduced new test cases. Then you may need to update +1. Usually some newly introduced tests will fail. Then you may need to update - [onnxruntime/test/onnx/main.cc](/onnxruntime/test/onnx/main.cc) - [onnxruntime/test/providers/cpu/model_tests.cc](/onnxruntime/test/providers/cpu/model_tests.cc) - [csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs](/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs) - [onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc](/onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc) +- [onnxruntime/test/testdata/onnx_backend_test_series_overrides.jsonc](/onnxruntime/test/testdata/onnx_backend_test_series_overrides.jsonc) diff --git a/js/node/test/test-runner.ts b/js/node/test/test-runner.ts index 609f98299e..5ba4767a28 100644 --- a/js/node/test/test-runner.ts +++ b/js/node/test/test-runner.ts @@ -5,7 +5,7 @@ import * as fs from 'fs-extra'; import {InferenceSession, Tensor} from 'onnxruntime-common'; import * as path from 'path'; -import {assertTensorEqual, loadTensorFromFile, shouldSkipModel} from './test-utils'; +import {assertTensorEqual, atol, loadTensorFromFile, rtol, shouldSkipModel} from './test-utils'; export function run(testDataFolder: string): void { const models = fs.readdirSync(testDataFolder); @@ -98,7 +98,7 @@ export function run(testDataFolder: string): void { let j = 0; for (const name of session.outputNames) { - assertTensorEqual(outputs[name], expectedOutputs[j++]!); + assertTensorEqual(outputs[name], expectedOutputs[j++]!, atol(model), rtol(model)); } } else { throw new TypeError('session is null'); diff --git a/js/node/test/test-utils.ts b/js/node/test/test-utils.ts index 4613fb7519..18ca9a5252 100644 --- a/js/node/test/test-utils.ts +++ b/js/node/test/test-utils.ts @@ -20,6 +20,13 @@ export const SQUEEZENET_OUTPUT0_DATA: number[] = require(path.join(TEST_DATA_ROO export const BACKEND_TEST_SERIES_FILTERS: {[name: string]: string[]} = jsonc.readSync(path.join(ORT_ROOT, 'onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc')); +const OVERRIDES: { + atol_default: number; rtol_default: number; atol_overrides: {[name: string]: number}; + rtol_overrides: {[name: string]: number}; +} = jsonc.readSync(path.join(ORT_ROOT, 'onnxruntime/test/testdata/onnx_backend_test_series_overrides.jsonc')); + +const ATOL_DEFAULT = OVERRIDES.atol_default; +const RTOL_DEFAULT = OVERRIDES.rtol_default; export const NUMERIC_TYPE_MAP = new Map Tensor.DataType>([ ['float32', Float32Array], @@ -83,9 +90,10 @@ export function warmup(): void { } export function assertFloatEqual( - actual: number[]|Float32Array|Float64Array, expected: number[]|Float32Array|Float64Array): void { - const THRESHOLD_ABSOLUTE_ERROR = 1.0e-4; - const THRESHOLD_RELATIVE_ERROR = 1.000001; + actual: number[]|Float32Array|Float64Array, expected: number[]|Float32Array|Float64Array, atol?: number, + rtol?: number): void { + const absolute_tol: number = atol ?? 1.0e-4; + const relative_tol: number = 1 + (rtol ?? 1.0e-6); assert.strictEqual(actual.length, expected.length); @@ -114,10 +122,10 @@ export function assertFloatEqual( // test fail // endif // - if (Math.abs(a - b) < THRESHOLD_ABSOLUTE_ERROR) { + if (Math.abs(a - b) < absolute_tol) { continue; // absolute error check pass } - if (a !== 0 && b !== 0 && a * b > 0 && a / b < THRESHOLD_RELATIVE_ERROR && b / a < THRESHOLD_RELATIVE_ERROR) { + if (a !== 0 && b !== 0 && a * b > 0 && a / b < relative_tol && b / a < relative_tol) { continue; // relative error check pass } @@ -126,12 +134,14 @@ export function assertFloatEqual( } } -export function assertDataEqual(type: Tensor.Type, actual: Tensor.DataType, expected: Tensor.DataType): void { +export function assertDataEqual( + type: Tensor.Type, actual: Tensor.DataType, expected: Tensor.DataType, atol?: number, rtol?: number): void { switch (type) { case 'float32': case 'float64': assertFloatEqual( - actual as number[] | Float32Array | Float64Array, expected as number[] | Float32Array | Float64Array); + actual as number[] | Float32Array | Float64Array, expected as number[] | Float32Array | Float64Array, atol, + rtol); break; case 'uint8': @@ -153,7 +163,7 @@ export function assertDataEqual(type: Tensor.Type, actual: Tensor.DataType, expe } // This function check whether 2 tensors should be considered as 'match' or not -export function assertTensorEqual(actual: Tensor, expected: Tensor): void { +export function assertTensorEqual(actual: Tensor, expected: Tensor, atol?: number, rtol?: number): void { assert(typeof actual === 'object'); assert(typeof expected === 'object'); @@ -168,7 +178,7 @@ export function assertTensorEqual(actual: Tensor, expected: Tensor): void { assert.strictEqual(actualType, expectedType); assert.deepStrictEqual(actualDims, expectedDims); - assertDataEqual(actualType, actual.data, expected.data); + assertDataEqual(actualType, actual.data, expected.data, atol, rtol); } export function loadTensorFromFile(pbFile: string): Tensor { @@ -274,3 +284,11 @@ export function shouldSkipModel(model: string, eps: string[]): boolean { return false; } + +export function atol(model: string): number { + return OVERRIDES.atol_overrides[model] ?? ATOL_DEFAULT; +} + +export function rtol(model: string): number { + return OVERRIDES.rtol_overrides[model] ?? RTOL_DEFAULT; +} diff --git a/onnxruntime/test/onnx/TestCase.cc b/onnxruntime/test/onnx/TestCase.cc index 389518dd68..18d187485a 100644 --- a/onnxruntime/test/onnx/TestCase.cc +++ b/onnxruntime/test/onnx/TestCase.cc @@ -5,6 +5,14 @@ #include "TestCase.h" +#include +#include +#include +#include +#include +#include +#include + #include "callback.h" #include "heap_buffer.h" #include "mem_buffer.h" @@ -22,13 +30,6 @@ #include "core/framework/TensorSeq.h" #include "re2/re2.h" -#include -#include -#include -#include -#include -#include - using namespace onnxruntime; using namespace onnxruntime::common; using google::protobuf::RepeatedPtrField; @@ -417,7 +418,7 @@ static bool read_config_file(const std::basic_string& path, std: return true; } -//load tensors from disk +// load tensors from disk template static void LoadTensor(const PATH_STRING_TYPE& pb_file, ONNX_NAMESPACE::TensorProto& input_pb) { int tensor_fd; @@ -432,7 +433,7 @@ static void LoadTensor(const PATH_STRING_TYPE& pb_file, ONNX_NAMESPACE::TensorPr } } -//load sequence tensors from disk +// load sequence tensors from disk template static void LoadSequenceTensor(const PATH_STRING_TYPE& pb_file, ONNX_NAMESPACE::SequenceProto& input_pb) { int tensor_fd; @@ -475,7 +476,7 @@ void OnnxTestCase::LoadTestData(size_t id, onnxruntime::test::HeapBuffer& b, test_data_dirs_[id], (is_input ? ORT_TSTR("inputs.pb") : ORT_TSTR("outputs.pb"))); int test_data_pb_fd; auto st = Env::Default().FileOpenRd(test_data_pb, test_data_pb_fd); - if (st.IsOK()) { //has an all-in-one input file + if (st.IsOK()) { // has an all-in-one input file std::ostringstream oss; { std::lock_guard l(m_); @@ -727,7 +728,7 @@ OnnxTestCase::OnnxTestCase(const std::string& test_case_name, _In_ std::unique_p void LoadTests(const std::vector>& input_paths, const std::vector>& whitelisted_test_cases, - double default_per_sample_tolerance, double default_relative_per_sample_tolerance, + const TestTolerances& tolerances, const std::unordered_set>& disabled_tests, const std::function)>& process_function) { std::vector> paths(input_paths); @@ -781,11 +782,37 @@ void LoadTests(const std::vector>& input_paths ORT_NOT_IMPLEMENTED(ToUTF8String(filename_str), " is not supported"); } + const auto tolerance_key = ToUTF8String(my_dir_name); + std::unique_ptr l = CreateOnnxTestCase(ToUTF8String(test_case_name), std::move(model_info), - default_per_sample_tolerance, - default_relative_per_sample_tolerance); + tolerances.absolute(tolerance_key), + tolerances.relative(tolerance_key)); process_function(std::move(l)); return true; }); } } + +TestTolerances::TestTolerances( + double absolute_default, double relative_default, + const Map& absolute_overrides, + const Map& relative_overrides) : absolute_default_(absolute_default), + relative_default_(relative_default), + absolute_overrides_(absolute_overrides), + relative_overrides_(relative_overrides) {} + +double TestTolerances::absolute(const std::string& name) const { + const auto iter = absolute_overrides_.find(name); + if (iter == absolute_overrides_.end()) { + return absolute_default_; + } + return iter->second; +} + +double TestTolerances::relative(const std::string& name) const { + const auto iter = relative_overrides_.find(name); + if (iter == relative_overrides_.end()) { + return relative_default_; + } + return iter->second; +} diff --git a/onnxruntime/test/onnx/TestCase.h b/onnxruntime/test/onnx/TestCase.h index b5348efcb4..2366f11bbb 100644 --- a/onnxruntime/test/onnx/TestCase.h +++ b/onnxruntime/test/onnx/TestCase.h @@ -24,8 +24,8 @@ class HeapBuffer; } } // namespace onnxruntime -//One test case is for one model file -//One test case can contain multiple test data(input/output pairs) +// One test case is for one model file +// One test case can contain multiple test data(input/output pairs) class ITestCase { public: virtual void LoadTestData(size_t id, onnxruntime::test::HeapBuffer& b, @@ -38,9 +38,9 @@ class ITestCase { virtual const std::string& GetTestCaseName() const = 0; virtual std::string GetTestCaseVersion() const = 0; - //a string to help identify the dataset + // a string to help identify the dataset virtual std::string GetDatasetDebugInfoString(size_t dataset_id) const = 0; - //The number of input/output pairs + // The number of input/output pairs virtual size_t GetDataCount() const = 0; virtual ~ITestCase() = default; virtual void GetPerSampleTolerance(double* value) const = 0; @@ -83,8 +83,26 @@ std::unique_ptr CreateOnnxTestCase(const std::string& test_case_name, double default_per_sample_tolerance, double default_relative_per_sample_tolerance); +class TestTolerances { + public: + typedef std::unordered_map Map; + TestTolerances( + double absolute_default, double relative_default, + const Map& absolute_overrides, + const Map& relative_overrides); + TestTolerances() = delete; + double absolute(const std::string& test_name) const; + double relative(const std::string& test_name) const; + + private: + double absolute_default_; + double relative_default_; + const Map absolute_overrides_; + const Map relative_overrides_; +}; + void LoadTests(const std::vector>& input_paths, const std::vector>& whitelisted_test_cases, - double default_per_sample_tolerance, double default_relative_per_sample_tolerance, + const TestTolerances& tolerances, const std::unordered_set>& disabled_tests, const std::function)>& process_function); diff --git a/onnxruntime/test/onnx/main.cc b/onnxruntime/test/onnx/main.cc index da99075142..5a469f966a 100644 --- a/onnxruntime/test/onnx/main.cc +++ b/onnxruntime/test/onnx/main.cc @@ -4,6 +4,8 @@ #include #include #include +#include +#include #ifdef _WIN32 #include "getopt.h" #else @@ -20,6 +22,7 @@ #include "core/optimizer/graph_transformer_level.h" #include "core/framework/session_options.h" #include "core/session/onnxruntime_session_options_config_keys.h" +#include "nlohmann/json.hpp" using namespace onnxruntime; @@ -56,6 +59,28 @@ void usage() { OrtGetApiBase()->GetVersionString()); } +static TestTolerances LoadTestTolerances(bool enable_cuda, bool enable_openvino) { + TestTolerances::Map absolute_overrides; + TestTolerances::Map relative_overrides; + std::ifstream overrides_ifstream(ConcatPathComponent( + ORT_TSTR("testdata"), ORT_TSTR("onnx_backend_test_series_overrides.jsonc"))); + if (!overrides_ifstream.good()) { + const double absolute = 1e-3; + // when cuda is enabled, set it to a larger value for resolving random MNIST test failure + // when openvino is enabled, set it to a larger value for resolving MNIST accuracy mismatch + const double relative = enable_cuda ? 0.017 : enable_openvino ? 0.009 + : 1e-3; + return TestTolerances(absolute, relative, absolute_overrides, relative_overrides); + } + const auto overrides_json = nlohmann::json::parse( + overrides_ifstream, + /*cb=*/nullptr, /*allow_exceptions=*/true, /*ignore_comments=*/true); + overrides_json["atol_overrides"].get_to(absolute_overrides); + overrides_json["rtol_overrides"].get_to(relative_overrides); + return TestTolerances( + overrides_json["atol_default"], overrides_json["rtol_default"], absolute_overrides, relative_overrides); +} + #ifdef _WIN32 int GetNumCpuCores() { SYSTEM_LOGICAL_PROCESSOR_INFORMATION buffer[256]; @@ -301,12 +326,6 @@ int real_main(int argc, char* argv[], Ort::Env& env) { std::vector> owned_tests; { - double per_sample_tolerance = 1e-3; - // when cuda is enabled, set it to a larger value for resolving random MNIST test failure - // when openvino is enabled, set it to a larger value for resolving MNIST accuracy mismatch - double relative_per_sample_tolerance = enable_cuda ? 0.017 : enable_openvino ? 0.009 - : 1e-3; - Ort::SessionOptions sf; if (enable_cpu_mem_arena) @@ -336,7 +355,7 @@ int real_main(int argc, char* argv[], Ort::Env& env) { } if (enable_openvino) { #ifdef USE_OPENVINO - //Setting default optimization level for OpenVINO can be overriden with -o option + // Setting default optimization level for OpenVINO can be overridden with -o option sf.SetGraphOptimizationLevel(ORT_DISABLE_ALL); sf.AppendExecutionProvider_OpenVINO(OrtOpenVINOProviderOptions{}); #else @@ -407,7 +426,7 @@ int real_main(int argc, char* argv[], Ort::Env& env) { } auto pos = token.find("|"); if (pos == std::string::npos || pos == 0 || pos == token.length()) { - ORT_THROW(R"(Use a '|' to separate the key and value for + ORT_THROW(R"(Use a '|' to separate the key and value for the run-time option you are trying to use.\n)"); } @@ -420,7 +439,7 @@ the run-time option you are trying to use.\n)"); snpe_option_keys.push_back("runtime"); values.push_back(value); } else { - ORT_THROW(R"(Wrong configuration value for the key 'runtime'. + ORT_THROW(R"(Wrong configuration value for the key 'runtime'. select from 'CPU', 'GPU_FP32', 'GPU', 'GPU_FLOAT16', 'DSP', 'AIP_FIXED_TF'. \n)"); } } else if (key == "priority") { @@ -432,14 +451,14 @@ select from 'CPU', 'GPU_FP32', 'GPU', 'GPU_FLOAT16', 'DSP', 'AIP_FIXED_TF'. \n)" snpe_option_keys.push_back("buffer_type"); values.push_back(value); } else { - ORT_THROW(R"(Wrong configuration value for the key 'buffer_type'. + ORT_THROW(R"(Wrong configuration value for the key 'buffer_type'. select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); } } else { ORT_THROW("Wrong key type entered. Choose from options: ['runtime', 'priority', 'buffer_type'] \n"); } } - for (auto &it : values) { + for (auto& it : values) { snpe_option_values.push_back(it.c_str()); } sf.AppendExecutionProvider_SNPE(snpe_option_keys.data(), snpe_option_values.data(), snpe_option_keys.size()); @@ -501,6 +520,7 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); sf.SetGraphOptimizationLevel(graph_optimization_level); } + // TODO: Get these from onnx_backend_test_series_filters.jsonc. // Permanently exclude following tests because ORT support only opset staring from 7, // Please make no more changes to the list static const ORTCHAR_T* immutable_broken_tests[] = @@ -565,13 +585,14 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); all_disabled_tests.insert(std::begin(dnnl_disabled_tests), std::end(dnnl_disabled_tests)); } #if !defined(__amd64__) && !defined(_M_AMD64) - //out of memory + // out of memory static const ORTCHAR_T* x86_disabled_tests[] = {ORT_TSTR("mlperf_ssd_resnet34_1200"), ORT_TSTR("mask_rcnn_keras"), ORT_TSTR("mask_rcnn"), ORT_TSTR("faster_rcnn"), ORT_TSTR("vgg19"), ORT_TSTR("coreml_VGG16_ImageNet")}; all_disabled_tests.insert(std::begin(x86_disabled_tests), std::end(x86_disabled_tests)); #endif std::vector tests; - LoadTests(data_dirs, whitelisted_test_cases, per_sample_tolerance, relative_per_sample_tolerance, + LoadTests(data_dirs, whitelisted_test_cases, + LoadTestTolerances(enable_cuda, enable_openvino), all_disabled_tests, [&owned_tests, &tests](std::unique_ptr l) { tests.push_back(l.get()); @@ -895,7 +916,7 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); broken_tests.insert({"tinyyolov3", "The parameter is incorrect"}); broken_tests.insert({"mlperf_ssd_mobilenet_300", "unknown error"}); broken_tests.insert({"mlperf_ssd_resnet34_1200", "unknown error"}); - broken_tests.insert({"tf_inception_v1", "flaky test"}); //TODO: Investigate cause for flakiness + broken_tests.insert({"tf_inception_v1", "flaky test"}); // TODO: Investigate cause for flakiness broken_tests.insert({"faster_rcnn", "Linux: faster_rcnn:output=6383:shape mismatch, expect {77} got {57}"}); broken_tests.insert({"split_zero_size_splits", "alloc failed"}); } diff --git a/onnxruntime/test/python/onnx_backend_test_series.py b/onnxruntime/test/python/onnx_backend_test_series.py index 90f152b605..629cb287e3 100644 --- a/onnxruntime/test/python/onnx_backend_test_series.py +++ b/onnxruntime/test/python/onnx_backend_test_series.py @@ -2,86 +2,128 @@ # Licensed under the MIT License. import argparse +import collections import json import os import platform import sys import unittest +from typing import Dict import numpy as np import onnx -import onnx.backend.test +import onnx.backend.test.case.test_case +import onnx.backend.test.runner -import onnxruntime.backend as c2 +import onnxruntime.backend as backend # pylint: disable=consider-using-from-import pytest_plugins = ("onnx.backend.test.report",) -class OrtBackendTest(onnx.backend.test.BackendTest): - def __init__(self, backend, parent_module=None): - super(OrtBackendTest, self).__init__(backend, parent_module) +class OrtBackendTest(onnx.backend.test.runner.Runner): + """ONNX test runner with ORT-specific behavior.""" + + # pylint: disable=too-few-public-methods + def __init__( + self, + rtol_overrides: Dict[str, float], + atol_overrides: Dict[str, float], + ): + self._rtol_overrides = rtol_overrides + self._atol_overrides = atol_overrides + + super().__init__(backend, parent_module=__name__) @classmethod def assert_similar_outputs(cls, ref_outputs, outputs, rtol, atol): + """Asserts ref_outputs and outputs match to within the given tolerances.""" + def assert_similar_array(ref_output, output): np.testing.assert_equal(ref_output.dtype, output.dtype) - if ref_output.dtype == np.object: + if ref_output.dtype == object: np.testing.assert_array_equal(ref_output, output) else: - np.testing.assert_allclose(ref_output, output, rtol=1e-3, atol=1e-5) + np.testing.assert_allclose(ref_output, output, rtol=rtol, atol=atol) np.testing.assert_equal(len(ref_outputs), len(outputs)) - for i in range(len(outputs)): + for i in range(len(outputs)): # pylint: disable=consider-using-enumerate if isinstance(outputs[i], list): for j in range(len(outputs[i])): assert_similar_array(ref_outputs[i][j], outputs[i][j]) else: assert_similar_array(ref_outputs[i], outputs[i]) + def _add_model_test(self, model_test: onnx.backend.test.case.test_case.TestCase, kind: str) -> None: + attrs = {} + # TestCase changed from a namedtuple to a dataclass in ONNX 1.12. + # We can just modify t_c.rtol and atol directly once ONNX 1.11 is no longer supported. + if hasattr(model_test, "_asdict"): + attrs = model_test._asdict() + else: + attrs = vars(model_test) + attrs["rtol"] = self._rtol_overrides[model_test.name] + attrs["atol"] = self._atol_overrides[model_test.name] -def create_backend_test(testname=None): - backend_test = OrtBackendTest(c2, __name__) + super()._add_model_test(onnx.backend.test.case.test_case.TestCase(**attrs), kind) + + +def load_jsonc(basename: str): + """Returns a deserialized object from the JSONC file in testdata/.""" + with open( + os.path.join( + os.path.dirname(os.path.realpath(__file__)), + "testdata", + basename, + ), + encoding="utf-8", + ) as f: # pylint: disable=invalid-name + lines = f.readlines() + lines = [x.split("//")[0] for x in lines] + return json.loads("\n".join(lines)) + + +def create_backend_test(test_name=None): + """Creates an OrtBackendTest and adds its TestCase's to global scope so unittest will find them.""" + + overrides = load_jsonc("onnx_backend_test_series_overrides.jsonc") + rtol_default = overrides["rtol_default"] + atol_default = overrides["atol_default"] + rtol_overrides = collections.defaultdict(lambda: rtol_default) + rtol_overrides.update(overrides["rtol_overrides"]) + atol_overrides = collections.defaultdict(lambda: atol_default) + atol_overrides.update(overrides["atol_overrides"]) + + backend_test = OrtBackendTest(rtol_overrides, atol_overrides) # Type not supported backend_test.exclude(r"(FLOAT16)") - if testname: - backend_test.include(testname + ".*") + if test_name: + backend_test.include(test_name + ".*") else: - # read filters data - with open( - os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "testdata", - "onnx_backend_test_series_filters.jsonc", - ) - ) as f: - filters_lines = f.readlines() - filters_lines = [x.split("//")[0] for x in filters_lines] - filters = json.loads("\n".join(filters_lines)) - + filters = load_jsonc("onnx_backend_test_series_filters.jsonc") current_failing_tests = filters["current_failing_tests"] if platform.architecture()[0] == "32bit": current_failing_tests += filters["current_failing_tests_x86"] - if c2.supports_device("DNNL"): + if backend.supports_device("DNNL"): current_failing_tests += filters["current_failing_tests_DNNL"] - if c2.supports_device("NNAPI"): + if backend.supports_device("NNAPI"): current_failing_tests += filters["current_failing_tests_NNAPI"] - if c2.supports_device("OPENVINO_GPU_FP32") or c2.supports_device("OPENVINO_GPU_FP16"): + if backend.supports_device("OPENVINO_GPU_FP32") or backend.supports_device("OPENVINO_GPU_FP16"): current_failing_tests += filters["current_failing_tests_OPENVINO_GPU"] - if c2.supports_device("OPENVINO_MYRIAD"): + if backend.supports_device("OPENVINO_MYRIAD"): current_failing_tests += filters["current_failing_tests_OPENVINO_GPU"] current_failing_tests += filters["current_failing_tests_OPENVINO_MYRIAD"] - if c2.supports_device("OPENVINO_CPU_FP32"): + if backend.supports_device("OPENVINO_CPU_FP32"): current_failing_tests += filters["current_failing_tests_OPENVINO_CPU_FP32"] - if c2.supports_device("MIGRAPHX"): + if backend.supports_device("MIGRAPHX"): current_failing_tests += [ "^test_constant_pad_cpu", "^test_round_cpu", @@ -114,9 +156,9 @@ def create_backend_test(testname=None): ] # Skip these tests for a "pure" DML onnxruntime python wheel. We keep these tests enabled for instances where both DML and CUDA - # EPs are available (Windows GPU CI pipeline has this config) - these test will pass because CUDA has higher precendence than DML + # EPs are available (Windows GPU CI pipeline has this config) - these test will pass because CUDA has higher precedence than DML # and the nodes are assigned to only the CUDA EP (which supports these tests) - if c2.supports_device("DML") and not c2.supports_device("GPU"): + if backend.supports_device("DML") and not backend.supports_device("GPU"): current_failing_tests += [ "^test_negative_log_likelihood_loss_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index_cpu", "^test_negative_log_likelihood_loss_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index_expanded_cpu", @@ -152,10 +194,9 @@ def create_backend_test(testname=None): # them visible to python.unittest. globals().update(backend_test.enable_report().test_cases) - return backend_test - def parse_args(): + """Returns args parsed from sys.argv.""" parser = argparse.ArgumentParser( os.path.basename(__file__), description="Run the ONNX backend tests using ONNXRuntime.", @@ -163,25 +204,25 @@ def parse_args(): # Add an argument to match a single test name, by adding the name to the 'include' filter. # Using -k with python unittest (https://docs.python.org/3/library/unittest.html#command-line-options) - # doesn't work as it filters on the test method name (Runner._add_model_test) rather than inidividual + # doesn't work as it filters on the test method name (Runner._add_model_test) rather than individual # test case names. parser.add_argument( "-t", "--test-name", - dest="testname", + dest="test_name", type=str, help="Only run tests that match this value. Matching is regex based, and '.*' is automatically appended", ) # parse just our args. python unittest has its own args and arg parsing, and that runs inside unittest.main() - args, left = parser.parse_known_args() - sys.argv = sys.argv[:1] + left + parsed, unknown = parser.parse_known_args() + sys.argv = sys.argv[:1] + unknown - return args + return parsed if __name__ == "__main__": args = parse_args() - backend_test = create_backend_test(args.testname) + create_backend_test(args.test_name) unittest.main() diff --git a/onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc b/onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc index fad26df3fe..95b3516312 100644 --- a/onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc +++ b/onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc @@ -1,4 +1,7 @@ { + // If possible, add a test to onnx_backend_test_series_overrides.jsonc instead of this file. + // This file results in skipping tests, that one allows looser tolerance. + // // Tests that are failing temporarily and should be fixed "current_failing_tests": [ "^test_adagrad", @@ -40,7 +43,7 @@ "^test_resize_downsample_scales_cubic_A_n0p5_exclude_outside_cpu", // NOT_IMPLEMENTED : Could not find an implementation for the node Resize(13) "^test_resize_downsample_scales_cubic_cpu", "^test_resize_downsample_scales_linear_cpu", - "^test_resize_downsample_scales_nearest_cpu", + "^test_resize_downsample_scales_nearest_cpu", "^test_resize_downsample_sizes_cubic_cpu", "^test_resize_downsample_sizes_linear_pytorch_half_pixel_cpu", "^test_resize_downsample_sizes_nearest_cpu", @@ -169,12 +172,12 @@ "^test_sce.*", "^test_nllloss.*", "^test_gather_negative_indices.*", - "^test_reduce_sum_do_not_keepdims*", // Does not support axes as input + "^test_reduce_sum_do_not_keepdims*", // Does not support axes as input "^test_reduce_sum_keepdims*", "^test_reduce_sum_default_axes_keepdims*", "^test_reduce_sum_negative_axes_keepdims*", "^test_reduce_sum_empty_axes_input_noop*", - "^test_unsqueeze_*", // Does not support axes as input + "^test_unsqueeze_*", // Does not support axes as input "^test_sequence_insert_at_back", "^test_sequence_insert_at_front", "^test_loop13_seq", @@ -204,15 +207,15 @@ "^test_sce.*", "^test_nllloss.*", "^test_upsample_nearest.*", - "^test_reduce_sum_do_not_keepdims*", // Does not support axes as input + "^test_reduce_sum_do_not_keepdims*", // Does not support axes as input "^test_reduce_sum_keepdims*", "^test_reduce_sum_default_axes_keepdims*", "^test_reduce_sum_negative_axes_keepdims*", "^test_reduce_sum_empty_axes_input_noop*", "^test_scatter_elements_with_negative_indices_cpu", "^test_sum_one_input_cpu", - "^test_unsqueeze_*", // Does not support axes as input - "^test_squeeze_*", // Does not support axes as input + "^test_unsqueeze_*", // Does not support axes as input + "^test_squeeze_*", // Does not support axes as input "^test_logsoftmax_*", // Does not support opset-13 yet "^test_softmax_*", // Does not support opset-13 yet "^test_pow", // Runs disabled pow tests from the "current_failing_tests" list at the top @@ -234,7 +237,6 @@ "^test_training_dropout_default_mask", // Runs but there's accuracy mismatch "^test_training_dropout_mask", // Runs but there's accuracy mismatch "^test_training_dropout_default" // Runs but there's accuracy mismatch - ], // ORT first supported opset 7, so models with nodes that require versions prior to opset 7 are not supported "tests_with_pre_opset7_dependencies": [ diff --git a/onnxruntime/test/testdata/onnx_backend_test_series_overrides.jsonc b/onnxruntime/test/testdata/onnx_backend_test_series_overrides.jsonc new file mode 100644 index 0000000000..921e491b63 --- /dev/null +++ b/onnxruntime/test/testdata/onnx_backend_test_series_overrides.jsonc @@ -0,0 +1,13 @@ +{ + "rtol_default": 1e-3, + "atol_default": 1e-5, + // Key: str, the name of the test as defined by ONNX without any device suffix. + // Val: float, max absolute difference between expected and actual. + "atol_overrides": { + "test_dft": 1e-4, + "test_dft_axis": 1e-4 + }, + // Key: str, the name of the test as defined by ONNX without any device suffix. + // Val: float, max relative difference between expected and actual. + "rtol_overrides": {} +} diff --git a/winml/test/model/model_tests.cpp b/winml/test/model/model_tests.cpp index 33fe868f93..8a9e937f7d 100644 --- a/winml/test/model/model_tests.cpp +++ b/winml/test/model/model_tests.cpp @@ -173,8 +173,6 @@ std::string GetTestDataPath() { static std::vector GetAllTestCases() { std::vector tests; std::vector> whitelistedTestCases; - double perSampleTolerance = 1e-3; - double relativePerSampleTolerance = 1e-3; std::unordered_set> allDisabledTests; std::vector> dataDirs; auto testDataPath = GetTestDataPath(); @@ -231,7 +229,7 @@ static std::vector GetAllTestCases() { #endif - WINML_EXPECT_NO_THROW(LoadTests(dataDirs, whitelistedTestCases, perSampleTolerance, relativePerSampleTolerance, + WINML_EXPECT_NO_THROW(LoadTests(dataDirs, whitelistedTestCases, TestTolerances(1e-3, 1e-3, {}, {}), allDisabledTests, [&tests](std::unique_ptr l) { tests.push_back(l.get());