mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-29 20:14:01 +00:00
Support per-test tolerances for ONNX tests (#11775)
Prior to this every test shared the same tolerances. This meant that if an ONNX test failed due to a small but acceptable difference in output, the only alternative was to disable the test entirely. In op set 17, the DFT operator is being added. Without this change, the tests for that operator fail because the output is off by about 5e-5. It's better to keep test coverage for this new op rather than disable the test entirely. Also prior to this change, the global tolerances were not shared between C++, JavaScript, and Python tests. Now they are. Also fix various minor issues raised by linters. Unblocks https://github.com/microsoft/onnxruntime/issues/11640.
This commit is contained in:
Gary Miguel
GitHub
parent
d936751aad
commit
e8b0d24071
13 changed files with 252 additions and 105 deletions
|
|
@ -434,6 +434,7 @@ if (onnxruntime_DISABLE_EXCEPTIONS)
|
|||
add_compile_definitions("ORT_NO_EXCEPTIONS")
|
||||
add_compile_definitions("MLAS_NO_EXCEPTION")
|
||||
add_compile_definitions("ONNX_NO_EXCEPTIONS")
|
||||
add_compile_definitions("JSON_NOEXCEPTION") # https://json.nlohmann.me/api/macros/json_noexception/
|
||||
|
||||
if (MSVC)
|
||||
string(REGEX REPLACE "/EHsc" "/EHs-c-" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
|
||||
|
|
|
|||
2
cmake/external/json
vendored
2
cmake/external/json
vendored
|
|
@ -1 +1 @@
|
|||
Subproject commit db78ac1d7716f56fc9f1b030b715f872f93964e4
|
||||
Subproject commit b5364faf9d732052506cefc933d3f4e4f04513a5
|
||||
|
|
@ -861,7 +861,7 @@ if (onnxruntime_BUILD_WEBASSEMBLY)
|
|||
endif()
|
||||
endif()
|
||||
|
||||
target_link_libraries(onnx_test_runner PRIVATE onnx_test_runner_common ${GETOPT_LIB_WIDE} ${onnx_test_libs})
|
||||
target_link_libraries(onnx_test_runner PRIVATE onnx_test_runner_common ${GETOPT_LIB_WIDE} ${onnx_test_libs} nlohmann_json::nlohmann_json)
|
||||
target_include_directories(onnx_test_runner PRIVATE ${ONNXRUNTIME_ROOT})
|
||||
if (onnxruntime_USE_ROCM)
|
||||
target_include_directories(onnx_test_runner PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/amdgpu/onnxruntime ${CMAKE_CURRENT_BINARY_DIR}/amdgpu/orttraining)
|
||||
|
|
|
|||
|
|
@ -1,9 +1,11 @@
|
|||
This is a note only for ONNX Runtime developers.
|
||||
# How to update ONNX
|
||||
|
||||
It's very often, you need to update the ONNX submodule to a newer version in the upstream. Please follow the steps below, don't miss any!
|
||||
This note is only for ONNX Runtime developers.
|
||||
|
||||
1. Update the ONNX subfolder
|
||||
```
|
||||
If you need to update the ONNX submodule to a different version, follow the steps below.
|
||||
|
||||
1. Update the ONNX submodule
|
||||
```sh
|
||||
cd cmake/external/onnx
|
||||
git remote update
|
||||
git reset --hard <commit_id>
|
||||
|
|
@ -15,22 +17,28 @@ git add onnx
|
|||
1. Update [cgmanifests/generated/cgmanifest.json](/cgmanifests/generated/cgmanifest.json).
|
||||
This file should be generated. See [cgmanifests/README](/cgmanifests/README.md) for instructions.
|
||||
|
||||
1. Update [tools/ci_build/github/linux/docker/scripts/requirements.txt](/tools/ci_build/github/linux/docker/scripts/requirements.txt) and [tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt](/tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt).
|
||||
Update the commit hash for `git+http://github.com/onnx/onnx.git@targetonnxcommithash#egg=onnx`.
|
||||
1. Update [tools/ci_build/github/linux/docker/scripts/requirements.txt](/tools/ci_build/github/linux/docker/scripts/requirements.txt)
|
||||
and [tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt](/tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt).
|
||||
Update the commit hash for `git+http://github.com/onnx/onnx.git@targetonnxcommithash#egg=onnx`.
|
||||
|
||||
1. If there is any change to `cmake/external/onnx/onnx/*.in.proto`, you need to regenerate OnnxMl.cs. [Building onnxruntime with Nuget](https://onnxruntime.ai/docs/build/inferencing.html#build-nuget-packages) will do this.
|
||||
1. If there is any change to `cmake/external/onnx/onnx/*.in.proto`, you need to regenerate OnnxMl.cs.
|
||||
[Building onnxruntime with Nuget](https://onnxruntime.ai/docs/build/inferencing.html#build-nuget-packages) will do
|
||||
this.
|
||||
|
||||
1. If you are updating ONNX from a released tag to a new commit, please tell Changming deploying the new test data along with other test models to our CI build machines. This is to ensure that our tests cover every ONNX opset.
|
||||
1. If you are updating ONNX from a released tag to a new commit, please ask Changming (@snnn) to deploy the new test
|
||||
data along with other test models to our CI build machines. This is to ensure that our tests cover every ONNX opset.
|
||||
|
||||
1. Send you PR, and **manually** queue a build for every packaging pipeline for your branch.
|
||||
|
||||
1. If there is a build failure in stage "Check out of dated documents" in WebAssembly CI pipeline, update ONNX Runtime Web WebGL operator support document:
|
||||
1. If there is a build failure in stage "Check out of dated documents" in WebAssembly CI pipeline, update ONNX Runtime
|
||||
Web WebGL operator support document:
|
||||
- Make sure Node.js is installed (see [Prerequisites](../js/README.md#Prerequisites) for instructions).
|
||||
- Follow step 1 in [js/Build](../js/README.md#Build-2) to install dependencies).
|
||||
- Follow instructions in [Generate document](../js/README.md#Generating-Document) to update document. Commit changes applied to file `docs/operators.md`.
|
||||
|
||||
1. Usually there would be some unitest failures, because you introduced new test cases. Then you may need to update
|
||||
1. Usually some newly introduced tests will fail. Then you may need to update
|
||||
- [onnxruntime/test/onnx/main.cc](/onnxruntime/test/onnx/main.cc)
|
||||
- [onnxruntime/test/providers/cpu/model_tests.cc](/onnxruntime/test/providers/cpu/model_tests.cc)
|
||||
- [csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs](/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs)
|
||||
- [onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc](/onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc)
|
||||
- [onnxruntime/test/testdata/onnx_backend_test_series_overrides.jsonc](/onnxruntime/test/testdata/onnx_backend_test_series_overrides.jsonc)
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import * as fs from 'fs-extra';
|
|||
import {InferenceSession, Tensor} from 'onnxruntime-common';
|
||||
import * as path from 'path';
|
||||
|
||||
import {assertTensorEqual, loadTensorFromFile, shouldSkipModel} from './test-utils';
|
||||
import {assertTensorEqual, atol, loadTensorFromFile, rtol, shouldSkipModel} from './test-utils';
|
||||
|
||||
export function run(testDataFolder: string): void {
|
||||
const models = fs.readdirSync(testDataFolder);
|
||||
|
|
@ -98,7 +98,7 @@ export function run(testDataFolder: string): void {
|
|||
|
||||
let j = 0;
|
||||
for (const name of session.outputNames) {
|
||||
assertTensorEqual(outputs[name], expectedOutputs[j++]!);
|
||||
assertTensorEqual(outputs[name], expectedOutputs[j++]!, atol(model), rtol(model));
|
||||
}
|
||||
} else {
|
||||
throw new TypeError('session is null');
|
||||
|
|
|
|||
|
|
@ -20,6 +20,13 @@ export const SQUEEZENET_OUTPUT0_DATA: number[] = require(path.join(TEST_DATA_ROO
|
|||
export const BACKEND_TEST_SERIES_FILTERS: {[name: string]: string[]} =
|
||||
jsonc.readSync(path.join(ORT_ROOT, 'onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc'));
|
||||
|
||||
const OVERRIDES: {
|
||||
atol_default: number; rtol_default: number; atol_overrides: {[name: string]: number};
|
||||
rtol_overrides: {[name: string]: number};
|
||||
} = jsonc.readSync(path.join(ORT_ROOT, 'onnxruntime/test/testdata/onnx_backend_test_series_overrides.jsonc'));
|
||||
|
||||
const ATOL_DEFAULT = OVERRIDES.atol_default;
|
||||
const RTOL_DEFAULT = OVERRIDES.rtol_default;
|
||||
|
||||
export const NUMERIC_TYPE_MAP = new Map<Tensor.Type, new (len: number) => Tensor.DataType>([
|
||||
['float32', Float32Array],
|
||||
|
|
@ -83,9 +90,10 @@ export function warmup(): void {
|
|||
}
|
||||
|
||||
export function assertFloatEqual(
|
||||
actual: number[]|Float32Array|Float64Array, expected: number[]|Float32Array|Float64Array): void {
|
||||
const THRESHOLD_ABSOLUTE_ERROR = 1.0e-4;
|
||||
const THRESHOLD_RELATIVE_ERROR = 1.000001;
|
||||
actual: number[]|Float32Array|Float64Array, expected: number[]|Float32Array|Float64Array, atol?: number,
|
||||
rtol?: number): void {
|
||||
const absolute_tol: number = atol ?? 1.0e-4;
|
||||
const relative_tol: number = 1 + (rtol ?? 1.0e-6);
|
||||
|
||||
assert.strictEqual(actual.length, expected.length);
|
||||
|
||||
|
|
@ -114,10 +122,10 @@ export function assertFloatEqual(
|
|||
// test fail
|
||||
// endif
|
||||
//
|
||||
if (Math.abs(a - b) < THRESHOLD_ABSOLUTE_ERROR) {
|
||||
if (Math.abs(a - b) < absolute_tol) {
|
||||
continue; // absolute error check pass
|
||||
}
|
||||
if (a !== 0 && b !== 0 && a * b > 0 && a / b < THRESHOLD_RELATIVE_ERROR && b / a < THRESHOLD_RELATIVE_ERROR) {
|
||||
if (a !== 0 && b !== 0 && a * b > 0 && a / b < relative_tol && b / a < relative_tol) {
|
||||
continue; // relative error check pass
|
||||
}
|
||||
|
||||
|
|
@ -126,12 +134,14 @@ export function assertFloatEqual(
|
|||
}
|
||||
}
|
||||
|
||||
export function assertDataEqual(type: Tensor.Type, actual: Tensor.DataType, expected: Tensor.DataType): void {
|
||||
export function assertDataEqual(
|
||||
type: Tensor.Type, actual: Tensor.DataType, expected: Tensor.DataType, atol?: number, rtol?: number): void {
|
||||
switch (type) {
|
||||
case 'float32':
|
||||
case 'float64':
|
||||
assertFloatEqual(
|
||||
actual as number[] | Float32Array | Float64Array, expected as number[] | Float32Array | Float64Array);
|
||||
actual as number[] | Float32Array | Float64Array, expected as number[] | Float32Array | Float64Array, atol,
|
||||
rtol);
|
||||
break;
|
||||
|
||||
case 'uint8':
|
||||
|
|
@ -153,7 +163,7 @@ export function assertDataEqual(type: Tensor.Type, actual: Tensor.DataType, expe
|
|||
}
|
||||
|
||||
// This function check whether 2 tensors should be considered as 'match' or not
|
||||
export function assertTensorEqual(actual: Tensor, expected: Tensor): void {
|
||||
export function assertTensorEqual(actual: Tensor, expected: Tensor, atol?: number, rtol?: number): void {
|
||||
assert(typeof actual === 'object');
|
||||
assert(typeof expected === 'object');
|
||||
|
||||
|
|
@ -168,7 +178,7 @@ export function assertTensorEqual(actual: Tensor, expected: Tensor): void {
|
|||
assert.strictEqual(actualType, expectedType);
|
||||
assert.deepStrictEqual(actualDims, expectedDims);
|
||||
|
||||
assertDataEqual(actualType, actual.data, expected.data);
|
||||
assertDataEqual(actualType, actual.data, expected.data, atol, rtol);
|
||||
}
|
||||
|
||||
export function loadTensorFromFile(pbFile: string): Tensor {
|
||||
|
|
@ -274,3 +284,11 @@ export function shouldSkipModel(model: string, eps: string[]): boolean {
|
|||
|
||||
return false;
|
||||
}
|
||||
|
||||
export function atol(model: string): number {
|
||||
return OVERRIDES.atol_overrides[model] ?? ATOL_DEFAULT;
|
||||
}
|
||||
|
||||
export function rtol(model: string): number {
|
||||
return OVERRIDES.rtol_overrides[model] ?? RTOL_DEFAULT;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,6 +5,14 @@
|
|||
|
||||
#include "TestCase.h"
|
||||
|
||||
#include <cctype>
|
||||
#include <fstream>
|
||||
#include <memory>
|
||||
#include <sstream>
|
||||
#include <map>
|
||||
#include <regex>
|
||||
#include <string>
|
||||
|
||||
#include "callback.h"
|
||||
#include "heap_buffer.h"
|
||||
#include "mem_buffer.h"
|
||||
|
|
@ -22,13 +30,6 @@
|
|||
#include "core/framework/TensorSeq.h"
|
||||
#include "re2/re2.h"
|
||||
|
||||
#include <cctype>
|
||||
#include <fstream>
|
||||
#include <memory>
|
||||
#include <sstream>
|
||||
#include <map>
|
||||
#include <regex>
|
||||
|
||||
using namespace onnxruntime;
|
||||
using namespace onnxruntime::common;
|
||||
using google::protobuf::RepeatedPtrField;
|
||||
|
|
@ -417,7 +418,7 @@ static bool read_config_file(const std::basic_string<PATH_CHAR_TYPE>& path, std:
|
|||
return true;
|
||||
}
|
||||
|
||||
//load tensors from disk
|
||||
// load tensors from disk
|
||||
template <typename PATH_STRING_TYPE>
|
||||
static void LoadTensor(const PATH_STRING_TYPE& pb_file, ONNX_NAMESPACE::TensorProto& input_pb) {
|
||||
int tensor_fd;
|
||||
|
|
@ -432,7 +433,7 @@ static void LoadTensor(const PATH_STRING_TYPE& pb_file, ONNX_NAMESPACE::TensorPr
|
|||
}
|
||||
}
|
||||
|
||||
//load sequence tensors from disk
|
||||
// load sequence tensors from disk
|
||||
template <typename PATH_STRING_TYPE>
|
||||
static void LoadSequenceTensor(const PATH_STRING_TYPE& pb_file, ONNX_NAMESPACE::SequenceProto& input_pb) {
|
||||
int tensor_fd;
|
||||
|
|
@ -475,7 +476,7 @@ void OnnxTestCase::LoadTestData(size_t id, onnxruntime::test::HeapBuffer& b,
|
|||
test_data_dirs_[id], (is_input ? ORT_TSTR("inputs.pb") : ORT_TSTR("outputs.pb")));
|
||||
int test_data_pb_fd;
|
||||
auto st = Env::Default().FileOpenRd(test_data_pb, test_data_pb_fd);
|
||||
if (st.IsOK()) { //has an all-in-one input file
|
||||
if (st.IsOK()) { // has an all-in-one input file
|
||||
std::ostringstream oss;
|
||||
{
|
||||
std::lock_guard<OrtMutex> l(m_);
|
||||
|
|
@ -727,7 +728,7 @@ OnnxTestCase::OnnxTestCase(const std::string& test_case_name, _In_ std::unique_p
|
|||
|
||||
void LoadTests(const std::vector<std::basic_string<PATH_CHAR_TYPE>>& input_paths,
|
||||
const std::vector<std::basic_string<PATH_CHAR_TYPE>>& whitelisted_test_cases,
|
||||
double default_per_sample_tolerance, double default_relative_per_sample_tolerance,
|
||||
const TestTolerances& tolerances,
|
||||
const std::unordered_set<std::basic_string<ORTCHAR_T>>& disabled_tests,
|
||||
const std::function<void(std::unique_ptr<ITestCase>)>& process_function) {
|
||||
std::vector<std::basic_string<PATH_CHAR_TYPE>> paths(input_paths);
|
||||
|
|
@ -781,11 +782,37 @@ void LoadTests(const std::vector<std::basic_string<PATH_CHAR_TYPE>>& input_paths
|
|||
ORT_NOT_IMPLEMENTED(ToUTF8String(filename_str), " is not supported");
|
||||
}
|
||||
|
||||
const auto tolerance_key = ToUTF8String(my_dir_name);
|
||||
|
||||
std::unique_ptr<ITestCase> l = CreateOnnxTestCase(ToUTF8String(test_case_name), std::move(model_info),
|
||||
default_per_sample_tolerance,
|
||||
default_relative_per_sample_tolerance);
|
||||
tolerances.absolute(tolerance_key),
|
||||
tolerances.relative(tolerance_key));
|
||||
process_function(std::move(l));
|
||||
return true;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
TestTolerances::TestTolerances(
|
||||
double absolute_default, double relative_default,
|
||||
const Map& absolute_overrides,
|
||||
const Map& relative_overrides) : absolute_default_(absolute_default),
|
||||
relative_default_(relative_default),
|
||||
absolute_overrides_(absolute_overrides),
|
||||
relative_overrides_(relative_overrides) {}
|
||||
|
||||
double TestTolerances::absolute(const std::string& name) const {
|
||||
const auto iter = absolute_overrides_.find(name);
|
||||
if (iter == absolute_overrides_.end()) {
|
||||
return absolute_default_;
|
||||
}
|
||||
return iter->second;
|
||||
}
|
||||
|
||||
double TestTolerances::relative(const std::string& name) const {
|
||||
const auto iter = relative_overrides_.find(name);
|
||||
if (iter == relative_overrides_.end()) {
|
||||
return relative_default_;
|
||||
}
|
||||
return iter->second;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24,8 +24,8 @@ class HeapBuffer;
|
|||
}
|
||||
} // namespace onnxruntime
|
||||
|
||||
//One test case is for one model file
|
||||
//One test case can contain multiple test data(input/output pairs)
|
||||
// One test case is for one model file
|
||||
// One test case can contain multiple test data(input/output pairs)
|
||||
class ITestCase {
|
||||
public:
|
||||
virtual void LoadTestData(size_t id, onnxruntime::test::HeapBuffer& b,
|
||||
|
|
@ -38,9 +38,9 @@ class ITestCase {
|
|||
|
||||
virtual const std::string& GetTestCaseName() const = 0;
|
||||
virtual std::string GetTestCaseVersion() const = 0;
|
||||
//a string to help identify the dataset
|
||||
// a string to help identify the dataset
|
||||
virtual std::string GetDatasetDebugInfoString(size_t dataset_id) const = 0;
|
||||
//The number of input/output pairs
|
||||
// The number of input/output pairs
|
||||
virtual size_t GetDataCount() const = 0;
|
||||
virtual ~ITestCase() = default;
|
||||
virtual void GetPerSampleTolerance(double* value) const = 0;
|
||||
|
|
@ -83,8 +83,26 @@ std::unique_ptr<ITestCase> CreateOnnxTestCase(const std::string& test_case_name,
|
|||
double default_per_sample_tolerance,
|
||||
double default_relative_per_sample_tolerance);
|
||||
|
||||
class TestTolerances {
|
||||
public:
|
||||
typedef std::unordered_map<std::string, double> Map;
|
||||
TestTolerances(
|
||||
double absolute_default, double relative_default,
|
||||
const Map& absolute_overrides,
|
||||
const Map& relative_overrides);
|
||||
TestTolerances() = delete;
|
||||
double absolute(const std::string& test_name) const;
|
||||
double relative(const std::string& test_name) const;
|
||||
|
||||
private:
|
||||
double absolute_default_;
|
||||
double relative_default_;
|
||||
const Map absolute_overrides_;
|
||||
const Map relative_overrides_;
|
||||
};
|
||||
|
||||
void LoadTests(const std::vector<std::basic_string<PATH_CHAR_TYPE>>& input_paths,
|
||||
const std::vector<std::basic_string<PATH_CHAR_TYPE>>& whitelisted_test_cases,
|
||||
double default_per_sample_tolerance, double default_relative_per_sample_tolerance,
|
||||
const TestTolerances& tolerances,
|
||||
const std::unordered_set<std::basic_string<ORTCHAR_T>>& disabled_tests,
|
||||
const std::function<void(std::unique_ptr<ITestCase>)>& process_function);
|
||||
|
|
|
|||
|
|
@ -4,6 +4,8 @@
|
|||
#include <set>
|
||||
#include <iostream>
|
||||
#include <fstream>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#ifdef _WIN32
|
||||
#include "getopt.h"
|
||||
#else
|
||||
|
|
@ -20,6 +22,7 @@
|
|||
#include "core/optimizer/graph_transformer_level.h"
|
||||
#include "core/framework/session_options.h"
|
||||
#include "core/session/onnxruntime_session_options_config_keys.h"
|
||||
#include "nlohmann/json.hpp"
|
||||
|
||||
using namespace onnxruntime;
|
||||
|
||||
|
|
@ -56,6 +59,28 @@ void usage() {
|
|||
OrtGetApiBase()->GetVersionString());
|
||||
}
|
||||
|
||||
static TestTolerances LoadTestTolerances(bool enable_cuda, bool enable_openvino) {
|
||||
TestTolerances::Map absolute_overrides;
|
||||
TestTolerances::Map relative_overrides;
|
||||
std::ifstream overrides_ifstream(ConcatPathComponent<ORTCHAR_T>(
|
||||
ORT_TSTR("testdata"), ORT_TSTR("onnx_backend_test_series_overrides.jsonc")));
|
||||
if (!overrides_ifstream.good()) {
|
||||
const double absolute = 1e-3;
|
||||
// when cuda is enabled, set it to a larger value for resolving random MNIST test failure
|
||||
// when openvino is enabled, set it to a larger value for resolving MNIST accuracy mismatch
|
||||
const double relative = enable_cuda ? 0.017 : enable_openvino ? 0.009
|
||||
: 1e-3;
|
||||
return TestTolerances(absolute, relative, absolute_overrides, relative_overrides);
|
||||
}
|
||||
const auto overrides_json = nlohmann::json::parse(
|
||||
overrides_ifstream,
|
||||
/*cb=*/nullptr, /*allow_exceptions=*/true, /*ignore_comments=*/true);
|
||||
overrides_json["atol_overrides"].get_to(absolute_overrides);
|
||||
overrides_json["rtol_overrides"].get_to(relative_overrides);
|
||||
return TestTolerances(
|
||||
overrides_json["atol_default"], overrides_json["rtol_default"], absolute_overrides, relative_overrides);
|
||||
}
|
||||
|
||||
#ifdef _WIN32
|
||||
int GetNumCpuCores() {
|
||||
SYSTEM_LOGICAL_PROCESSOR_INFORMATION buffer[256];
|
||||
|
|
@ -301,12 +326,6 @@ int real_main(int argc, char* argv[], Ort::Env& env) {
|
|||
|
||||
std::vector<std::unique_ptr<ITestCase>> owned_tests;
|
||||
{
|
||||
double per_sample_tolerance = 1e-3;
|
||||
// when cuda is enabled, set it to a larger value for resolving random MNIST test failure
|
||||
// when openvino is enabled, set it to a larger value for resolving MNIST accuracy mismatch
|
||||
double relative_per_sample_tolerance = enable_cuda ? 0.017 : enable_openvino ? 0.009
|
||||
: 1e-3;
|
||||
|
||||
Ort::SessionOptions sf;
|
||||
|
||||
if (enable_cpu_mem_arena)
|
||||
|
|
@ -336,7 +355,7 @@ int real_main(int argc, char* argv[], Ort::Env& env) {
|
|||
}
|
||||
if (enable_openvino) {
|
||||
#ifdef USE_OPENVINO
|
||||
//Setting default optimization level for OpenVINO can be overriden with -o option
|
||||
// Setting default optimization level for OpenVINO can be overridden with -o option
|
||||
sf.SetGraphOptimizationLevel(ORT_DISABLE_ALL);
|
||||
sf.AppendExecutionProvider_OpenVINO(OrtOpenVINOProviderOptions{});
|
||||
#else
|
||||
|
|
@ -407,7 +426,7 @@ int real_main(int argc, char* argv[], Ort::Env& env) {
|
|||
}
|
||||
auto pos = token.find("|");
|
||||
if (pos == std::string::npos || pos == 0 || pos == token.length()) {
|
||||
ORT_THROW(R"(Use a '|' to separate the key and value for
|
||||
ORT_THROW(R"(Use a '|' to separate the key and value for
|
||||
the run-time option you are trying to use.\n)");
|
||||
}
|
||||
|
||||
|
|
@ -420,7 +439,7 @@ the run-time option you are trying to use.\n)");
|
|||
snpe_option_keys.push_back("runtime");
|
||||
values.push_back(value);
|
||||
} else {
|
||||
ORT_THROW(R"(Wrong configuration value for the key 'runtime'.
|
||||
ORT_THROW(R"(Wrong configuration value for the key 'runtime'.
|
||||
select from 'CPU', 'GPU_FP32', 'GPU', 'GPU_FLOAT16', 'DSP', 'AIP_FIXED_TF'. \n)");
|
||||
}
|
||||
} else if (key == "priority") {
|
||||
|
|
@ -432,14 +451,14 @@ select from 'CPU', 'GPU_FP32', 'GPU', 'GPU_FLOAT16', 'DSP', 'AIP_FIXED_TF'. \n)"
|
|||
snpe_option_keys.push_back("buffer_type");
|
||||
values.push_back(value);
|
||||
} else {
|
||||
ORT_THROW(R"(Wrong configuration value for the key 'buffer_type'.
|
||||
ORT_THROW(R"(Wrong configuration value for the key 'buffer_type'.
|
||||
select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)");
|
||||
}
|
||||
} else {
|
||||
ORT_THROW("Wrong key type entered. Choose from options: ['runtime', 'priority', 'buffer_type'] \n");
|
||||
}
|
||||
}
|
||||
for (auto &it : values) {
|
||||
for (auto& it : values) {
|
||||
snpe_option_values.push_back(it.c_str());
|
||||
}
|
||||
sf.AppendExecutionProvider_SNPE(snpe_option_keys.data(), snpe_option_values.data(), snpe_option_keys.size());
|
||||
|
|
@ -501,6 +520,7 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)");
|
|||
sf.SetGraphOptimizationLevel(graph_optimization_level);
|
||||
}
|
||||
|
||||
// TODO: Get these from onnx_backend_test_series_filters.jsonc.
|
||||
// Permanently exclude following tests because ORT support only opset staring from 7,
|
||||
// Please make no more changes to the list
|
||||
static const ORTCHAR_T* immutable_broken_tests[] =
|
||||
|
|
@ -565,13 +585,14 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)");
|
|||
all_disabled_tests.insert(std::begin(dnnl_disabled_tests), std::end(dnnl_disabled_tests));
|
||||
}
|
||||
#if !defined(__amd64__) && !defined(_M_AMD64)
|
||||
//out of memory
|
||||
// out of memory
|
||||
static const ORTCHAR_T* x86_disabled_tests[] = {ORT_TSTR("mlperf_ssd_resnet34_1200"), ORT_TSTR("mask_rcnn_keras"), ORT_TSTR("mask_rcnn"), ORT_TSTR("faster_rcnn"), ORT_TSTR("vgg19"), ORT_TSTR("coreml_VGG16_ImageNet")};
|
||||
all_disabled_tests.insert(std::begin(x86_disabled_tests), std::end(x86_disabled_tests));
|
||||
#endif
|
||||
|
||||
std::vector<ITestCase*> tests;
|
||||
LoadTests(data_dirs, whitelisted_test_cases, per_sample_tolerance, relative_per_sample_tolerance,
|
||||
LoadTests(data_dirs, whitelisted_test_cases,
|
||||
LoadTestTolerances(enable_cuda, enable_openvino),
|
||||
all_disabled_tests,
|
||||
[&owned_tests, &tests](std::unique_ptr<ITestCase> l) {
|
||||
tests.push_back(l.get());
|
||||
|
|
@ -895,7 +916,7 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)");
|
|||
broken_tests.insert({"tinyyolov3", "The parameter is incorrect"});
|
||||
broken_tests.insert({"mlperf_ssd_mobilenet_300", "unknown error"});
|
||||
broken_tests.insert({"mlperf_ssd_resnet34_1200", "unknown error"});
|
||||
broken_tests.insert({"tf_inception_v1", "flaky test"}); //TODO: Investigate cause for flakiness
|
||||
broken_tests.insert({"tf_inception_v1", "flaky test"}); // TODO: Investigate cause for flakiness
|
||||
broken_tests.insert({"faster_rcnn", "Linux: faster_rcnn:output=6383:shape mismatch, expect {77} got {57}"});
|
||||
broken_tests.insert({"split_zero_size_splits", "alloc failed"});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,86 +2,128 @@
|
|||
# Licensed under the MIT License.
|
||||
|
||||
import argparse
|
||||
import collections
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import sys
|
||||
import unittest
|
||||
from typing import Dict
|
||||
|
||||
import numpy as np
|
||||
import onnx
|
||||
import onnx.backend.test
|
||||
import onnx.backend.test.case.test_case
|
||||
import onnx.backend.test.runner
|
||||
|
||||
import onnxruntime.backend as c2
|
||||
import onnxruntime.backend as backend # pylint: disable=consider-using-from-import
|
||||
|
||||
pytest_plugins = ("onnx.backend.test.report",)
|
||||
|
||||
|
||||
class OrtBackendTest(onnx.backend.test.BackendTest):
|
||||
def __init__(self, backend, parent_module=None):
|
||||
super(OrtBackendTest, self).__init__(backend, parent_module)
|
||||
class OrtBackendTest(onnx.backend.test.runner.Runner):
|
||||
"""ONNX test runner with ORT-specific behavior."""
|
||||
|
||||
# pylint: disable=too-few-public-methods
|
||||
def __init__(
|
||||
self,
|
||||
rtol_overrides: Dict[str, float],
|
||||
atol_overrides: Dict[str, float],
|
||||
):
|
||||
self._rtol_overrides = rtol_overrides
|
||||
self._atol_overrides = atol_overrides
|
||||
|
||||
super().__init__(backend, parent_module=__name__)
|
||||
|
||||
@classmethod
|
||||
def assert_similar_outputs(cls, ref_outputs, outputs, rtol, atol):
|
||||
"""Asserts ref_outputs and outputs match to within the given tolerances."""
|
||||
|
||||
def assert_similar_array(ref_output, output):
|
||||
np.testing.assert_equal(ref_output.dtype, output.dtype)
|
||||
if ref_output.dtype == np.object:
|
||||
if ref_output.dtype == object:
|
||||
np.testing.assert_array_equal(ref_output, output)
|
||||
else:
|
||||
np.testing.assert_allclose(ref_output, output, rtol=1e-3, atol=1e-5)
|
||||
np.testing.assert_allclose(ref_output, output, rtol=rtol, atol=atol)
|
||||
|
||||
np.testing.assert_equal(len(ref_outputs), len(outputs))
|
||||
for i in range(len(outputs)):
|
||||
for i in range(len(outputs)): # pylint: disable=consider-using-enumerate
|
||||
if isinstance(outputs[i], list):
|
||||
for j in range(len(outputs[i])):
|
||||
assert_similar_array(ref_outputs[i][j], outputs[i][j])
|
||||
else:
|
||||
assert_similar_array(ref_outputs[i], outputs[i])
|
||||
|
||||
def _add_model_test(self, model_test: onnx.backend.test.case.test_case.TestCase, kind: str) -> None:
|
||||
attrs = {}
|
||||
# TestCase changed from a namedtuple to a dataclass in ONNX 1.12.
|
||||
# We can just modify t_c.rtol and atol directly once ONNX 1.11 is no longer supported.
|
||||
if hasattr(model_test, "_asdict"):
|
||||
attrs = model_test._asdict()
|
||||
else:
|
||||
attrs = vars(model_test)
|
||||
attrs["rtol"] = self._rtol_overrides[model_test.name]
|
||||
attrs["atol"] = self._atol_overrides[model_test.name]
|
||||
|
||||
def create_backend_test(testname=None):
|
||||
backend_test = OrtBackendTest(c2, __name__)
|
||||
super()._add_model_test(onnx.backend.test.case.test_case.TestCase(**attrs), kind)
|
||||
|
||||
|
||||
def load_jsonc(basename: str):
|
||||
"""Returns a deserialized object from the JSONC file in testdata/<basename>."""
|
||||
with open(
|
||||
os.path.join(
|
||||
os.path.dirname(os.path.realpath(__file__)),
|
||||
"testdata",
|
||||
basename,
|
||||
),
|
||||
encoding="utf-8",
|
||||
) as f: # pylint: disable=invalid-name
|
||||
lines = f.readlines()
|
||||
lines = [x.split("//")[0] for x in lines]
|
||||
return json.loads("\n".join(lines))
|
||||
|
||||
|
||||
def create_backend_test(test_name=None):
|
||||
"""Creates an OrtBackendTest and adds its TestCase's to global scope so unittest will find them."""
|
||||
|
||||
overrides = load_jsonc("onnx_backend_test_series_overrides.jsonc")
|
||||
rtol_default = overrides["rtol_default"]
|
||||
atol_default = overrides["atol_default"]
|
||||
rtol_overrides = collections.defaultdict(lambda: rtol_default)
|
||||
rtol_overrides.update(overrides["rtol_overrides"])
|
||||
atol_overrides = collections.defaultdict(lambda: atol_default)
|
||||
atol_overrides.update(overrides["atol_overrides"])
|
||||
|
||||
backend_test = OrtBackendTest(rtol_overrides, atol_overrides)
|
||||
|
||||
# Type not supported
|
||||
backend_test.exclude(r"(FLOAT16)")
|
||||
|
||||
if testname:
|
||||
backend_test.include(testname + ".*")
|
||||
if test_name:
|
||||
backend_test.include(test_name + ".*")
|
||||
else:
|
||||
# read filters data
|
||||
with open(
|
||||
os.path.join(
|
||||
os.path.dirname(os.path.realpath(__file__)),
|
||||
"testdata",
|
||||
"onnx_backend_test_series_filters.jsonc",
|
||||
)
|
||||
) as f:
|
||||
filters_lines = f.readlines()
|
||||
filters_lines = [x.split("//")[0] for x in filters_lines]
|
||||
filters = json.loads("\n".join(filters_lines))
|
||||
|
||||
filters = load_jsonc("onnx_backend_test_series_filters.jsonc")
|
||||
current_failing_tests = filters["current_failing_tests"]
|
||||
|
||||
if platform.architecture()[0] == "32bit":
|
||||
current_failing_tests += filters["current_failing_tests_x86"]
|
||||
|
||||
if c2.supports_device("DNNL"):
|
||||
if backend.supports_device("DNNL"):
|
||||
current_failing_tests += filters["current_failing_tests_DNNL"]
|
||||
|
||||
if c2.supports_device("NNAPI"):
|
||||
if backend.supports_device("NNAPI"):
|
||||
current_failing_tests += filters["current_failing_tests_NNAPI"]
|
||||
|
||||
if c2.supports_device("OPENVINO_GPU_FP32") or c2.supports_device("OPENVINO_GPU_FP16"):
|
||||
if backend.supports_device("OPENVINO_GPU_FP32") or backend.supports_device("OPENVINO_GPU_FP16"):
|
||||
current_failing_tests += filters["current_failing_tests_OPENVINO_GPU"]
|
||||
|
||||
if c2.supports_device("OPENVINO_MYRIAD"):
|
||||
if backend.supports_device("OPENVINO_MYRIAD"):
|
||||
current_failing_tests += filters["current_failing_tests_OPENVINO_GPU"]
|
||||
current_failing_tests += filters["current_failing_tests_OPENVINO_MYRIAD"]
|
||||
|
||||
if c2.supports_device("OPENVINO_CPU_FP32"):
|
||||
if backend.supports_device("OPENVINO_CPU_FP32"):
|
||||
current_failing_tests += filters["current_failing_tests_OPENVINO_CPU_FP32"]
|
||||
|
||||
if c2.supports_device("MIGRAPHX"):
|
||||
if backend.supports_device("MIGRAPHX"):
|
||||
current_failing_tests += [
|
||||
"^test_constant_pad_cpu",
|
||||
"^test_round_cpu",
|
||||
|
|
@ -114,9 +156,9 @@ def create_backend_test(testname=None):
|
|||
]
|
||||
|
||||
# Skip these tests for a "pure" DML onnxruntime python wheel. We keep these tests enabled for instances where both DML and CUDA
|
||||
# EPs are available (Windows GPU CI pipeline has this config) - these test will pass because CUDA has higher precendence than DML
|
||||
# EPs are available (Windows GPU CI pipeline has this config) - these test will pass because CUDA has higher precedence than DML
|
||||
# and the nodes are assigned to only the CUDA EP (which supports these tests)
|
||||
if c2.supports_device("DML") and not c2.supports_device("GPU"):
|
||||
if backend.supports_device("DML") and not backend.supports_device("GPU"):
|
||||
current_failing_tests += [
|
||||
"^test_negative_log_likelihood_loss_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index_cpu",
|
||||
"^test_negative_log_likelihood_loss_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index_expanded_cpu",
|
||||
|
|
@ -152,10 +194,9 @@ def create_backend_test(testname=None):
|
|||
# them visible to python.unittest.
|
||||
globals().update(backend_test.enable_report().test_cases)
|
||||
|
||||
return backend_test
|
||||
|
||||
|
||||
def parse_args():
|
||||
"""Returns args parsed from sys.argv."""
|
||||
parser = argparse.ArgumentParser(
|
||||
os.path.basename(__file__),
|
||||
description="Run the ONNX backend tests using ONNXRuntime.",
|
||||
|
|
@ -163,25 +204,25 @@ def parse_args():
|
|||
|
||||
# Add an argument to match a single test name, by adding the name to the 'include' filter.
|
||||
# Using -k with python unittest (https://docs.python.org/3/library/unittest.html#command-line-options)
|
||||
# doesn't work as it filters on the test method name (Runner._add_model_test) rather than inidividual
|
||||
# doesn't work as it filters on the test method name (Runner._add_model_test) rather than individual
|
||||
# test case names.
|
||||
parser.add_argument(
|
||||
"-t",
|
||||
"--test-name",
|
||||
dest="testname",
|
||||
dest="test_name",
|
||||
type=str,
|
||||
help="Only run tests that match this value. Matching is regex based, and '.*' is automatically appended",
|
||||
)
|
||||
|
||||
# parse just our args. python unittest has its own args and arg parsing, and that runs inside unittest.main()
|
||||
args, left = parser.parse_known_args()
|
||||
sys.argv = sys.argv[:1] + left
|
||||
parsed, unknown = parser.parse_known_args()
|
||||
sys.argv = sys.argv[:1] + unknown
|
||||
|
||||
return args
|
||||
return parsed
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = parse_args()
|
||||
|
||||
backend_test = create_backend_test(args.testname)
|
||||
create_backend_test(args.test_name)
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -1,4 +1,7 @@
|
|||
{
|
||||
// If possible, add a test to onnx_backend_test_series_overrides.jsonc instead of this file.
|
||||
// This file results in skipping tests, that one allows looser tolerance.
|
||||
//
|
||||
// Tests that are failing temporarily and should be fixed
|
||||
"current_failing_tests": [
|
||||
"^test_adagrad",
|
||||
|
|
@ -40,7 +43,7 @@
|
|||
"^test_resize_downsample_scales_cubic_A_n0p5_exclude_outside_cpu", // NOT_IMPLEMENTED : Could not find an implementation for the node Resize(13)
|
||||
"^test_resize_downsample_scales_cubic_cpu",
|
||||
"^test_resize_downsample_scales_linear_cpu",
|
||||
"^test_resize_downsample_scales_nearest_cpu",
|
||||
"^test_resize_downsample_scales_nearest_cpu",
|
||||
"^test_resize_downsample_sizes_cubic_cpu",
|
||||
"^test_resize_downsample_sizes_linear_pytorch_half_pixel_cpu",
|
||||
"^test_resize_downsample_sizes_nearest_cpu",
|
||||
|
|
@ -169,12 +172,12 @@
|
|||
"^test_sce.*",
|
||||
"^test_nllloss.*",
|
||||
"^test_gather_negative_indices.*",
|
||||
"^test_reduce_sum_do_not_keepdims*", // Does not support axes as input
|
||||
"^test_reduce_sum_do_not_keepdims*", // Does not support axes as input
|
||||
"^test_reduce_sum_keepdims*",
|
||||
"^test_reduce_sum_default_axes_keepdims*",
|
||||
"^test_reduce_sum_negative_axes_keepdims*",
|
||||
"^test_reduce_sum_empty_axes_input_noop*",
|
||||
"^test_unsqueeze_*", // Does not support axes as input
|
||||
"^test_unsqueeze_*", // Does not support axes as input
|
||||
"^test_sequence_insert_at_back",
|
||||
"^test_sequence_insert_at_front",
|
||||
"^test_loop13_seq",
|
||||
|
|
@ -204,15 +207,15 @@
|
|||
"^test_sce.*",
|
||||
"^test_nllloss.*",
|
||||
"^test_upsample_nearest.*",
|
||||
"^test_reduce_sum_do_not_keepdims*", // Does not support axes as input
|
||||
"^test_reduce_sum_do_not_keepdims*", // Does not support axes as input
|
||||
"^test_reduce_sum_keepdims*",
|
||||
"^test_reduce_sum_default_axes_keepdims*",
|
||||
"^test_reduce_sum_negative_axes_keepdims*",
|
||||
"^test_reduce_sum_empty_axes_input_noop*",
|
||||
"^test_scatter_elements_with_negative_indices_cpu",
|
||||
"^test_sum_one_input_cpu",
|
||||
"^test_unsqueeze_*", // Does not support axes as input
|
||||
"^test_squeeze_*", // Does not support axes as input
|
||||
"^test_unsqueeze_*", // Does not support axes as input
|
||||
"^test_squeeze_*", // Does not support axes as input
|
||||
"^test_logsoftmax_*", // Does not support opset-13 yet
|
||||
"^test_softmax_*", // Does not support opset-13 yet
|
||||
"^test_pow", // Runs disabled pow tests from the "current_failing_tests" list at the top
|
||||
|
|
@ -234,7 +237,6 @@
|
|||
"^test_training_dropout_default_mask", // Runs but there's accuracy mismatch
|
||||
"^test_training_dropout_mask", // Runs but there's accuracy mismatch
|
||||
"^test_training_dropout_default" // Runs but there's accuracy mismatch
|
||||
|
||||
],
|
||||
// ORT first supported opset 7, so models with nodes that require versions prior to opset 7 are not supported
|
||||
"tests_with_pre_opset7_dependencies": [
|
||||
|
|
|
|||
13
onnxruntime/test/testdata/onnx_backend_test_series_overrides.jsonc
vendored
Normal file
13
onnxruntime/test/testdata/onnx_backend_test_series_overrides.jsonc
vendored
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
{
|
||||
"rtol_default": 1e-3,
|
||||
"atol_default": 1e-5,
|
||||
// Key: str, the name of the test as defined by ONNX without any device suffix.
|
||||
// Val: float, max absolute difference between expected and actual.
|
||||
"atol_overrides": {
|
||||
"test_dft": 1e-4,
|
||||
"test_dft_axis": 1e-4
|
||||
},
|
||||
// Key: str, the name of the test as defined by ONNX without any device suffix.
|
||||
// Val: float, max relative difference between expected and actual.
|
||||
"rtol_overrides": {}
|
||||
}
|
||||
|
|
@ -173,8 +173,6 @@ std::string GetTestDataPath() {
|
|||
static std::vector<ITestCase*> GetAllTestCases() {
|
||||
std::vector<ITestCase*> tests;
|
||||
std::vector<std::basic_string<PATH_CHAR_TYPE>> whitelistedTestCases;
|
||||
double perSampleTolerance = 1e-3;
|
||||
double relativePerSampleTolerance = 1e-3;
|
||||
std::unordered_set<std::basic_string<ORTCHAR_T>> allDisabledTests;
|
||||
std::vector<std::basic_string<PATH_CHAR_TYPE>> dataDirs;
|
||||
auto testDataPath = GetTestDataPath();
|
||||
|
|
@ -231,7 +229,7 @@ static std::vector<ITestCase*> GetAllTestCases() {
|
|||
#endif
|
||||
|
||||
|
||||
WINML_EXPECT_NO_THROW(LoadTests(dataDirs, whitelistedTestCases, perSampleTolerance, relativePerSampleTolerance,
|
||||
WINML_EXPECT_NO_THROW(LoadTests(dataDirs, whitelistedTestCases, TestTolerances(1e-3, 1e-3, {}, {}),
|
||||
allDisabledTests,
|
||||
[&tests](std::unique_ptr<ITestCase> l) {
|
||||
tests.push_back(l.get());
|
||||
|
|
|
|||
Loading…
Reference in a new issue