saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-26 19:52:38 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Model tests refactor (#11194)

Browse source 
* Update model test

* update comment

* create map to hold OnnxModelInfo so test doesn't need to reload the model again

* revert the code and use GTEST_SKIP() to skip test

* fix bug

* revert LATEST_ONNX_OPSET_SUPPORTED_BY_TENSORRT
This commit is contained in:
Chi Lo 2022-04-20 10:14:28 -07:00 committed by GitHub
parent af249943a1
commit cb46d79108
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 17 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
onnxruntime/test/providers/cpu/model_tests.cc
View file

@ -52,6 +52,11 @@ struct BrokenTest {
#ifdef GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST
GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(ModelTest);
#endif
void SkipTest() {
GTEST_SKIP() << "Skipping single test";
}
TEST_P(ModelTest, Run) {
std::basic_string<ORTCHAR_T> param = GetParam();
size_t pos = param.find(ORT_TSTR("_"));
@ -72,16 +77,19 @@ TEST_P(ModelTest, Run) {
// them is enabled here to save CI build time.
// Besides saving CI build time, TRT isn’t able to support full ONNX ops spec and therefore some testcases will fail.
// That's one of reasons we skip those testcases and only test latest ONNX opsets.
SkipTest();
return;
}
if (model_info->GetONNXOpSetVersion() == 10 && provider_name == "dnnl") {
// DNNL can run most of the model tests, but only part of
// them is enabled here to save CI build time.
SkipTest();
return;
}
#ifndef ENABLE_TRAINING
if (model_info->HasDomain(ONNX_NAMESPACE::AI_ONNX_TRAINING_DOMAIN) ||
model_info->HasDomain(ONNX_NAMESPACE::AI_ONNX_PREVIEW_TRAINING_DOMAIN)) {
SkipTest();
return;
}
#endif
@ -396,7 +404,6 @@ TEST_P(ModelTest, Run) {
// TensorRT EP CI uses Nvidia Tesla M60 which doesn't support fp16.
broken_tests_keyword_set.insert({"FLOAT16"});
}
if (provider_name == "dml") {
@ -547,14 +554,16 @@ TEST_P(ModelTest, Run) {
if (iter != broken_tests.end() &&
(model_version == TestModelInfo::unknown_version || iter->broken_versions_.empty() ||
iter->broken_versions_.find(model_version) != iter->broken_versions_.end())) {
SkipTest();
return;
}
for (auto iter2 = broken_tests_keyword_set.begin(); iter2 != broken_tests_keyword_set.end(); ++iter2) {
std::string keyword = *iter2;
if (ToUTF8String(test_case_name).find(keyword) != std::string::npos) {
return;
}
std::string keyword = *iter2;
if (ToUTF8String(test_case_name).find(keyword) != std::string::npos) {
SkipTest();
return;
}
}
}
bool is_single_node = !model_info->GetNodeName().empty();
@ -567,7 +576,6 @@ TEST_P(ModelTest, Run) {
if (provider_name == "cpu" && is_single_node)
use_single_thread.push_back(true);
std::unique_ptr<ITestCase> l = CreateOnnxTestCase(ToUTF8String(test_case_name), std::move(model_info),
per_sample_tolerance, relative_per_sample_tolerance);
@ -599,7 +607,7 @@ TEST_P(ModelTest, Run) {
1000,
1,
1 << 30,
1, // enable fp16
1, // enable fp16
0,
nullptr,
0,
@ -995,7 +1003,7 @@ TEST_P(ModelTest, Run) {
return v;
}
auto ExpandModelName = [](const ::testing::TestParamInfo<ModelTest::ParamType>& info) {
auto ExpandModelName = [](const ::testing::TestParamInfo<ModelTest::ParamType>& info) {
// use info.param here to generate the test suffix
std::basic_string<ORTCHAR_T> name = info.param;
@ -1034,4 +1042,4 @@ auto ExpandModelName = [](const ::testing::TestParamInfo<ModelTest::ParamType>&
INSTANTIATE_TEST_SUITE_P(ModelTests, ModelTest, testing::ValuesIn(GetParameterStrings()), ExpandModelName);
} // namespace test
} // namespace onnxruntime
} // namespace onnxruntime