mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-06-04 23:59:56 +00:00
Exclude test by onnx version tag (#1073)
* add version filter to failed tests * exclude test from backend * exclude shrink from opset 9 * fix compile err * exclude certain version of constant shape * enable flatten test * fix compile err * comment mvn test * disable constantofshape test in x86 * disable x86 test * get model version from imported opset * test linux x86 case * disable nonzero opset 10 * make mutex const * test filter by commit id * adjust substr offset * Limit test platform * remove change impacting TFModleInfo.h * refactoring * refactoring * test x86 pipeline with filter * add comment * restrict version extraction on non-win * restrict version extraction on non-win * add tag * exclude case from backend test * remove dup * remove dup * make script runnable * hard code adsolute path * refactor log * fix x86 compile err * fix x86 compile err * fix x86 compile err * sync with latest tensorrt * switch to regex * fix cpu pipeline err * test filter * disable nonzero from all versions
This commit is contained in:
RandySheriffH
GitHub
parent
f9f6818e4c
commit
4757933afe
10 changed files with 168 additions and 117 deletions
|
|
@ -177,6 +177,7 @@ using PATH_STRING_TYPE = std::basic_string<PATH_CHAR_TYPE>;
|
|||
class OnnxModelInfo : public TestModelInfo {
|
||||
private:
|
||||
std::string node_name_;
|
||||
std::string onnx_commit_tag_ = "";
|
||||
std::vector<ONNX_NAMESPACE::ValueInfoProto> input_value_info_;
|
||||
std::vector<ONNX_NAMESPACE::ValueInfoProto> output_value_info_;
|
||||
|
||||
|
|
@ -203,7 +204,14 @@ class OnnxModelInfo : public TestModelInfo {
|
|||
if (!model_pb.ParseFromZeroCopyStream(&f)) {
|
||||
ORT_THROW("Failed to load model because protobuf parsing failed.");
|
||||
}
|
||||
|
||||
#ifdef __GNUG__
|
||||
std::smatch match;
|
||||
std::string url_string{model_url};
|
||||
const std::regex onnx_tag_regex("onnx[0-9a-z]{3}"); //e.g. onnx141, onnx150, onnxtip
|
||||
if (std::regex_search(url_string, match, onnx_tag_regex)) {
|
||||
onnx_commit_tag_ = match[0].str();
|
||||
}
|
||||
#endif
|
||||
const ONNX_NAMESPACE::GraphProto& graph = model_pb.graph();
|
||||
if (graph.node().size() == 1) {
|
||||
node_name_ = graph.node()[0].op_type();
|
||||
|
|
@ -221,6 +229,7 @@ class OnnxModelInfo : public TestModelInfo {
|
|||
}
|
||||
|
||||
const PATH_CHAR_TYPE* GetModelUrl() const override { return model_url_.c_str(); }
|
||||
std::string GetModelVersion() const override { return onnx_commit_tag_; }
|
||||
|
||||
const std::string& GetNodeName() const override { return node_name_; }
|
||||
const ONNX_NAMESPACE::ValueInfoProto* GetOutputInfoFromModel(size_t i) const override {
|
||||
|
|
@ -407,6 +416,9 @@ class OnnxTestCase : public ITestCase {
|
|||
const std::string& GetTestCaseName() const override {
|
||||
return test_case_name_;
|
||||
}
|
||||
std::string GetTestCaseVersion() const override {
|
||||
return model_info_->GetModelVersion();
|
||||
}
|
||||
void LoadTestData(size_t id, HeapBuffer& b, std::unordered_map<std::string, OrtValue*>&, bool is_input) override;
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ class ITestCase {
|
|||
virtual const ONNX_NAMESPACE::ValueInfoProto* GetOutputInfoFromModel(size_t i) const = 0;
|
||||
|
||||
virtual const std::string& GetTestCaseName() const = 0;
|
||||
virtual std::string GetTestCaseVersion() const = 0;
|
||||
//a string to help identify the dataset
|
||||
virtual std::string GetDatasetDebugInfoString(size_t dataset_id) = 0;
|
||||
//The number of input/output pairs
|
||||
|
|
@ -52,6 +53,7 @@ class TestModelInfo {
|
|||
virtual int GetOutputCount() const = 0;
|
||||
virtual const std::string& GetInputName(size_t i) const = 0;
|
||||
virtual const std::string& GetOutputName(size_t i) const = 0;
|
||||
virtual std::string GetModelVersion() const {return "";}
|
||||
virtual ~TestModelInfo() = default;
|
||||
|
||||
static TestModelInfo* LoadOnnxModel(_In_ const PATH_CHAR_TYPE* model_url);
|
||||
|
|
|
|||
|
|
@ -8,6 +8,11 @@
|
|||
#include "TestResultStat.h"
|
||||
|
||||
namespace {
|
||||
|
||||
void operator << (std::ostringstream& oss, const std::pair<std::string, std::string>& p) {
|
||||
oss << p.first << " of " << p.second;
|
||||
}
|
||||
|
||||
template <typename T1>
|
||||
std::string containerToStr(const T1& input) {
|
||||
std::ostringstream oss;
|
||||
|
|
|
|||
|
|
@ -33,12 +33,12 @@ class TestResultStat {
|
|||
failed_kernels.insert(s);
|
||||
}
|
||||
|
||||
void AddFailedTest(const std::string& s) {
|
||||
void AddFailedTest(const std::pair<std::string, std::string>& p) {
|
||||
std::lock_guard<onnxruntime::OrtMutex> l(m_);
|
||||
failed_test_cases.insert(s);
|
||||
failed_test_cases.push_back(p);
|
||||
}
|
||||
|
||||
std::unordered_set<std::string> GetFailedTest() {
|
||||
const std::vector< std::pair<std::string, std::string> >& GetFailedTest() const {
|
||||
std::lock_guard<onnxruntime::OrtMutex> l(m_);
|
||||
return failed_test_cases;
|
||||
}
|
||||
|
|
@ -61,19 +61,19 @@ class TestResultStat {
|
|||
}
|
||||
|
||||
for(const auto& s:result.failed_kernels) {
|
||||
AddNotImplementedKernels(s);
|
||||
AddFailedKernels(s);
|
||||
}
|
||||
|
||||
for(const auto& s:result.failed_test_cases) {
|
||||
AddNotImplementedKernels(s);
|
||||
for(const auto& p:result.failed_test_cases) {
|
||||
AddFailedTest(p);
|
||||
}
|
||||
|
||||
return *this;
|
||||
}
|
||||
|
||||
private:
|
||||
onnxruntime::OrtMutex m_;
|
||||
mutable onnxruntime::OrtMutex m_;
|
||||
std::unordered_set<std::string> not_implemented_kernels;
|
||||
std::unordered_set<std::string> failed_kernels;
|
||||
std::unordered_set<std::string> failed_test_cases;
|
||||
std::vector< std::pair<std::string, std::string> > failed_test_cases; // pairs of test name and version
|
||||
};
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
// Licensed under the MIT License.
|
||||
|
||||
#include <core/session/onnxruntime_cxx_api.h>
|
||||
#include <set>
|
||||
#include <iostream>
|
||||
#include <fstream>
|
||||
#ifdef _WIN32
|
||||
|
|
@ -286,7 +287,18 @@ int real_main(int argc, char* argv[], Ort::Env& env) {
|
|||
fwrite(res.c_str(), 1, res.size(), stdout);
|
||||
}
|
||||
// clang-format off
|
||||
std::map<std::string, std::string> broken_tests{
|
||||
|
||||
struct BrokenTest
|
||||
{
|
||||
std::string test_name_;
|
||||
std::string reason_;
|
||||
std::set<std::string> broken_versions_ = {}; // apply to all versions if empty
|
||||
bool operator < (const struct BrokenTest& test) const {
|
||||
return strcmp(test_name_.c_str(), test.test_name_.c_str()) < 0;
|
||||
}
|
||||
};
|
||||
|
||||
std::set<BrokenTest> broken_tests = {
|
||||
{"AvgPool1d", "disable reason"},
|
||||
{"AvgPool1d_stride", "disable reason"},
|
||||
{"AvgPool2d", "disable reason"},
|
||||
|
|
@ -299,8 +311,13 @@ int real_main(int argc, char* argv[], Ort::Env& env) {
|
|||
{"BatchNorm2d_momentum_eval", "disable reason"},
|
||||
{"BatchNorm3d_eval", "disable reason"},
|
||||
{"BatchNorm3d_momentum_eval", "disable reason"},
|
||||
#if defined(__GNUG__) && !defined(__LP64__)
|
||||
{"constantofshape_float_ones", "test data bug"},
|
||||
{"constantofshape_int_zeros", "test data bug"},
|
||||
#else
|
||||
{"constantofshape_float_ones", "test data bug", {"onnx141","onnx150"}},
|
||||
{"constantofshape_int_zeros", "test data bug", {"onnx141","onnx150"}},
|
||||
#endif
|
||||
{"GLU", "disable reason"},
|
||||
{"GLU_dim", "disable reason"},
|
||||
{"Linear", "disable reason"},
|
||||
|
|
@ -314,11 +331,6 @@ int real_main(int argc, char* argv[], Ort::Env& env) {
|
|||
{"Softsign", "disable reason"},
|
||||
{"convtranspose_1d", "disable reason"},
|
||||
{"convtranspose_3d", "disable reason"},
|
||||
{"flatten_axis0", "disable reason"},
|
||||
{"flatten_axis1", "disable reason"},
|
||||
{"flatten_axis2", "disable reason"},
|
||||
{"flatten_axis3", "disable reason"},
|
||||
{"flatten_default_axis", "disable reason"},
|
||||
{"gemm_broadcast", "disable reason"},
|
||||
{"gemm_nobroadcast", "disable reason"},
|
||||
{"matmul_2d", "disable reason"},
|
||||
|
|
@ -342,7 +354,7 @@ int real_main(int argc, char* argv[], Ort::Env& env) {
|
|||
{"tf_nasnet_large", "disable temporarily"},
|
||||
{"tf_nasnet_mobile", "disable temporarily"},
|
||||
{"tf_pnasnet_large", "disable temporarily"},
|
||||
{"shrink", "test case is wrong"},
|
||||
{"shrink", "test case is wrong", {"onnx141"}},
|
||||
{"maxpool_2d_precomputed_strides", "ShapeInferenceError"},
|
||||
{"averagepool_2d_precomputed_strides", "ShapeInferenceError"},
|
||||
{"maxpool_with_argmax_2d_precomputed_strides", "ShapeInferenceError"},
|
||||
|
|
@ -355,73 +367,77 @@ int real_main(int argc, char* argv[], Ort::Env& env) {
|
|||
};
|
||||
|
||||
#ifdef USE_NGRAPH
|
||||
broken_tests["dequantizelinear"] = "ambiguity in scalar dimensions [] vs [1]";
|
||||
broken_tests["qlinearconv"] = "ambiguity in scalar dimensions [] vs [1]";
|
||||
broken_tests["quantizelinear"] = "ambiguity in scalar dimensions [] vs [1]";
|
||||
broken_tests["tiny_yolov2"] = "temporarily disable due to graph resolve failure.";
|
||||
broken_tests["operator_repeat_dim_overflow"] = "temporarily disable due to graph resolve failure.";
|
||||
broken_tests.insert({"dequantizelinear", "ambiguity in scalar dimensions [] vs [1]"});
|
||||
broken_tests.insert({"qlinearconv", "ambiguity in scalar dimensions [] vs [1]"});
|
||||
broken_tests.insert({"quantizelinear", "ambiguity in scalar dimensions [] vs [1]"});
|
||||
broken_tests.insert({"tiny_yolov2", "temporarily disable due to graph resolve failure."});
|
||||
broken_tests.insert({"operator_repeat_dim_overflow", "temporarily disable due to graph resolve failure."});
|
||||
#endif
|
||||
|
||||
#ifdef USE_CUDA
|
||||
broken_tests["mxnet_arcface"] = "result mismatch";
|
||||
broken_tests["tf_inception_v1"] = "flaky test"; //TODO: Investigate cause for flakiness
|
||||
broken_tests.insert({"mxnet_arcface", "result mismatch"});
|
||||
broken_tests.insert({"tf_inception_v1", "flaky test"}); //TODO: Investigate cause for flakiness
|
||||
#endif
|
||||
// clang-format on
|
||||
|
||||
#if defined(_WIN32) && !defined(_WIN64)
|
||||
broken_tests["vgg19"] = "failed: bad allocation";
|
||||
broken_tests.insert({"vgg19", "failed: bad allocation"});
|
||||
#endif
|
||||
|
||||
#if defined(__GNUG__) && !defined(__LP64__)
|
||||
broken_tests["nonzero_example"] = "failed: type mismatch";
|
||||
broken_tests.insert({"nonzero_example", "failed: type mismatch", {"onnx123","onnx130","onnx141","onnx150","onnxtip"}});
|
||||
broken_tests.insert({"slice_neg_steps", "failed: type mismatch"});
|
||||
broken_tests.insert({"mod_float_mixed_sign_example", "failed: type mismatch"});
|
||||
#endif
|
||||
|
||||
#ifdef DISABLE_CONTRIB_OPS
|
||||
broken_tests["coreml_SqueezeNet_ImageNet"] = "This model uses contrib ops.";
|
||||
broken_tests["keras2coreml_Permute_ImageNet"] = "This model uses contrib ops.";
|
||||
broken_tests["keras2coreml_ReLU_ImageNet"] = "This model uses contrib ops.";
|
||||
broken_tests["keras2coreml_Padding-Upsampling-Normalizer_ImageNet"] = "This model uses contrib ops.";
|
||||
broken_tests["tiny_yolov2"] = "This model uses contrib ops.";
|
||||
broken_tests["keras2coreml_Pooling_ImageNet"] = "This model uses contrib ops.";
|
||||
broken_tests["keras2coreml_Padding_ImageNet"] = "This model uses contrib ops.";
|
||||
broken_tests["keras2coreml_Normalizer_ImageNet"] = "This model uses contrib ops.";
|
||||
broken_tests["keras2coreml_linear_sklearn_load_breast_cancer"] = "This model uses contrib ops.";
|
||||
broken_tests["keras2coreml_linear_ImageNet_small"] = "This model uses contrib ops.";
|
||||
broken_tests["keras2coreml_linear_ImageNet_large"] = "This model uses contrib ops.";
|
||||
broken_tests["keras2coreml_linear_ImageNet"] = "This model uses contrib ops.";
|
||||
broken_tests["keras2coreml_leakyrelu_ImageNet"] = "This model uses contrib ops.";
|
||||
broken_tests["keras2coreml_hard_sigmoid_ImageNet"] = "This model uses contrib ops.";
|
||||
broken_tests["keras2coreml_elu_ImageNet"] = "This model uses contrib ops.";
|
||||
broken_tests["keras2coreml_Dense_ImageNet"] = "This model uses contrib ops.";
|
||||
broken_tests["keras2coreml_Conv2D_ImageNet"] = "This model uses contrib ops.";
|
||||
broken_tests["coreml_VGG16_ImageNet"] = "This model uses contrib ops.";
|
||||
broken_tests["coreml_Resnet50_ImageNet"] = "This model uses contrib ops.";
|
||||
broken_tests["coreml_Inceptionv3_ImageNet"] = "This model uses contrib ops.";
|
||||
broken_tests["coreml_FNS-Candy_ImageNet"] = "This model uses contrib ops.";
|
||||
broken_tests["coreml_AgeNet_ImageNet"] = "This model uses contrib ops.";
|
||||
broken_tests["keras2coreml_thresholdedrelu_ImageNet_large"] = "This model uses contrib ops.";
|
||||
broken_tests["keras2coreml_thresholdedrelu_ImageNet_small"] = "This model uses contrib ops.";
|
||||
broken_tests["keras2coreml_thresholdedrelu_sklearn_load_breast_cancer"] = "This model uses contrib ops.";
|
||||
broken_tests["thresholdedrelu"] = "This model uses contrib ops.";
|
||||
broken_tests["thresholdedrelu_default"] = "This model uses contrib ops.";
|
||||
broken_tests["dynamic_slice_default_axes"] = "This model uses contrib ops.";
|
||||
broken_tests["thresholdedrelu_example"] = "This model uses contrib ops.";
|
||||
broken_tests["dynamic_slice_neg failed"] = "This model uses contrib ops.";
|
||||
broken_tests["dynamic_slice_start_out_of_bounds"] = "This model uses contrib ops.";
|
||||
broken_tests["dynamic_slice"] = "This model uses contrib ops.";
|
||||
broken_tests["dynamic_slice_end_out_of_bounds"] = "This model uses contrib ops.";
|
||||
broken_tests["dynamic_slice_neg"] = "This model uses contrib ops.";
|
||||
broken_tests["mvn"] = "This model uses contrib ops.";
|
||||
broken_tests.insert({"coreml_SqueezeNet_ImageNet", "This model uses contrib ops."});
|
||||
broken_tests.insert({"keras2coreml_Permute_ImageNet", "This model uses contrib ops."});
|
||||
broken_tests.insert({"keras2coreml_ReLU_ImageNet", "This model uses contrib ops."});
|
||||
broken_tests.insert({"keras2coreml_Padding-Upsampling-Normalizer_ImageNet", "This model uses contrib ops."});
|
||||
broken_tests.insert({"tiny_yolov2", "This model uses contrib ops."});
|
||||
broken_tests.insert({"keras2coreml_Pooling_ImageNet", "This model uses contrib ops."});
|
||||
broken_tests.insert({"keras2coreml_Padding_ImageNet", "This model uses contrib ops."});
|
||||
broken_tests.insert({"keras2coreml_Normalizer_ImageNet", "This model uses contrib ops."});
|
||||
broken_tests.insert({"keras2coreml_linear_sklearn_load_breast_cancer", "This model uses contrib ops."});
|
||||
broken_tests.insert({"keras2coreml_linear_ImageNet_small", "This model uses contrib ops."});
|
||||
broken_tests.insert({"keras2coreml_linear_ImageNet_large", "This model uses contrib ops."});
|
||||
broken_tests.insert({"keras2coreml_linear_ImageNet", "This model uses contrib ops."});
|
||||
broken_tests.insert({"keras2coreml_leakyrelu_ImageNet", "This model uses contrib ops."});
|
||||
broken_tests.insert({"keras2coreml_hard_sigmoid_ImageNet", "This model uses contrib ops."});
|
||||
broken_tests.insert({"keras2coreml_elu_ImageNet", "This model uses contrib ops."});
|
||||
broken_tests.insert({"keras2coreml_Dense_ImageNet", "This model uses contrib ops."});
|
||||
broken_tests.insert({"keras2coreml_Conv2D_ImageNet", "This model uses contrib ops."});
|
||||
broken_tests.insert({"coreml_VGG16_ImageNet", "This model uses contrib ops."});
|
||||
broken_tests.insert({"coreml_Resnet50_ImageNet", "This model uses contrib ops."});
|
||||
broken_tests.insert({"coreml_Inceptionv3_ImageNet", "This model uses contrib ops."});
|
||||
broken_tests.insert({"coreml_FNS-Candy_ImageNet", "This model uses contrib ops."});
|
||||
broken_tests.insert({"coreml_AgeNet_ImageNet", "This model uses contrib ops."});
|
||||
broken_tests.insert({"keras2coreml_thresholdedrelu_ImageNet_large", "This model uses contrib ops."});
|
||||
broken_tests.insert({"keras2coreml_thresholdedrelu_ImageNet_small", "This model uses contrib ops."});
|
||||
broken_tests.insert({"keras2coreml_thresholdedrelu_sklearn_load_breast_cancer", "This model uses contrib ops."});
|
||||
broken_tests.insert({"thresholdedrelu", "This model uses contrib ops."});
|
||||
broken_tests.insert({"thresholdedrelu_default", "This model uses contrib ops."});
|
||||
broken_tests.insert({"dynamic_slice_default_axes", "This model uses contrib ops."});
|
||||
broken_tests.insert({"thresholdedrelu_example", "This model uses contrib ops."});
|
||||
broken_tests.insert({"dynamic_slice_neg failed", "This model uses contrib ops."});
|
||||
broken_tests.insert({"dynamic_slice_start_out_of_bounds", "This model uses contrib ops."});
|
||||
broken_tests.insert({"dynamic_slice", "This model uses contrib ops."});
|
||||
broken_tests.insert({"dynamic_slice_end_out_of_bounds", "This model uses contrib ops."});
|
||||
broken_tests.insert({"dynamic_slice_neg", "This model uses contrib ops."});
|
||||
broken_tests.insert({"mvn", "This model uses contrib ops.", {"onnx130"}});
|
||||
#endif
|
||||
|
||||
int result = 0;
|
||||
for (const std::string& s : stat.GetFailedTest()) {
|
||||
if (broken_tests.find(s) == broken_tests.end()) {
|
||||
fprintf(stderr, "test %s failed, please fix it\n", s.c_str());
|
||||
for (const auto& p: stat.GetFailedTest()) {
|
||||
BrokenTest t = {p.first, ""};
|
||||
auto iter = broken_tests.find(t);
|
||||
if (iter == broken_tests.end() ||
|
||||
(p.second != "" && !iter->broken_versions_.empty() && iter->broken_versions_.find(p.second) == iter->broken_versions_.end())) {
|
||||
fprintf(stderr, "test %s failed, please fix it\n", p.first.c_str());
|
||||
result = -1;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
#ifdef _WIN32
|
||||
|
|
|
|||
|
|
@ -192,13 +192,13 @@ Status RunTests(TestEnv& env, int p_models, int concurrent_runs, size_t repeat_c
|
|||
}
|
||||
for (size_t i = 0; i != env.tests.size(); ++i) {
|
||||
if (!results[i]) {
|
||||
stat.AddFailedTest(env.tests[i]->GetTestCaseName());
|
||||
stat.AddFailedTest(std::pair<std::string,std::string>(env.tests[i]->GetTestCaseName(), env.tests[i]->GetTestCaseVersion()));
|
||||
continue;
|
||||
}
|
||||
const TestCaseResult& r = *results[i];
|
||||
for (const EXECUTE_RESULT res : r.GetExcutionResult()) {
|
||||
if (res != EXECUTE_RESULT::SUCCESS && res != EXECUTE_RESULT::NOT_SUPPORT) {
|
||||
stat.AddFailedTest(env.tests[i]->GetTestCaseName());
|
||||
stat.AddFailedTest(std::pair<std::string,std::string>(env.tests[i]->GetTestCaseName(),env.tests[i]->GetTestCaseVersion()));
|
||||
}
|
||||
switch (res) {
|
||||
case EXECUTE_RESULT::SUCCESS:
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import sys
|
|||
import os
|
||||
import platform
|
||||
import unittest
|
||||
|
||||
import onnx
|
||||
import onnx.backend.test
|
||||
|
||||
import numpy as np
|
||||
|
|
@ -14,6 +14,23 @@ import onnxruntime.backend as c2
|
|||
|
||||
pytest_plugins = 'onnx.backend.test.report',
|
||||
|
||||
def GetVersionTag():
|
||||
version2tag = {}
|
||||
file_path = '/data/onnx/version2tag'
|
||||
if os.path.isfile(file_path):
|
||||
with open(file_path, 'r') as f:
|
||||
for line in f.readlines():
|
||||
fields = line.strip().split(':')
|
||||
version2tag[fields[0]] = fields[1]
|
||||
print ("version2tag map", version2tag)
|
||||
if onnx.version.git_version in version2tag:
|
||||
return version2tag[onnx.version.git_version]
|
||||
else: return "unknown"
|
||||
|
||||
version_tag = GetVersionTag()
|
||||
print ("onnx version:", onnx.__version__)
|
||||
print ("git version:", onnx.version.git_version)
|
||||
print ("VERSION TAG:", version_tag)
|
||||
|
||||
class OrtBackendTest(onnx.backend.test.BackendTest):
|
||||
|
||||
|
|
@ -92,12 +109,18 @@ def create_backend_test(testname=None):
|
|||
backend_test.include(testname + '.*')
|
||||
else:
|
||||
# Tests that are failing temporarily and should be fixed
|
||||
current_failing_tests = ('^test_cast_FLOAT_to_STRING_cpu.*',
|
||||
'^test_constantofshape_*.*',
|
||||
current_failing_tests = ('^test_cast_STRING_to_FLOAT_cpu.*',
|
||||
'^test_cast_FLOAT_to_STRING_cpu.*',
|
||||
'^test_dequantizelinear_cpu.*',
|
||||
'^test_shrink_cpu.*',
|
||||
'^test_qlinearconv_cpu.*',
|
||||
'^test_quantizelinear_cpu.*')
|
||||
'^test_quantizelinear_cpu.*',
|
||||
'^test_gru_seq_length_cpu.*')
|
||||
global version_tag
|
||||
if version_tag == 'onnx141' or onnx.__version__ == '1.4.1':
|
||||
current_failing_tests = current_failing_tests + ('^test_shrink_cpu.*', '^test_constantofshape_*.*',)
|
||||
if version_tag == 'onnx150' or onnx.__version__ == '1.5.0':
|
||||
current_failing_tests = current_failing_tests + ('^test_constantofshape_*.*',)
|
||||
|
||||
# Failing for nGraph.
|
||||
if c2.supports_device('NGRAPH'):
|
||||
current_failing_tests = current_failing_tests + ('|^test_operator_repeat_dim_overflow_cpu.*',)
|
||||
|
|
@ -107,6 +130,7 @@ def create_backend_test(testname=None):
|
|||
unsupported_usages_filters()
|
||||
|
||||
backend_test.exclude('(' + '|'.join(filters) + ')')
|
||||
print ('excluded tests:', filters)
|
||||
|
||||
# import all test cases at global scope to make
|
||||
# them visible to python.unittest.
|
||||
|
|
|
|||
|
|
@ -34,29 +34,7 @@ fi
|
|||
if [ "$INSTALLED_PYTHON_VERSION" = "3.4" ];then
|
||||
echo "Python 3.5 and above is needed for running onnx tests!" 1>&2
|
||||
else
|
||||
#Install ONNX
|
||||
#5af210ca8a1c73aa6bae8754c9346ec54d0a756e is v1.2.3
|
||||
#bae6333e149a59a3faa9c4d9c44974373dcf5256 is v1.3.0
|
||||
#9e55ace55aad1ada27516038dfbdc66a8a0763db is v1.4.1
|
||||
#7d7bc83d29a328233d3e8affa4c4ea8b3e3599ef is v1.5.0
|
||||
#5d0975f43c1224edce17177783b859fca4b8e2b1 is master
|
||||
for onnx_version in "5af210ca8a1c73aa6bae8754c9346ec54d0a756e" "bae6333e149a59a3faa9c4d9c44974373dcf5256" "9e55ace55aad1ada27516038dfbdc66a8a0763db" "7d7bc83d29a328233d3e8affa4c4ea8b3e3599ef" "5d0975f43c1224edce17177783b859fca4b8e2b1"; do
|
||||
if [ -z ${lastest_onnx_version+x} ]; then
|
||||
echo "first pass";
|
||||
else
|
||||
echo "deleting old onnx-${lastest_onnx_version}";
|
||||
pip3 uninstall -y onnx
|
||||
fi
|
||||
lastest_onnx_version=$onnx_version
|
||||
aria2c -q -d /tmp/src https://github.com/onnx/onnx/archive/$onnx_version.tar.gz
|
||||
tar -xf /tmp/src/onnx-$onnx_version.tar.gz -C /tmp/src
|
||||
cd /tmp/src/onnx-$onnx_version
|
||||
git clone https://github.com/pybind/pybind11.git third_party/pybind11
|
||||
python3 setup.py bdist_wheel
|
||||
pip3 install -q dist/*
|
||||
mkdir -p /data/onnx/$onnx_version
|
||||
backend-test-tools generate-data -o /data/onnx/$onnx_version
|
||||
done
|
||||
source /tmp/scripts/install_onnx.sh
|
||||
fi
|
||||
|
||||
#The last onnx version will be kept
|
||||
|
|
|
|||
|
|
@ -28,29 +28,7 @@ INSTALLED_PYTHON_VERSION=$(python3 -c 'import sys; version=sys.version_info[:2];
|
|||
if [ "$INSTALLED_PYTHON_VERSION" = "3.7" ];then
|
||||
pip3 install --upgrade setuptools
|
||||
else
|
||||
#Install ONNX
|
||||
#5af210ca8a1c73aa6bae8754c9346ec54d0a756e is v1.2.3
|
||||
#bae6333e149a59a3faa9c4d9c44974373dcf5256 is v1.3.0
|
||||
#9e55ace55aad1ada27516038dfbdc66a8a0763db is v1.4.1
|
||||
#7d7bc83d29a328233d3e8affa4c4ea8b3e3599ef is v1.5.0
|
||||
#5d0975f43c1224edce17177783b859fca4b8e2b1 is master
|
||||
for onnx_version in "5af210ca8a1c73aa6bae8754c9346ec54d0a756e" "bae6333e149a59a3faa9c4d9c44974373dcf5256" "9e55ace55aad1ada27516038dfbdc66a8a0763db" "7d7bc83d29a328233d3e8affa4c4ea8b3e3599ef" "5d0975f43c1224edce17177783b859fca4b8e2b1"; do
|
||||
if [ -z ${lastest_onnx_version+x} ]; then
|
||||
echo "first pass";
|
||||
else
|
||||
echo "deleting old onnx-${lastest_onnx_version}";
|
||||
pip3 uninstall -y onnx
|
||||
fi
|
||||
lastest_onnx_version=$onnx_version
|
||||
aria2c -q -d /tmp/src https://github.com/onnx/onnx/archive/$onnx_version.tar.gz
|
||||
tar -xf /tmp/src/onnx-$onnx_version.tar.gz -C /tmp/src
|
||||
cd /tmp/src/onnx-$onnx_version
|
||||
git clone https://github.com/pybind/pybind11.git third_party/pybind11
|
||||
python3 setup.py bdist_wheel
|
||||
pip3 install onnx
|
||||
mkdir -p /data/onnx/$onnx_version
|
||||
backend-test-tools generate-data -o /data/onnx/$onnx_version
|
||||
done
|
||||
source /tmp/scripts/install_onnx.sh x86
|
||||
fi
|
||||
|
||||
#The last onnx version will be kept
|
||||
|
|
|
|||
36
tools/ci_build/github/linux/docker/scripts/install_onnx.sh
Executable file
36
tools/ci_build/github/linux/docker/scripts/install_onnx.sh
Executable file
|
|
@ -0,0 +1,36 @@
|
|||
#!/bin/bash
|
||||
#Install ONNX
|
||||
#5af210ca8a1c73aa6bae8754c9346ec54d0a756e is v1.2.3
|
||||
#bae6333e149a59a3faa9c4d9c44974373dcf5256 is v1.3.0
|
||||
#9e55ace55aad1ada27516038dfbdc66a8a0763db is v1.4.1
|
||||
#7d7bc83d29a328233d3e8affa4c4ea8b3e3599ef is v1.5.0
|
||||
#5d0975f43c1224edce17177783b859fca4b8e2b1 is master
|
||||
declare -A version2tag
|
||||
version2tag+=(["5af210ca8a1c73aa6bae8754c9346ec54d0a756e"]="onnx123"
|
||||
["bae6333e149a59a3faa9c4d9c44974373dcf5256"]="onnx130"
|
||||
["9e55ace55aad1ada27516038dfbdc66a8a0763db"]="onnx141"
|
||||
["7d7bc83d29a328233d3e8affa4c4ea8b3e3599ef"]="onnx150"
|
||||
["5d0975f43c1224edce17177783b859fca4b8e2b1"]="onnxtip")
|
||||
for onnx_version in ${!version2tag[@]}; do
|
||||
if [ -z ${lastest_onnx_version+x} ]; then
|
||||
echo "first pass";
|
||||
else
|
||||
echo "deleting old onnx-${lastest_onnx_version}";
|
||||
pip3 uninstall -y onnx
|
||||
fi
|
||||
lastest_onnx_version=$onnx_version
|
||||
aria2c -q -d /tmp/src https://github.com/onnx/onnx/archive/$onnx_version.tar.gz
|
||||
tar -xf /tmp/src/onnx-$onnx_version.tar.gz -C /tmp/src
|
||||
cd /tmp/src/onnx-$onnx_version
|
||||
git clone https://github.com/pybind/pybind11.git third_party/pybind11
|
||||
python3 setup.py bdist_wheel
|
||||
if [ $1 = "x86" ]; then
|
||||
pip3 install -q onnx
|
||||
else
|
||||
pip3 install -q dist/*
|
||||
fi
|
||||
mkdir -p /data/onnx/${version2tag[$onnx_version]}
|
||||
backend-test-tools generate-data -o /data/onnx/${version2tag[$onnx_version]}
|
||||
echo $onnx_version":"${version2tag[$onnx_version]} >> /data/onnx/version2tag
|
||||
done
|
||||
|
||||
Loading…
Reference in a new issue