mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-05-17 21:10:43 +00:00
151 lines
13 KiB
C++
151 lines
13 KiB
C++
#pragma once
|
|
#include "common.h"
|
|
|
|
//Need to file bugs for failing tests and add to reason. Before that happens, default reasons will be used.
|
|
static const std::string disabledTestDefaultReason = "Model not working on CPU and GPU. Please file bug and replace this reason message.";
|
|
static const std::string disabledGpuTestDefaultReason = "Model not working on GPU. Please file bug and replace this reason message.";
|
|
|
|
// {"model test name", "reason for why it is happening and bug filed for it."}
|
|
std::unordered_map<std::string, std::string> disabledTests(
|
|
{
|
|
// Disabled cases common to both CPU&GPU (no _CPU/_GPU suffix):
|
|
|
|
// Tier 3 models
|
|
{"mxnet_arcface_opset8", disabledTestDefaultReason},
|
|
{"XGBoost_XGClassifier_sklearn_load_wine_opset7", disabledTestDefaultReason},
|
|
{"XGBoost_XGClassifier_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason},
|
|
{"XGBoost_XGClassifier_OpenML_312_scene_opset7", disabledTestDefaultReason},
|
|
{"XGBoost_XGClassifier_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason},
|
|
{"scikit_SVC_sklearn_load_wine_opset7", disabledTestDefaultReason},
|
|
{"scikit_SVC_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason},
|
|
{"scikit_SVC_OpenML_312_scene_opset7", disabledTestDefaultReason},
|
|
{"scikit_SVC_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason},
|
|
{"scikit_Scaler_LogisticRegression_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason},
|
|
{"scikit_Scaler_LogisticRegression_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason},
|
|
{"scikit_RandomForestClassifier_sklearn_load_wine_opset7", disabledTestDefaultReason},
|
|
{"scikit_RandomForestClassifier_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason},
|
|
{"scikit_RandomForestClassifier_OpenML_312_scene_opset7", disabledTestDefaultReason},
|
|
{"scikit_Nu_SVC_sklearn_load_wine_opset7", disabledTestDefaultReason},
|
|
{"scikit_Nu_SVC_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason},
|
|
{"scikit_Nu_SVC_OpenML_312_scene_opset7", disabledTestDefaultReason},
|
|
{"scikit_Nu_SVC_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason},
|
|
{"scikit_Normalizer_RandomForestClassifier_sklearn_load_wine_opset7", disabledTestDefaultReason},
|
|
{"scikit_Normalizer_LinearSVC_sklearn_load_wine_opset7", disabledTestDefaultReason},
|
|
{"scikit_LogisticRegression_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason},
|
|
{"scikit_LogisticRegression_OpenML_31_credit_opset7", disabledTestDefaultReason},
|
|
{"scikit_LogisticRegression_OpenML_312_scene_opset7", disabledTestDefaultReason},
|
|
{"scikit_LogisticRegression_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason},
|
|
{"scikit_LinearSVC_sklearn_load_wine_opset7", disabledTestDefaultReason},
|
|
{"scikit_LabelEncoder_OpenML_3_chess_opset7", disabledTestDefaultReason},
|
|
{"scikit_LabelEncoder_BikeSharing_opset7", disabledTestDefaultReason},
|
|
{"scikit_Imputer_LogisticRegression_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason},
|
|
{"scikit_Imputer_LogisticRegression_OpenML_1464_blood_transfusion_missing_opset7", disabledTestDefaultReason},
|
|
{"scikit_Imputer_GradientBoostingClassifier_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason},
|
|
{"scikit_Imputer_GradientBoostingClassifier_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason},
|
|
{"scikit_GradientBoostingClassifier_sklearn_load_wine_opset7", disabledTestDefaultReason},
|
|
{"scikit_GradientBoostingClassifier_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason},
|
|
{"scikit_GradientBoostingClassifier_OpenML_312_scene_opset7", disabledTestDefaultReason},
|
|
{"scikit_GradientBoostingClassifier_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason},
|
|
{"scikit_DictVectorizer_sklearn_load_Iris_missing_opset7", disabledTestDefaultReason},
|
|
{"scikit_DictVectorizer_sklearn_load_digits_opset7", disabledTestDefaultReason},
|
|
{"scikit_DictVectorizer_sklearn_load_diabetes_missing_opset7", disabledTestDefaultReason},
|
|
{"scikit_DictVectorizer_OpenML_31_credit_opset7", disabledTestDefaultReason},
|
|
{"scikit_DictVectorizer_RandomForestRegressor_sklearn_load_diabetes_opset7", disabledTestDefaultReason},
|
|
{"scikit_DictVectorizer_RandomForestClassifier_sklearn_load_wine_opset7", disabledTestDefaultReason},
|
|
{"scikit_DictVectorizer_LinearSVC_sklearn_load_wine_opset7", disabledTestDefaultReason},
|
|
{"scikit_DictVectorizer_LinearRegression_sklearn_load_diabetes_opset7", disabledTestDefaultReason},
|
|
{"scikit_DictVectorizer_GradientBoostingRegressor_sklearn_load_boston_opset7", disabledTestDefaultReason},
|
|
{"scikit_DecisionTreeClassifier_sklearn_load_wine_opset7", disabledTestDefaultReason},
|
|
{"scikit_DecisionTreeClassifier_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason},
|
|
{"scikit_DecisionTreeClassifier_OpenML_312_scene_opset7", disabledTestDefaultReason},
|
|
{"scikit_DecisionTreeClassifier_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason},
|
|
{"scikit_Binarization_DecisionTreeClassifier_OpenML_1492_plants_opset7", disabledTestDefaultReason},
|
|
{"scikit_Binarization_DecisionTreeClassifier_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason},
|
|
{"libsvm_Nu_SVC_sklearn_load_wine_opset7", disabledTestDefaultReason},
|
|
{"libsvm_Nu_SVC_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason},
|
|
{"libsvm_Nu_SVC_OpenML_312_scene_opset7", disabledTestDefaultReason},
|
|
{"libsvm_Nu_SVC_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason},
|
|
{"coreml_VGG16_ImageNet_opset7", disabledTestDefaultReason},
|
|
{"coreml_SVC_sklearn_load_wine_opset7", disabledTestDefaultReason},
|
|
{"coreml_SVC_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason},
|
|
{"coreml_SVC_OpenML_312_scene_opset7", disabledTestDefaultReason},
|
|
{"coreml_SVC_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason},
|
|
{"coreml_SqueezeNet_ImageNet_opset7", disabledTestDefaultReason},
|
|
{"coreml_Scaler_LogisticRegression_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason},
|
|
{"coreml_Scaler_LogisticRegression_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason},
|
|
{"coreml_Resnet50_ImageNet_opset7", disabledTestDefaultReason},
|
|
{"coreml_RandomForestClassifier_sklearn_load_wine_opset7", disabledTestDefaultReason},
|
|
{"coreml_RandomForestClassifier_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason},
|
|
{"coreml_RandomForestClassifier_OpenML_312_scene_opset7", disabledTestDefaultReason},
|
|
{"coreml_RandomForestClassifier_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason},
|
|
{"coreml_Normalizer_RandomForestClassifier_sklearn_load_wine_opset7", disabledTestDefaultReason},
|
|
{"coreml_Normalizer_LinearSVC_sklearn_load_wine_opset7", disabledTestDefaultReason},
|
|
{"coreml_LogisticRegression_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason},
|
|
{"coreml_LogisticRegression_OpenML_31_credit_opset7", disabledTestDefaultReason},
|
|
{"coreml_LogisticRegression_OpenML_312_scene_opset7", disabledTestDefaultReason},
|
|
{"coreml_LogisticRegression_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason},
|
|
{"coreml_LinearSVC_sklearn_load_wine_opset7", disabledTestDefaultReason},
|
|
{"coreml_LinearSVC_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason},
|
|
{"coreml_LinearSVC_OpenML_312_scene_opset7", disabledTestDefaultReason},
|
|
{"coreml_LinearSVC_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason},
|
|
{"coreml_Inceptionv3_ImageNet_opset7", disabledTestDefaultReason},
|
|
{"coreml_Imputer_LogisticRegression_OpenML_1464_blood_transfusion_missing_opset7", disabledTestDefaultReason},
|
|
{"coreml_Imputer_GradientBoostingClassifier_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason},
|
|
{"coreml_Imputer_GradientBoostingClassifier_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason},
|
|
{"coreml_GradientBoostingClassifier_sklearn_load_wine_opset7", disabledTestDefaultReason},
|
|
{"coreml_GradientBoostingClassifier_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason},
|
|
{"coreml_GradientBoostingClassifier_OpenML_312_scene_opset7", disabledTestDefaultReason},
|
|
{"coreml_GradientBoostingClassifier_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason},
|
|
{"coreml_GradientBoostingClassifier_Criteo_opset7", disabledTestDefaultReason},
|
|
{"coreml_GradientBoostingClassifier_BingClick_opset7", disabledTestDefaultReason},
|
|
{"coreml_DictVectorizer_sklearn_load_Iris_missing_opset7", disabledTestDefaultReason},
|
|
{"coreml_DictVectorizer_sklearn_load_digits_opset7", disabledTestDefaultReason},
|
|
{"coreml_DictVectorizer_sklearn_load_diabetes_missing_opset7", disabledTestDefaultReason},
|
|
{"coreml_DictVectorizer_OpenML_31_credit_opset7", disabledTestDefaultReason},
|
|
{"coreml_DictVectorizer_RandomForestRegressor_sklearn_load_diabetes_opset7", disabledTestDefaultReason},
|
|
{"coreml_DictVectorizer_RandomForestClassifier_sklearn_load_wine_opset7", disabledTestDefaultReason},
|
|
{"coreml_DictVectorizer_LinearSVC_sklearn_load_wine_opset7", disabledTestDefaultReason},
|
|
{"coreml_DictVectorizer_LinearRegression_sklearn_load_diabetes_opset7", disabledTestDefaultReason},
|
|
{"coreml_DictVectorizer_GradientBoostingRegressor_sklearn_load_boston_opset7", disabledTestDefaultReason},
|
|
{"coreml_DecisionTreeClassifier_sklearn_load_wine_opset7", disabledTestDefaultReason},
|
|
{"coreml_DecisionTreeClassifier_sklearn_load_breast_cancer_opset7", disabledTestDefaultReason},
|
|
{"coreml_DecisionTreeClassifier_OpenML_312_scene_opset7", disabledTestDefaultReason},
|
|
{"coreml_DecisionTreeClassifier_OpenML_1464_blood_transfusion_opset7", disabledTestDefaultReason},
|
|
{"coreml_AgeNet_ImageNet_opset7", disabledTestDefaultReason},
|
|
|
|
// GPU specific cases:
|
|
|
|
// ONNX zoo models
|
|
{"mask_rcnn_opset10_GPU", "Bug 31005388: mask_rcnn opset 10 onnx zoo model fails to evaluate on DirectML https://microsoft.visualstudio.com/OS/_workitems/edit/31005388"},
|
|
{"faster_rcnn_opset10_GPU", "Bug 31005511: Failed to extract tensor data from evaluate result of faster_rcnn opset 10 model in DirectML https://microsoft.visualstudio.com/OS/_workitems/edit/31005511"},
|
|
|
|
// Tier 2 models
|
|
{"fp16_test_tiny_yolov2_opset7_GPU", "Bug 31005780: Result of fp16_test_tiny_yolov2_opset7 and fp16_coreml_FNS_Candy_opset7 models on DirectML aren't as accurate as on CPU https://microsoft.visualstudio.com/OS/_workitems/edit/31005780"},
|
|
{"fp16_tiny_yolov2_opset8_GPU", "Bug 31005780: Result of fp16_test_tiny_yolov2_opset7 and fp16_coreml_FNS_Candy_opset7 models on DirectML aren't as accurate as on CPU https://microsoft.visualstudio.com/OS/_workitems/edit/31005780"},
|
|
{"fp16_coreml_FNS_Candy_opset7_GPU", "Bug 31005780: Result of fp16_test_tiny_yolov2_opset7 and fp16_coreml_FNS_Candy_opset7 models on DirectML aren't as accurate as on CPU https://microsoft.visualstudio.com/OS/_workitems/edit/31005780"},
|
|
{"mlperf_ssd_mobilenet_300_opset10_GPU", "Bug 31005624: mlperf_ssd_mobilenet_300 opset 10 model fails to evaluate in DirectML https://microsoft.visualstudio.com/OS/_workitems/edit/31005624"},
|
|
}
|
|
);
|
|
|
|
/*
|
|
model name -> (adapter name regex, skipped test reason)
|
|
*/
|
|
std::unordered_map<std::string, std::pair<std::string, std::string>> disabledGpuAdapterTests(
|
|
{
|
|
// e.g. {"fp16_inception_v1_opset7_GPU", std::make_pair("NVIDIA", "Bug 31144419: Results of fp16_inception_v1 opset7 and opset8 aren't accurate enough on AMD Radeon VII & Intel(R) UHD Graphics 630 & NVIDIA https://microsoft.visualstudio.com/OS/_workitems/edit/31144419")},
|
|
// {"candy_opset9", std::make_pair("(Intel\\(R\\) (UHD )?Graphics)|(Adreno)", "Bug 31652854: Results of candy_opset9 aren't accurate enough on Intel Graphics and Qualcomm Adreno 685 https://microsoft.visualstudio.com/OS/_workitems/edit/31652854")},
|
|
}
|
|
);
|
|
|
|
/*
|
|
Override the default tolerances for these test cases (can be tailored to only CPU or GPU with suffix).
|
|
test name -> absolute difference sampleTolerance
|
|
*/
|
|
std::unordered_map<std::string, double> sampleTolerancePerTests(
|
|
{
|
|
{"fp16_inception_v1_opset7_GPU", 0.005},
|
|
{"fp16_inception_v1_opset8_GPU", 0.005},
|
|
{"candy_opset9_GPU", 0.00150000}, // Intel(R) UHD Graphics 630 (29.20.100.9020) AP machine has inaccurate GPU results for FNS Candy opset 9 https://microsoft.visualstudio.com/OS/_workitems/edit/30696168/
|
|
{"fp16_tiny_yolov2_opset8_GPU", 0.109000}, // Intel(R) UHD Graphics 630 (29.20.100.9020) AP machine has inaccurate GPU results for FNS Candy opset 9 https://microsoft.visualstudio.com/OS/_workitems/edit/30696168/
|
|
}
|
|
);
|