mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-30 20:18:08 +00:00
Clean up build.py (#2446)
This commit is contained in:
Changming Sun
GitHub
parent
d49cbf6e08
commit
0341ee9060
5 changed files with 51 additions and 114 deletions
|
|
@ -32,9 +32,9 @@ class CPUExecutionProvider : public IExecutionProvider {
|
|||
std::numeric_limits<size_t>::max()};
|
||||
|
||||
#ifdef USE_JEMALLOC
|
||||
#if defined(USE_MIMALLOC)
|
||||
#error jemalloc and mimalloc should not both be enabled
|
||||
#endif
|
||||
#if defined(USE_MIMALLOC)
|
||||
#error jemalloc and mimalloc should not both be enabled
|
||||
#endif
|
||||
|
||||
ORT_UNUSED_PARAMETER(info);
|
||||
//JEMalloc already has memory pool, so just use device allocator.
|
||||
|
|
@ -42,12 +42,16 @@ class CPUExecutionProvider : public IExecutionProvider {
|
|||
std::shared_ptr<IArenaAllocator>(
|
||||
onnxruntime::make_unique<DummyArena>(device_info.factory(0))));
|
||||
#else
|
||||
//Disable Arena allocator for x86_32 build because it may run into infinite loop when integer overflow happens
|
||||
#if defined(__amd64__) || defined(_M_AMD64)
|
||||
if (info.create_arena)
|
||||
InsertAllocator(CreateAllocator(device_info));
|
||||
else
|
||||
#endif
|
||||
InsertAllocator(
|
||||
std::shared_ptr<IArenaAllocator>(
|
||||
onnxruntime::make_unique<DummyArena>(device_info.factory(0))));
|
||||
|
||||
#endif
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -166,14 +166,6 @@ TEST(TensorTest, StringTensorTest) {
|
|||
EXPECT_EQ(tensor_data[1], "b");
|
||||
string_ptr = new_data;
|
||||
}
|
||||
// on msvc, check does the ~string be called when release tensor
|
||||
// It may be not stable as access to a deleted pointer could have
|
||||
// undefined behavior. If we find it is failure on other platform
|
||||
// go ahead to remove it.
|
||||
#ifdef _MSC_VER
|
||||
EXPECT_EQ(string_ptr->size(), 0);
|
||||
EXPECT_EQ((string_ptr + 1)->size(), 0);
|
||||
#endif
|
||||
}
|
||||
|
||||
TEST(TensorTest, ConvertToString) {
|
||||
|
|
|
|||
|
|
@ -254,7 +254,6 @@ class OnnxModelInfo : public TestModelInfo {
|
|||
const std::string& GetOutputName(size_t i) const override { return output_value_info_[i].name(); }
|
||||
};
|
||||
|
||||
template <typename PATH_CHAR_TYPE>
|
||||
static void SortTensorFileNames(std::vector<std::basic_string<PATH_CHAR_TYPE>>& input_pb_files) {
|
||||
if (input_pb_files.size() <= 1) return;
|
||||
std::sort(input_pb_files.begin(), input_pb_files.end(),
|
||||
|
|
@ -265,10 +264,15 @@ static void SortTensorFileNames(std::vector<std::basic_string<PATH_CHAR_TYPE>>&
|
|||
int right1 = ExtractFileNo(rightname);
|
||||
return left1 < right1;
|
||||
});
|
||||
|
||||
for (size_t i = 0; i != input_pb_files.size(); ++i) {
|
||||
int fileno = ExtractFileNo(GetLastComponent(input_pb_files[i]));
|
||||
if (static_cast<size_t>(fileno) != i) {
|
||||
ORT_THROW("illegal input file name:", ToMBString(input_pb_files[i]));
|
||||
std::basic_ostringstream<PATH_CHAR_TYPE> oss;
|
||||
oss << input_pb_files[0];
|
||||
for (size_t j = 1; j != input_pb_files.size(); ++j)
|
||||
oss << ORT_TSTR(" ") << input_pb_files[j];
|
||||
ORT_THROW("illegal input file name:", ToMBString(oss.str()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -364,22 +364,13 @@ int real_main(int argc, char* argv[], Ort::Env& env) {
|
|||
if (enable_dml) {
|
||||
all_disabled_tests.insert(std::begin(dml_disabled_tests), std::end(dml_disabled_tests));
|
||||
}
|
||||
|
||||
#if (defined(_WIN32) && !defined(_WIN64)) || (defined(__GNUG__) && !defined(__LP64__))
|
||||
// Minimize mem consumption
|
||||
LoadTests(data_dirs, whitelisted_test_cases, per_sample_tolerance, relative_per_sample_tolerance,
|
||||
[&stat, &sf, &all_disabled_tests, &env](ITestCase* l) {
|
||||
std::unique_ptr<ITestCase> test_case_ptr(l);
|
||||
if (all_disabled_tests.find(l->GetTestCaseName()) != all_disabled_tests.end()) {
|
||||
return;
|
||||
}
|
||||
TestResultStat per_case_stat;
|
||||
std::vector<ITestCase*> per_case_tests = {l};
|
||||
TestEnv per_case_args(per_case_tests, per_case_stat, env, sf);
|
||||
RunTests(per_case_args, 1, 1, 1, GetDefaultThreadPool(Env::Default()));
|
||||
stat += per_case_stat;
|
||||
});
|
||||
#if defined(__amd64__) || defined(_M_AMD64)
|
||||
#else
|
||||
//out of memory
|
||||
static const char* x86_disabled_tests[] = {"mlperf_ssd_resnet34_1200", "mask_rcnn_keras", "mask_rcnn", "faster_rcnn", "vgg19"};
|
||||
all_disabled_tests.insert(std::begin(x86_disabled_tests), std::end(x86_disabled_tests));
|
||||
#endif
|
||||
|
||||
std::vector<ITestCase*> tests;
|
||||
LoadTests(data_dirs, whitelisted_test_cases, per_sample_tolerance, relative_per_sample_tolerance,
|
||||
[&tests](ITestCase* l) { tests.push_back(l); });
|
||||
|
|
@ -403,7 +394,6 @@ int real_main(int argc, char* argv[], Ort::Env& env) {
|
|||
for (ITestCase* l : tests) {
|
||||
delete l;
|
||||
}
|
||||
#endif
|
||||
std::string res = stat.ToString();
|
||||
fwrite(res.c_str(), 1, res.size(), stdout);
|
||||
}
|
||||
|
|
@ -569,13 +559,6 @@ int real_main(int argc, char* argv[], Ort::Env& env) {
|
|||
broken_tests.insert({"vgg19", "failed: bad allocation"});
|
||||
#endif
|
||||
|
||||
#if defined(__GNUG__) && !defined(__LP64__)
|
||||
broken_tests.insert(
|
||||
{"nonzero_example", "failed: type mismatch", {"onnx123", "onnx130", "onnx141", "onnx150", "onnxtip"}});
|
||||
broken_tests.insert({"slice_neg_steps", "failed: type mismatch"});
|
||||
broken_tests.insert({"mod_float_mixed_sign_example", "failed: type mismatch"});
|
||||
#endif
|
||||
|
||||
#ifdef DISABLE_CONTRIB_OPS
|
||||
broken_tests.insert({"mask_rcnn_keras", "This model uses contrib ops."});
|
||||
broken_tests.insert({"coreml_SqueezeNet_ImageNet", "This model uses contrib ops."});
|
||||
|
|
|
|||
|
|
@ -65,8 +65,6 @@ Use the individual flags to only run the specified stages.
|
|||
parser.add_argument("--enable_onnx_tests", action='store_true',
|
||||
help='''When running the Test phase, run onnx_test_running against available test data directories.''')
|
||||
parser.add_argument("--path_to_protoc_exe", help="Path to protoc exe. ")
|
||||
parser.add_argument("--download_test_data", action="store_true",
|
||||
help='''Downloads test data without running the tests''')
|
||||
parser.add_argument("--test_data_url", help="Test data URL.")
|
||||
parser.add_argument("--test_data_checksum", help="Test data checksum (MD5 digest).")
|
||||
|
||||
|
|
@ -143,7 +141,6 @@ Use the individual flags to only run the specified stages.
|
|||
parser.add_argument("--use_eigenthreadpool", action="store_true", help="Build with eigenthreadpool")
|
||||
parser.add_argument("--enable_msinternal", action="store_true", help="Enable for Microsoft internal builds only.")
|
||||
parser.add_argument("--llvm_path", help="Path to llvm dir")
|
||||
parser.add_argument("--azure_sas_key", help="Azure storage sas key, starts with '?'")
|
||||
parser.add_argument("--use_brainslice", action="store_true", help="Build with brain slice")
|
||||
parser.add_argument("--brain_slice_package_path", help="Path to brain slice packages")
|
||||
parser.add_argument("--brain_slice_package_name", help="Name of brain slice packages")
|
||||
|
|
@ -260,53 +257,9 @@ def check_md5(filename, expected_md5):
|
|||
return False
|
||||
return True
|
||||
|
||||
#the last part of src_url should be unique, across all the builds
|
||||
def download_test_data(build_dir, src_url, expected_md5, azure_sas_key):
|
||||
cache_dir = os.path.join(expanduser("~"), '.cache','onnxruntime')
|
||||
os.makedirs(cache_dir, exist_ok=True)
|
||||
local_zip_file = os.path.join(cache_dir, os.path.basename(src_url))
|
||||
if not check_md5(local_zip_file, expected_md5):
|
||||
log.info("Downloading test data")
|
||||
if azure_sas_key:
|
||||
src_url += azure_sas_key
|
||||
# try to avoid logging azure_sas_key
|
||||
if shutil.which('aria2c'):
|
||||
result = subprocess.run(['aria2c','-x', '5', '-j',' 5', '-q', src_url, '-d', cache_dir])
|
||||
if result.returncode != 0:
|
||||
raise BuildError("aria2c exited with code {}.".format(result.returncode))
|
||||
elif shutil.which('curl'):
|
||||
result = subprocess.run(['curl', '-s', src_url, '-o', local_zip_file])
|
||||
if result.returncode != 0:
|
||||
raise BuildError("curl exited with code {}.".format(result.returncode))
|
||||
else:
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
try:
|
||||
urllib.request.urlretrieve(src_url, local_zip_file)
|
||||
except urllib.error.URLError:
|
||||
raise BuildError("urllib.request.urlretrieve() failed.")
|
||||
models_dir = os.path.join(build_dir,'models')
|
||||
if os.path.exists(models_dir):
|
||||
log.info('deleting %s' % models_dir)
|
||||
shutil.rmtree(models_dir)
|
||||
if shutil.which('unzip'):
|
||||
run_subprocess(['unzip','-qd', models_dir, local_zip_file])
|
||||
elif shutil.which('7z'): # 7-Zip
|
||||
run_subprocess(['7z','x', local_zip_file, '-y', '-o' + models_dir])
|
||||
elif shutil.which('7za'): # 7-Zip standalone
|
||||
run_subprocess(['7za', 'x', local_zip_file, '-y', '-o' + models_dir])
|
||||
else:
|
||||
#TODO: use python for unzip
|
||||
log.error("No unzip tool for use")
|
||||
return False
|
||||
return True
|
||||
|
||||
def setup_test_data(build_dir, configs, test_data_url, test_data_checksum, azure_sas_key):
|
||||
if test_data_url is not None:
|
||||
"""Sets up the test data, downloading it if needed."""
|
||||
if not download_test_data(build_dir, test_data_url, test_data_checksum, azure_sas_key):
|
||||
raise BuildError("Failed to set up test data.")
|
||||
|
||||
def setup_test_data(build_dir, configs):
|
||||
# create a shortcut for test models if there is a 'models' folder in build_dir
|
||||
if is_windows():
|
||||
src_model_dir = os.path.join(build_dir, 'models')
|
||||
|
|
@ -620,7 +573,7 @@ def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs, enab
|
|||
if onnxml_test:
|
||||
run_subprocess([sys.executable, 'onnxruntime_test_python_keras.py'], cwd=cwd, dll_path=dll_path)
|
||||
|
||||
def run_onnx_tests(build_dir, configs, onnx_test_data_dir, provider, enable_multi_device_test, enable_parallel_executor_test, num_parallel_models):
|
||||
def run_onnx_tests(build_dir, configs, onnx_test_data_dir, provider, enable_multi_device_test, enable_parallel_executor_test, num_parallel_models, num_parallel_tests=0):
|
||||
for config in configs:
|
||||
cwd = get_config_build_dir(build_dir, config)
|
||||
if is_windows():
|
||||
|
|
@ -632,12 +585,9 @@ def run_onnx_tests(build_dir, configs, onnx_test_data_dir, provider, enable_mult
|
|||
cmd = []
|
||||
if provider:
|
||||
cmd += ["-e", provider]
|
||||
if provider == 'mkldnn':
|
||||
cmd += ['-c', '1']
|
||||
if provider == 'openvino':
|
||||
cmd += ['-c', '1']
|
||||
if provider == 'nuphar':
|
||||
cmd += ['-c', '1']
|
||||
|
||||
if num_parallel_tests != 0:
|
||||
cmd += ['-c', str(num_parallel_tests)]
|
||||
|
||||
if num_parallel_models > 0:
|
||||
cmd += ["-j", str(num_parallel_models)]
|
||||
|
|
@ -677,8 +627,7 @@ def mkldnn_run_onnx_tests(build_dir, configs, onnx_test_data_dir):
|
|||
# /data/onnx
|
||||
run_subprocess([exe] + onnxdata_cmd, cwd=cwd)
|
||||
run_subprocess([exe,'-x'] + onnxdata_cmd, cwd=cwd)
|
||||
|
||||
# models/opset7, models/opset8, models/opset9
|
||||
|
||||
if config != 'Debug' and os.path.exists(model_dir):
|
||||
opset7_model_dir = os.path.join(model_dir, 'opset7')
|
||||
opset7_cmd = cmd_base + [opset7_model_dir]
|
||||
|
|
@ -686,16 +635,20 @@ def mkldnn_run_onnx_tests(build_dir, configs, onnx_test_data_dir):
|
|||
opset8_cmd = cmd_base + [opset8_model_dir]
|
||||
opset9_model_dir = os.path.join(model_dir, 'opset9')
|
||||
opset9_cmd = cmd_base + [opset9_model_dir]
|
||||
opset10_model_dir = os.path.join(model_dir, 'opset10')
|
||||
opset10_cmd = cmd_base + [opset10_model_dir]
|
||||
run_subprocess([exe] + opset7_cmd, cwd=cwd)
|
||||
run_subprocess([exe, '-x'] + opset7_cmd, cwd=cwd)
|
||||
run_subprocess([exe] + opset8_cmd, cwd=cwd)
|
||||
run_subprocess([exe, '-x'] + opset8_cmd, cwd=cwd)
|
||||
run_subprocess([exe] + opset9_cmd, cwd=cwd)
|
||||
run_subprocess([exe, '-x'] + opset9_cmd, cwd=cwd)
|
||||
run_subprocess([exe] + opset10_cmd, cwd=cwd)
|
||||
run_subprocess([exe, '-x'] + opset10_cmd, cwd=cwd)
|
||||
|
||||
|
||||
# nuphar temporary function for running python tests separately as it requires ONNX 1.5.0
|
||||
def nuphar_run_python_tests(build_dir, configs, azure_sas_key):
|
||||
def nuphar_run_python_tests(build_dir, configs):
|
||||
for config in configs:
|
||||
if config == 'Debug':
|
||||
continue
|
||||
|
|
@ -957,11 +910,8 @@ def main():
|
|||
if (not args.skip_submodule_sync):
|
||||
update_submodules(source_dir)
|
||||
|
||||
if args.enable_onnx_tests or args.download_test_data:
|
||||
if args.download_test_data:
|
||||
if not args.test_data_url or not args.test_data_checksum:
|
||||
raise UsageError("The test_data_url and test_data_checksum arguments are required.")
|
||||
setup_test_data(build_dir, configs, args.test_data_url, args.test_data_checksum, args.azure_sas_key)
|
||||
if args.enable_onnx_tests:
|
||||
setup_test_data(build_dir, configs)
|
||||
|
||||
if args.path_to_protoc_exe:
|
||||
path_to_protoc_exe = args.path_to_protoc_exe
|
||||
|
|
@ -995,29 +945,33 @@ def main():
|
|||
if not is_windows():
|
||||
onnx_test_data_dir = os.path.join(source_dir, "cmake", "external", "onnx", "onnx", "backend", "test", "data", "simple")
|
||||
run_onnx_tests(build_dir, configs, onnx_test_data_dir, 'tensorrt', args.enable_multi_device_test, False, 1)
|
||||
elif args.use_cuda:
|
||||
run_onnx_tests(build_dir, configs, onnx_test_data_dir, 'cuda', args.enable_multi_device_test, False, 2)
|
||||
elif args.x86 or platform.system() == 'Darwin':
|
||||
run_onnx_tests(build_dir, configs, onnx_test_data_dir, None, args.enable_multi_device_test, False, 1)
|
||||
elif args.use_ngraph:
|
||||
|
||||
if args.use_cuda:
|
||||
run_onnx_tests(build_dir, configs, onnx_test_data_dir, 'cuda', args.enable_multi_device_test, False, 2)
|
||||
|
||||
if args.use_ngraph:
|
||||
run_onnx_tests(build_dir, configs, onnx_test_data_dir, 'ngraph', args.enable_multi_device_test, True, 1)
|
||||
elif args.use_openvino:
|
||||
run_onnx_tests(build_dir, configs, onnx_test_data_dir, 'openvino', args.enable_multi_device_test, False, 1)
|
||||
|
||||
if args.use_openvino:
|
||||
run_onnx_tests(build_dir, configs, onnx_test_data_dir, 'openvino', args.enable_multi_device_test, False, 1, 1)
|
||||
# TODO: parallel executor test fails on MacOS
|
||||
elif args.use_nuphar:
|
||||
run_onnx_tests(build_dir, configs, onnx_test_data_dir, 'nuphar', args.enable_multi_device_test, False, 1)
|
||||
else:
|
||||
run_onnx_tests(build_dir, configs, onnx_test_data_dir, None, args.enable_multi_device_test, True, 0)
|
||||
if args.use_nuphar:
|
||||
run_onnx_tests(build_dir, configs, onnx_test_data_dir, 'nuphar', args.enable_multi_device_test, False, 1, 1)
|
||||
|
||||
if args.use_dml:
|
||||
run_onnx_tests(build_dir, configs, onnx_test_data_dir, 'dml', args.enable_multi_device_test, False, 1)
|
||||
run_onnx_tests(build_dir, configs, onnx_test_data_dir, 'dml', args.enable_multi_device_test, False, 1)
|
||||
|
||||
if args.use_mkldnn:
|
||||
mkldnn_run_onnx_tests(build_dir, configs, onnx_test_data_dir)
|
||||
#It could run out of memory because of memory leak
|
||||
#if args.use_mkldnn:
|
||||
# mkldnn_run_onnx_tests(build_dir, configs, onnx_test_data_dir)
|
||||
|
||||
run_onnx_tests(build_dir, configs, onnx_test_data_dir, None, args.enable_multi_device_test, False,
|
||||
1 if args.x86 or platform.system() == 'Darwin' else 0,
|
||||
1 if args.x86 or platform.system() == 'Darwin' else 0)
|
||||
|
||||
# run nuphar python tests last, as it installs ONNX 1.5.0
|
||||
if args.enable_pybind and not args.skip_onnx_tests and args.use_nuphar:
|
||||
nuphar_run_python_tests(build_dir, configs, args.azure_sas_key)
|
||||
nuphar_run_python_tests(build_dir, configs)
|
||||
|
||||
if args.build_server:
|
||||
split_server_binary_and_symbol(build_dir, configs)
|
||||
|
|
|
|||
Loading…
Reference in a new issue