From 2ad626752acb8226097b27fefdb1e227c01c3bd6 Mon Sep 17 00:00:00 2001 From: Klein Hu Date: Mon, 20 May 2019 18:40:52 -0700 Subject: [PATCH] Use local ort python package in server model tests (#1027) * Use local ort python package in server model tests * Create symlink for onnxruntime during test * Using generated _pb2.py in the build folder * Generate onnx_ml_pb2.py from ONNX CMakeList.txt * Update model tests for python package path * Only use onnx python package from build * Revert some changes for pb2.py generation --- cmake/onnxruntime_unittests.cmake | 2 +- .../integration_tests/model_zoo_data_prep.py | 15 ++++++++------- .../integration_tests/model_zoo_tests.py | 6 +++++- .../server/integration_tests/test_main.py | 6 +++++- tools/ci_build/build.py | 19 ++++++++++++------- 5 files changed, 31 insertions(+), 17 deletions(-) diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index 53f9524b23..3189b671fd 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -623,7 +623,7 @@ if (onnxruntime_BUILD_SERVER) COMMAND ${CMAKE_COMMAND} -E copy ${onnxruntime_integration_test_server_src} ${CMAKE_CURRENT_BINARY_DIR}/server_test/ - COMMAND ${CMAKE_COMMAND} -E copy + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/onnx_ml_pb2.py ${CMAKE_CURRENT_BINARY_DIR}/server_test/ COMMAND ${CMAKE_COMMAND} -E copy diff --git a/onnxruntime/test/server/integration_tests/model_zoo_data_prep.py b/onnxruntime/test/server/integration_tests/model_zoo_data_prep.py index bee920effb..f5767d9748 100644 --- a/onnxruntime/test/server/integration_tests/model_zoo_data_prep.py +++ b/onnxruntime/test/server/integration_tests/model_zoo_data_prep.py @@ -4,16 +4,10 @@ import os import sys import shutil - -import onnx -import onnxruntime import json from google.protobuf.json_format import MessageToJson -import predict_pb2 -import onnx_ml_pb2 - # Current models only have one input and one output def get_io_name(model_file_name): sess = onnxruntime.InferenceSession(model_file_name) @@ -44,7 +38,7 @@ def gen_output_pb(pb_full_path, output_name, response_file_path): def tensor2dict(full_path): - t = onnx.TensorProto() + t = onnx_ml_pb2.TensorProto() with open(full_path, 'rb') as f: t.ParseFromString(f.read()) @@ -140,6 +134,13 @@ if __name__ == '__main__': model_zoo = os.path.realpath(sys.argv[1]) test_data = os.path.realpath(sys.argv[2]) + sys.path.append(os.path.realpath(sys.argv[3])) + sys.path.append(os.path.realpath(sys.argv[4])) + + import onnxruntime + import predict_pb2 + import onnx_ml_pb2 + os.makedirs(test_data, exist_ok=True) gen_req_resp(model_zoo, test_data) \ No newline at end of file diff --git a/onnxruntime/test/server/integration_tests/model_zoo_tests.py b/onnxruntime/test/server/integration_tests/model_zoo_tests.py index 6906d9badf..ab1e5514c5 100644 --- a/onnxruntime/test/server/integration_tests/model_zoo_tests.py +++ b/onnxruntime/test/server/integration_tests/model_zoo_tests.py @@ -4,7 +4,6 @@ import unittest import random import os -import test_util import sys @@ -101,6 +100,11 @@ class ModelZooTests(unittest.TestCase): if __name__ == '__main__': + sys.path.append(sys.argv[4]) + sys.path.append(sys.argv[5]) + + import test_util + loader = unittest.TestLoader() test_classes = [ModelZooTests] diff --git a/onnxruntime/test/server/integration_tests/test_main.py b/onnxruntime/test/server/integration_tests/test_main.py index 83ff585fb5..39706f040b 100644 --- a/onnxruntime/test/server/integration_tests/test_main.py +++ b/onnxruntime/test/server/integration_tests/test_main.py @@ -4,9 +4,13 @@ import sys import random import unittest -import function_tests if __name__ == '__main__': + sys.path.append(sys.argv[4]) + sys.path.append(sys.argv[5]) + + import function_tests + loader = unittest.TestLoader() test_classes = [function_tests.HttpJsonPayloadTests, function_tests.HttpProtobufPayloadTests, function_tests.HttpEndpointTests] diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index a041238401..2d291a71f8 100755 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -589,27 +589,32 @@ def run_server_tests(build_dir, configs): config_build_dir = get_config_build_dir(build_dir, config) if is_windows(): server_app_path = os.path.join(config_build_dir, config, 'onnxruntime_server.exe') + python_package_path = os.path.join(config_build_dir, config) else: server_app_path = os.path.join(config_build_dir, 'onnxruntime_server') + python_package_path = config_build_dir server_test_folder = os.path.join(config_build_dir, 'server_test') server_test_data_folder = os.path.join(os.path.join(config_build_dir, 'testdata'), 'server') - run_subprocess([sys.executable, 'test_main.py', server_app_path, server_test_data_folder, server_test_data_folder], cwd=server_test_folder, dll_path=None) + run_subprocess([sys.executable, 'test_main.py', server_app_path, server_test_data_folder, server_test_data_folder, python_package_path, server_test_folder], cwd=server_test_folder, dll_path=None) def run_server_model_tests(build_dir, configs): for config in configs: config_build_dir = get_config_build_dir(build_dir, config) + server_test_folder = os.path.join(config_build_dir, 'server_test') + server_test_data_folder = os.path.join(config_build_dir, 'server_test_data') + if is_windows(): server_app_path = os.path.join(config_build_dir, config, 'onnxruntime_server.exe') test_raw_data_folder = os.path.join(config_build_dir, 'models') + python_package_path = os.path.join(config_build_dir, config) else: server_app_path = os.path.join(config_build_dir, 'onnxruntime_server') test_raw_data_folder = os.path.join(build_dir, 'models') - - server_test_folder = os.path.join(config_build_dir, 'server_test') - server_test_data_folder = os.path.join(config_build_dir, 'server_test_data') - run_subprocess([sys.executable, 'model_zoo_data_prep.py', test_raw_data_folder, server_test_data_folder], cwd=server_test_folder, dll_path=None) - run_subprocess([sys.executable, 'model_zoo_tests.py', server_app_path, test_raw_data_folder, server_test_data_folder], cwd=server_test_folder, dll_path=None) + python_package_path = config_build_dir + + run_subprocess([sys.executable, 'model_zoo_data_prep.py', test_raw_data_folder, server_test_data_folder, python_package_path, server_test_folder], cwd=server_test_folder, dll_path=None) + run_subprocess([sys.executable, 'model_zoo_tests.py', server_app_path, test_raw_data_folder, server_test_data_folder, python_package_path, server_test_folder], cwd=server_test_folder, dll_path=None) def build_python_wheel(source_dir, build_dir, configs, use_cuda, use_ngraph, use_tensorrt, nightly_build = False): @@ -708,7 +713,7 @@ def main(): if args.use_tensorrt: args.use_cuda = True - if args.build_wheel: + if args.build_wheel or args.enable_server_model_tests: args.enable_pybind = True if args.build_csharp: