mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-30 20:18:08 +00:00
Enable ONNX Runtime Server Model Tests (#1002)
* Enable model tests for ORT Server in build script * Nightly build pipeline definition * Force prune docker images * Clean up containers before prune images
This commit is contained in:
Klein Hu
Pranav Sharma
parent
f7e57a3d16
commit
406770c484
5 changed files with 170 additions and 46 deletions
|
|
@ -82,7 +82,11 @@ def gen_output_json(pb_full_path, output_name, json_file_path):
|
|||
json.dump(resp, outfile)
|
||||
|
||||
|
||||
def gen_req_resp(model_zoo, test_data, copy_model=True):
|
||||
def gen_req_resp(model_zoo, test_data, copy_model=False):
|
||||
skip_list = [
|
||||
('opset8', 'mxnet_arcface') # REASON: Known issue
|
||||
]
|
||||
|
||||
opsets = [name for name in os.listdir(model_zoo) if os.path.isdir(os.path.join(model_zoo, name))]
|
||||
for opset in opsets:
|
||||
os.makedirs(os.path.join(test_data, opset), exist_ok=True)
|
||||
|
|
@ -92,15 +96,35 @@ def gen_req_resp(model_zoo, test_data, copy_model=True):
|
|||
|
||||
models = [name for name in os.listdir(current_model_folder) if os.path.isdir(os.path.join(current_model_folder, name))]
|
||||
for model in models:
|
||||
print("Working on Opset: {0}, Model: {1}".format(opset, model))
|
||||
if (opset, model) in skip_list:
|
||||
print(" SKIP!!")
|
||||
continue
|
||||
|
||||
os.makedirs(os.path.join(current_data_folder, model), exist_ok=True)
|
||||
|
||||
src_folder = os.path.join(current_model_folder, model)
|
||||
dst_folder = os.path.join(current_data_folder, model)
|
||||
|
||||
if copy_model:
|
||||
shutil.copy2(os.path.join(src_folder, 'model.onnx'), dst_folder)
|
||||
onnx_file_path = ''
|
||||
for fname in os.listdir(src_folder):
|
||||
if not fname.startswith(".") and fname.endswith(".onnx") and os.path.isfile(os.path.join(src_folder, fname)):
|
||||
onnx_file_path = os.path.join(src_folder, fname)
|
||||
break
|
||||
|
||||
iname, oname = get_io_name(os.path.join(src_folder, 'model.onnx'))
|
||||
if onnx_file_path == '':
|
||||
raise FileNotFoundError('Could not find any *.onnx file in {0}'.format(src_folder))
|
||||
|
||||
if copy_model:
|
||||
# Copy model file
|
||||
target_file_path = os.path.join(dst_folder, "model.onnx")
|
||||
shutil.copy2(onnx_file_path, target_file_path)
|
||||
|
||||
for fname in os.listdir(src_folder):
|
||||
if not fname.endswith(".onnx") and os.path.isfile(os.path.join(src_folder, fname)):
|
||||
shutil.copy2(os.path.join(src_folder, fname), dst_folder)
|
||||
|
||||
iname, oname = get_io_name(onnx_file_path)
|
||||
model_test_data = [name for name in os.listdir(src_folder) if os.path.isdir(os.path.join(src_folder, name))]
|
||||
for test in model_test_data:
|
||||
src = os.path.join(src_folder, test)
|
||||
|
|
@ -118,3 +142,4 @@ if __name__ == '__main__':
|
|||
|
||||
os.makedirs(test_data, exist_ok=True)
|
||||
gen_req_resp(model_zoo, test_data)
|
||||
|
||||
|
|
@ -7,6 +7,7 @@ import os
|
|||
import test_util
|
||||
import sys
|
||||
|
||||
|
||||
class ModelZooTests(unittest.TestCase):
|
||||
server_ip = '127.0.0.1'
|
||||
server_port = 54321
|
||||
|
|
@ -19,10 +20,12 @@ class ModelZooTests(unittest.TestCase):
|
|||
need_data_cleanup = False
|
||||
model_zoo_model_path = '' # Required
|
||||
model_zoo_test_data_path = '' # Required
|
||||
supported_opsets = ['opset_7', 'opset_8', 'opset_9']
|
||||
skipped_models = []
|
||||
supported_opsets = ['opset7', 'opset8', 'opset9', 'opset_7', 'opset_8', 'opset_9']
|
||||
skipped_models = [
|
||||
('opset7', 'tf_inception_v2'), # Known issue
|
||||
]
|
||||
|
||||
def test_models_from_model_zoo(self):
|
||||
def __test_model(self, model_path, data_paths):
|
||||
json_request_headers = {
|
||||
'Content-Type': 'application/json',
|
||||
'Accept': 'application/json'
|
||||
|
|
@ -32,6 +35,44 @@ class ModelZooTests(unittest.TestCase):
|
|||
'Accept': 'application/octet-stream'
|
||||
}
|
||||
|
||||
server_app_proc = None
|
||||
try:
|
||||
onnx_file_path = ''
|
||||
for fname in os.listdir(model_path):
|
||||
if not fname.startswith(".") and fname.endswith(".onnx") and os.path.isfile(os.path.join(model_path, fname)):
|
||||
onnx_file_path = os.path.join(model_path, fname)
|
||||
break
|
||||
|
||||
if onnx_file_path == '':
|
||||
raise FileNotFoundError('Could not find any *.onnx file in {0}'.format(model_path))
|
||||
|
||||
cmd = [self.server_app_path, '--http_port', str(self.server_port), '--model_path', onnx_file_path, '--log_level', self.log_level]
|
||||
test_util.test_log(cmd)
|
||||
server_app_proc = test_util.launch_server_app(cmd, self.server_ip, self.server_port,
|
||||
self.server_ready_in_seconds)
|
||||
|
||||
test_util.test_log('[{0}] Run tests...'.format(model_path))
|
||||
for test in data_paths:
|
||||
test_util.test_log('[{0}] Current: {0}'.format(model_path, test))
|
||||
|
||||
test_util.test_log('[{0}] JSON payload testing ....'.format(model_path))
|
||||
url = self.url_pattern.format(self.server_ip, self.server_port, 'default_model', 12345)
|
||||
with open(os.path.join(test, 'request.json')) as f:
|
||||
request_payload = f.read()
|
||||
resp = test_util.make_http_request(url, json_request_headers, request_payload)
|
||||
test_util.json_response_validation(self, resp, os.path.join(test, 'response.json'))
|
||||
|
||||
test_util.test_log('[{0}] Protobuf payload testing ....'.format(model_path))
|
||||
url = self.url_pattern.format(self.server_ip, self.server_port, 'default_model', 54321)
|
||||
with open(os.path.join(test, 'request.pb'), 'rb') as f:
|
||||
request_payload = f.read()
|
||||
resp = test_util.make_http_request(url, pb_request_headers, request_payload)
|
||||
test_util.pb_response_validation(self, resp, os.path.join(test, 'response.pb'))
|
||||
finally:
|
||||
test_util.shutdown_server_app(server_app_proc, self.server_off_in_seconds)
|
||||
|
||||
|
||||
def test_models_from_model_zoo(self):
|
||||
model_data_map = {}
|
||||
for opset in self.supported_opsets:
|
||||
test_data_folder = os.path.join(self.model_zoo_test_data_path, opset)
|
||||
|
|
@ -39,9 +80,10 @@ class ModelZooTests(unittest.TestCase):
|
|||
|
||||
if os.path.isdir(test_data_folder):
|
||||
for name in os.listdir(test_data_folder):
|
||||
if name in self.skipped_models:
|
||||
if (opset, name) in self.skipped_models:
|
||||
test_util.test_log(" Skip {0}:{1}".format(opset, name))
|
||||
continue
|
||||
|
||||
|
||||
if os.path.isdir(os.path.join(test_data_folder, name)):
|
||||
current_dir = os.path.join(test_data_folder, name)
|
||||
model_data_map[os.path.join(model_file_folder, name)] = [os.path.join(current_dir, name) for name in os.listdir(current_dir) if os.path.isdir(os.path.join(current_dir, name))]
|
||||
|
|
@ -55,31 +97,7 @@ class ModelZooTests(unittest.TestCase):
|
|||
|
||||
self.server_port = random.randint(30000, 40000)
|
||||
for model_path, data_paths in model_data_map.items():
|
||||
server_app_proc = None
|
||||
try:
|
||||
cmd = [self.server_app_path, '--http_port', str(self.server_port), '--model_path', os.path.join(model_path, 'model.onnx'), '--log_level', self.log_level]
|
||||
test_util.test_log(cmd)
|
||||
server_app_proc = test_util.launch_server_app(cmd, self.server_ip, self.server_port, self.server_ready_in_seconds)
|
||||
|
||||
test_util.test_log('[{0}] Run tests...'.format(model_path))
|
||||
for test in data_paths:
|
||||
test_util.test_log('[{0}] Current: {0}'.format(model_path, test))
|
||||
|
||||
test_util.test_log('[{0}] JSON payload testing ....'.format(model_path))
|
||||
url = self.url_pattern.format(self.server_ip, self.server_port, 'default_model', 12345)
|
||||
with open(os.path.join(test, 'request.json')) as f:
|
||||
request_payload = f.read()
|
||||
resp = test_util.make_http_request(url, json_request_headers, request_payload)
|
||||
test_util.json_response_validation(self, resp, os.path.join(test, 'response.json'))
|
||||
|
||||
test_util.test_log('[{0}] Protobuf payload testing ....'.format(model_path))
|
||||
url = self.url_pattern.format(self.server_ip, self.server_port, 'default_model', 54321)
|
||||
with open(os.path.join(test, 'request.pb'), 'rb') as f:
|
||||
request_payload = f.read()
|
||||
resp = test_util.make_http_request(url, pb_request_headers, request_payload)
|
||||
test_util.pb_response_validation(self, resp, os.path.join(test, 'response.pb'))
|
||||
finally:
|
||||
test_util.shutdown_server_app(server_app_proc, self.server_off_in_seconds)
|
||||
self.__test_model(model_path, data_paths)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
|||
|
|
@ -95,6 +95,9 @@ def launch_server_app(cmd, server_ip, server_port, wait_server_ready_in_seconds)
|
|||
test_log('Wait up to {0} second(s) for server initialization'.format(wait_server_ready_in_seconds))
|
||||
wait_service_up(server_ip, server_port, wait_server_ready_in_seconds)
|
||||
|
||||
# Additional sleep to make sure the server is ready.
|
||||
time.sleep(1)
|
||||
|
||||
return server_app_proc
|
||||
|
||||
|
||||
|
|
@ -143,12 +146,21 @@ def json_response_validation(cls, resp, expected_resp_json_file):
|
|||
for x in actual_response['outputs'][output]['dims']:
|
||||
count = count * int(x)
|
||||
|
||||
actual_array = decode_base64_string(actual_response['outputs'][output]['rawData'], '{0}f'.format(count))
|
||||
expected_array = decode_base64_string(expected_result['outputs'][output]['rawData'], '{0}f'.format(count))
|
||||
cls.assertEqual(len(actual_array), len(expected_array))
|
||||
cls.assertEqual(len(actual_array), count)
|
||||
for i in range(0, count):
|
||||
cls.assertTrue(compare_floats(actual_array[i], expected_array[i], rel_tol=0.001))
|
||||
if actual_response['outputs'][output]['dataType'] == 10 or actual_response['outputs'][output]['dataType'] == 16:
|
||||
actual_array = numpy.frombuffer(base64.b64decode(actual_response['outputs'][output]['rawData']), dtype=numpy.float16)
|
||||
expected_array = numpy.frombuffer(base64.b64decode(expected_result['outputs'][output]['rawData']), dtype=numpy.float16)
|
||||
cls.assertEqual(len(actual_array), len(expected_array))
|
||||
cls.assertEqual(len(actual_array), count)
|
||||
for i in range(0, count):
|
||||
cls.assertTrue(compare_floats(actual_array[i], expected_array[i], rel_tol=0.05, abs_tol=0.05))
|
||||
elif actual_response['outputs'][output]['dataType'] == 1:
|
||||
actual_array = decode_base64_string(actual_response['outputs'][output]['rawData'], '{0}f'.format(count))
|
||||
expected_array = decode_base64_string(expected_result['outputs'][output]['rawData'], '{0}f'.format(count))
|
||||
cls.assertEqual(len(actual_array), len(expected_array))
|
||||
cls.assertEqual(len(actual_array), count)
|
||||
for i in range(0, count):
|
||||
cls.assertTrue(compare_floats(actual_array[i], expected_array[i], rel_tol=0.001))
|
||||
|
||||
|
||||
|
||||
def pb_response_validation(cls, resp, expected_resp_pb_file):
|
||||
|
|
@ -171,9 +183,17 @@ def pb_response_validation(cls, resp, expected_resp_pb_file):
|
|||
cls.assertEqual(actual_result.outputs[k].dims[i], expected_result.outputs[k].dims[i])
|
||||
count = count * int(actual_result.outputs[k].dims[i])
|
||||
|
||||
actual_array = numpy.frombuffer(actual_result.outputs[k].raw_data, dtype=numpy.float32)
|
||||
expected_array = numpy.frombuffer(expected_result.outputs[k].raw_data, dtype=numpy.float32)
|
||||
cls.assertEqual(len(actual_array), len(expected_array))
|
||||
cls.assertEqual(len(actual_array), count)
|
||||
for i in range(0, count):
|
||||
cls.assertTrue(compare_floats(actual_array[i], expected_array[i], rel_tol=0.001))
|
||||
if actual_result.outputs[k].data_type == 10 or actual_result.outputs[k].data_type == 16:
|
||||
actual_array = numpy.frombuffer(actual_result.outputs[k].raw_data, dtype=numpy.float16)
|
||||
expected_array = numpy.frombuffer(expected_result.outputs[k].raw_data, dtype=numpy.float16)
|
||||
cls.assertEqual(len(actual_array), len(expected_array))
|
||||
cls.assertEqual(len(actual_array), count)
|
||||
for i in range(0, count):
|
||||
cls.assertTrue(compare_floats(actual_array[i], expected_array[i], rel_tol=0.05, abs_tol=0.05))
|
||||
elif actual_result.outputs[k].data_type == 1:
|
||||
actual_array = numpy.frombuffer(actual_result.outputs[k].raw_data, dtype=numpy.float32)
|
||||
expected_array = numpy.frombuffer(expected_result.outputs[k].raw_data, dtype=numpy.float32)
|
||||
cls.assertEqual(len(actual_array), len(expected_array))
|
||||
cls.assertEqual(len(actual_array), count)
|
||||
for i in range(0, count):
|
||||
cls.assertTrue(compare_floats(actual_array[i], expected_array[i], rel_tol=0.001))
|
||||
|
|
@ -98,6 +98,7 @@ Use the individual flags to only run the specified stages.
|
|||
# Build ONNX Runtime server
|
||||
parser.add_argument("--build_server", action='store_true', help="Build server application for the ONNXRuntime.")
|
||||
parser.add_argument("--enable_server_tests", action='store_true', help="Run server application tests.")
|
||||
parser.add_argument("--enable_server_model_tests", action='store_true', help="Run server model tests.")
|
||||
|
||||
# Build options
|
||||
parser.add_argument("--cmake_extra_defines", nargs="+",
|
||||
|
|
@ -592,6 +593,23 @@ def run_server_tests(build_dir, configs):
|
|||
server_test_data_folder = os.path.join(os.path.join(config_build_dir, 'testdata'), 'server')
|
||||
run_subprocess([sys.executable, 'test_main.py', server_app_path, server_test_data_folder, server_test_data_folder], cwd=server_test_folder, dll_path=None)
|
||||
|
||||
|
||||
def run_server_model_tests(build_dir, configs):
|
||||
for config in configs:
|
||||
config_build_dir = get_config_build_dir(build_dir, config)
|
||||
if is_windows():
|
||||
server_app_path = os.path.join(config_build_dir, config, 'onnxruntime_server.exe')
|
||||
test_raw_data_folder = os.path.join(config_build_dir, 'models')
|
||||
else:
|
||||
server_app_path = os.path.join(config_build_dir, 'onnxruntime_server')
|
||||
test_raw_data_folder = os.path.join(build_dir, 'models')
|
||||
|
||||
server_test_folder = os.path.join(config_build_dir, 'server_test')
|
||||
server_test_data_folder = os.path.join(config_build_dir, 'server_test_data')
|
||||
run_subprocess([sys.executable, 'model_zoo_data_prep.py', test_raw_data_folder, server_test_data_folder], cwd=server_test_folder, dll_path=None)
|
||||
run_subprocess([sys.executable, 'model_zoo_tests.py', server_app_path, test_raw_data_folder, server_test_data_folder], cwd=server_test_folder, dll_path=None)
|
||||
|
||||
|
||||
def build_python_wheel(source_dir, build_dir, configs, use_cuda, use_ngraph, use_tensorrt, nightly_build = False):
|
||||
for config in configs:
|
||||
cwd = get_config_build_dir(build_dir, config)
|
||||
|
|
@ -802,6 +820,9 @@ def main():
|
|||
if args.build_server and args.enable_server_tests:
|
||||
run_server_tests(build_dir, configs)
|
||||
|
||||
if args.build_server and args.enable_server_model_tests:
|
||||
run_server_model_tests(build_dir, configs)
|
||||
|
||||
if args.build:
|
||||
if args.build_wheel:
|
||||
nightly_build = bool(os.getenv('NIGHTLY_BUILD') == '1')
|
||||
|
|
|
|||
|
|
@ -0,0 +1,40 @@
|
|||
jobs:
|
||||
- job: Debug_Build_with_Model_Tests
|
||||
pool: Linux-CPU
|
||||
steps:
|
||||
- template: templates/set-test-data-variables-step.yml
|
||||
|
||||
- task: CmdLine@2
|
||||
displayName: 'Prune docker images'
|
||||
inputs:
|
||||
script: |
|
||||
docker container prune -f
|
||||
docker image prune -f
|
||||
workingDirectory: $(Build.BinariesDirectory)
|
||||
continueOnError: true
|
||||
condition: always()
|
||||
|
||||
- task: CmdLine@2
|
||||
displayName: 'Download azcopy'
|
||||
inputs:
|
||||
script: |
|
||||
curl -so azcopy.tar.gz -L 'https://aka.ms/downloadazcopy-v10-linux'
|
||||
tar -zxvf azcopy.tar.gz --strip 1
|
||||
workingDirectory: $(Build.BinariesDirectory)
|
||||
|
||||
- task: PythonScript@0
|
||||
displayName: 'Download test data'
|
||||
inputs:
|
||||
scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py'
|
||||
arguments: --test_data_url $(TestDataUrl)
|
||||
pythonInterpreter: '/usr/bin/python3'
|
||||
workingDirectory: $(Build.BinariesDirectory)
|
||||
|
||||
- script: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -x "--config Debug --build_server --use_openmp --use_full_protobuf --enable_server_model_tests"'
|
||||
displayName: 'Run build script with model tests'
|
||||
|
||||
- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
|
||||
displayName: 'Component Detection'
|
||||
condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI'))
|
||||
|
||||
- template: templates/clean-agent-build-directory-step.yml
|
||||
Loading…
Reference in a new issue