saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-30 20:18:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

drop GIL before Run to enable multiple python threads to execute in parallel (#1042)

Browse source 
* drop GIL before Run to allow multiple python threads to execute in parallel.

* revert setup.py change
This commit is contained in:
jywu-msft 2019-05-15 18:11:14 -07:00 committed by GitHub
parent d3c0cd24fc
commit 303f3c1278
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 33 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
onnxruntime/python/onnxruntime_pybind_state.cc
View file

@ -361,8 +361,6 @@ void addOpSchemaSubmodule(py::module& m){
#endif //onnxruntime_PYBIND_EXPORT_OPSCHEMA
void addObjectMethods(py::module& m) {
// allow unit tests to redirect std::cout and std::cerr to sys.stdout and sys.stderr
py::add_ostream_redirect(m, "onnxruntime_ostream_redirect");
py::class_<SessionOptions>(m, "SessionOptions", R"pbdoc(Configuration information for a session.)pbdoc")
.def(py::init())
.def_readwrite("enable_cpu_mem_arena", &SessionOptions::enable_cpu_mem_arena,
@ -512,10 +510,14 @@ including arg name, arg type (contains both type and shape).)pbdoc")
std::vector<MLValue> fetches;
common::Status status;
if (run_options != nullptr) {
status = sess->Run(*run_options, feeds, output_names, &fetches);
} else {
status = sess->Run(feeds, output_names, &fetches);
{
// release GIL to allow multiple python threads to invoke Run() in parallel.
py::gil_scoped_release release;
if (run_options != nullptr) {
status = sess->Run(*run_options, feeds, output_names, &fetches);
} else {
status = sess->Run(feeds, output_names, &fetches);
}
}
if (!status.IsOK()) {

51
onnxruntime/test/python/onnxruntime_test_python.py
View file

@ -4,10 +4,9 @@
# -*- coding: UTF-8 -*-
import unittest
import os
import sys
import numpy as np
import onnxruntime as onnxrt
from onnxruntime.capi._pybind_state import onnxruntime_ostream_redirect
import threading
class TestInferenceSession(unittest.TestCase):
@ -25,6 +24,13 @@ class TestInferenceSession(unittest.TestCase):
return res
raise FileNotFoundError("Unable to find '{0}' or '{1}' or '{2}'".format(name, rel, res))
def run_model(self, session_object, run_options):
x = np.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=np.float32)
input_name = session_object.get_inputs()[0].name
res = session_object.run([], {input_name: x}, run_options=run_options)
output_expected = np.array([[1.0, 4.0], [9.0, 16.0], [25.0, 36.0]], dtype=np.float32)
np.testing.assert_allclose(output_expected, res[0], rtol=1e-05, atol=1e-08)
def testRunModel(self):
sess = onnxrt.InferenceSession(self.get_name("mul_1.pb"))
x = np.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=np.float32)
@ -72,6 +78,22 @@ class TestInferenceSession(unittest.TestCase):
output_expected = np.array([[5.0], [11.0], [17.0]], dtype=np.float32)
np.testing.assert_allclose(output_expected, res[0], rtol=1e-05, atol=1e-08)
def testRunModelMultipleThreads(self):
so = onnxrt.SessionOptions()
so.session_log_verbosity_level = 1
so.session_logid = "MultiThreadsTest"
sess = onnxrt.InferenceSession(self.get_name("mul_1.pb"), sess_options=so)
ro1 = onnxrt.RunOptions()
ro1.run_tag = "thread1"
t1 = threading.Thread(target=self.run_model, args = (sess, ro1))
ro2 = onnxrt.RunOptions()
ro2.run_tag = "thread2"
t2 = threading.Thread(target=self.run_model, args = (sess, ro2))
t1.start()
t2.start()
t1.join()
t2.join()
def testRunDevice(self):
device = onnxrt.get_device()
self.assertTrue('CPU' in device or 'GPU' in device)
@ -315,29 +337,6 @@ class TestInferenceSession(unittest.TestCase):
self.assertEqual('', modelmeta.domain)
self.assertEqual('', modelmeta.description)
def testConfigureSessionVerbosityLevel(self):
so = onnxrt.SessionOptions()
so.session_log_verbosity_level = 1
# use onnxruntime_ostream_redirect to redirect c++ stdout/stderr to python sys.stdout and sys.stderr
with onnxruntime_ostream_redirect(stdout=True, stderr=True):
sess = onnxrt.InferenceSession(self.get_name("matmul_1.pb"), sess_options=so)
output = sys.stderr.getvalue()
self.assertTrue('[I:onnxruntime:InferenceSession, inference_session' in output)
def testConfigureRunVerbosityLevel(self):
ro = onnxrt.RunOptions()
ro.run_log_verbosity_level = 1
ro.run_tag = "testtag123"
# use onnxruntime_ostream_redirect to redirect c++ stdout/stderr to python sys.stdout and sys.stderr
with onnxruntime_ostream_redirect(stdout=True, stderr=True):
sess = onnxrt.InferenceSession(self.get_name("mul_1.pb"))
x = np.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=np.float32)
sess.run([], {'X': x}, run_options=ro)
output = sys.stderr.getvalue()
self.assertTrue('[I:onnxruntime:testtag123,' in output)
def testProfilerWithSessionOptions(self):
so = onnxrt.SessionOptions()
so.enable_profiling = True
@ -459,4 +458,4 @@ class TestInferenceSession(unittest.TestCase):
if __name__ == '__main__':
unittest.main(module=__name__, buffer=True)
unittest.main()