From 56bb503c2f26474b6613bcb2a198691a11dcef38 Mon Sep 17 00:00:00 2001 From: Ashwini Khade Date: Mon, 30 Dec 2019 12:31:44 -0800 Subject: [PATCH] Change default optimization level to All (from Basic) (#2745) * change default optimization level to All (from Basic) * fix test * fix c# test --- csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs | 4 ++-- csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs | 2 +- csharp/tools/Microsoft.ML.OnnxRuntime.PerfTool/Program.cs | 2 +- docs/ONNX_Runtime_Graph_Optimizations.md | 4 +++- docs/ONNX_Runtime_Perf_Tuning.md | 4 ++-- onnxruntime/core/framework/session_options.h | 2 +- onnxruntime/python/onnxruntime_pybind_state.cc | 6 +++--- onnxruntime/test/onnx/main.cc | 2 +- onnxruntime/test/perftest/test_configuration.h | 2 +- onnxruntime/test/python/onnxruntime_test_python.py | 6 +++--- 10 files changed, 18 insertions(+), 16 deletions(-) diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs b/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs index 70d48877da..a21e16295b 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs @@ -345,7 +345,7 @@ namespace Microsoft.ML.OnnxRuntime private int _interOpNumThreads = 0; // set to what is set in C++ SessionOptions by default; /// - /// Sets the graph optimization level for the session. Default is set to ORT_ENABLE_BASIC. + /// Sets the graph optimization level for the session. Default is set to ORT_ENABLE_ALL. /// public GraphOptimizationLevel GraphOptimizationLevel { @@ -359,7 +359,7 @@ namespace Microsoft.ML.OnnxRuntime _graphOptimizationLevel = value; } } - private GraphOptimizationLevel _graphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_BASIC; + private GraphOptimizationLevel _graphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_ALL; /// /// Sets the execution mode for the session. Default is set to ORT_SEQUENTIAL. diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs index 29d3181568..a18e4faf08 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs @@ -41,7 +41,7 @@ namespace Microsoft.ML.OnnxRuntime.Tests Assert.Equal(LogLevel.Verbose, opt.LogVerbosityLevel); Assert.Equal(0, opt.IntraOpNumThreads); Assert.Equal(0, opt.InterOpNumThreads); - Assert.Equal(GraphOptimizationLevel.ORT_ENABLE_BASIC, opt.GraphOptimizationLevel); + Assert.Equal(GraphOptimizationLevel.ORT_ENABLE_ALL, opt.GraphOptimizationLevel); // try setting options opt.ExecutionMode = ExecutionMode.ORT_PARALLEL; diff --git a/csharp/tools/Microsoft.ML.OnnxRuntime.PerfTool/Program.cs b/csharp/tools/Microsoft.ML.OnnxRuntime.PerfTool/Program.cs index de589114e7..e409f413e9 100644 --- a/csharp/tools/Microsoft.ML.OnnxRuntime.PerfTool/Program.cs +++ b/csharp/tools/Microsoft.ML.OnnxRuntime.PerfTool/Program.cs @@ -33,7 +33,7 @@ namespace Microsoft.ML.OnnxRuntime.PerfTool public bool ParallelExecution { get; set; } = false; [Option('o', "optimization_level", Required = false, HelpText = "Optimization Level. Default is 1, partial optimization.")] - public GraphOptimizationLevel OptimizationLevel { get; set; } = GraphOptimizationLevel.ORT_ENABLE_BASIC; + public GraphOptimizationLevel OptimizationLevel { get; set; } = GraphOptimizationLevel.ORT_ENABLE_ALL; } class Program diff --git a/docs/ONNX_Runtime_Graph_Optimizations.md b/docs/ONNX_Runtime_Graph_Optimizations.md index 0c347f87e4..2d124420db 100644 --- a/docs/ONNX_Runtime_Graph_Optimizations.md +++ b/docs/ONNX_Runtime_Graph_Optimizations.md @@ -15,9 +15,11 @@ Graph optimizations are divided in three levels: The optimizations belonging to one level are performed after the optimizations of the previous level have been applied (e.g., extended optimizations are applied after basic optimizations have been applied). +**All optimizations are enabled by default.** + ### Basic Graph Optimizations -These are semantics-preserving graph rewrites which remove redundant nodes and redundant computation. These optimizations are enabled by default. They run before graph partitioning and thus apply to all the execution providers. Available basic graph optimizations are as follows: +These are semantics-preserving graph rewrites which remove redundant nodes and redundant computation. They run before graph partitioning and thus apply to all the execution providers. Available basic graph optimizations are as follows: * Constant Folding: Statically computes parts of the graph that rely only on constant initializers. This eliminates the need to compute them during runtime. diff --git a/docs/ONNX_Runtime_Perf_Tuning.md b/docs/ONNX_Runtime_Perf_Tuning.md index 2788ae041b..e2659700de 100644 --- a/docs/ONNX_Runtime_Perf_Tuning.md +++ b/docs/ONNX_Runtime_Perf_Tuning.md @@ -86,9 +86,9 @@ sess_options.graph_optimization_level = rt.GraphOptimizationLevel.ORT_ENABLE_ALL * Sequential vs Parallel Execution * `sess_options.execution_mode = rt.ExecutionMode.ORT_SEQUENTIAL` controls whether then operators in the graph should run sequentially or in parallel. Usually when a model has many branches, setting this option to false will provide better performance. * When `sess_options.execution_mode = rt.ExecutionMode.ORT_PARALLEL`, you can set `sess_options.inter_op_num_threads` to control the - number of threads used to parallelize the execution of the graph (across nodes). -* sess_options.graph_optimization_level = rt.GraphOptimizationLevel.ORT_ENABLE_ALL. Default is ORT_ENABLE_BASIC(1). Please see [onnxruntime_c_api.h](../include/onnxruntime/core/session/onnxruntime_c_api.h#L241) (enum GraphOptimizationLevel) for the full list of all optimization levels. For details regarding available optimizations and usage please refer to the [Graph Optimizations Doc](../docs/ONNX_Runtime_Graph_Optimizations.md). + +* sess_options.graph_optimization_level = rt.GraphOptimizationLevel.ORT_ENABLE_ALL. Default is ORT_ENABLE_All. Please see [onnxruntime_c_api.h](../include/onnxruntime/core/session/onnxruntime_c_api.h#L241) (enum GraphOptimizationLevel) for the full list of all optimization levels. For details regarding available optimizations and usage please refer to the [Graph Optimizations Doc](../docs/ONNX_Runtime_Graph_Optimizations.md). ### MKL_DNN/nGraph/MKL_ML Execution Provider MKL_DNN, MKL_ML and nGraph all depends on openmp for parallization. For those execution providers, we need to use the openmp enviroment variable to tune the performance. diff --git a/onnxruntime/core/framework/session_options.h b/onnxruntime/core/framework/session_options.h index aaefc50cc6..1c02f9f0eb 100644 --- a/onnxruntime/core/framework/session_options.h +++ b/onnxruntime/core/framework/session_options.h @@ -52,7 +52,7 @@ struct SessionOptions { unsigned max_num_graph_transformation_steps = 10; // TODO choose a good default here? // set graph optimization level - TransformerLevel graph_optimization_level = TransformerLevel::Level1; + TransformerLevel graph_optimization_level = TransformerLevel::Level3; // controls the size of the thread pool used to parallelize the execution of tasks within individual nodes (ops) int intra_op_num_threads = 0; diff --git a/onnxruntime/python/onnxruntime_pybind_state.cc b/onnxruntime/python/onnxruntime_pybind_state.cc index e48c03323c..ba6035ba2d 100644 --- a/onnxruntime/python/onnxruntime_pybind_state.cc +++ b/onnxruntime/python/onnxruntime_pybind_state.cc @@ -587,7 +587,7 @@ Applies to session load, initialization, etc. Default is 0.)pbdoc") .def_property( "graph_optimization_level", [](const SessionOptions* options) -> GraphOptimizationLevel { - GraphOptimizationLevel retval = ORT_ENABLE_BASIC; + GraphOptimizationLevel retval = ORT_ENABLE_ALL; switch (options->graph_optimization_level) { case onnxruntime::TransformerLevel::Default: retval = ORT_DISABLE_ALL; @@ -602,8 +602,8 @@ Applies to session load, initialization, etc. Default is 0.)pbdoc") retval = ORT_ENABLE_ALL; break; default: - retval = ORT_ENABLE_BASIC; - LOGS_DEFAULT(WARNING) << "Got invalid graph optimization level; defaulting to ORT_ENABLE_BASIC"; + retval = ORT_ENABLE_ALL; + LOGS_DEFAULT(WARNING) << "Got invalid graph optimization level; defaulting to ORT_ENABLE_ALL"; break; } return retval; diff --git a/onnxruntime/test/onnx/main.cc b/onnxruntime/test/onnx/main.cc index 6fa0b30456..915212d818 100644 --- a/onnxruntime/test/onnx/main.cc +++ b/onnxruntime/test/onnx/main.cc @@ -41,7 +41,7 @@ void usage() { "Default: 'cpu'.\n" "\t-x: Use parallel executor, default (without -x): sequential executor.\n" "\t-d [device_id]: Specifies the device id for multi-device (e.g. GPU). The value should > 0\n" - "\t-o [optimization level]: Default is 1. Valid values are 0 (disable), 1 (basic), 2 (extended), 99 (all).\n" + "\t-o [optimization level]: Default is 99. Valid values are 0 (disable), 1 (basic), 2 (extended), 99 (all).\n" "\t\tPlease see onnxruntime_c_api.h (enum GraphOptimizationLevel) for the full list of all optimization levels. " "\n" "\t-h: help\n" diff --git a/onnxruntime/test/perftest/test_configuration.h b/onnxruntime/test/perftest/test_configuration.h index 9d0bf46774..36186e129f 100644 --- a/onnxruntime/test/perftest/test_configuration.h +++ b/onnxruntime/test/perftest/test_configuration.h @@ -47,7 +47,7 @@ struct RunConfig { ExecutionMode execution_mode{ExecutionMode::ORT_SEQUENTIAL}; int intra_op_num_threads{0}; int inter_op_num_threads{0}; - GraphOptimizationLevel optimization_level{ORT_ENABLE_EXTENDED}; + GraphOptimizationLevel optimization_level{ORT_ENABLE_ALL}; std::basic_string optimized_model_path; }; diff --git a/onnxruntime/test/python/onnxruntime_test_python.py b/onnxruntime/test/python/onnxruntime_test_python.py index 0a0060767e..b8d5263f03 100644 --- a/onnxruntime/test/python/onnxruntime_test_python.py +++ b/onnxruntime/test/python/onnxruntime_test_python.py @@ -521,10 +521,10 @@ class TestInferenceSession(unittest.TestCase): def testGraphOptimizationLevel(self): opt = onnxrt.SessionOptions() - self.assertEqual(opt.graph_optimization_level, onnxrt.GraphOptimizationLevel.ORT_ENABLE_BASIC) - # default should be basic optimization - opt.graph_optimization_level = onnxrt.GraphOptimizationLevel.ORT_ENABLE_ALL self.assertEqual(opt.graph_optimization_level, onnxrt.GraphOptimizationLevel.ORT_ENABLE_ALL) + # default should be all optimization + opt.graph_optimization_level = onnxrt.GraphOptimizationLevel.ORT_ENABLE_EXTENDED + self.assertEqual(opt.graph_optimization_level, onnxrt.GraphOptimizationLevel.ORT_ENABLE_EXTENDED) sess = onnxrt.InferenceSession(self.get_name("logicaland.onnx"), sess_options=opt) a = np.array([[True, True], [False, False]], dtype=np.bool) b = np.array([[True, False], [True, False]], dtype=np.bool)