diff --git a/csharp/OnnxRuntime.CSharp.proj b/csharp/OnnxRuntime.CSharp.proj index 56c37d9f7b..9651738fa3 100644 --- a/csharp/OnnxRuntime.CSharp.proj +++ b/csharp/OnnxRuntime.CSharp.proj @@ -56,7 +56,7 @@ CMake creates a target to this project - + diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs index 7cbcdad809..8813011450 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs @@ -92,6 +92,7 @@ namespace Microsoft.ML.OnnxRuntime public IntPtr AllocatorFree; public IntPtr AllocatorGetInfo; public IntPtr GetAllocatorWithDefaultOptions; + public IntPtr AddFreeDimensionOverride; public IntPtr GetValue; public IntPtr GetValueCount; public IntPtr CreateValue; @@ -180,6 +181,7 @@ namespace Microsoft.ML.OnnxRuntime OrtGetAllocatorWithDefaultOptions = (DOrtGetAllocatorWithDefaultOptions)Marshal.GetDelegateForFunctionPointer(api_.GetAllocatorWithDefaultOptions, typeof(DOrtGetAllocatorWithDefaultOptions)); OrtAllocatorFree = (DOrtAllocatorFree)Marshal.GetDelegateForFunctionPointer(api_.AllocatorFree, typeof(DOrtAllocatorFree)); OrtAllocatorGetInfo = (DOrtAllocatorGetInfo)Marshal.GetDelegateForFunctionPointer(api_.AllocatorGetInfo, typeof(DOrtAllocatorGetInfo)); + OrtAddFreeDimensionOverride = (DOrtAddFreeDimensionOverride)Marshal.GetDelegateForFunctionPointer(api_.AddFreeDimensionOverride, typeof(DOrtAddFreeDimensionOverride)); OrtGetValue = (DOrtGetValue)Marshal.GetDelegateForFunctionPointer(api_.GetValue, typeof(DOrtGetValue)); OrtGetValueType = (DOrtGetValueType)Marshal.GetDelegateForFunctionPointer(api_.GetValueType, typeof(DOrtGetValueType)); @@ -190,7 +192,7 @@ namespace Microsoft.ML.OnnxRuntime OrtCreateTensorWithDataAsOrtValue = (DOrtCreateTensorWithDataAsOrtValue)Marshal.GetDelegateForFunctionPointer(api_.CreateTensorWithDataAsOrtValue, typeof(DOrtCreateTensorWithDataAsOrtValue)); OrtGetTensorMutableData = (DOrtGetTensorMutableData)Marshal.GetDelegateForFunctionPointer(api_.GetTensorMutableData, typeof(DOrtGetTensorMutableData)); OrtFillStringTensor = (DOrtFillStringTensor)Marshal.GetDelegateForFunctionPointer(api_.FillStringTensor, typeof(DOrtFillStringTensor)); - OrtGetStringTensorContent = (DOrtGetStringTensorContent)Marshal.GetDelegateForFunctionPointer(api_.GetStringTensorContent, typeof(DOrtGetStringTensorContent)); + OrtGetStringTensorContent = (DOrtGetStringTensorContent)Marshal.GetDelegateForFunctionPointer(api_.GetStringTensorContent, typeof(DOrtGetStringTensorContent)); OrtGetStringTensorDataLength = (DOrtGetStringTensorDataLength)Marshal.GetDelegateForFunctionPointer(api_.GetStringTensorDataLength, typeof(DOrtGetStringTensorDataLength)); OrtCastTypeInfoToTensorInfo = (DOrtCastTypeInfoToTensorInfo)Marshal.GetDelegateForFunctionPointer(api_.CastTypeInfoToTensorInfo, typeof(DOrtCastTypeInfoToTensorInfo)); OrtGetTensorTypeAndShape = (DOrtGetTensorTypeAndShape)Marshal.GetDelegateForFunctionPointer(api_.GetTensorTypeAndShape, typeof(DOrtGetTensorTypeAndShape)); @@ -352,7 +354,7 @@ namespace Microsoft.ML.OnnxRuntime public delegate IntPtr /*(OrtStatus*)*/ DOrtSetInterOpNumThreads(IntPtr /* OrtSessionOptions* */ options, int interOpNumThreads); public static DOrtSetInterOpNumThreads OrtSetInterOpNumThreads; - + public delegate IntPtr /*(OrtStatus*)*/ DOrtSetSessionGraphOptimizationLevel(IntPtr /* OrtSessionOptions* */ options, GraphOptimizationLevel graphOptimizationLevel); public static DOrtSetSessionGraphOptimizationLevel OrtSetSessionGraphOptimizationLevel; @@ -402,6 +404,9 @@ namespace Microsoft.ML.OnnxRuntime //[DllImport(nativeLib, CharSet = charSet)] //public static extern void OrtAddCustomOp(IntPtr /*(OrtSessionOptions*)*/ options, string custom_op_path); + public delegate IntPtr /*(OrtStatus*)*/DOrtAddFreeDimensionOverride(IntPtr /*(OrtSessionOptions*) */ options, string /*(const char*)*/ symbolic_dim, int dim_override); + public static DOrtAddFreeDimensionOverride OrtAddFreeDimensionOverride; + #endregion #region RunOptions API diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index 310cd8f1a0..2039a1b246 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -504,6 +504,11 @@ struct OrtApi { // Always returns the same instance on every invocation. OrtStatus*(ORT_API_CALL* GetAllocatorWithDefaultOptions)(_Outptr_ OrtAllocator** out)NO_EXCEPTION; + // Override symbolic dimensions with actual values if known at session initialization time to enable + // optimizations that can take advantage of fixed values (such as memory planning, etc) + OrtStatus*(ORT_API_CALL* OrtAddFreeDimensionOverride)(_Inout_ OrtSessionOptions* options, + _In_ const char* symbolic_dim, _In_ int64_t dim_override)NO_EXCEPTION; + /** * APIs to support non-tensor types - map and sequence. * Currently only the following types are supported diff --git a/onnxruntime/core/session/abi_session_options.cc b/onnxruntime/core/session/abi_session_options.cc index 4bc617f36c..2df36ba1a9 100644 --- a/onnxruntime/core/session/abi_session_options.cc +++ b/onnxruntime/core/session/abi_session_options.cc @@ -143,3 +143,9 @@ ORT_API_STATUS_IMPL(OrtApis::SetInterOpNumThreads, _In_ OrtSessionOptions* optio options->value.inter_op_num_threads = inter_op_num_threads; return nullptr; } + +ORT_API_STATUS_IMPL(OrtApis::OrtAddFreeDimensionOverride, _Inout_ OrtSessionOptions* options, + _In_ const char* symbolic_dim, _In_ int64_t dim_override) { + options->value.free_dimension_overrides.push_back(onnxruntime::FreeDimensionOverride{symbolic_dim, dim_override}); + return nullptr; +} diff --git a/onnxruntime/core/session/inference_session.h b/onnxruntime/core/session/inference_session.h index 8f704c8b91..76b9a241b8 100644 --- a/onnxruntime/core/session/inference_session.h +++ b/onnxruntime/core/session/inference_session.h @@ -46,8 +46,7 @@ namespace logging { class LoggingManager; } -struct FreeDimensionOverride -{ +struct FreeDimensionOverride { std::string dimension_denotation; int64_t dimension_override; }; @@ -281,7 +280,7 @@ class InferenceSession { */ std::pair GetModelInputs() const; - /** + /** * Get all definitions of the model for overridable initializers. * This does not include weights. Use this to get the name/type/shapes of the overridable initializers. * @return pair.first = OK; FAIL otherwise. pair.second is non-NULL when pair.first = OK. diff --git a/onnxruntime/core/session/onnxruntime_c_api.cc b/onnxruntime/core/session/onnxruntime_c_api.cc index 2169674410..5710023b0f 100644 --- a/onnxruntime/core/session/onnxruntime_c_api.cc +++ b/onnxruntime/core/session/onnxruntime_c_api.cc @@ -1204,6 +1204,7 @@ static constexpr OrtApi ort_api_1 = { &OrtApis::AllocatorFree, &OrtApis::AllocatorGetInfo, &OrtApis::GetAllocatorWithDefaultOptions, + &OrtApis::OrtAddFreeDimensionOverride, &OrtApis::GetValue, &OrtApis::GetValueCount, &OrtApis::CreateValue, diff --git a/onnxruntime/core/session/ort_apis.h b/onnxruntime/core/session/ort_apis.h index 29a47b5906..3a97753b88 100644 --- a/onnxruntime/core/session/ort_apis.h +++ b/onnxruntime/core/session/ort_apis.h @@ -104,6 +104,7 @@ ORT_API_STATUS_IMPL(GetTensorShapeElementCount, _In_ const OrtTensorTypeAndShape ORT_API_STATUS_IMPL(GetTensorTypeAndShape, _In_ const OrtValue* value, _Outptr_ OrtTensorTypeAndShapeInfo** out); ORT_API_STATUS_IMPL(GetTypeInfo, _In_ const OrtValue* value, _Outptr_ OrtTypeInfo** out); ORT_API_STATUS_IMPL(GetValueType, _In_ const OrtValue* value, _Out_ enum ONNXType* out); +ORT_API_STATUS_IMPL(OrtAddFreeDimensionOverride, _Inout_ OrtSessionOptions* options, _In_ const char* symbolic_dim, _In_ int64_t dim_override); ORT_API_STATUS_IMPL(CreateMemoryInfo, _In_ const char* name1, enum OrtAllocatorType type, int id1, enum OrtMemType mem_type1, _Outptr_ OrtMemoryInfo** out); ORT_API_STATUS_IMPL(CreateCpuMemoryInfo, enum OrtAllocatorType type, enum OrtMemType mem_type1, _Outptr_ OrtMemoryInfo** out) diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index 1f0d5ee65f..50ccf6ced6 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -92,7 +92,11 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device session_options.DisableSequentialExecution(); fprintf(stdout, "Setting intra_op_num_threads to %d\n", performance_test_config.run_config.intra_op_num_threads); session_options.SetIntraOpNumThreads(performance_test_config.run_config.intra_op_num_threads); - fprintf(stdout, "Setting inter_op_num_threads to %d\n", performance_test_config.run_config.inter_op_num_threads); + + if (!performance_test_config.run_config.enable_sequential_execution) { + fprintf(stdout, "Setting inter_op_num_threads to %d\n", performance_test_config.run_config.inter_op_num_threads); + } + session_options.SetInterOpNumThreads(performance_test_config.run_config.inter_op_num_threads); // Set optimization level. session_options.SetGraphOptimizationLevel(performance_test_config.run_config.optimization_level);