From efa393e596e4141ef8f92a32e2536e689d12d45a Mon Sep 17 00:00:00 2001 From: Sheil Kumar Date: Mon, 27 Jul 2020 09:56:49 -0700 Subject: [PATCH 1/6] WinML should dynamically link against onnxruntime.dll and only system32 for inbox builds (#4615) * Dynamically link onnxruntime.dll * fixes * add preceeding backslash to onnxruntime.dll for inbox builds * remove /d * loadlibrary -> loadlibraryex * use loadlibrary system32 option Co-authored-by: Sheil Kumar --- cmake/winml.cmake | 4 +- winml/lib/Api.Ort/OnnxruntimeEngine.cpp | 10 ---- winml/lib/Api.Ort/OnnxruntimeEnvironment.cpp | 51 +++++++++++++++++++- winml/lib/Api.Ort/OnnxruntimeEnvironment.h | 3 ++ 4 files changed, 56 insertions(+), 12 deletions(-) diff --git a/cmake/winml.cmake b/cmake/winml.cmake index 26bb827911..66d12b3be5 100644 --- a/cmake/winml.cmake +++ b/cmake/winml.cmake @@ -190,6 +190,9 @@ target_compile_options(winml_lib_ort PRIVATE /GR- /await /wd4238) target_compile_definitions(winml_lib_ort PRIVATE WINML_ROOT_NS=${winml_root_ns}) target_compile_definitions(winml_lib_ort PRIVATE PLATFORM_WINDOWS) target_compile_definitions(winml_lib_ort PRIVATE _SCL_SECURE_NO_WARNINGS) # remove warnings about unchecked iterators +if (onnxruntime_WINML_NAMESPACE_OVERRIDE STREQUAL "Windows") + target_compile_definitions(winml_lib_ort PRIVATE "BUILD_INBOX=1") +endif() # Specify the usage of a precompiled header target_precompiled_header(winml_lib_ort pch.h) @@ -622,7 +625,6 @@ add_dependencies(winml_dll winml_api_native) add_dependencies(winml_dll winml_api_native_internal) # Link libraries -target_link_libraries(winml_dll PRIVATE onnxruntime) target_link_libraries(winml_dll PRIVATE re2) target_link_libraries(winml_dll PRIVATE wil) target_link_libraries(winml_dll PRIVATE winml_lib_api) diff --git a/winml/lib/Api.Ort/OnnxruntimeEngine.cpp b/winml/lib/Api.Ort/OnnxruntimeEngine.cpp index 4a71e40d89..7b011ec795 100644 --- a/winml/lib/Api.Ort/OnnxruntimeEngine.cpp +++ b/winml/lib/Api.Ort/OnnxruntimeEngine.cpp @@ -13,16 +13,6 @@ using namespace _winml; -static const OrtApi* GetVersionedOrtApi() { - static const uint32_t ort_version = 2; - const auto ort_api_base = OrtGetApiBase(); - return ort_api_base->GetApi(ort_version); -} - -static const WinmlAdapterApi* GetVersionedWinmlAdapterApi() { - return OrtGetWinMLAdapter(GetVersionedOrtApi()); -} - static ONNXTensorElementDataType ONNXTensorElementDataTypeFromTensorKind(winml::TensorKind kind) { switch (kind) { diff --git a/winml/lib/Api.Ort/OnnxruntimeEnvironment.cpp b/winml/lib/Api.Ort/OnnxruntimeEnvironment.cpp index af0188e827..8940f9642d 100644 --- a/winml/lib/Api.Ort/OnnxruntimeEnvironment.cpp +++ b/winml/lib/Api.Ort/OnnxruntimeEnvironment.cpp @@ -7,10 +7,59 @@ #include "core/platform/windows/TraceLoggingConfig.h" #include +#include + using namespace _winml; static bool debug_output_ = false; +static HRESULT GetOnnxruntimeLibrary(HMODULE& module) { + DWORD flags = 0; +#ifdef BUILD_INBOX + flags = LOAD_LIBRARY_SEARCH_SYSTEM32; +#endif + + auto out_module = LoadLibraryExA("onnxruntime.dll", nullptr, flags); + if (out_module == nullptr) { + return HRESULT_FROM_WIN32(GetLastError()); + } + module = out_module; + return S_OK; +} + +const OrtApi* _winml::GetVersionedOrtApi() { + HMODULE onnxruntime_dll; + FAIL_FAST_IF_FAILED(GetOnnxruntimeLibrary(onnxruntime_dll)); + + using OrtGetApiBaseSignature = decltype(OrtGetApiBase); + auto ort_get_api_base_fn = reinterpret_cast(GetProcAddress(onnxruntime_dll, "OrtGetApiBase")); + if (ort_get_api_base_fn == nullptr) { + FAIL_FAST_HR(HRESULT_FROM_WIN32(GetLastError())); + } + + const auto ort_api_base = ort_get_api_base_fn(); + + static const uint32_t ort_version = 2; + return ort_api_base->GetApi(ort_version); +} + +static const WinmlAdapterApi* GetVersionedWinmlAdapterApi(const OrtApi* ort_api) { + HMODULE onnxruntime_dll; + FAIL_FAST_IF_FAILED(GetOnnxruntimeLibrary(onnxruntime_dll)); + + using OrtGetWinMLAdapterSignature = decltype(OrtGetWinMLAdapter); + auto ort_get_winml_adapter_fn = reinterpret_cast(GetProcAddress(onnxruntime_dll, "OrtGetWinMLAdapter")); + if (ort_get_winml_adapter_fn == nullptr) { + FAIL_FAST_HR(HRESULT_FROM_WIN32(GetLastError())); + } + + return ort_get_winml_adapter_fn(ort_api); +} + +const WinmlAdapterApi* _winml::GetVersionedWinmlAdapterApi() { + return GetVersionedWinmlAdapterApi(GetVersionedOrtApi()); +} + static void __stdcall WinmlOrtLoggingCallback(void* param, OrtLoggingLevel severity, const char* category, const char* logger_id, const char* code_location, const char* message) noexcept { UNREFERENCED_PARAMETER(param); @@ -128,7 +177,7 @@ OnnxruntimeEnvironment::OnnxruntimeEnvironment(const OrtApi* ort_api) : ort_env_ ort_env_ = UniqueOrtEnv(ort_env, ort_api->ReleaseEnv); // Configure the environment with the winml logger - auto winml_adapter_api = OrtGetWinMLAdapter(ort_api); + auto winml_adapter_api = GetVersionedWinmlAdapterApi(ort_api); THROW_IF_NOT_OK_MSG(winml_adapter_api->EnvConfigureCustomLoggerAndProfiler(ort_env_.get(), &WinmlOrtLoggingCallback, &WinmlOrtProfileEventCallback, nullptr, OrtLoggingLevel::ORT_LOGGING_LEVEL_VERBOSE, "Default", &ort_env), diff --git a/winml/lib/Api.Ort/OnnxruntimeEnvironment.h b/winml/lib/Api.Ort/OnnxruntimeEnvironment.h index 5d47266cf9..10df3aec6c 100644 --- a/winml/lib/Api.Ort/OnnxruntimeEnvironment.h +++ b/winml/lib/Api.Ort/OnnxruntimeEnvironment.h @@ -19,6 +19,9 @@ class OnnxruntimeEnvironment { UniqueOrtEnv ort_env_; }; +const OrtApi* GetVersionedOrtApi(); +const WinmlAdapterApi* GetVersionedWinmlAdapterApi(); + } // namespace _winml #pragma warning(pop) \ No newline at end of file From c08e5f55e9eabd4992f7ee5b33f70eb3560ad37f Mon Sep 17 00:00:00 2001 From: albertomagni-ms <49027342+albertomagni-ms@users.noreply.github.com> Date: Mon, 27 Jul 2020 22:42:58 +0100 Subject: [PATCH 2/6] Fix misleading indentation (#4629) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Adjust indentation of statement, without this fix GCC 7.5 errors out with: "this ‘if’ clause does not guard this statement, but the latter is misleadingly indented as if it were guarded by the ‘if’" * Add braces around the if-statement for improved clarity. Co-authored-by: Alberto Magni --- orttraining/orttraining/training_ops/cpu/tensor/split.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/orttraining/orttraining/training_ops/cpu/tensor/split.cc b/orttraining/orttraining/training_ops/cpu/tensor/split.cc index 8332d58c0f..d14c10951c 100644 --- a/orttraining/orttraining/training_ops/cpu/tensor/split.cc +++ b/orttraining/orttraining/training_ops/cpu/tensor/split.cc @@ -49,15 +49,15 @@ Status PrepareForTrainingCompute(const TensorShape& input_shape, int num_outputs // populate split_sizes with the same size for each output split_sizes = std::vector(static_cast(num_outputs), split_dim_size / num_outputs); } else { - if (split_sizes_values.size() != static_cast(num_outputs) || split_size_sum != split_dim_size) + if (split_sizes_values.size() != static_cast(num_outputs) || split_size_sum != split_dim_size) { return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Cannot split using values in 'split' input. Axis=", axis_value, " Input shape=", input_shape, " NumOutputs=", num_outputs, " Num entries in 'split' (must equal number of outputs) was ", split_sizes_values.size(), " Sum of sizes in 'split' (must equal size of selected axis) was ", split_size_sum); - - split_sizes = split_sizes_values; + } + split_sizes = split_sizes_values; } return Status::OK(); From d73e01e5b90be91150a3551eb6fa963c30f9fcaa Mon Sep 17 00:00:00 2001 From: Xiang Zhang Date: Mon, 27 Jul 2020 20:06:11 -0700 Subject: [PATCH 3/6] remove ENABLE_TELEMETRY macro (#4633) --- include/onnxruntime/core/platform/windows/TraceLoggingConfig.h | 2 +- .../github/azure-pipelines/templates/telemetry-steps.yml | 2 +- tools/ci_build/github/azure-pipelines/templates/win-ci-2019.yml | 2 +- tools/ci_build/github/azure-pipelines/templates/win-ci-arm.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/onnxruntime/core/platform/windows/TraceLoggingConfig.h b/include/onnxruntime/core/platform/windows/TraceLoggingConfig.h index 77114ecdf8..2ee92a8938 100644 --- a/include/onnxruntime/core/platform/windows/TraceLoggingConfig.h +++ b/include/onnxruntime/core/platform/windows/TraceLoggingConfig.h @@ -23,7 +23,7 @@ Environment: // Configuration macro for use in TRACELOGGING_DEFINE_PROVIDER. The definition // in this file configures the provider as a normal (non-telemetry) provider. -#ifndef ENABLE_TELEMETRY +#ifndef TraceLoggingOptionMicrosoftTelemetry #define TraceLoggingOptionMicrosoftTelemetry() \ TraceLoggingOptionGroup(0000000000, 00000, 00000, 0000, 0000, 0000, 0000, 0000, 000, 0000, 0000) // Empty definition for TraceLoggingOptionMicrosoftTelemetry diff --git a/tools/ci_build/github/azure-pipelines/templates/telemetry-steps.yml b/tools/ci_build/github/azure-pipelines/templates/telemetry-steps.yml index ed8b7f8f45..c309dd9fc5 100644 --- a/tools/ci_build/github/azure-pipelines/templates/telemetry-steps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/telemetry-steps.yml @@ -12,7 +12,7 @@ steps: - powershell: | $length = $env:TELEMETRYGUID.length - $fileContent = "#define ENABLE_TELEMETRY`n#define TraceLoggingOptionMicrosoftTelemetry() \ + $fileContent = "#define TraceLoggingOptionMicrosoftTelemetry() \ TraceLoggingOptionGroup("+$env:TELEMETRYGUID.substring(1, $length-2)+")" New-Item -Path "$(Build.SourcesDirectory)\include\onnxruntime\core\platform\windows\TraceLoggingConfigPrivate.h" -ItemType "file" -Value "$fileContent" -Force displayName: 'Create TraceLoggingConfigPrivate.h For WinML Telemetry' diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci-2019.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci-2019.yml index 9fa4134d12..8b22a8cadd 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-ci-2019.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-ci-2019.yml @@ -54,7 +54,7 @@ jobs: if($env:TELEMETRYGUID) { $length = $env:TELEMETRYGUID.length - $fileContent = "#define ENABLE_TELEMETRY`n#define TraceLoggingOptionMicrosoftTelemetry() \ + $fileContent = "#define TraceLoggingOptionMicrosoftTelemetry() \ TraceLoggingOptionGroup("+$env:TELEMETRYGUID.substring(1, $length-2)+")" New-Item -Path "$(Build.SourcesDirectory)\include\onnxruntime\core\platform\windows\TraceLoggingConfigPrivate.h" -ItemType "file" -Value "$fileContent" -Force Write-Output "Enabling TELEMETRY" diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci-arm.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci-arm.yml index 947983f9f0..d2269c532d 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-ci-arm.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-ci-arm.yml @@ -34,7 +34,7 @@ jobs: if($env:TELEMETRYGUID) { $length = $env:TELEMETRYGUID.length - $fileContent = "#define ENABLE_TELEMETRY`n#define TraceLoggingOptionMicrosoftTelemetry() \ + $fileContent = "#define TraceLoggingOptionMicrosoftTelemetry() \ TraceLoggingOptionGroup("+$env:TELEMETRYGUID.substring(1, $length-2)+")" New-Item -Path "$(Build.SourcesDirectory)\include\onnxruntime\core\platform\windows\TraceLoggingConfigPrivate.h" -ItemType "file" -Value "$fileContent" -Force Write-Output "Enabling TELEMETRY" From 73c99f8269dae08de9ee3af47a84514e8ae0c03d Mon Sep 17 00:00:00 2001 From: Tiago Koji Castro Shibata Date: Mon, 27 Jul 2020 20:24:11 -0700 Subject: [PATCH 4/6] Set WINVER (#4636) --- cmake/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 374d28e3ee..703afeacfe 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -162,6 +162,7 @@ if(NOT WIN32) set(onnxruntime_ENABLE_INSTRUMENT OFF) endif() else() + add_compile_definitions(WINVER=0x0601) # Support Windows 7 and newer if(WINDOWS_STORE) # cmake/external/protobuf/src/google/protobuf/compiler/subprocess.cc and onnxruntime/core/platform/windows/env.cc call a bunch of Win32 APIs. # For now, we'll set the API family to desktop globally to expose Win32 symbols in headers; this must be fixed! From a06cf6a3b361ec0003fe9151530bbddabccf6abe Mon Sep 17 00:00:00 2001 From: Yufeng Li Date: Mon, 27 Jul 2020 23:56:33 -0700 Subject: [PATCH 5/6] Show quantization model size in benchmark of transformer (#4626) * Show quantization model size in benchmark of transformer * refine model size calculation --- .../python/tools/transformers/onnx_exporter.py | 10 ++++++---- .../python/tools/transformers/quantize_helper.py | 14 +++++++++++++- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/onnxruntime/python/tools/transformers/onnx_exporter.py b/onnxruntime/python/tools/transformers/onnx_exporter.py index 61e2f93124..1e22f5f4ac 100644 --- a/onnxruntime/python/tools/transformers/onnx_exporter.py +++ b/onnxruntime/python/tools/transformers/onnx_exporter.py @@ -127,15 +127,17 @@ def optimize_onnx_model_by_ort(onnx_model_path, ort_model_path, use_gpu, overwri def optimize_onnx_model(onnx_model_path, optimized_model_path, model_type, num_attention_heads, hidden_size, use_gpu, - fp16, use_raw_attention_mask, overwrite, model_fusion_statistics): + precision, use_raw_attention_mask, overwrite, model_fusion_statistics): if overwrite or not os.path.exists(optimized_model_path): from optimizer import optimize_model from onnx_model_bert import BertOptimizationOptions optimization_options = BertOptimizationOptions(model_type) if use_raw_attention_mask: optimization_options.use_raw_attention_mask() - if fp16: + if Precision.FLOAT16 == precision: optimization_options.enable_gelu_approximation = True + if Precision.INT8 == precision: + optimization_options.enable_embed_layer_norm = False # Use script to optimize model. # Use opt_level <= 1 for models to be converted to fp16, because some fused op (like FusedGemm) has only fp32 and no fp16. @@ -150,7 +152,7 @@ def optimize_onnx_model(onnx_model_path, optimized_model_path, model_type, num_a only_onnxruntime=False) model_fusion_statistics[optimized_model_path] = opt_model.get_fused_operator_statistics() - if fp16: + if Precision.FLOAT16 == precision: opt_model.convert_model_float32_to_float16() opt_model.save_model_to_file(optimized_model_path) else: @@ -215,7 +217,7 @@ def export_onnx_model(model_name, opset_version, use_external_data_format, model optimized_model_path = get_onnx_file_path(onnx_dir, model_name, len(input_names), True, use_gpu, precision, False, use_external_data_format) optimize_onnx_model(onnx_model_path, optimized_model_path, model_type, config.num_attention_heads, - config.hidden_size, use_gpu, precision == Precision.FLOAT16, use_raw_attention_mask, + config.hidden_size, use_gpu, precision, use_raw_attention_mask, overwrite, model_fusion_statistics) onnx_model_path = optimized_model_path diff --git a/onnxruntime/python/tools/transformers/quantize_helper.py b/onnxruntime/python/tools/transformers/quantize_helper.py index 4d073f427e..3ba683ee1e 100644 --- a/onnxruntime/python/tools/transformers/quantize_helper.py +++ b/onnxruntime/python/tools/transformers/quantize_helper.py @@ -7,6 +7,7 @@ import logging import torch import onnx +import os from transformers.modeling_utils import Conv1D logger = logging.getLogger(__name__) @@ -34,6 +35,12 @@ def conv1d_to_linear(model): conv1d_to_linear(module) +def _get_size_of_pytorch_model(model): + torch.save(model.state_dict(), "temp.p") + size = os.path.getsize("temp.p")/(1024*1024) + os.remove('temp.p') + return size + class QuantizeHelper: @staticmethod def quantize_torch_model(model, dtype=torch.qint8): @@ -43,11 +50,15 @@ class QuantizeHelper: TODO: mix of in-place and return, but results are different ''' conv1d_to_linear(model) - return torch.quantization.quantize_dynamic(model, {torch.nn.Linear}, dtype=dtype) + quantized_model = torch.quantization.quantize_dynamic(model, {torch.nn.Linear}, dtype=dtype) + logger.info(f'Size of full precision Torch model(MB):{_get_size_of_pytorch_model(model)}') + logger.info(f'Size of quantized Torch model(MB):{_get_size_of_pytorch_model(quantized_model)}') + return quantized_model @staticmethod def quantize_onnx_model(onnx_model_path, quantized_model_path): from onnxruntime.quantization import quantize, QuantizationMode + logger.info(f'Size of full precision ONNX model(MB):{os.path.getsize(onnx_model_path)/(1024*1024)}') onnx_opt_model = onnx.load(onnx_model_path) quantized_onnx_model = quantize(onnx_opt_model, quantization_mode=QuantizationMode.IntegerOps, @@ -55,3 +66,4 @@ class QuantizeHelper: force_fusions=True) onnx.save(quantized_onnx_model, quantized_model_path) logger.info(f"quantized model saved to:{quantized_model_path}") + logger.info(f'Size of quantized ONNX model(MB):{os.path.getsize(quantized_model_path)/(1024*1024)}') From 73ad92e7732944edf44b24a49efdf6e4c130a435 Mon Sep 17 00:00:00 2001 From: "M. Zeeshan Siddiqui" Date: Tue, 28 Jul 2020 04:37:11 -0700 Subject: [PATCH 6/6] Change ignore_index to 0 in Bert-Loss. (#4640) --- orttraining/orttraining/core/graph/loss_func/bert_loss.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orttraining/orttraining/core/graph/loss_func/bert_loss.cc b/orttraining/orttraining/core/graph/loss_func/bert_loss.cc index 3b8b60ab56..2f3b3f4e90 100644 --- a/orttraining/orttraining/core/graph/loss_func/bert_loss.cc +++ b/orttraining/orttraining/core/graph/loss_func/bert_loss.cc @@ -122,7 +122,7 @@ GraphAugmenter::GraphDefs BertLoss::operator()(const Graph& graph, const LossFun "Reshape_label")); std::vector attrs; - attrs.push_back(ONNX_NAMESPACE::MakeAttribute("ignore_index", static_cast(-1))); + attrs.push_back(ONNX_NAMESPACE::MakeAttribute("ignore_index", static_cast(0))); attrs.push_back(ONNX_NAMESPACE::MakeAttribute("reduction", "mean")); new_nodes.emplace_back(NodeDef("SoftmaxCrossEntropyLoss",